Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(workflow-recipes): keep the token seen until the finalizer observes it gone (R4-L1)

Alfredo Lopez1386b2d0e629
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.json
index 70cef38b7..1ea1beae1 100644
--- a/workflow-recipes/package-lock.json
+++ b/workflow-recipes/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.434",
+ "version": "0.1.435",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "workload-recipes-controller",
- "version": "0.1.434",
+ "version": "0.1.435",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@aws-sdk/client-s3": "^3.1009.0",
diff --git a/workflow-recipes/package.json b/workflow-recipes/package.json
index 842b6cde0..8e420e39f 100644
--- a/workflow-recipes/package.json
+++ b/workflow-recipes/package.json
@@ -1,6 +1,6 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.434",
+ "version": "0.1.435",
"description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface",
"main": "dist/main.js",
"engines": {
diff --git a/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.property.test.ts b/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.property.test.ts
index b6acb347c..930e15f20 100644
--- a/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.property.test.ts
+++ b/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.property.test.ts
@@ -12,15 +12,16 @@ import {
* caller can see (which events happened, in which order) and asserts
* implications on the answers. Cases no invariant speaks about stay free.
*
- * I1 No token ADDED since the process started or the recipe was deleted:
- * no Secret delete.
+ * I1 No token ADDED since the process started: no Secret delete. A
+ * finalizer start (`forgetRecipe`) does not clear the token-seen bit.
* I2 After an ADDED with no later recorded delete or provisioning, a pass
* at or above the watermark (the highest generation recorded or
- * provisioned since the recipe was last deleted) deletes.
+ * provisioned since the last finalizer start) deletes. A finalizer
+ * start with the token seen arms every generation.
* I3 A pass below the highest generation that provisioned the token does
* not delete it.
- * I4 A delete whose DELETE was sent before an ADDED (or a recipe deletion)
- * records nothing; with neither in between it is recorded.
+ * I4 A delete whose DELETE was sent before an ADDED records nothing; with
+ * no ADDED in between it is recorded, across a finalizer start too.
* I5 The NetworkPolicy side skips the recorded pass until the TTL expires,
* and deletes for a different uid or a higher generation.
*/
@@ -36,7 +37,7 @@ type Op =
| { kind: 'endDelete'; failed: boolean }
| { kind: 'invalidateSecret' }
| { kind: 'addedElsewhere' }
- | { kind: 'invalidate' }
+ | { kind: 'forgetRecipe' }
const generationArb = fc.integer({ min: 0, max: MAX_GENERATION })
@@ -47,7 +48,7 @@ const opArb: fc.Arbitrary<Op> = fc.oneof(
fc.boolean().map(failed => ({ kind: 'endDelete' as const, failed })),
fc.constant({ kind: 'invalidateSecret' as const }),
fc.constant({ kind: 'addedElsewhere' as const }),
- fc.constant({ kind: 'invalidate' as const })
+ fc.constant({ kind: 'forgetRecipe' as const })
)
function ref(generation: number, uid = UID): OAuthBrokerLedgerRecipe {
@@ -57,9 +58,9 @@ function ref(generation: number, uid = UID): OAuthBrokerLedgerRecipe {
/** What a caller observed; never the ledger's internal entry. */
interface Observed {
tokenSeen: boolean
- /** Highest generation recorded or provisioned since the last recipe deletion. */
+ /** Highest generation recorded or provisioned since the last finalizer start. */
watermark: number
- /** Highest generation provisioned since the last recipe deletion. */
+ /** Highest generation provisioned since the last finalizer start. */
provisioned: number
/** An ADDED landed after the last recorded delete or provisioning. */
addedAfterLastWrite: boolean
@@ -139,18 +140,23 @@ function step(ledger: OAuthBrokerDeleteLedger, seen: Observed, op: Op): void {
case 'addedElsewhere':
ledger.invalidateSecret('other')
return
- case 'invalidate': {
- ledger.invalidate(NAME)
- const inFlight = seen.inFlight
+ case 'forgetRecipe': {
+ // The finalizer starts: both entries go, the token-seen bit stays, and
+ // an in-flight DELETE is not raced (the epoch did not move).
+ ledger.forgetRecipe(NAME)
+ const { tokenSeen, inFlight } = seen
Object.assign(seen, freshObserved())
- if (inFlight) seen.inFlight = { ...inFlight, raced: true }
+ seen.tokenSeen = tokenSeen
+ // With no entry left, every generation deletes while the token is seen.
+ seen.addedAfterLastWrite = tokenSeen
+ seen.inFlight = inFlight
return
}
}
}
describe('OAuthBrokerDeleteLedger observable properties', () => {
- it('I1-I4 hold over provision, pass, beginDelete/endDelete, invalidateSecret and invalidate', () => {
+ it('I1-I4 hold over provision, pass, beginDelete/endDelete, invalidateSecret and forgetRecipe', () => {
fc.assert(
fc.property(fc.array(opArb, { maxLength: 40 }), ops => {
const ledger = new OAuthBrokerDeleteLedger()
diff --git a/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.test.ts b/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.test.ts
index deeb563a8..c7db5ff9b 100644
--- a/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.test.ts
+++ b/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.test.ts
@@ -180,7 +180,7 @@ describe('OAuthBrokerDeleteLedger', () => {
expect(ledger.shouldDeleteSecret(ref('r2', 4))).toBe(false)
})
- it('invalidate drops both sides for the named recipe and no other', () => {
+ it('forgetRecipe drops both entries for the named recipe and no other, and keeps the token seen', () => {
const ledger = new OAuthBrokerDeleteLedger()
tokenSeen(ledger, 'r')
tokenSeen(ledger, 'r2')
@@ -189,29 +189,69 @@ describe('OAuthBrokerDeleteLedger', () => {
recordSecret(ledger, ref('r2', 4))
ledger.recordPolicyDelete(ref('r2', 4), 0)
- ledger.invalidate('r')
- // The recipe is gone with its token: nothing to delete until a new ADDED.
- expect(ledger.shouldDeleteSecret(ref('r', 5))).toBe(false)
+ ledger.forgetRecipe('r')
+ // Deleting the recipe does not delete its token: the watermark went, the
+ // epoch stayed, so a generation below the old watermark deletes.
+ expect(ledger.shouldDeleteSecret(ref('r', 3))).toBe(true)
+ expect(ledger.shouldDeleteSecret(ref('r', 1, 'uid-recreated'))).toBe(true)
expect(ledger.shouldDeletePolicy(ref('r', 4), 1)).toBe(true)
+ // The other recipe keeps both entries.
expect(ledger.shouldDeleteSecret(ref('r2', 4))).toBe(false)
expect(ledger.shouldDeletePolicy(ref('r2', 4), 1)).toBe(false)
+ })
- // The watermark went too: after the next ADDED a generation below the
- // old one deletes.
+ it('Table B: noteSecretGone with the epoch unchanged clears the token-seen bit until the next ADDED', () => {
+ const ledger = new OAuthBrokerDeleteLedger()
tokenSeen(ledger, 'r')
- expect(ledger.shouldDeleteSecret(ref('r', 3))).toBe(true)
+ tokenSeen(ledger, 'r2')
+ ledger.forgetRecipe('r')
+ const epochBeforeDelete = ledger.secretEpoch('r')
+ expect(epochBeforeDelete).toBeGreaterThan(0)
+ expect(ledger.noteSecretGone('r', epochBeforeDelete)).toBe(true)
+
+ expect(ledger.secretEpoch('r')).toBe(0)
+ expect(ledger.shouldDeleteSecret(ref('r', 1, 'uid-recreated'))).toBe(false)
+ // Liveness witness: another recipe is untouched, and the next ADDED re-arms.
+ expect(ledger.shouldDeleteSecret(ref('r2', 1))).toBe(true)
+ tokenSeen(ledger, 'r')
+ expect(ledger.shouldDeleteSecret(ref('r', 1, 'uid-recreated'))).toBe(true)
})
- it('a DELETE in flight across a recipe deletion that followed an ADDED is not recorded', () => {
+ it('Table B: noteSecretGone after an ADDED raced the finalizer DELETE clears nothing', () => {
const ledger = new OAuthBrokerDeleteLedger()
- ledger.invalidateSecret('r')
+ tokenSeen(ledger, 'r')
+ ledger.forgetRecipe('r')
const epochBeforeDelete = ledger.secretEpoch('r')
- ledger.invalidate('r')
- expect(ledger.recordSecretDelete(ref('r', 4), epochBeforeDelete)).toBe(false)
+ // The recreated recipe's token ADDED lands while the DELETE is in flight.
+ ledger.invalidateSecret('r')
+ expect(ledger.noteSecretGone('r', epochBeforeDelete)).toBe(false)
+ expect(ledger.secretEpoch('r')).toBeGreaterThan(epochBeforeDelete)
+ expect(ledger.shouldDeleteSecret(ref('r', 1, 'uid-recreated'))).toBe(true)
+ })
- // Nothing was recorded: after the next ADDED a lower generation deletes,
- // which a watermark at 4 would have skipped.
+ it('Table B: a finalizer DELETE that failed or was never sent keeps the token seen', () => {
+ const ledger = new OAuthBrokerDeleteLedger()
tokenSeen(ledger, 'r')
- expect(ledger.shouldDeleteSecret(ref('r', 3))).toBe(true)
+ recordSecret(ledger, ref('r', 4, 'uid-old'))
+ const epochBeforeForget = ledger.secretEpoch('r')
+ // Finalizer start; the DELETE then fails (or is never sent), so
+ // noteSecretGone is not called.
+ ledger.forgetRecipe('r')
+ expect(ledger.secretEpoch('r')).toBe(epochBeforeForget)
+ expect(ledger.shouldDeleteSecret(ref('r', 1, 'uid-recreated'))).toBe(true)
+ })
+
+ it('a DELETE in flight across a finalizer start that followed an ADDED is recorded against the old uid only', () => {
+ const ledger = new OAuthBrokerDeleteLedger()
+ ledger.invalidateSecret('r')
+ const epochBeforeDelete = ledger.secretEpoch('r')
+ ledger.forgetRecipe('r')
+ // The epoch did not move, so the in-flight DELETE of the old recipe is
+ // recorded for its own uid.
+ expect(ledger.recordSecretDelete(ref('r', 4, 'uid-old'), epochBeforeDelete)).toBe(true)
+ expect(ledger.shouldDeleteSecret(ref('r', 4, 'uid-old'))).toBe(false)
+
+ // A recipe recreated under the same name (new uid) still deletes.
+ expect(ledger.shouldDeleteSecret(ref('r', 1, 'uid-new'))).toBe(true)
})
})
diff --git a/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.ts b/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.ts
index 34e12a1e1..0b62df6a4 100644
--- a/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.ts
+++ b/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.ts
@@ -14,7 +14,9 @@
* missed.
*
* The Secret side deletes only after the Secret watch observed the recipe's
- * token (a non-zero epoch): a recipe that never had one gets no DELETE.
+ * token (a non-zero epoch): a recipe that never had one gets no DELETE. The
+ * epoch survives recipe deletion and returns to 0 only when the finalizer's
+ * own token DELETE is observed 2xx/404 with no ADDED in between.
*
* The Secret watermark rises on a recorded delete and on
* `noteSecretProvisioned`, which the backgroundAccess branch calls for the
@@ -80,9 +82,10 @@ export class OAuthBrokerDeleteLedger {
private epochClock = 0
shouldDeleteSecret(recipe: OAuthBrokerLedgerRecipe): boolean {
- // No token ADDED since this process started or the recipe was deleted:
- // the Secret watch's initial list replays ADDED for every token that
- // exists, so there is no token to reap and a DELETE would only 404.
+ // Epoch 0: no token ADDED since this process started, or the finalizer
+ // observed the token gone (2xx/404) with no ADDED since. The Secret
+ // watch's initial list replays ADDED for every token that exists, so
+ // there is no token to reap and a DELETE would only 404.
if (this.secretEpoch(recipe.name) === 0) return false
const entry = this.secrets.get(recipe.name)
if (entry === undefined || entry.uid !== recipe.uid) return true
@@ -149,12 +152,25 @@ export class OAuthBrokerDeleteLedger {
}
/**
- * Recipe deletion drops both sides. The epoch entry goes too; the clock is
- * global, so a DELETE in flight that read a non-zero epoch still mismatches.
+ * Finalizer start: drops the Secret and NetworkPolicy entries and keeps the
+ * epoch. Deleting a recipe does not delete its token; only the finalizer's
+ * own DELETE, observed through `noteSecretGone`, may clear the epoch. A
+ * DELETE that fails, is never sent, or races an ADDED leaves the token
+ * seen, so a recipe recreated under the same name still reaps it.
*/
- invalidate(recipeName: string): void {
+ forgetRecipe(recipeName: string): void {
this.secrets.delete(recipeName)
this.policies.delete(recipeName)
+ }
+
+ /**
+ * The finalizer's token DELETE returned 2xx or 404. Clears the epoch only
+ * when it is unchanged since `epochBeforeDelete` was read; returns false,
+ * clearing nothing, when an ADDED landed while the DELETE was in flight.
+ */
+ noteSecretGone(recipeName: string, epochBeforeDelete: number): boolean {
+ if (this.secretEpoch(recipeName) !== epochBeforeDelete) return false
this.secretEpochs.delete(recipeName)
+ return true
}
}
diff --git a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
index 9f3a2be89..668da9157 100644
--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
@@ -3976,7 +3976,7 @@ export class WorkflowRecipeReconciler {
createLogger('wrc', recipe.metadata.name).info('Deleting recipe resources', { name })
this.secretReverseIndex?.delete(name)
- this.oauthBrokerDeleteLedger.invalidate(name)
+ this.oauthBrokerDeleteLedger.forgetRecipe(name)
// ─── Workflow Delete (Stage 1) ────────────────────────────────────
const isWorkflow = recipe.spec.steps !== undefined && recipe.spec.steps.length > 0
@@ -4246,16 +4246,9 @@ export class WorkflowRecipeReconciler {
}
// The OAuth broker token Secret + its egress NetworkPolicy (Path B) carry
- // no ownerReference, so they are reaped explicitly here. safeDelete
- // swallows 404 when the recipe never used background OAuth.
- await this.safeDelete(
- () =>
- this.coreApi.deleteNamespacedSecret({
- name: rb.oauthBrokerTokenSecretName(recipe.metadata.name),
- namespace: this.config.sandboxNamespace,
- }),
- `Secret "${rb.oauthBrokerTokenSecretName(recipe.metadata.name)}" in ${this.config.sandboxNamespace}`
- )
+ // no ownerReference, so they are reaped explicitly here. A 404 is logged
+ // as already gone when the recipe never used background OAuth.
+ await this.deleteOAuthBrokerTokenOnFinalize(recipe.metadata.name)
await this.safeDelete(
() =>
this.networkingApi.deleteNamespacedNetworkPolicy({
@@ -4679,6 +4672,45 @@ export class WorkflowRecipeReconciler {
})
}
+ /**
+ * Finalizer reap of the oauth-broker token Secret. Logs exactly as
+ * `safeDelete` does and swallows a failure the same way (#949), but hands
+ * the observed outcome to the ledger: only a 2xx/404 with no token ADDED
+ * while the DELETE was in flight clears the token-seen epoch. A failed
+ * DELETE keeps it, so a recipe recreated under the same name still reaps
+ * the leftover token.
+ */
+ private async deleteOAuthBrokerTokenOnFinalize(recipeName: string): Promise<void> {
+ const secretName = rb.oauthBrokerTokenSecretName(recipeName)
+ const namespace = this.config.sandboxNamespace
+ const label = `Secret "${secretName}" in ${namespace}`
+ const epochBeforeDelete = this.oauthBrokerDeleteLedger.secretEpoch(recipeName)
+ const outcome = await observeNamespacedDelete(() =>
+ this.coreApi.deleteNamespacedSecret({ name: secretName, namespace })
+ )
+ const log = createLogger('wrc', 'workflow-recipes')
+ switch (outcome.kind) {
+ case 'deleted':
+ log.info('Deleted resource', { label })
+ break
+ case 'gone':
+ log.info('Resource already gone', { label })
+ break
+ case 'failed':
+ log.error('Failed to delete resource', { label, err: outcome.error })
+ return
+ default: {
+ const _exhaustive: never = outcome
+ return _exhaustive
+ }
+ }
+ if (!this.oauthBrokerDeleteLedger.noteSecretGone(recipeName, epochBeforeDelete)) {
+ log.info('oauth-broker-token Secret re-added during finalizer delete; keeping it seen', {
+ recipe: recipeName,
+ })
+ }
+ }
+
private async safeDelete(deleteFn: () => Promise<unknown>, label: string): Promise<void> {
try {
await deleteFn()