Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(workflow-recipes): keep the token seen until the finalizer observes it gone (R4-L1)
Alfredo Lopez1386b2d0e629
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.jsonindex 70cef38b7..1ea1beae1 100644--- a/workflow-recipes/package-lock.json+++ b/workflow-recipes/package-lock.json@@ -1,12 +1,12 @@{"name": "workload-recipes-controller",- "version": "0.1.434",+ "version": "0.1.435","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "workload-recipes-controller",- "version": "0.1.434",+ "version": "0.1.435","license": "SEE LICENSE IN LICENSE","dependencies": {"@aws-sdk/client-s3": "^3.1009.0",diff --git a/workflow-recipes/package.json b/workflow-recipes/package.jsonindex 842b6cde0..8e420e39f 100644--- a/workflow-recipes/package.json+++ b/workflow-recipes/package.json@@ -1,6 +1,6 @@{"name": "workload-recipes-controller",- "version": "0.1.434",+ "version": "0.1.435","description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface","main": "dist/main.js","engines": {diff --git a/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.property.test.ts b/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.property.test.tsindex b6acb347c..930e15f20 100644--- a/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.property.test.ts+++ b/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.property.test.ts@@ -12,15 +12,16 @@ import {* caller can see (which events happened, in which order) and asserts* implications on the answers. Cases no invariant speaks about stay free.*- * I1 No token ADDED since the process started or the recipe was deleted:- * no Secret delete.+ * I1 No token ADDED since the process started: no Secret delete. A+ * finalizer start (`forgetRecipe`) does not clear the token-seen bit.* I2 After an ADDED with no later recorded delete or provisioning, a pass* at or above the watermark (the highest generation recorded or- * provisioned since the recipe was last deleted) deletes.+ * provisioned since the last finalizer start) deletes. A finalizer+ * start with the token seen arms every generation.* I3 A pass below the highest generation that provisioned the token does* not delete it.- * I4 A delete whose DELETE was sent before an ADDED (or a recipe deletion)- * records nothing; with neither in between it is recorded.+ * I4 A delete whose DELETE was sent before an ADDED records nothing; with+ * no ADDED in between it is recorded, across a finalizer start too.* I5 The NetworkPolicy side skips the recorded pass until the TTL expires,* and deletes for a different uid or a higher generation.*/@@ -36,7 +37,7 @@ type Op =| { kind: 'endDelete'; failed: boolean }| { kind: 'invalidateSecret' }| { kind: 'addedElsewhere' }- | { kind: 'invalidate' }+ | { kind: 'forgetRecipe' }const generationArb = fc.integer({ min: 0, max: MAX_GENERATION })@@ -47,7 +48,7 @@ const opArb: fc.Arbitrary<Op> = fc.oneof(fc.boolean().map(failed => ({ kind: 'endDelete' as const, failed })),fc.constant({ kind: 'invalidateSecret' as const }),fc.constant({ kind: 'addedElsewhere' as const }),- fc.constant({ kind: 'invalidate' as const })+ fc.constant({ kind: 'forgetRecipe' as const }))function ref(generation: number, uid = UID): OAuthBrokerLedgerRecipe {@@ -57,9 +58,9 @@ function ref(generation: number, uid = UID): OAuthBrokerLedgerRecipe {/** What a caller observed; never the ledger's internal entry. */interface Observed {tokenSeen: boolean- /** Highest generation recorded or provisioned since the last recipe deletion. */+ /** Highest generation recorded or provisioned since the last finalizer start. */watermark: number- /** Highest generation provisioned since the last recipe deletion. */+ /** Highest generation provisioned since the last finalizer start. */provisioned: number/** An ADDED landed after the last recorded delete or provisioning. */addedAfterLastWrite: boolean@@ -139,18 +140,23 @@ function step(ledger: OAuthBrokerDeleteLedger, seen: Observed, op: Op): void {case 'addedElsewhere':ledger.invalidateSecret('other')return- case 'invalidate': {- ledger.invalidate(NAME)- const inFlight = seen.inFlight+ case 'forgetRecipe': {+ // The finalizer starts: both entries go, the token-seen bit stays, and+ // an in-flight DELETE is not raced (the epoch did not move).+ ledger.forgetRecipe(NAME)+ const { tokenSeen, inFlight } = seenObject.assign(seen, freshObserved())- if (inFlight) seen.inFlight = { ...inFlight, raced: true }+ seen.tokenSeen = tokenSeen+ // With no entry left, every generation deletes while the token is seen.+ seen.addedAfterLastWrite = tokenSeen+ seen.inFlight = inFlightreturn}}}describe('OAuthBrokerDeleteLedger observable properties', () => {- it('I1-I4 hold over provision, pass, beginDelete/endDelete, invalidateSecret and invalidate', () => {+ it('I1-I4 hold over provision, pass, beginDelete/endDelete, invalidateSecret and forgetRecipe', () => {fc.assert(fc.property(fc.array(opArb, { maxLength: 40 }), ops => {const ledger = new OAuthBrokerDeleteLedger()diff --git a/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.test.ts b/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.test.tsindex deeb563a8..c7db5ff9b 100644--- a/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.test.ts+++ b/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.test.ts@@ -180,7 +180,7 @@ describe('OAuthBrokerDeleteLedger', () => {expect(ledger.shouldDeleteSecret(ref('r2', 4))).toBe(false)})- it('invalidate drops both sides for the named recipe and no other', () => {+ it('forgetRecipe drops both entries for the named recipe and no other, and keeps the token seen', () => {const ledger = new OAuthBrokerDeleteLedger()tokenSeen(ledger, 'r')tokenSeen(ledger, 'r2')@@ -189,29 +189,69 @@ describe('OAuthBrokerDeleteLedger', () => {recordSecret(ledger, ref('r2', 4))ledger.recordPolicyDelete(ref('r2', 4), 0)- ledger.invalidate('r')- // The recipe is gone with its token: nothing to delete until a new ADDED.- expect(ledger.shouldDeleteSecret(ref('r', 5))).toBe(false)+ ledger.forgetRecipe('r')+ // Deleting the recipe does not delete its token: the watermark went, the+ // epoch stayed, so a generation below the old watermark deletes.+ expect(ledger.shouldDeleteSecret(ref('r', 3))).toBe(true)+ expect(ledger.shouldDeleteSecret(ref('r', 1, 'uid-recreated'))).toBe(true)expect(ledger.shouldDeletePolicy(ref('r', 4), 1)).toBe(true)+ // The other recipe keeps both entries.expect(ledger.shouldDeleteSecret(ref('r2', 4))).toBe(false)expect(ledger.shouldDeletePolicy(ref('r2', 4), 1)).toBe(false)+ })- // The watermark went too: after the next ADDED a generation below the- // old one deletes.+ it('Table B: noteSecretGone with the epoch unchanged clears the token-seen bit until the next ADDED', () => {+ const ledger = new OAuthBrokerDeleteLedger()tokenSeen(ledger, 'r')- expect(ledger.shouldDeleteSecret(ref('r', 3))).toBe(true)+ tokenSeen(ledger, 'r2')+ ledger.forgetRecipe('r')+ const epochBeforeDelete = ledger.secretEpoch('r')+ expect(epochBeforeDelete).toBeGreaterThan(0)+ expect(ledger.noteSecretGone('r', epochBeforeDelete)).toBe(true)++ expect(ledger.secretEpoch('r')).toBe(0)+ expect(ledger.shouldDeleteSecret(ref('r', 1, 'uid-recreated'))).toBe(false)+ // Liveness witness: another recipe is untouched, and the next ADDED re-arms.+ expect(ledger.shouldDeleteSecret(ref('r2', 1))).toBe(true)+ tokenSeen(ledger, 'r')+ expect(ledger.shouldDeleteSecret(ref('r', 1, 'uid-recreated'))).toBe(true)})- it('a DELETE in flight across a recipe deletion that followed an ADDED is not recorded', () => {+ it('Table B: noteSecretGone after an ADDED raced the finalizer DELETE clears nothing', () => {const ledger = new OAuthBrokerDeleteLedger()- ledger.invalidateSecret('r')+ tokenSeen(ledger, 'r')+ ledger.forgetRecipe('r')const epochBeforeDelete = ledger.secretEpoch('r')- ledger.invalidate('r')- expect(ledger.recordSecretDelete(ref('r', 4), epochBeforeDelete)).toBe(false)+ // The recreated recipe's token ADDED lands while the DELETE is in flight.+ ledger.invalidateSecret('r')+ expect(ledger.noteSecretGone('r', epochBeforeDelete)).toBe(false)+ expect(ledger.secretEpoch('r')).toBeGreaterThan(epochBeforeDelete)+ expect(ledger.shouldDeleteSecret(ref('r', 1, 'uid-recreated'))).toBe(true)+ })- // Nothing was recorded: after the next ADDED a lower generation deletes,- // which a watermark at 4 would have skipped.+ it('Table B: a finalizer DELETE that failed or was never sent keeps the token seen', () => {+ const ledger = new OAuthBrokerDeleteLedger()tokenSeen(ledger, 'r')- expect(ledger.shouldDeleteSecret(ref('r', 3))).toBe(true)+ recordSecret(ledger, ref('r', 4, 'uid-old'))+ const epochBeforeForget = ledger.secretEpoch('r')+ // Finalizer start; the DELETE then fails (or is never sent), so+ // noteSecretGone is not called.+ ledger.forgetRecipe('r')+ expect(ledger.secretEpoch('r')).toBe(epochBeforeForget)+ expect(ledger.shouldDeleteSecret(ref('r', 1, 'uid-recreated'))).toBe(true)+ })++ it('a DELETE in flight across a finalizer start that followed an ADDED is recorded against the old uid only', () => {+ const ledger = new OAuthBrokerDeleteLedger()+ ledger.invalidateSecret('r')+ const epochBeforeDelete = ledger.secretEpoch('r')+ ledger.forgetRecipe('r')+ // The epoch did not move, so the in-flight DELETE of the old recipe is+ // recorded for its own uid.+ expect(ledger.recordSecretDelete(ref('r', 4, 'uid-old'), epochBeforeDelete)).toBe(true)+ expect(ledger.shouldDeleteSecret(ref('r', 4, 'uid-old'))).toBe(false)++ // A recipe recreated under the same name (new uid) still deletes.+ expect(ledger.shouldDeleteSecret(ref('r', 1, 'uid-new'))).toBe(true)})})diff --git a/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.ts b/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.tsindex 34e12a1e1..0b62df6a4 100644--- a/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.ts+++ b/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.ts@@ -14,7 +14,9 @@* missed.** The Secret side deletes only after the Secret watch observed the recipe's- * token (a non-zero epoch): a recipe that never had one gets no DELETE.+ * token (a non-zero epoch): a recipe that never had one gets no DELETE. The+ * epoch survives recipe deletion and returns to 0 only when the finalizer's+ * own token DELETE is observed 2xx/404 with no ADDED in between.** The Secret watermark rises on a recorded delete and on* `noteSecretProvisioned`, which the backgroundAccess branch calls for the@@ -80,9 +82,10 @@ export class OAuthBrokerDeleteLedger {private epochClock = 0shouldDeleteSecret(recipe: OAuthBrokerLedgerRecipe): boolean {- // No token ADDED since this process started or the recipe was deleted:- // the Secret watch's initial list replays ADDED for every token that- // exists, so there is no token to reap and a DELETE would only 404.+ // Epoch 0: no token ADDED since this process started, or the finalizer+ // observed the token gone (2xx/404) with no ADDED since. The Secret+ // watch's initial list replays ADDED for every token that exists, so+ // there is no token to reap and a DELETE would only 404.if (this.secretEpoch(recipe.name) === 0) return falseconst entry = this.secrets.get(recipe.name)if (entry === undefined || entry.uid !== recipe.uid) return true@@ -149,12 +152,25 @@ export class OAuthBrokerDeleteLedger {}/**- * Recipe deletion drops both sides. The epoch entry goes too; the clock is- * global, so a DELETE in flight that read a non-zero epoch still mismatches.+ * Finalizer start: drops the Secret and NetworkPolicy entries and keeps the+ * epoch. Deleting a recipe does not delete its token; only the finalizer's+ * own DELETE, observed through `noteSecretGone`, may clear the epoch. A+ * DELETE that fails, is never sent, or races an ADDED leaves the token+ * seen, so a recipe recreated under the same name still reaps it.*/- invalidate(recipeName: string): void {+ forgetRecipe(recipeName: string): void {this.secrets.delete(recipeName)this.policies.delete(recipeName)+ }++ /**+ * The finalizer's token DELETE returned 2xx or 404. Clears the epoch only+ * when it is unchanged since `epochBeforeDelete` was read; returns false,+ * clearing nothing, when an ADDED landed while the DELETE was in flight.+ */+ noteSecretGone(recipeName: string, epochBeforeDelete: number): boolean {+ if (this.secretEpoch(recipeName) !== epochBeforeDelete) return falsethis.secretEpochs.delete(recipeName)+ return true}}diff --git a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts b/workflow-recipes/src/reconciler/workflowRecipeReconciler.tsindex 9f3a2be89..668da9157 100644--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts@@ -3976,7 +3976,7 @@ export class WorkflowRecipeReconciler {createLogger('wrc', recipe.metadata.name).info('Deleting recipe resources', { name })this.secretReverseIndex?.delete(name)- this.oauthBrokerDeleteLedger.invalidate(name)+ this.oauthBrokerDeleteLedger.forgetRecipe(name)// ─── Workflow Delete (Stage 1) ────────────────────────────────────const isWorkflow = recipe.spec.steps !== undefined && recipe.spec.steps.length > 0@@ -4246,16 +4246,9 @@ export class WorkflowRecipeReconciler {}// The OAuth broker token Secret + its egress NetworkPolicy (Path B) carry- // no ownerReference, so they are reaped explicitly here. safeDelete- // swallows 404 when the recipe never used background OAuth.- await this.safeDelete(- () =>- this.coreApi.deleteNamespacedSecret({- name: rb.oauthBrokerTokenSecretName(recipe.metadata.name),- namespace: this.config.sandboxNamespace,- }),- `Secret "${rb.oauthBrokerTokenSecretName(recipe.metadata.name)}" in ${this.config.sandboxNamespace}`- )+ // no ownerReference, so they are reaped explicitly here. A 404 is logged+ // as already gone when the recipe never used background OAuth.+ await this.deleteOAuthBrokerTokenOnFinalize(recipe.metadata.name)await this.safeDelete(() =>this.networkingApi.deleteNamespacedNetworkPolicy({@@ -4679,6 +4672,45 @@ export class WorkflowRecipeReconciler {})}+ /**+ * Finalizer reap of the oauth-broker token Secret. Logs exactly as+ * `safeDelete` does and swallows a failure the same way (#949), but hands+ * the observed outcome to the ledger: only a 2xx/404 with no token ADDED+ * while the DELETE was in flight clears the token-seen epoch. A failed+ * DELETE keeps it, so a recipe recreated under the same name still reaps+ * the leftover token.+ */+ private async deleteOAuthBrokerTokenOnFinalize(recipeName: string): Promise<void> {+ const secretName = rb.oauthBrokerTokenSecretName(recipeName)+ const namespace = this.config.sandboxNamespace+ const label = `Secret "${secretName}" in ${namespace}`+ const epochBeforeDelete = this.oauthBrokerDeleteLedger.secretEpoch(recipeName)+ const outcome = await observeNamespacedDelete(() =>+ this.coreApi.deleteNamespacedSecret({ name: secretName, namespace })+ )+ const log = createLogger('wrc', 'workflow-recipes')+ switch (outcome.kind) {+ case 'deleted':+ log.info('Deleted resource', { label })+ break+ case 'gone':+ log.info('Resource already gone', { label })+ break+ case 'failed':+ log.error('Failed to delete resource', { label, err: outcome.error })+ return+ default: {+ const _exhaustive: never = outcome+ return _exhaustive+ }+ }+ if (!this.oauthBrokerDeleteLedger.noteSecretGone(recipeName, epochBeforeDelete)) {+ log.info('oauth-broker-token Secret re-added during finalizer delete; keeping it seen', {+ recipe: recipeName,+ })+ }+ }+private async safeDelete(deleteFn: () => Promise<unknown>, label: string): Promise<void> {try {await deleteFn()