Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

test(workflow-recipes): close the ledger property gaps with I6, recreation and NP sequences (R3-L1)

Alfredo Lopezff4d9126254e
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.json
index 1ea1beae1..32bf2efa6 100644
--- a/workflow-recipes/package-lock.json
+++ b/workflow-recipes/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.435",
+ "version": "0.1.436",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "workload-recipes-controller",
- "version": "0.1.435",
+ "version": "0.1.436",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@aws-sdk/client-s3": "^3.1009.0",
diff --git a/workflow-recipes/package.json b/workflow-recipes/package.json
index 8e420e39f..e5d71c39f 100644
--- a/workflow-recipes/package.json
+++ b/workflow-recipes/package.json
@@ -1,6 +1,6 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.435",
+ "version": "0.1.436",
"description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface",
"main": "dist/main.js",
"engines": {
diff --git a/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.property.test.ts b/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.property.test.ts
index 930e15f20..c405aa949 100644
--- a/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.property.test.ts
+++ b/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.property.test.ts
@@ -11,25 +11,34 @@ import {
* The harness does not rebuild the ledger's entries: it records only what a
* caller can see (which events happened, in which order) and asserts
* implications on the answers. Cases no invariant speaks about stay free.
+ * "Current uid" is the uid of the live recipe object under the name.
*
- * I1 No token ADDED since the process started: no Secret delete. A
- * finalizer start (`forgetRecipe`) does not clear the token-seen bit.
- * I2 After an ADDED with no later recorded delete or provisioning, a pass
- * at or above the watermark (the highest generation recorded or
- * provisioned since the last finalizer start) deletes. A finalizer
- * start with the token seen arms every generation.
- * I3 A pass below the highest generation that provisioned the token does
- * not delete it.
- * I4 A delete whose DELETE was sent before an ADDED records nothing; with
- * no ADDED in between it is recorded, across a finalizer start too.
- * I5 The NetworkPolicy side skips the recorded pass until the TTL expires,
+ * I1 No token ADDED since the process started, or since the finalizer
+ * observed the token gone: no Secret delete. A finalizer start
+ * (`forgetRecipe`) alone does not clear the token-seen bit.
+ * I2 Token seen, and armed (an ADDED since the last write for the current
+ * uid, or no write for it since the last finalizer start or
+ * recreation): a pass at or above the watermark deletes.
+ * I3 A pass below the highest generation that provisioned the token for
+ * the current uid does not delete it.
+ * I4 A delete whose DELETE was sent before an ADDED, or before the
+ * finalizer observed the token gone, records nothing; otherwise it is
+ * recorded, across a finalizer start too.
+ * I5 The NetworkPolicy side skips a covered pass until the TTL expires,
* and deletes for a different uid or a higher generation.
+ * I6 Idempotence (#759 anti-storm): after a recorded delete or
+ * provisioning for the current uid, with no ADDED since, a pass at or
+ * below the watermark does not delete.
+ * I7 The finalizer's token DELETE result: 2xx/404 with no ADDED in flight
+ * clears the token-seen bit; a raced ADDED, a failed DELETE or one
+ * never sent keeps it.
*/
const NAME = 'r'
-const UID = 'uid-1'
const MAX_GENERATION = 6
+type FinalizeOutcome = 'deleted' | 'gone' | 'failed' | 'notAttempted'
+
type Op =
| { kind: 'provision'; generation: number }
| { kind: 'pass'; generation: number }
@@ -38,6 +47,8 @@ type Op =
| { kind: 'invalidateSecret' }
| { kind: 'addedElsewhere' }
| { kind: 'forgetRecipe' }
+ | { kind: 'recreate' }
+ | { kind: 'finalize'; outcome: FinalizeOutcome; racedAdded: boolean }
const generationArb = fc.integer({ min: 0, max: MAX_GENERATION })
@@ -48,32 +59,65 @@ const opArb: fc.Arbitrary<Op> = fc.oneof(
fc.boolean().map(failed => ({ kind: 'endDelete' as const, failed })),
fc.constant({ kind: 'invalidateSecret' as const }),
fc.constant({ kind: 'addedElsewhere' as const }),
- fc.constant({ kind: 'forgetRecipe' as const })
+ fc.constant({ kind: 'forgetRecipe' as const }),
+ fc.constant({ kind: 'recreate' as const }),
+ fc
+ .record({
+ outcome: fc.constantFrom<FinalizeOutcome>('deleted', 'gone', 'failed', 'notAttempted'),
+ racedAdded: fc.boolean(),
+ })
+ .map(({ outcome, racedAdded }) => ({ kind: 'finalize' as const, outcome, racedAdded }))
)
-function ref(generation: number, uid = UID): OAuthBrokerLedgerRecipe {
+function ref(generation: number, uid: string): OAuthBrokerLedgerRecipe {
return { name: NAME, uid, generation }
}
/** What a caller observed; never the ledger's internal entry. */
interface Observed {
+ uid: string
+ /** Every uid the name has carried, for the records-nothing check. */
+ uids: string[]
tokenSeen: boolean
- /** Highest generation recorded or provisioned since the last finalizer start. */
+ /** Highest generation recorded or provisioned for the current uid since the last finalizer start. */
watermark: number
- /** Highest generation provisioned since the last finalizer start. */
+ /** Highest generation provisioned for the current uid since the last finalizer start. */
provisioned: number
- /** An ADDED landed after the last recorded delete or provisioning. */
- addedAfterLastWrite: boolean
- inFlight?: { generation: number; epoch: number; raced: boolean }
+ /** An ADDED, a finalizer start or a recreation landed after the last write for the current uid. */
+ armed: boolean
+ inFlight?: { uid: string; generation: number; epoch: number; raced: boolean }
}
function freshObserved(): Observed {
- return { tokenSeen: false, watermark: -1, provisioned: -1, addedAfterLastWrite: false }
+ return {
+ uid: 'uid-1',
+ uids: ['uid-1'],
+ tokenSeen: false,
+ watermark: -1,
+ provisioned: -1,
+ armed: true,
+ }
}
-function allAnswers(ledger: OAuthBrokerDeleteLedger): boolean[] {
+/** The ledger no longer holds a write for the current uid. */
+function forgetCurrentUid(seen: Observed): void {
+ seen.watermark = -1
+ seen.provisioned = -1
+ seen.armed = true
+}
+
+/** A write (recorded delete or provisioning) for the current uid. */
+function noteWrite(seen: Observed, generation: number, provisioned: boolean): void {
+ if (provisioned) seen.provisioned = Math.max(seen.provisioned, generation)
+ // A write below the watermark leaves the ledger's entry as it is.
+ if (generation < seen.watermark) return
+ seen.watermark = generation
+ seen.armed = false
+}
+
+function allAnswers(ledger: OAuthBrokerDeleteLedger, seen: Observed): boolean[] {
const answers: boolean[] = []
- for (const uid of [UID, 'uid-2']) {
+ for (const uid of [...seen.uids, 'uid-never']) {
for (let generation = 0; generation <= MAX_GENERATION; generation++) {
answers.push(ledger.shouldDeleteSecret(ref(generation, uid)))
}
@@ -82,22 +126,33 @@ function allAnswers(ledger: OAuthBrokerDeleteLedger): boolean[] {
}
function checkPass(ledger: OAuthBrokerDeleteLedger, seen: Observed, generation: number): void {
- const answer = ledger.shouldDeleteSecret(ref(generation))
+ const answer = ledger.shouldDeleteSecret(ref(generation, seen.uid))
const where = `gen${generation} observed=${JSON.stringify(seen)}`
if (!seen.tokenSeen) expect(answer, `I1 ${where}`).toBe(false)
if (generation < seen.provisioned) expect(answer, `I3 ${where}`).toBe(false)
- if (seen.tokenSeen && seen.addedAfterLastWrite && generation >= seen.watermark) {
+ if (seen.tokenSeen && seen.armed && generation >= seen.watermark) {
expect(answer, `I2 ${where}`).toBe(true)
}
+ if (!seen.armed && generation <= seen.watermark) {
+ expect(answer, `I6 ${where}`).toBe(false)
+ }
+}
+
+function added(ledger: OAuthBrokerDeleteLedger, seen: Observed): void {
+ ledger.invalidateSecret(NAME)
+ seen.tokenSeen = true
+ seen.armed = true
+ if (seen.inFlight) seen.inFlight.raced = true
}
function step(ledger: OAuthBrokerDeleteLedger, seen: Observed, op: Op): void {
switch (op.kind) {
case 'provision':
- ledger.noteSecretProvisioned(ref(op.generation))
- seen.provisioned = Math.max(seen.provisioned, op.generation)
- seen.watermark = Math.max(seen.watermark, op.generation)
- seen.addedAfterLastWrite = false
+ // The per-recipe queue serialises passes for one name: the recreated
+ // recipe cannot provision while the old uid's pass holds a DELETE.
+ if (seen.inFlight && seen.inFlight.uid !== seen.uid) return
+ ledger.noteSecretProvisioned(ref(op.generation, seen.uid))
+ noteWrite(seen, op.generation, true)
return
case 'pass':
checkPass(ledger, seen, op.generation)
@@ -105,8 +160,9 @@ function step(ledger: OAuthBrokerDeleteLedger, seen: Observed, op: Op): void {
case 'beginDelete':
// The WRC sends the DELETE only when the ledger allows it, and the
// per-recipe queue keeps one in flight.
- if (seen.inFlight || !ledger.shouldDeleteSecret(ref(op.generation))) return
+ if (seen.inFlight || !ledger.shouldDeleteSecret(ref(op.generation, seen.uid))) return
seen.inFlight = {
+ uid: seen.uid,
generation: op.generation,
epoch: ledger.secretEpoch(NAME),
raced: false,
@@ -118,45 +174,70 @@ function step(ledger: OAuthBrokerDeleteLedger, seen: Observed, op: Op): void {
seen.inFlight = undefined
// A failed DELETE is never recorded.
if (op.failed) return
- const before = allAnswers(ledger)
- const recorded = ledger.recordSecretDelete(ref(inFlight.generation), inFlight.epoch)
+ const before = allAnswers(ledger, seen)
+ const recorded = ledger.recordSecretDelete(
+ ref(inFlight.generation, inFlight.uid),
+ inFlight.epoch
+ )
if (inFlight.raced) {
expect(recorded, `I4 raced gen${inFlight.generation}`).toBe(false)
- expect(allAnswers(ledger), `I4 records nothing gen${inFlight.generation}`).toEqual(before)
+ expect(allAnswers(ledger, seen), `I4 records nothing gen${inFlight.generation}`).toEqual(
+ before
+ )
return
}
// Liveness witness for I4: an undisturbed delete is recorded.
expect(recorded, `I4 undisturbed gen${inFlight.generation}`).toBe(true)
- seen.watermark = Math.max(seen.watermark, inFlight.generation)
- seen.addedAfterLastWrite = false
+ if (inFlight.uid === seen.uid) {
+ noteWrite(seen, inFlight.generation, false)
+ } else {
+ // The old uid's entry replaced whatever the name held; the current
+ // uid has no write in the ledger any more.
+ forgetCurrentUid(seen)
+ }
return
}
case 'invalidateSecret':
- ledger.invalidateSecret(NAME)
- seen.tokenSeen = true
- seen.addedAfterLastWrite = true
- if (seen.inFlight) seen.inFlight.raced = true
+ added(ledger, seen)
return
case 'addedElsewhere':
ledger.invalidateSecret('other')
return
- case 'forgetRecipe': {
+ case 'forgetRecipe':
// The finalizer starts: both entries go, the token-seen bit stays, and
// an in-flight DELETE is not raced (the epoch did not move).
ledger.forgetRecipe(NAME)
- const { tokenSeen, inFlight } = seen
- Object.assign(seen, freshObserved())
- seen.tokenSeen = tokenSeen
- // With no entry left, every generation deletes while the token is seen.
- seen.addedAfterLastWrite = tokenSeen
- seen.inFlight = inFlight
+ forgetCurrentUid(seen)
+ return
+ case 'recreate': {
+ // The recipe is recreated under the same name, whether or not its
+ // finalizer ran in this process.
+ const uid = `uid-${seen.uids.length + 1}`
+ seen.uids.push(uid)
+ seen.uid = uid
+ forgetCurrentUid(seen)
+ return
+ }
+ case 'finalize': {
+ ledger.forgetRecipe(NAME)
+ forgetCurrentUid(seen)
+ if (op.outcome === 'notAttempted') return
+ const epochBeforeDelete = ledger.secretEpoch(NAME)
+ if (op.racedAdded) added(ledger, seen)
+ if (op.outcome === 'failed') return
+ const cleared = ledger.noteSecretGone(NAME, epochBeforeDelete)
+ expect(cleared, `I7 ${op.outcome} racedAdded=${op.racedAdded}`).toBe(!op.racedAdded)
+ if (!cleared) return
+ seen.tokenSeen = false
+ // An ensure DELETE in flight read a non-zero epoch; it no longer matches.
+ if (seen.inFlight) seen.inFlight.raced = true
return
}
}
}
describe('OAuthBrokerDeleteLedger observable properties', () => {
- it('I1-I4 hold over provision, pass, beginDelete/endDelete, invalidateSecret and forgetRecipe', () => {
+ it('I1-I4, I6 and I7 hold over provision, pass, delete, ADDED, finalizer and recreation sequences', () => {
fc.assert(
fc.property(fc.array(opArb, { maxLength: 40 }), ops => {
const ledger = new OAuthBrokerDeleteLedger()
@@ -176,10 +257,29 @@ describe('OAuthBrokerDeleteLedger observable properties', () => {
fc.assert(
fc.property(fc.integer({ min: 1, max: MAX_GENERATION }), provisioned => {
const ledger = new OAuthBrokerDeleteLedger()
- ledger.noteSecretProvisioned(ref(provisioned))
+ ledger.noteSecretProvisioned(ref(provisioned, 'uid-1'))
+ ledger.invalidateSecret(NAME)
+ for (let generation = 0; generation <= MAX_GENERATION; generation++) {
+ expect(ledger.shouldDeleteSecret(ref(generation, 'uid-1'))).toBe(
+ generation >= provisioned
+ )
+ }
+ }),
+ { numRuns: 500 }
+ )
+ })
+
+ it('I6/I2 are reachable: a recorded delete disarms its generation, a recreation re-arms every generation', () => {
+ fc.assert(
+ fc.property(generationArb, recorded => {
+ const ledger = new OAuthBrokerDeleteLedger()
ledger.invalidateSecret(NAME)
+ expect(ledger.recordSecretDelete(ref(recorded, 'uid-1'), ledger.secretEpoch(NAME))).toBe(
+ true
+ )
for (let generation = 0; generation <= MAX_GENERATION; generation++) {
- expect(ledger.shouldDeleteSecret(ref(generation))).toBe(generation >= provisioned)
+ expect(ledger.shouldDeleteSecret(ref(generation, 'uid-1'))).toBe(generation > recorded)
+ expect(ledger.shouldDeleteSecret(ref(generation, 'uid-2'))).toBe(true)
}
}),
{ numRuns: 500 }
@@ -190,12 +290,12 @@ describe('OAuthBrokerDeleteLedger observable properties', () => {
fc.assert(
fc.property(
fc.record({
- uid: fc.constantFrom(UID, 'uid-2'),
+ uid: fc.constantFrom('uid-1', 'uid-2'),
generation: generationArb,
recordedAt: fc.integer({ min: 0, max: 10 * OAUTH_BROKER_NP_TTL_MS }),
}),
fc.record({
- uid: fc.constantFrom(UID, 'uid-2'),
+ uid: fc.constantFrom('uid-1', 'uid-2'),
generation: generationArb,
elapsed: fc.oneof(
fc.integer({ min: 0, max: OAUTH_BROKER_NP_TTL_MS - 1 }),
@@ -219,4 +319,51 @@ describe('OAuthBrokerDeleteLedger observable properties', () => {
{ numRuns: 5000 }
)
})
+
+ it('I5 over recordPolicyDelete sequences: a late lower generation keeps the watermark, a new uid replaces it', () => {
+ fc.assert(
+ fc.property(
+ fc.array(
+ fc.record({
+ uid: fc.constantFrom('uid-1', 'uid-2'),
+ generation: generationArb,
+ advance: fc.integer({ min: 0, max: OAUTH_BROKER_NP_TTL_MS / 2 }),
+ }),
+ { minLength: 1, maxLength: 12 }
+ ),
+ fc.integer({ min: 0, max: 2 * OAUTH_BROKER_NP_TTL_MS }),
+ (records, elapsed) => {
+ const ledger = new OAuthBrokerDeleteLedger()
+ let now = 0
+ let observed: { uid: string; generation: number; at: number } | undefined
+ for (const record of records) {
+ now += record.advance
+ ledger.recordPolicyDelete(ref(record.generation, record.uid), now)
+ observed =
+ observed !== undefined && observed.uid === record.uid
+ ? {
+ uid: record.uid,
+ generation: Math.max(observed.generation, record.generation),
+ at: now,
+ }
+ : { uid: record.uid, generation: record.generation, at: now }
+ for (const uid of ['uid-1', 'uid-2']) {
+ for (let generation = 0; generation <= MAX_GENERATION; generation++) {
+ const at = now + elapsed
+ const covered =
+ uid === observed.uid &&
+ generation <= observed.generation &&
+ at < observed.at + OAUTH_BROKER_NP_TTL_MS
+ expect(
+ ledger.shouldDeletePolicy(ref(generation, uid), at),
+ `I5 seq ${JSON.stringify({ records, elapsed, uid, generation })}`
+ ).toBe(!covered)
+ }
+ }
+ }
+ }
+ ),
+ { numRuns: 2000 }
+ )
+ })
})