Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(workflow-recipes): identity-gate the grant-update listener reconnect (#375)

Alfredo Lopez6f78b28ff8bd
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.json
index af8ad5a5d..c4ae4b6de 100644
--- a/workflow-recipes/package-lock.json
+++ b/workflow-recipes/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.290",
+ "version": "0.1.291",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "workload-recipes-controller",
- "version": "0.1.290",
+ "version": "0.1.291",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@aws-sdk/client-s3": "^3.1009.0",
diff --git a/workflow-recipes/package.json b/workflow-recipes/package.json
index b577ff64f..29782fa82 100644
--- a/workflow-recipes/package.json
+++ b/workflow-recipes/package.json
@@ -1,6 +1,6 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.290",
+ "version": "0.1.291",
"description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface",
"main": "dist/main.js",
"engines": {
diff --git a/workflow-recipes/src/reconciler/grantUpdateListener.test.ts b/workflow-recipes/src/reconciler/grantUpdateListener.test.ts
index f1040c079..a25a60d6c 100644
--- a/workflow-recipes/src/reconciler/grantUpdateListener.test.ts
+++ b/workflow-recipes/src/reconciler/grantUpdateListener.test.ts
@@ -185,6 +185,183 @@ describe('createGrantUpdateListener', () => {
})
})
+// ── Finding 1 (#205-class): sustained multi-generation reconnect regime ──
+// A LISTEN session behind pgbouncer/cloud LB is closed repeatedly under load
+// (GKE produces this; minikube never does). A LATE 'error' from an
+// already-superseded client must release ONLY itself and must NOT spawn a
+// second attach loop that orphans the current live session (duplicate dispatch
+// + one leaked pooled connection per occurrence → monotonic pool exhaustion).
+describe('createGrantUpdateListener — sustained reconnect regime (issue #375 Finding 1)', () => {
+ interface FakeGenClient {
+ released: boolean
+ on: (event: string, handler: (...args: unknown[]) => void) => FakeGenClient
+ query: ReturnType<typeof vi.fn>
+ release: ReturnType<typeof vi.fn>
+ emitError: (err: Error) => void
+ emitNotification: (msg: { channel: string; payload?: string }) => void
+ }
+
+ /** A fresh fake client per connect(), tracking created + per-client released. */
+ function makeFakeClientFactory() {
+ const created: FakeGenClient[] = []
+ function connect(): FakeGenClient {
+ const handlers: Record<string, ((...args: unknown[]) => void)[]> = {}
+ const client: FakeGenClient = {
+ released: false,
+ on(event, handler) {
+ ;(handlers[event] ??= []).push(handler)
+ return client
+ },
+ query: vi.fn(async () => ({ rows: [], rowCount: 0 })),
+ release: vi.fn(() => {
+ client.released = true
+ }),
+ emitError(err: Error) {
+ for (const h of handlers.error ?? []) h(err)
+ },
+ emitNotification(msg) {
+ // A released client is back in the pool and must not dispatch.
+ if (client.released) return
+ for (const h of handlers.notification ?? []) h(msg)
+ },
+ }
+ created.push(client)
+ return client
+ }
+ return {
+ created,
+ connect,
+ /** Live (checked-out, unreleased) sessions right now. */
+ liveCount: () => created.filter(c => !c.released).length,
+ current: () => created[created.length - 1],
+ /** Simulate a real pg NOTIFY broadcast to EVERY live session. */
+ broadcast(msg: { channel: string; payload?: string }) {
+ for (const c of created) c.emitNotification(msg)
+ },
+ }
+ }
+
+ /** A resolver-queue sleep seam so backoff can be driven deterministically. */
+ function resolverQueueSleep() {
+ const resolvers: Array<() => void> = []
+ let calls = 0
+ const sleep = (_ms: number) =>
+ new Promise<void>(resolve => {
+ calls += 1
+ resolvers.push(resolve)
+ })
+ return { sleep, resolvers, sleepCalls: () => calls }
+ }
+
+ const flush = async () => {
+ for (let i = 0; i < 12; i += 1) await Promise.resolve()
+ }
+
+ const NOTIFY = {
+ channel: PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL,
+ payload: JSON.stringify({ recipeNamespace: 'sandbox-recipes', recipeName: 'research' }),
+ }
+
+ it('T1: holds ≤1 live session and exactly-once dispatch across 25 drop→reattach cycles with late superseded errors', async () => {
+ const factory = makeFakeClientFactory()
+ const { sleep, resolvers, sleepCalls } = resolverQueueSleep()
+ const onGrantUpdate = vi.fn()
+ const listener = createGrantUpdateListener({
+ pool: {} as never,
+ onGrantUpdate,
+ logger: makeLogger(),
+ connect: async () => factory.connect() as never,
+ sleep,
+ })
+
+ // Drain every queued backoff sleep and let the reconnect attaches settle.
+ const settle = async () => {
+ await flush()
+ while (resolvers.length > 0) {
+ const resume = resolvers.shift() as () => void
+ resume()
+ await flush()
+ }
+ }
+
+ await listener.start()
+ expect(factory.liveCount()).toBe(1)
+
+ const K = 25
+ for (let i = 0; i < K; i += 1) {
+ const dropped = factory.current()
+ // 1) the current session drops → schedules a reconnect.
+ dropped.emitError(new Error(`drop-${i}`))
+ // 2) the reconnect completes → a new session becomes current.
+ await settle()
+ // 3) a LATE, second 'error' from the now-SUPERSEDED (already-released)
+ // generation — the storm's signature interleaving. It must NOT drive a
+ // second attach loop that orphans the current live session.
+ dropped.emitError(new Error(`late-drop-${i}`))
+ await settle()
+
+ // The invariant #205 never asserted: at most ONE live session, ever.
+ expect(factory.liveCount(), `cycle ${i}: live sessions`).toBeLessThanOrEqual(1)
+
+ // A real pg NOTIFY reaches every live session; with no orphan it dispatches
+ // exactly once (an orphaned session would double-dispatch).
+ onGrantUpdate.mockClear()
+ factory.broadcast(NOTIFY)
+ expect(onGrantUpdate, `cycle ${i}: dispatch count`).toHaveBeenCalledTimes(1)
+ }
+
+ // No runaway parallel reconnect loops: ~one scheduled sleep per real drop.
+ expect(sleepCalls()).toBeLessThanOrEqual(K + 2)
+
+ await listener.stop()
+ expect(factory.liveCount(), 'after stop(): no leaked sessions').toBe(0)
+ })
+
+ it('T2: one full reconnect cycle recovers notification flow', async () => {
+ const factory = makeFakeClientFactory()
+ const { sleep, resolvers } = resolverQueueSleep()
+ const onGrantUpdate = vi.fn()
+ const listener = createGrantUpdateListener({
+ pool: {} as never,
+ onGrantUpdate,
+ logger: makeLogger(),
+ connect: async () => factory.connect() as never,
+ sleep,
+ })
+
+ await listener.start()
+ const clientA = factory.current()
+ expect(factory.liveCount()).toBe(1)
+
+ // Drop A → resolve the backoff → a NEW client B attaches.
+ clientA.emitError(new Error('drop'))
+ await flush()
+ expect(resolvers.length).toBe(1)
+ ;(resolvers.shift() as () => void)()
+ await flush()
+
+ const clientB = factory.current()
+ expect(clientB).not.toBe(clientA)
+ expect(clientA.released).toBe(true)
+ // B issued its own LISTEN and is the sole live session.
+ expect(clientB.query).toHaveBeenCalledWith(
+ expect.stringMatching(/LISTEN\s+plugin_workload_sdk_grant_update/)
+ )
+ expect(factory.liveCount()).toBe(1)
+
+ // A NOTIFY on the recovered session dispatches exactly once.
+ factory.broadcast(NOTIFY)
+ expect(onGrantUpdate).toHaveBeenCalledTimes(1)
+ expect(onGrantUpdate).toHaveBeenCalledWith({
+ recipeNamespace: 'sandbox-recipes',
+ recipeName: 'research',
+ })
+
+ await listener.stop()
+ expect(factory.liveCount()).toBe(0)
+ })
+})
+
describe('createGrantUpdateListener — reconnect / stop resilience (issue #375 H1-H3)', () => {
/** A sleep seam that never resolves, so we can count reconnect scheduling without real timers. */
function pendingSleep() {
diff --git a/workflow-recipes/src/reconciler/grantUpdateListener.ts b/workflow-recipes/src/reconciler/grantUpdateListener.ts
index f4a456d7f..b3aeb681c 100644
--- a/workflow-recipes/src/reconciler/grantUpdateListener.ts
+++ b/workflow-recipes/src/reconciler/grantUpdateListener.ts
@@ -91,9 +91,14 @@ export function createGrantUpdateListener(opts: GrantUpdateListenerOptions): Gra
// consumer has the same latent patterns this module hardens against —
// (1) a LISTEN rejection AND an 'error' emission on the same drop can start two
// reconnect loops, (2) stop() cannot cancel an attach already awaiting
- // connect(), and (3) a LISTEN failure without an 'error' event leaks the
- // checked-out client. Fixing dbRunProcessor is out of #375 scope; this listener
- // intentionally improves on it and the guards below are the reference.
+ // connect(), (3) a LISTEN failure without an 'error' event leaks the
+ // checked-out client, and (4) a LATE error from a superseded generation drives
+ // a reconnect that orphans the live session (duplicate dispatch + monotonic
+ // pool leak under a sustained reconnect storm — the #205 blind-spot shape).
+ // Fixing dbRunProcessor is out of #375 scope; this listener intentionally
+ // improves on it and the guards below (esp. the identity gate on the 'error'
+ // handler + the non-overwrite guard before assigning `listenClient`) are the
+ // reference for the follow-up.
let listenClient: PoolClient | null = null
let stopped = true
// H1: collapse concurrent reconnect triggers into ONE in-flight reconnect.
@@ -132,13 +137,27 @@ export function createGrantUpdateListener(opts: GrantUpdateListenerOptions): Gra
}
})
attached.on('error', (err: Error) => {
- log.warn('listen session error — will reconnect', { err: err.message })
try {
attached.release(err)
} catch {
/* idempotent */
}
- if (listenClient === attached) listenClient = null
+ // Finding 1 (#205-class): IDENTITY GATE. Only the CURRENT session's error
+ // may drive a reconnect. A LATE error from a superseded generation — its
+ // successor has already attached and become `listenClient` — must release
+ // ONLY itself and return. If it scheduled a reconnect it would spawn a
+ // second attach loop whose success overwrites the live session, orphaning
+ // it: still checked out from the shared pool with a live LISTEN handler →
+ // duplicate NOTIFY dispatch + one leaked connection per occurrence →
+ // monotonic pool exhaustion under a reconnect storm (GKE regime).
+ if (listenClient !== attached) {
+ log.debug('superseded listen session error — released self, no reconnect', {
+ err: err.message,
+ })
+ return
+ }
+ log.warn('listen session error — will reconnect', { err: err.message })
+ listenClient = null
scheduleReconnect()
})
await attached.query(`LISTEN ${PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL}`)
@@ -152,6 +171,17 @@ export function createGrantUpdateListener(opts: GrantUpdateListenerOptions): Gra
}
return
}
+ // Finding 1: never overwrite a live `listenClient` without releasing the
+ // prior one. Defense-in-depth alongside the identity gate above — a stray
+ // interleaving that reaches here with a different session still checked out
+ // must not leak it.
+ if (listenClient && listenClient !== attached) {
+ try {
+ listenClient.release()
+ } catch {
+ /* idempotent */
+ }
+ }
listenClient = attached
log.info('LISTEN plugin_workload_sdk_grant_update attached')
} catch (err) {