Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(workflow-recipes): identity-gate the grant-update listener reconnect (#375)
Alfredo Lopez6f78b28ff8bd
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.jsonindex af8ad5a5d..c4ae4b6de 100644--- a/workflow-recipes/package-lock.json+++ b/workflow-recipes/package-lock.json@@ -1,12 +1,12 @@{"name": "workload-recipes-controller",- "version": "0.1.290",+ "version": "0.1.291","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "workload-recipes-controller",- "version": "0.1.290",+ "version": "0.1.291","license": "SEE LICENSE IN LICENSE","dependencies": {"@aws-sdk/client-s3": "^3.1009.0",diff --git a/workflow-recipes/package.json b/workflow-recipes/package.jsonindex b577ff64f..29782fa82 100644--- a/workflow-recipes/package.json+++ b/workflow-recipes/package.json@@ -1,6 +1,6 @@{"name": "workload-recipes-controller",- "version": "0.1.290",+ "version": "0.1.291","description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface","main": "dist/main.js","engines": {diff --git a/workflow-recipes/src/reconciler/grantUpdateListener.test.ts b/workflow-recipes/src/reconciler/grantUpdateListener.test.tsindex f1040c079..a25a60d6c 100644--- a/workflow-recipes/src/reconciler/grantUpdateListener.test.ts+++ b/workflow-recipes/src/reconciler/grantUpdateListener.test.ts@@ -185,6 +185,183 @@ describe('createGrantUpdateListener', () => {})})+// ── Finding 1 (#205-class): sustained multi-generation reconnect regime ──+// A LISTEN session behind pgbouncer/cloud LB is closed repeatedly under load+// (GKE produces this; minikube never does). A LATE 'error' from an+// already-superseded client must release ONLY itself and must NOT spawn a+// second attach loop that orphans the current live session (duplicate dispatch+// + one leaked pooled connection per occurrence → monotonic pool exhaustion).+describe('createGrantUpdateListener — sustained reconnect regime (issue #375 Finding 1)', () => {+ interface FakeGenClient {+ released: boolean+ on: (event: string, handler: (...args: unknown[]) => void) => FakeGenClient+ query: ReturnType<typeof vi.fn>+ release: ReturnType<typeof vi.fn>+ emitError: (err: Error) => void+ emitNotification: (msg: { channel: string; payload?: string }) => void+ }++ /** A fresh fake client per connect(), tracking created + per-client released. */+ function makeFakeClientFactory() {+ const created: FakeGenClient[] = []+ function connect(): FakeGenClient {+ const handlers: Record<string, ((...args: unknown[]) => void)[]> = {}+ const client: FakeGenClient = {+ released: false,+ on(event, handler) {+ ;(handlers[event] ??= []).push(handler)+ return client+ },+ query: vi.fn(async () => ({ rows: [], rowCount: 0 })),+ release: vi.fn(() => {+ client.released = true+ }),+ emitError(err: Error) {+ for (const h of handlers.error ?? []) h(err)+ },+ emitNotification(msg) {+ // A released client is back in the pool and must not dispatch.+ if (client.released) return+ for (const h of handlers.notification ?? []) h(msg)+ },+ }+ created.push(client)+ return client+ }+ return {+ created,+ connect,+ /** Live (checked-out, unreleased) sessions right now. */+ liveCount: () => created.filter(c => !c.released).length,+ current: () => created[created.length - 1],+ /** Simulate a real pg NOTIFY broadcast to EVERY live session. */+ broadcast(msg: { channel: string; payload?: string }) {+ for (const c of created) c.emitNotification(msg)+ },+ }+ }++ /** A resolver-queue sleep seam so backoff can be driven deterministically. */+ function resolverQueueSleep() {+ const resolvers: Array<() => void> = []+ let calls = 0+ const sleep = (_ms: number) =>+ new Promise<void>(resolve => {+ calls += 1+ resolvers.push(resolve)+ })+ return { sleep, resolvers, sleepCalls: () => calls }+ }++ const flush = async () => {+ for (let i = 0; i < 12; i += 1) await Promise.resolve()+ }++ const NOTIFY = {+ channel: PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL,+ payload: JSON.stringify({ recipeNamespace: 'sandbox-recipes', recipeName: 'research' }),+ }++ it('T1: holds ≤1 live session and exactly-once dispatch across 25 drop→reattach cycles with late superseded errors', async () => {+ const factory = makeFakeClientFactory()+ const { sleep, resolvers, sleepCalls } = resolverQueueSleep()+ const onGrantUpdate = vi.fn()+ const listener = createGrantUpdateListener({+ pool: {} as never,+ onGrantUpdate,+ logger: makeLogger(),+ connect: async () => factory.connect() as never,+ sleep,+ })++ // Drain every queued backoff sleep and let the reconnect attaches settle.+ const settle = async () => {+ await flush()+ while (resolvers.length > 0) {+ const resume = resolvers.shift() as () => void+ resume()+ await flush()+ }+ }++ await listener.start()+ expect(factory.liveCount()).toBe(1)++ const K = 25+ for (let i = 0; i < K; i += 1) {+ const dropped = factory.current()+ // 1) the current session drops → schedules a reconnect.+ dropped.emitError(new Error(`drop-${i}`))+ // 2) the reconnect completes → a new session becomes current.+ await settle()+ // 3) a LATE, second 'error' from the now-SUPERSEDED (already-released)+ // generation — the storm's signature interleaving. It must NOT drive a+ // second attach loop that orphans the current live session.+ dropped.emitError(new Error(`late-drop-${i}`))+ await settle()++ // The invariant #205 never asserted: at most ONE live session, ever.+ expect(factory.liveCount(), `cycle ${i}: live sessions`).toBeLessThanOrEqual(1)++ // A real pg NOTIFY reaches every live session; with no orphan it dispatches+ // exactly once (an orphaned session would double-dispatch).+ onGrantUpdate.mockClear()+ factory.broadcast(NOTIFY)+ expect(onGrantUpdate, `cycle ${i}: dispatch count`).toHaveBeenCalledTimes(1)+ }++ // No runaway parallel reconnect loops: ~one scheduled sleep per real drop.+ expect(sleepCalls()).toBeLessThanOrEqual(K + 2)++ await listener.stop()+ expect(factory.liveCount(), 'after stop(): no leaked sessions').toBe(0)+ })++ it('T2: one full reconnect cycle recovers notification flow', async () => {+ const factory = makeFakeClientFactory()+ const { sleep, resolvers } = resolverQueueSleep()+ const onGrantUpdate = vi.fn()+ const listener = createGrantUpdateListener({+ pool: {} as never,+ onGrantUpdate,+ logger: makeLogger(),+ connect: async () => factory.connect() as never,+ sleep,+ })++ await listener.start()+ const clientA = factory.current()+ expect(factory.liveCount()).toBe(1)++ // Drop A → resolve the backoff → a NEW client B attaches.+ clientA.emitError(new Error('drop'))+ await flush()+ expect(resolvers.length).toBe(1)+ ;(resolvers.shift() as () => void)()+ await flush()++ const clientB = factory.current()+ expect(clientB).not.toBe(clientA)+ expect(clientA.released).toBe(true)+ // B issued its own LISTEN and is the sole live session.+ expect(clientB.query).toHaveBeenCalledWith(+ expect.stringMatching(/LISTEN\s+plugin_workload_sdk_grant_update/)+ )+ expect(factory.liveCount()).toBe(1)++ // A NOTIFY on the recovered session dispatches exactly once.+ factory.broadcast(NOTIFY)+ expect(onGrantUpdate).toHaveBeenCalledTimes(1)+ expect(onGrantUpdate).toHaveBeenCalledWith({+ recipeNamespace: 'sandbox-recipes',+ recipeName: 'research',+ })++ await listener.stop()+ expect(factory.liveCount()).toBe(0)+ })+})+describe('createGrantUpdateListener — reconnect / stop resilience (issue #375 H1-H3)', () => {/** A sleep seam that never resolves, so we can count reconnect scheduling without real timers. */function pendingSleep() {diff --git a/workflow-recipes/src/reconciler/grantUpdateListener.ts b/workflow-recipes/src/reconciler/grantUpdateListener.tsindex f4a456d7f..b3aeb681c 100644--- a/workflow-recipes/src/reconciler/grantUpdateListener.ts+++ b/workflow-recipes/src/reconciler/grantUpdateListener.ts@@ -91,9 +91,14 @@ export function createGrantUpdateListener(opts: GrantUpdateListenerOptions): Gra// consumer has the same latent patterns this module hardens against —// (1) a LISTEN rejection AND an 'error' emission on the same drop can start two// reconnect loops, (2) stop() cannot cancel an attach already awaiting- // connect(), and (3) a LISTEN failure without an 'error' event leaks the- // checked-out client. Fixing dbRunProcessor is out of #375 scope; this listener- // intentionally improves on it and the guards below are the reference.+ // connect(), (3) a LISTEN failure without an 'error' event leaks the+ // checked-out client, and (4) a LATE error from a superseded generation drives+ // a reconnect that orphans the live session (duplicate dispatch + monotonic+ // pool leak under a sustained reconnect storm — the #205 blind-spot shape).+ // Fixing dbRunProcessor is out of #375 scope; this listener intentionally+ // improves on it and the guards below (esp. the identity gate on the 'error'+ // handler + the non-overwrite guard before assigning `listenClient`) are the+ // reference for the follow-up.let listenClient: PoolClient | null = nulllet stopped = true// H1: collapse concurrent reconnect triggers into ONE in-flight reconnect.@@ -132,13 +137,27 @@ export function createGrantUpdateListener(opts: GrantUpdateListenerOptions): Gra}})attached.on('error', (err: Error) => {- log.warn('listen session error — will reconnect', { err: err.message })try {attached.release(err)} catch {/* idempotent */}- if (listenClient === attached) listenClient = null+ // Finding 1 (#205-class): IDENTITY GATE. Only the CURRENT session's error+ // may drive a reconnect. A LATE error from a superseded generation — its+ // successor has already attached and become `listenClient` — must release+ // ONLY itself and return. If it scheduled a reconnect it would spawn a+ // second attach loop whose success overwrites the live session, orphaning+ // it: still checked out from the shared pool with a live LISTEN handler →+ // duplicate NOTIFY dispatch + one leaked connection per occurrence →+ // monotonic pool exhaustion under a reconnect storm (GKE regime).+ if (listenClient !== attached) {+ log.debug('superseded listen session error — released self, no reconnect', {+ err: err.message,+ })+ return+ }+ log.warn('listen session error — will reconnect', { err: err.message })+ listenClient = nullscheduleReconnect()})await attached.query(`LISTEN ${PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL}`)@@ -152,6 +171,17 @@ export function createGrantUpdateListener(opts: GrantUpdateListenerOptions): Gra}return}+ // Finding 1: never overwrite a live `listenClient` without releasing the+ // prior one. Defense-in-depth alongside the identity gate above — a stray+ // interleaving that reaches here with a different session still checked out+ // must not leak it.+ if (listenClient && listenClient !== attached) {+ try {+ listenClient.release()+ } catch {+ /* idempotent */+ }+ }listenClient = attachedlog.info('LISTEN plugin_workload_sdk_grant_update attached')} catch (err) {