Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(control-api): hold an error listener on the withTransaction client (R3-H3)
Diego Zacariasb5524e9f7946
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex 364f3ea20..4b100e852 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.658",+ "version": "0.1.659","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.658",+ "version": "0.1.659","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex 0adc48882..84b5488db 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.658",+ "version": "0.1.659","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/db.ts b/control-api/src/db.tsindex ea547c73c..289053c39 100644--- a/control-api/src/db.ts+++ b/control-api/src/db.ts@@ -6554,12 +6554,32 @@ export async function assertDbReady(db: DbClient = pool): Promise<void> {}export async function withTransaction<T>(- work: (db: DbTransactionClient) => Promise<T>+ work: (db: DbTransactionClient) => Promise<T>,+ // Injectable only so the carrier's client lifecycle can be unit-tested against+ // a fake Pool; production always uses the module-level core pool.+ txPool: Pool = pool): Promise<T> {- const client = (await pool.connect()) as PoolClient+ const client = (await txPool.connect()) as PoolClientlet transactionStarted = falselet commitSent = falselet releaseError: Error | boolean | undefined++ // pg-pool detaches its own idle 'error' listener from a client while it is+ // checked out, so an out-of-band backend error on the borrowed client (e.g.+ // Postgres terminating the backend on idle_in_transaction_session_timeout,+ // 25P03, while the transaction awaits an external POST with no query running)+ // arrives with NO 'error' listener. Node's EventEmitter then THROWS in that+ // tick — an async event outside any await stack, so no try/catch here or in a+ // caller can catch it — and with no process-level handler it exits the process.+ // The carrier owns the client for the whole checkout, so it holds the listener:+ // capture the async error and fold it into releaseError so pg-pool DESTROYS the+ // poisoned connection instead of returning it to the pool.+ let asyncClientError: Error | undefined+ const onClientError = (err: unknown): void => {+ asyncClientError ??= err instanceof Error ? err : new Error(String(err))+ }+ client.on('error', onClientError)+try {await client.query('BEGIN')transactionStarted = true@@ -6582,6 +6602,14 @@ export async function withTransaction<T>(}throw error} finally {+ client.removeListener('error', onClientError)+ // A client that emitted 'error' mid-checkout is poisoned; destroy it. Keep an+ // existing releaseError (the primary failure from work/commit/rollback) and+ // adopt the async one only when the sync path saw none, so a truthy value+ // always reaches release() and pg-pool never recycles the bad connection.+ if (asyncClientError && !releaseError) {+ releaseError = asyncClientError+ }client.release(releaseError)}}diff --git a/control-api/test/db.withTransaction.test.ts b/control-api/test/db.withTransaction.test.tsindex 2af1d82c4..b541e2e9b 100644--- a/control-api/test/db.withTransaction.test.ts+++ b/control-api/test/db.withTransaction.test.ts@@ -2,6 +2,10 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'const clientQuery = vi.fn()const clientRelease = vi.fn()+// withTransaction attaches/detaches an 'error' listener on the borrowed client+// for its whole checkout; the fake client must expose EventEmitter surface.+const clientOn = vi.fn()+const clientRemoveListener = vi.fn()const mockConnect = vi.fn()const mockPoolCtor = vi.fn(function MockPool() {return {@@ -21,6 +25,8 @@ describe('db.withTransaction', () => {mockConnect.mockResolvedValue({query: clientQuery,release: clientRelease,+ on: clientOn,+ removeListener: clientRemoveListener,})clientQuery.mockResolvedValue({ rows: [], rowCount: 0 })})diff --git a/control-api/test/db.withTransactionAsyncClientError.test.ts b/control-api/test/db.withTransactionAsyncClientError.test.tsnew file mode 100644index 000000000..f53929a80--- /dev/null+++ b/control-api/test/db.withTransactionAsyncClientError.test.ts@@ -0,0 +1,98 @@+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'+import { EventEmitter } from 'node:events'+import type { Pool } from 'pg'+import { withTransaction } from '../src/db.js'++// R3-H3: the proactive OAuth-refresh carrier holds an open, idle transaction+// across an external POST. If idle_in_transaction_session_timeout fires, Postgres+// terminates the backend (25P03) and node-postgres emits an 'error' event on the+// BORROWED client — for which pg-pool has removed its own listener while the+// client is checked out. With no listener, Node's EventEmitter throws in that+// tick, off any await stack, which the cron's try/catch cannot see → the process+// exits. This test drives that exact out-of-band 'error' through a REAL+// EventEmitter (the actual producer of the "emit 'error' with no listener throws"+// semantics) so the assertion — not a setup crash — is what distinguishes the+// pre-fix behavior (uncaught → process would exit) from the fix (captured, client+// destroyed).++type FakeClient = EventEmitter & {+ query: ReturnType<typeof vi.fn>+ release: ReturnType<typeof vi.fn>+}++function createFakeClient(): FakeClient {+ const client = new EventEmitter() as FakeClient+ client.query = vi.fn().mockResolvedValue({ rows: [], rowCount: 0 })+ client.release = vi.fn()+ return client+}++function createDeferred(): { promise: Promise<void>; resolve: () => void } {+ let resolve!: () => void+ const promise = new Promise<void>(res => {+ resolve = res+ })+ return { promise, resolve }+}++const tick = (): Promise<void> => new Promise(res => setTimeout(res, 0))++describe('db.withTransaction — async out-of-band client error (R3-H3)', () => {+ const uncaught: unknown[] = []+ // Record uncaughtException instead of letting it crash the worker: on the+ // pre-fix code the emitted 'error' has no listener and surfaces here; the+ // assertion below then fails. This handler must be installed so the observable+ // difference is an assertion, not a nondeterministic worker abort.+ const onUncaught = (err: unknown): void => {+ uncaught.push(err)+ }++ beforeEach(() => {+ uncaught.length = 0+ process.on('uncaughtException', onUncaught)+ })++ afterEach(() => {+ process.removeListener('uncaughtException', onUncaught)+ vi.clearAllMocks()+ })++ it('captures the async client error, destroys the client, and never crashes the process', async () => {+ const client = createFakeClient()+ const fakePool = {+ connect: vi.fn().mockResolvedValue(client),+ } as unknown as Pool++ const terminated = new Error('terminating connection due to idle-in-transaction timeout')+ // Mimic 25P03 delivered by node-postgres on the checked-out client.+ ;(terminated as Error & { code?: string }).code = '25P03'++ const externalCall = createDeferred()++ const resultPromise = withTransaction(async () => {+ // The transaction is open and idle here (no query running), awaiting an+ // external POST. Fire the backend error out-of-band on the next tick — on a+ // stack independent of this await, exactly as node-postgres delivers 25P03.+ setTimeout(() => client.emit('error', terminated), 0)+ await externalCall.promise+ return 'done'+ }, fakePool)++ // Let the out-of-band 'error' (delay 0) fire and be processed BEFORE we let+ // work finish: a later delay guarantees the emit runs first, so on the pre-fix+ // code it surfaces as an uncaughtException here while work is still parked.+ await new Promise(res => setTimeout(res, 10))+ externalCall.resolve()++ const result = await resultPromise+ // Give any pending uncaughtException a tick to surface.+ await tick()++ expect(result).toBe('done')+ // Observable outcome (T4): no uncaught error escaped, and the poisoned client+ // was released WITH an Error so pg-pool destroys it rather than recycling it.+ expect(uncaught).toEqual([])+ expect(client.release).toHaveBeenCalledTimes(1)+ expect(client.release).toHaveBeenCalledWith(expect.any(Error))+ })+})