Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(control-api): hold an error listener on the withTransaction client (R3-H3)

Diego Zacariasb5524e9f7946
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index 364f3ea20..4b100e852 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.658",
+ "version": "0.1.659",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.658",
+ "version": "0.1.659",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index 0adc48882..84b5488db 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.658",
+ "version": "0.1.659",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/db.ts b/control-api/src/db.ts
index ea547c73c..289053c39 100644
--- a/control-api/src/db.ts
+++ b/control-api/src/db.ts
@@ -6554,12 +6554,32 @@ export async function assertDbReady(db: DbClient = pool): Promise<void> {
}
export async function withTransaction<T>(
- work: (db: DbTransactionClient) => Promise<T>
+ work: (db: DbTransactionClient) => Promise<T>,
+ // Injectable only so the carrier's client lifecycle can be unit-tested against
+ // a fake Pool; production always uses the module-level core pool.
+ txPool: Pool = pool
): Promise<T> {
- const client = (await pool.connect()) as PoolClient
+ const client = (await txPool.connect()) as PoolClient
let transactionStarted = false
let commitSent = false
let releaseError: Error | boolean | undefined
+
+ // pg-pool detaches its own idle 'error' listener from a client while it is
+ // checked out, so an out-of-band backend error on the borrowed client (e.g.
+ // Postgres terminating the backend on idle_in_transaction_session_timeout,
+ // 25P03, while the transaction awaits an external POST with no query running)
+ // arrives with NO 'error' listener. Node's EventEmitter then THROWS in that
+ // tick — an async event outside any await stack, so no try/catch here or in a
+ // caller can catch it — and with no process-level handler it exits the process.
+ // The carrier owns the client for the whole checkout, so it holds the listener:
+ // capture the async error and fold it into releaseError so pg-pool DESTROYS the
+ // poisoned connection instead of returning it to the pool.
+ let asyncClientError: Error | undefined
+ const onClientError = (err: unknown): void => {
+ asyncClientError ??= err instanceof Error ? err : new Error(String(err))
+ }
+ client.on('error', onClientError)
+
try {
await client.query('BEGIN')
transactionStarted = true
@@ -6582,6 +6602,14 @@ export async function withTransaction<T>(
}
throw error
} finally {
+ client.removeListener('error', onClientError)
+ // A client that emitted 'error' mid-checkout is poisoned; destroy it. Keep an
+ // existing releaseError (the primary failure from work/commit/rollback) and
+ // adopt the async one only when the sync path saw none, so a truthy value
+ // always reaches release() and pg-pool never recycles the bad connection.
+ if (asyncClientError && !releaseError) {
+ releaseError = asyncClientError
+ }
client.release(releaseError)
}
}
diff --git a/control-api/test/db.withTransaction.test.ts b/control-api/test/db.withTransaction.test.ts
index 2af1d82c4..b541e2e9b 100644
--- a/control-api/test/db.withTransaction.test.ts
+++ b/control-api/test/db.withTransaction.test.ts
@@ -2,6 +2,10 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'
const clientQuery = vi.fn()
const clientRelease = vi.fn()
+// withTransaction attaches/detaches an 'error' listener on the borrowed client
+// for its whole checkout; the fake client must expose EventEmitter surface.
+const clientOn = vi.fn()
+const clientRemoveListener = vi.fn()
const mockConnect = vi.fn()
const mockPoolCtor = vi.fn(function MockPool() {
return {
@@ -21,6 +25,8 @@ describe('db.withTransaction', () => {
mockConnect.mockResolvedValue({
query: clientQuery,
release: clientRelease,
+ on: clientOn,
+ removeListener: clientRemoveListener,
})
clientQuery.mockResolvedValue({ rows: [], rowCount: 0 })
})
diff --git a/control-api/test/db.withTransactionAsyncClientError.test.ts b/control-api/test/db.withTransactionAsyncClientError.test.ts
new file mode 100644
index 000000000..f53929a80
--- /dev/null
+++ b/control-api/test/db.withTransactionAsyncClientError.test.ts
@@ -0,0 +1,98 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import { EventEmitter } from 'node:events'
+import type { Pool } from 'pg'
+import { withTransaction } from '../src/db.js'
+
+// R3-H3: the proactive OAuth-refresh carrier holds an open, idle transaction
+// across an external POST. If idle_in_transaction_session_timeout fires, Postgres
+// terminates the backend (25P03) and node-postgres emits an 'error' event on the
+// BORROWED client — for which pg-pool has removed its own listener while the
+// client is checked out. With no listener, Node's EventEmitter throws in that
+// tick, off any await stack, which the cron's try/catch cannot see → the process
+// exits. This test drives that exact out-of-band 'error' through a REAL
+// EventEmitter (the actual producer of the "emit 'error' with no listener throws"
+// semantics) so the assertion — not a setup crash — is what distinguishes the
+// pre-fix behavior (uncaught → process would exit) from the fix (captured, client
+// destroyed).
+
+type FakeClient = EventEmitter & {
+ query: ReturnType<typeof vi.fn>
+ release: ReturnType<typeof vi.fn>
+}
+
+function createFakeClient(): FakeClient {
+ const client = new EventEmitter() as FakeClient
+ client.query = vi.fn().mockResolvedValue({ rows: [], rowCount: 0 })
+ client.release = vi.fn()
+ return client
+}
+
+function createDeferred(): { promise: Promise<void>; resolve: () => void } {
+ let resolve!: () => void
+ const promise = new Promise<void>(res => {
+ resolve = res
+ })
+ return { promise, resolve }
+}
+
+const tick = (): Promise<void> => new Promise(res => setTimeout(res, 0))
+
+describe('db.withTransaction — async out-of-band client error (R3-H3)', () => {
+ const uncaught: unknown[] = []
+ // Record uncaughtException instead of letting it crash the worker: on the
+ // pre-fix code the emitted 'error' has no listener and surfaces here; the
+ // assertion below then fails. This handler must be installed so the observable
+ // difference is an assertion, not a nondeterministic worker abort.
+ const onUncaught = (err: unknown): void => {
+ uncaught.push(err)
+ }
+
+ beforeEach(() => {
+ uncaught.length = 0
+ process.on('uncaughtException', onUncaught)
+ })
+
+ afterEach(() => {
+ process.removeListener('uncaughtException', onUncaught)
+ vi.clearAllMocks()
+ })
+
+ it('captures the async client error, destroys the client, and never crashes the process', async () => {
+ const client = createFakeClient()
+ const fakePool = {
+ connect: vi.fn().mockResolvedValue(client),
+ } as unknown as Pool
+
+ const terminated = new Error('terminating connection due to idle-in-transaction timeout')
+ // Mimic 25P03 delivered by node-postgres on the checked-out client.
+ ;(terminated as Error & { code?: string }).code = '25P03'
+
+ const externalCall = createDeferred()
+
+ const resultPromise = withTransaction(async () => {
+ // The transaction is open and idle here (no query running), awaiting an
+ // external POST. Fire the backend error out-of-band on the next tick — on a
+ // stack independent of this await, exactly as node-postgres delivers 25P03.
+ setTimeout(() => client.emit('error', terminated), 0)
+ await externalCall.promise
+ return 'done'
+ }, fakePool)
+
+ // Let the out-of-band 'error' (delay 0) fire and be processed BEFORE we let
+ // work finish: a later delay guarantees the emit runs first, so on the pre-fix
+ // code it surfaces as an uncaughtException here while work is still parked.
+ await new Promise(res => setTimeout(res, 10))
+ externalCall.resolve()
+
+ const result = await resultPromise
+ // Give any pending uncaughtException a tick to surface.
+ await tick()
+
+ expect(result).toBe('done')
+ // Observable outcome (T4): no uncaught error escaped, and the poisoned client
+ // was released WITH an Error so pg-pool destroys it rather than recycling it.
+ expect(uncaught).toEqual([])
+ expect(client.release).toHaveBeenCalledTimes(1)
+ expect(client.release).toHaveBeenCalledWith(expect.any(Error))
+ })
+})