Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(control-api): bound abandoned transaction sessions

Alfredo Lopez2842a38018b7
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index a74a93fe0..b87a81497 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.705",
+ "version": "0.1.706",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.705",
+ "version": "0.1.706",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index 13c95663f..ff1079edf 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.705",
+ "version": "0.1.706",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/db.ts b/control-api/src/db.ts
index 3a04e3fb8..ae81db69a 100644
--- a/control-api/src/db.ts
+++ b/control-api/src/db.ts
@@ -6728,6 +6728,8 @@ export async function withTransaction<T>(
}
let transactionStarted = false
let commitSent = false
+ let transactionFinished = false
+ let originalIdleTimeout: string | undefined
let releaseError: Error | boolean | undefined
// pg-pool detaches its own idle 'error' listener from a client while it is
@@ -6747,9 +6749,9 @@ export async function withTransaction<T>(
client.on('error', onClientError)
let termination: Promise<void> | undefined
- const onAbort = (): void => {
+ const terminateClient = (reason: Error | boolean): void => {
+ releaseError ??= reason
if (termination) return
- releaseError = signal?.reason instanceof Error ? signal.reason : true
termination = client.end().catch(error => {
releaseError = error instanceof Error ? error : true
})
@@ -6761,11 +6763,58 @@ export async function withTransaction<T>(
// finish the caller ahead of that event.
client.connection.stream.destroy()
}
+ const onAbort = (): void => {
+ terminateClient(signal?.reason instanceof Error ? signal.reason : true)
+ }
signal?.addEventListener('abort', onAbort, { once: true })
if (signal?.aborted) onAbort()
try {
signal?.throwIfAborted()
+ if (signal) {
+ // Read normalized milliseconds from this backend, not from client startup
+ // options: a role/database default or a reused session can be different.
+ const settings = await client.query<{
+ statement_timeout_ms: string | null
+ idle_timeout_ms: string | null
+ }>(`SELECT
+ (SELECT setting FROM pg_settings WHERE name = 'statement_timeout') AS statement_timeout_ms,
+ (SELECT setting FROM pg_settings WHERE name = 'idle_in_transaction_session_timeout') AS idle_timeout_ms`)
+ signal.throwIfAborted()
+ const setting = settings.rows[0]
+ const statementTimeout = setting?.statement_timeout_ms
+ const idleTimeout = setting?.idle_timeout_ms
+ const statementMs = Number(statementTimeout)
+ const idleMs = Number(idleTimeout)
+ // These are the existing createBoundedPgPoolForConnection startup bounds.
+ // An unbounded or invalid effective server setting cannot supply a finite
+ // remote cleanup backstop, so this cancellable transaction must fail loud.
+ if (
+ typeof statementTimeout !== 'string' ||
+ !/^\d+$/.test(statementTimeout) ||
+ !Number.isInteger(statementMs) ||
+ statementMs < 100 ||
+ statementMs > 30_000 ||
+ typeof idleTimeout !== 'string' ||
+ !/^\d+$/.test(idleTimeout) ||
+ !Number.isSafeInteger(idleMs)
+ ) {
+ throw new Error('Invalid effective PostgreSQL cancellation timeout bounds')
+ }
+ if (idleMs === 0 || idleMs > statementMs) {
+ // This must be SESSION scope before BEGIN. AbortTransaction restores a
+ // SET LOCAL value before entering idle-in-transaction (aborted), which
+ // would remove the very backstop needed when a blackhole hides our FIN.
+ // Record the original before sending: a missing set_config reply is also
+ // uncertain, and that checkout must be ended rather than returned healthy.
+ originalIdleTimeout = idleTimeout
+ await client.query('SELECT set_config($1, $2, false)', [
+ 'idle_in_transaction_session_timeout',
+ statementTimeout,
+ ])
+ signal.throwIfAborted()
+ }
+ }
await client.query('BEGIN')
transactionStarted = true
if (signal) {
@@ -6795,6 +6844,7 @@ export async function withTransaction<T>(
signal?.throwIfAborted()
commitSent = true
await client.query('COMMIT')
+ transactionFinished = true
signal?.throwIfAborted()
return result
} catch (error) {
@@ -6803,6 +6853,7 @@ export async function withTransaction<T>(
} else {
try {
await client.query('ROLLBACK')
+ transactionFinished = true
} catch (rollbackError) {
releaseError = rollbackError instanceof Error ? rollbackError : true
}
@@ -6813,6 +6864,36 @@ export async function withTransaction<T>(
throw error
} finally {
// Work (including its own finally) and any rollback have already unwound.
+ // Only an acknowledged, healthy transaction may restore session state and
+ // return to the pool. Lost COMMIT/ROLLBACK replies leave the server backstop
+ // installed; local transport completion is not remote cleanup observation.
+ if (
+ originalIdleTimeout !== undefined &&
+ transactionFinished &&
+ !releaseError &&
+ !asyncClientError &&
+ !signal?.aborted
+ ) {
+ try {
+ await client.query('SELECT set_config($1, $2, false)', [
+ 'idle_in_transaction_session_timeout',
+ originalIdleTimeout,
+ ])
+ } catch (restoreError) {
+ // Do not replace the work outcome or relabel an acknowledged COMMIT as
+ // a rollback. A session that failed restoration is physically ended.
+ releaseError = restoreError instanceof Error ? restoreError : true
+ }
+ }
+ if (signal && (releaseError || asyncClientError || signal.aborted)) {
+ terminateClient(
+ signal.aborted
+ ? signal.reason instanceof Error
+ ? signal.reason
+ : true
+ : (releaseError ?? asyncClientError ?? true)
+ )
+ }
// Keep both listeners and the checkout until physical termination completes.
if (termination) await termination
signal?.removeEventListener('abort', onAbort)
diff --git a/control-api/test/helpers.realPostgresCancellation.test.ts b/control-api/test/helpers.realPostgresCancellation.test.ts
index a59539726..fa265f4de 100644
--- a/control-api/test/helpers.realPostgresCancellation.test.ts
+++ b/control-api/test/helpers.realPostgresCancellation.test.ts
@@ -2,7 +2,10 @@ import { expect, it } from 'vitest'
import { once } from 'node:events'
import { type Socket, createConnection, createServer } from 'node:net'
import { setImmediate as nextTurn } from 'node:timers/promises'
-import { createPostgresCommitReplyBlackhole } from './helpers/realPostgresCancellation.js'
+import {
+ createPostgresCommitReplyBlackhole,
+ createPostgresDisconnectBlackhole,
+} from './helpers/realPostgresCancellation.js'
async function observed(check: () => boolean): Promise<void> {
const deadline = Date.now() + 1_000
@@ -90,3 +93,62 @@ it('refuses an encrypted protocol prerequisite instead of pretending to observe
)
).rejects.toThrow('plaintext PostgreSQL connection')
})
+
+it('hides frontend Terminate and FIN as well as replies without closing the backend socket', async () => {
+ const sockets = new Set<Socket>()
+ let backendSocket: Socket | undefined
+ let received = Buffer.alloc(0)
+ let ended = false
+ const backend = createServer({ allowHalfOpen: true }, socket => {
+ backendSocket = socket
+ sockets.add(socket)
+ socket.on('data', chunk => {
+ received = Buffer.concat([received, chunk])
+ socket.write('visible-reply')
+ })
+ socket.on('end', () => {
+ ended = true
+ })
+ socket.on('error', () => {})
+ })
+ backend.listen(0, '127.0.0.1')
+ await once(backend, 'listening')
+ const address = backend.address()
+ if (!address || typeof address === 'string') throw new Error('missing fixture backend address')
+ const proxy = await createPostgresDisconnectBlackhole(
+ ['postgresql:', `//[email protected]:${address.port}/fixture`].join('')
+ )
+ const target = new URL(proxy.connectionString)
+ const client = createConnection({ host: target.hostname, port: Number(target.port) })
+ let replies = Buffer.alloc(0)
+ client.on('data', chunk => {
+ replies = Buffer.concat([replies, chunk])
+ })
+ client.on('error', () => {})
+ try {
+ await once(client, 'connect')
+ const startup = Buffer.alloc(8)
+ startup.writeInt32BE(8)
+ startup.writeInt32BE(196608, 4)
+ client.write(startup)
+ await observed(() => replies.toString() === 'visible-reply')
+ proxy.dropRepliesAndDisconnects()
+ const terminate = Buffer.from([88, 0, 0, 0, 4])
+ client.end(terminate)
+ await observed(() => proxy.suppressedDisconnects === 1)
+ expect(proxy.discardedFrontendBytes).toBe(terminate.length)
+ expect(received).toEqual(startup)
+ expect(ended).toBe(false)
+ expect(backendSocket?.destroyed).toBe(false)
+ backendSocket!.write('hidden-reply')
+ await observed(() => proxy.discardedReplyBytes === Buffer.byteLength('hidden-reply'))
+ expect(replies.toString()).toBe('visible-reply')
+ } finally {
+ client.destroy()
+ await proxy.close()
+ for (const socket of sockets) socket.destroy()
+ await new Promise<void>((resolve, reject) =>
+ backend.close(error => (error ? reject(error) : resolve()))
+ )
+ }
+})
diff --git a/control-api/test/helpers/realPostgresCancellation.ts b/control-api/test/helpers/realPostgresCancellation.ts
index a2aa3c6cf..98bc625be 100644
--- a/control-api/test/helpers/realPostgresCancellation.ts
+++ b/control-api/test/helpers/realPostgresCancellation.ts
@@ -3,6 +3,15 @@ import { type Socket, createConnection, createServer } from 'node:net'
/** Observe a real COMMIT while discarding its reply on the owned local PG lane. */
export async function createPostgresCommitReplyBlackhole(connectionString: string) {
+ return createPostgresProtocolBlackhole(connectionString, true)
+}
+
+/** Hide replies and client disconnects without ending the real backend socket. */
+export async function createPostgresDisconnectBlackhole(connectionString: string) {
+ return createPostgresProtocolBlackhole(connectionString, false)
+}
+
+async function createPostgresProtocolBlackhole(connectionString: string, dropCommitReply: boolean) {
const target = new URL(connectionString)
const sslMode = target.searchParams.get('sslmode')
if (sslMode && sslMode !== 'disable') {
@@ -14,23 +23,39 @@ export async function createPostgresCommitReplyBlackhole(connectionString: strin
const commitForwarded = new Promise<void>(resolve => {
commit = resolve
})
- let armed = true
+ let armed = dropCommitReply
let dropping = false
let discardedReplyBytes = 0
- const server = createServer(front => {
+ let discardedFrontendBytes = 0
+ let suppressDisconnects = false
+ let suppressedDisconnects = 0
+ const server = createServer({ allowHalfOpen: true }, front => {
const host = target.hostname.replace(/^\[|\]$/g, '')
- const back = createConnection({ host, port: Number(target.port || 5432) })
+ const back = createConnection({ host, port: Number(target.port || 5432), allowHalfOpen: true })
for (const socket of [front, back]) {
sockets.add(socket)
socket.once('close', () => sockets.delete(socket))
}
- front.on('error', () => back.destroy())
+ front.on('error', () => {
+ if (suppressDisconnects) suppressedDisconnects++
+ else back.destroy()
+ })
back.on('error', () => front.destroy())
- front.on('end', () => back.end())
+ front.on('end', () => {
+ if (suppressDisconnects) suppressedDisconnects++
+ else back.end()
+ })
back.on('end', () => front.end())
let startup = true
let buffered = Buffer.alloc(0)
front.on('data', chunk => {
+ // pg may send Terminate before FIN on an idle checkout. A network fault
+ // that hides disconnects must suppress both; forwarding either would let
+ // the test pass because of EOF instead of the server timeout backstop.
+ if (suppressDisconnects) {
+ discardedFrontendBytes += chunk.length
+ return
+ }
buffered = Buffer.concat([buffered, chunk])
if (startup && buffered.length >= 4 && buffered.length >= buffered.readInt32BE(0)) {
buffered = buffered.subarray(buffered.readInt32BE(0))
@@ -70,8 +95,20 @@ export async function createPostgresCommitReplyBlackhole(connectionString: strin
get discardedReplyBytes() {
return discardedReplyBytes
},
+ get discardedFrontendBytes() {
+ return discardedFrontendBytes
+ },
+ get suppressedDisconnects() {
+ return suppressedDisconnects
+ },
+ dropRepliesAndDisconnects: () => {
+ armed = false
+ dropping = true
+ suppressDisconnects = true
+ },
allowReplies: () => {
dropping = false
+ suppressDisconnects = false
},
close: async () => {
for (const socket of sockets) socket.destroy()
diff --git a/control-api/test/llmProviderAttemptAuthorizer.cancellation.realPostgres.integration.test.ts b/control-api/test/llmProviderAttemptAuthorizer.cancellation.realPostgres.integration.test.ts
index acc57ea8c..1db5d7fd5 100644
--- a/control-api/test/llmProviderAttemptAuthorizer.cancellation.realPostgres.integration.test.ts
+++ b/control-api/test/llmProviderAttemptAuthorizer.cancellation.realPostgres.integration.test.ts
@@ -6,7 +6,10 @@ import { Pool } from 'pg'
import { type DbClient, initDb, withTransaction } from '../src/db.js'
import { reserveInDangerZone } from '../src/services/budgets/reservations.js'
import { insertLlmProviderAttempt } from '../src/services/llmProviderAttemptStore.js'
-import { createPostgresCommitReplyBlackhole } from './helpers/realPostgresCancellation.js'
+import {
+ createPostgresCommitReplyBlackhole,
+ createPostgresDisconnectBlackhole,
+} from './helpers/realPostgresCancellation.js'
import { endPoolAndWaitForClients } from './helpers/realPostgresTeardown.js'
const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL
@@ -245,4 +248,101 @@ realPostgres('scoped transaction cancellation mechanism and durable outcome', ()
await proxy.close()
}
}, 10_000)
+ it.each(['idle', 'active-then-aborted'] as const)(
+ 'bounds the remote %s transaction after local cleanup when a protocol blackhole hides both replies and disconnects',
+ async mode => {
+ const proxy = await createPostgresDisconnectBlackhole(connectionString)
+ const isolated = new Pool({
+ connectionString: proxy.connectionString,
+ ssl: false,
+ max: 1,
+ connectionTimeoutMillis: 2_000,
+ statement_timeout: 2_000,
+ options: '-c idle_in_transaction_session_timeout=0',
+ })
+ const invocationId = randomUUID()
+ const lock = mode === 'idle' ? 806303 : 806304
+ const entered = deferred<number>()
+ const controller = new AbortController()
+ const reason = Object.assign(new Error('authorize_timeout'), { code: 'authorize_timeout' })
+ try {
+ const outcome = withTransaction(
+ async db => {
+ const identity = await db.query('SELECT pg_backend_pid() AS pid')
+ const setting = await db.query(
+ "SELECT setting FROM pg_settings WHERE name = 'idle_in_transaction_session_timeout'"
+ )
+ expect((setting.rows[0] as { setting: string }).setting).toBe('2000')
+ await writeAuthorizeRows(db, invocationId)
+ await db.query('SELECT pg_advisory_xact_lock($1)', [lock])
+ entered.resolve((identity.rows[0] as { pid: number }).pid)
+ if (mode === 'active-then-aborted') await db.query('SELECT pg_sleep(30)')
+ else await once(controller.signal, 'abort')
+ },
+ isolated,
+ { signal: controller.signal }
+ ).then(
+ value => ({ value }),
+ error => ({ error })
+ )
+ const pid = await Promise.race([
+ entered.promise,
+ outcome.then(early => {
+ if ('error' in early) throw early.error
+ throw new Error('transaction ended before the physical fixture was ready')
+ }),
+ ])
+ const backendState = async () => {
+ const result = await observer.query<{ state: string; query: string }>(
+ 'SELECT state, query FROM pg_stat_activity WHERE pid = $1 AND datname = $2',
+ [pid, database]
+ )
+ return result.rows[0]
+ }
+ await observe(async () => {
+ const backend = await backendState()
+ return mode === 'idle'
+ ? backend?.state === 'idle in transaction'
+ : backend?.state === 'active' && backend.query === 'SELECT pg_sleep(30)'
+ })
+ proxy.dropRepliesAndDisconnects()
+ controller.abort(reason)
+ expect(((await outcome) as { error: unknown }).error).toBe(reason)
+ expect(isolated.totalCount).toBe(0)
+ await observe(async () => proxy.suppressedDisconnects > 0)
+ // Local completion cannot establish remote cleanup. The backend still
+ // exists with its socket held by the fault proxy, independently observed.
+ expect(await backendState()).toBeDefined()
+ if (mode === 'active-then-aborted') {
+ // The server statement timeout first enters TRANS_ABORT. A SET LOCAL
+ // idle limit would have reverted to zero here and leave this backend
+ // alive indefinitely; the pre-BEGIN session backstop must remain.
+ await observe(
+ async () => (await backendState())?.state === 'idle in transaction (aborted)',
+ 4_000
+ )
+ expect(proxy.discardedReplyBytes).toBeGreaterThan(0)
+ }
+ await observe(async () => !(await backendState()), 6_000)
+ await lockAvailable(lock)
+ expect(await rowCounts(invocationId)).toEqual({ attempts: 0, tickets: 0, reservations: 0 })
+ proxy.allowReplies()
+
+ // With max one, a fresh physical backend still serves and commits the
+ // real writer after both the local checkout and remote backend ended.
+ const later = randomUUID()
+ await withTransaction(db => writeAuthorizeRows(db, later), isolated, {
+ signal: new AbortController().signal,
+ })
+ expect(await rowCounts(later)).toEqual({ attempts: 1, tickets: 1, reservations: 1 })
+ await pool.query('DELETE FROM budget_pending_reservations WHERE task_ref = $1', [later])
+ } finally {
+ controller.abort(reason)
+ proxy.allowReplies()
+ await endPoolAndWaitForClients(isolated)
+ await proxy.close()
+ }
+ },
+ 20_000
+ )
})
diff --git a/control-api/test/llmProviderAttemptAuthorizer.cancellation.test.ts b/control-api/test/llmProviderAttemptAuthorizer.cancellation.test.ts
index 55b72e154..5d5290759 100644
--- a/control-api/test/llmProviderAttemptAuthorizer.cancellation.test.ts
+++ b/control-api/test/llmProviderAttemptAuthorizer.cancellation.test.ts
@@ -309,12 +309,43 @@ function protocolMessage(type: string, body: Buffer): Buffer {
return Buffer.concat([header, body])
}
-async function postgresPeer(handshake = true) {
+function textRows(columns: string[], values: Array<string | null>): Buffer[] {
+ const count = Buffer.alloc(2)
+ count.writeInt16BE(columns.length)
+ const fields = columns.map(name => {
+ const description = Buffer.alloc(18)
+ description.writeInt32BE(25, 6)
+ description.writeInt16BE(-1, 10)
+ description.writeInt32BE(-1, 12)
+ return Buffer.concat([Buffer.from(`${name}\0`), description])
+ })
+ const row = values.map(value => {
+ const bytes = value === null ? undefined : Buffer.from(value)
+ const length = Buffer.alloc(4)
+ length.writeInt32BE(bytes?.length ?? -1)
+ return bytes ? Buffer.concat([length, bytes]) : length
+ })
+ return [
+ protocolMessage('T', Buffer.concat([count, ...fields])),
+ protocolMessage('D', Buffer.concat([count, ...row])),
+ ]
+}
+
+async function postgresPeer(
+ handshake = true,
+ settings: { statement?: string | null; idle?: string | null } = {}
+) {
const connected = deferred()
const queries: string[] = []
+ const sessionChanges: Array<{ sql: string; values: Array<string | null> }> = []
+ const lifecycle: string[] = []
const sockets = new Set<Socket>()
+ const stalledReplies: Array<() => void> = []
let stalledSql = ''
+ let stalledOccurrence = 1
let failedSql = ''
+ let failedOccurrence = 1
+ const occurrences = new Map<string, number>()
const server = createTcpServer({ allowHalfOpen: true }, socket => {
sockets.add(socket)
socket.on('error', () => {})
@@ -322,6 +353,60 @@ async function postgresPeer(handshake = true) {
connected.resolve()
let startup = true
let buffered = Buffer.alloc(0)
+ let statement = ''
+ let parameters: Array<string | null> = []
+ let state = 'I'
+ let waiting = false
+ let pendingSync = false
+ let idleTimeout = settings.idle === undefined ? '0' : settings.idle
+ const statementTimeout = settings.statement === undefined ? '15000' : settings.statement
+ const ready = () => socket.write(protocolMessage('Z', Buffer.from(state)))
+ const describe = (sql: string): [string[], Array<string | null>] => {
+ if (sql.includes('FROM pg_settings')) {
+ return [
+ ['statement_timeout_ms', 'idle_timeout_ms'],
+ [statementTimeout, idleTimeout],
+ ]
+ }
+ if (sql === 'SELECT set_config($1, $2, false)') return [['set_config'], [parameters[1]]]
+ return [[], []]
+ }
+ const execute = (sql: string, simple: boolean) => {
+ queries.push(sql)
+ const occurrence = (occurrences.get(sql) ?? 0) + 1
+ occurrences.set(sql, occurrence)
+ if (sql === 'SELECT set_config($1, $2, false)') {
+ sessionChanges.push({ sql, values: [...parameters] })
+ }
+ const complete = () => {
+ if (sql === failedSql && occurrence === failedOccurrence) {
+ if (state === 'T') state = 'E'
+ socket.write(
+ protocolMessage('E', Buffer.from('SERROR\0C42704\0Mfixture statement failure\0\0'))
+ )
+ } else {
+ if (sql === 'BEGIN') state = 'T'
+ if (sql === 'ROLLBACK' || sql === 'COMMIT') state = 'I'
+ if (sql === 'SELECT set_config($1, $2, false)') idleTimeout = parameters[1]
+ const [columns, values] = describe(sql)
+ if (columns.length) {
+ const [description, row] = textRows(columns, values)
+ if (simple) socket.write(description)
+ socket.write(row)
+ }
+ socket.write(protocolMessage('C', Buffer.from(`${sql.split(' ')[0]}\0`)))
+ }
+ waiting = false
+ if (simple || pendingSync) {
+ pendingSync = false
+ ready()
+ }
+ }
+ if (sql === stalledSql && occurrence === stalledOccurrence) {
+ waiting = true
+ stalledReplies.push(complete)
+ } else complete()
+ }
socket.on('data', chunk => {
buffered = Buffer.concat([buffered, chunk])
if (startup) {
@@ -341,28 +426,38 @@ async function postgresPeer(handshake = true) {
const length = buffered.readInt32BE(1)
const body = buffered.subarray(5, length + 1)
buffered = buffered.subarray(length + 1)
- if (type !== 'Q') continue
- const sql = body.toString('utf8', 0, body.length - 1)
- queries.push(sql)
- if (sql === failedSql) {
+ if (type === 'Q') {
+ parameters = []
+ execute(body.toString('utf8', 0, body.length - 1), true)
+ } else if (type === 'P') {
+ const start = body.indexOf(0) + 1
+ statement = body.toString('utf8', start, body.indexOf(0, start))
+ socket.write(protocolMessage('1', Buffer.alloc(0)))
+ } else if (type === 'B') {
+ let offset = body.indexOf(0) + 1
+ offset = body.indexOf(0, offset) + 1
+ const formats = body.readInt16BE(offset)
+ offset += 2 + formats * 2
+ const count = body.readInt16BE(offset)
+ offset += 2
+ parameters = []
+ for (let index = 0; index < count; index++) {
+ const size = body.readInt32BE(offset)
+ offset += 4
+ parameters.push(size < 0 ? null : body.toString('utf8', offset, offset + size))
+ if (size >= 0) offset += size
+ }
+ socket.write(protocolMessage('2', Buffer.alloc(0)))
+ } else if (type === 'D') {
+ const [columns, values] = describe(statement)
socket.write(
- Buffer.concat([
- protocolMessage(
- 'E',
- Buffer.from('SERROR\0C42704\0Munsupported connection check parameter\0\0')
- ),
- protocolMessage('Z', Buffer.from('E')),
- ])
+ columns.length ? textRows(columns, values)[0] : protocolMessage('n', Buffer.alloc(0))
)
- continue
+ } else if (type === 'E') execute(statement, false)
+ else if (type === 'S') {
+ if (waiting) pendingSync = true
+ else ready()
}
- if (sql === stalledSql) continue
- socket.write(
- Buffer.concat([
- protocolMessage('C', Buffer.from(`${sql.split(' ')[0]}\0`)),
- protocolMessage('Z', Buffer.from(sql === 'BEGIN' ? 'T' : 'I')),
- ])
- )
}
})
// This deliberate half-open peer never sends EOF in response to Terminate.
@@ -382,6 +477,8 @@ async function postgresPeer(handshake = true) {
idleTimeoutMillis: 1_000,
})
pool.on('error', () => {})
+ pool.on('connect', client => client.once('end', () => lifecycle.push('end')))
+ pool.on('remove', () => lifecycle.push('remove'))
cleanup.push(async () => {
for (const socket of sockets) socket.destroy()
await pool.end()
@@ -392,12 +489,19 @@ async function postgresPeer(handshake = true) {
return {
connected,
queries,
+ sessionChanges,
+ lifecycle,
pool,
- stall: (sql: string) => {
+ stall: (sql: string, occurrence = 1) => {
stalledSql = sql
+ stalledOccurrence = occurrence
+ },
+ resume: () => {
+ for (const reply of stalledReplies.splice(0)) reply()
},
- fail: (sql: string) => {
+ fail: (sql: string, occurrence = 1) => {
failedSql = sql
+ failedOccurrence = occurrence
},
}
}
@@ -425,9 +529,12 @@ describe('scoped transaction cancellation with the native pg driver and sockets'
).rejects.toMatchObject({ code: '42704' })
expect(work).not.toHaveBeenCalled()
expect(fixture.queries).toEqual([
+ expect.stringContaining('FROM pg_settings'),
+ 'SELECT set_config($1, $2, false)',
'BEGIN',
"SET LOCAL client_connection_check_interval = '100ms'",
'ROLLBACK',
+ 'SELECT set_config($1, $2, false)',
])
})
@@ -586,6 +693,8 @@ describe('scoped transaction cancellation with the native pg driver and sockets'
controller.abort(reason)
expect(((await within(outcome)) as { error: unknown }).error).toBe(reason)
expect(fixture.queries).toEqual([
+ expect.stringContaining('FROM pg_settings'),
+ 'SELECT set_config($1, $2, false)',
'BEGIN',
"SET LOCAL client_connection_check_interval = '100ms'",
'COMMIT',
@@ -593,3 +702,162 @@ describe('scoped transaction cancellation with the native pg driver and sockets'
expect(fixture.pool.totalCount).toBe(0)
})
})
+
+describe('cancellable transaction server idle backstop with native pool max one', () => {
+ const settingSql = 'SELECT set_config($1, $2, false)'
+
+ it.each(['0', '45000'])(
+ 'bounds an effective idle timeout of %s before BEGIN and restores it after acknowledged COMMIT',
+ async idle => {
+ const fixture = await postgresPeer(true, { idle })
+ await expect(
+ withTransaction(async () => 'committed', fixture.pool, {
+ signal: new AbortController().signal,
+ })
+ ).resolves.toBe('committed')
+ expect(fixture.sessionChanges).toEqual([
+ { sql: settingSql, values: ['idle_in_transaction_session_timeout', '15000'] },
+ { sql: settingSql, values: ['idle_in_transaction_session_timeout', idle] },
+ ])
+ expect(fixture.queries.indexOf(settingSql)).toBeLessThan(fixture.queries.indexOf('BEGIN'))
+ expect(fixture.queries.lastIndexOf(settingSql)).toBeGreaterThan(
+ fixture.queries.indexOf('COMMIT')
+ )
+ expect(fixture.pool.totalCount).toBe(1)
+ expect(fixture.pool.idleCount).toBe(1)
+ const inspected = await fixture.pool.query(
+ "SELECT setting FROM pg_settings WHERE name = 'idle_in_transaction_session_timeout'"
+ )
+ expect(inspected.rows[0].idle_timeout_ms).toBe(idle)
+ }
+ )
+
+ it('preserves a smaller positive server idle limit without changing the session', async () => {
+ const fixture = await postgresPeer(true, { idle: '300' })
+ await expect(
+ withTransaction(async () => 'committed', fixture.pool, {
+ signal: new AbortController().signal,
+ })
+ ).resolves.toBe('committed')
+ expect(fixture.sessionChanges).toEqual([])
+ expect(fixture.pool.idleCount).toBe(1)
+ })
+
+ it.each(['0', '99', '30001', 'not-a-number', null])(
+ 'fails before BEGIN for effective statement timeout %s and physically evicts the checkout',
+ async statement => {
+ const fixture = await postgresPeer(true, { statement })
+ const work = vi.fn()
+ const removed = vi.fn()
+ fixture.pool.on('remove', removed)
+ await expect(
+ withTransaction(work, fixture.pool, { signal: new AbortController().signal })
+ ).rejects.toThrow('Invalid effective PostgreSQL cancellation timeout bounds')
+ expect(work).not.toHaveBeenCalled()
+ expect(fixture.queries).toHaveLength(1)
+ expect(fixture.queries).not.toContain('BEGIN')
+ expect(fixture.pool.totalCount).toBe(0)
+ expect(removed).toHaveBeenCalledOnce()
+ expect(fixture.lifecycle).toEqual(['end', 'remove'])
+ }
+ )
+
+ it('restores the session after acknowledged ROLLBACK without replacing the primary work error', async () => {
+ const fixture = await postgresPeer()
+ const primary = new Error('work failed')
+ await expect(
+ withTransaction(
+ async () => {
+ throw primary
+ },
+ fixture.pool,
+ { signal: new AbortController().signal }
+ )
+ ).rejects.toBe(primary)
+ expect(fixture.queries.indexOf('ROLLBACK')).toBeLessThan(
+ fixture.queries.lastIndexOf(settingSql)
+ )
+ expect(fixture.sessionChanges.at(-1)?.values).toEqual([
+ 'idle_in_transaction_session_timeout',
+ '0',
+ ])
+ expect(fixture.pool.idleCount).toBe(1)
+ })
+
+ it('poisons and physically ends a failed session setup without running work', async () => {
+ const fixture = await postgresPeer()
+ fixture.fail(settingSql)
+ const work = vi.fn()
+ const removed = vi.fn()
+ fixture.pool.on('remove', removed)
+ await expect(
+ withTransaction(work, fixture.pool, { signal: new AbortController().signal })
+ ).rejects.toMatchObject({ code: '42704' })
+ expect(work).not.toHaveBeenCalled()
+ expect(fixture.queries).not.toContain('BEGIN')
+ expect(fixture.sessionChanges).toHaveLength(1)
+ expect(fixture.pool.totalCount).toBe(0)
+ expect(removed).toHaveBeenCalledOnce()
+ expect(fixture.lifecycle).toEqual(['end', 'remove'])
+ })
+
+ it('evicts a failed restoration while preserving the acknowledged COMMIT outcome', async () => {
+ const fixture = await postgresPeer()
+ fixture.fail(settingSql, 2)
+ const removed = vi.fn()
+ fixture.pool.on('remove', removed)
+ await expect(
+ withTransaction(async () => 'committed', fixture.pool, {
+ signal: new AbortController().signal,
+ })
+ ).resolves.toBe('committed')
+ expect(fixture.queries).toContain('COMMIT')
+ expect(fixture.queries).not.toContain('ROLLBACK')
+ expect(fixture.pool.totalCount).toBe(0)
+ expect(removed).toHaveBeenCalledOnce()
+ expect(fixture.lifecycle).toEqual(['end', 'remove'])
+ await expect(withTransaction(async () => 'recovered', fixture.pool)).resolves.toBe('recovered')
+ })
+
+ it('evicts a failed restoration after ROLLBACK and keeps the original work failure', async () => {
+ const fixture = await postgresPeer()
+ fixture.fail(settingSql, 2)
+ const primary = new Error('work failed')
+ await expect(
+ withTransaction(
+ async () => {
+ throw primary
+ },
+ fixture.pool,
+ { signal: new AbortController().signal }
+ )
+ ).rejects.toBe(primary)
+ expect(fixture.queries).toContain('ROLLBACK')
+ expect(fixture.pool.totalCount).toBe(0)
+ expect(fixture.lifecycle).toEqual(['end', 'remove'])
+ })
+
+ it('retains the sole pool checkout until restoration finishes before serving the next borrower', async () => {
+ const fixture = await postgresPeer()
+ fixture.stall(settingSql, 2)
+ let settled = false
+ const first = withTransaction(async () => 'first', fixture.pool, {
+ signal: new AbortController().signal,
+ }).finally(() => {
+ settled = true
+ })
+ await observed(() => fixture.sessionChanges.length === 2)
+ const borrowed = (fixture.pool as unknown as { _clients: PoolClient[] })._clients[0]
+ const release = vi.spyOn(borrowed, 'release')
+ const next = withTransaction(async () => 'next', fixture.pool)
+ await observed(() => fixture.pool.waitingCount === 1)
+ expect(settled).toBe(false)
+ expect(release).not.toHaveBeenCalled()
+ expect(fixture.pool.totalCount).toBe(1)
+ fixture.resume()
+ await expect(first).resolves.toBe('first')
+ await expect(next).resolves.toBe('next')
+ expect(fixture.pool.totalCount).toBe(1)
+ expect(fixture.pool.idleCount).toBe(1)
+ })
+})
diff --git a/docs/architecture/codex-subscription-transport-contract.md b/docs/architecture/codex-subscription-transport-contract.md
index 49e2d2284..38ef9303d 100644
--- a/docs/architecture/codex-subscription-transport-contract.md
+++ b/docs/architecture/codex-subscription-transport-contract.md
@@ -495,24 +495,24 @@ behavior changes:
visual entries (running or queued) of the gate's 10. A token whose `sub`
is missing, empty or not a string is refused 401 before any gate or
share is taken. When the share is full, the proxy answers 503
- `provider_unavailable` and logs `codex_proxy_admission_refused` with
- `reason: visual_host_share`. The share is released together with the
- visual slot: on read-deadline expiry, on client close before hand-off,
- and when the handler's finally block unwinds.
+ `visual_host_share` and logs `codex_proxy_admission_refused` with
+ `reason: visual_host_share`. The share is released with the visual slot
+ when the parser completion callback clears retained body references, or
+ when the stream handler's finally block unwinds. A deadline or response
+ close requests cancellation and does not release a live parser owner.
**Visual body read deadline and slot lifetime.** Once the visual gate
grants a slot, the body must be read and parsed within
`BODY_READ_DEADLINE_MS` (10 s). Otherwise the proxy answers 408
- `request_timeout` with `connection: close` (the rest of the body is never
- read, so the connection cannot be reused) and frees the slot and the
- principal share. After the grant, the slot is also freed when the response
- closes for any other reason, which covers every refusal before the stream
- starts. That close hook releases only while the slot is still held by the
- admission code: a kept visual body hands the release to the handler, which
- frees it in its outer `finally`, and a demoted body releases through the
- admission path before it joins the ordinary stream gate. A visual-gate
- refusal (queue full, wait exceeded) is answered 503 `provider_unavailable`
- and logged as `codex_proxy_admission_refused` with `reason: visual_gate`.
+ `request_timeout` with `connection: close` and starts the bounded
+ `VISUAL_READ_CLOSE_GRACE_MS` destruction backstop. The parser callback
+ clears retained body references before releasing the slot and principal
+ share; neither the deadline nor response close grants capacity early.
+ A kept visual body transfers the release to the handler's outer `finally`;
+ a demoted body releases through admission after acquiring its byte budget,
+ before joining the ordinary stream gate. A visual-gate capacity refusal
+ (queue full, wait exceeded) answers 503 `visual_gate` and logs
+ `codex_proxy_admission_refused` with `reason: visual_gate`.
A client that leaves while queued is not gate saturation: its place is freed
silently, with no log and no response.
diff --git a/scripts/ci/pinned-vitest-cases/control-api.json b/scripts/ci/pinned-vitest-cases/control-api.json
index e69bd8f49..3bb4f5d29 100644
--- a/scripts/ci/pinned-vitest-cases/control-api.json
+++ b/scripts/ci/pinned-vitest-cases/control-api.json
@@ -222,5 +222,61 @@
{
"suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",
"fullName": "scoped transaction cancellation with the native pg driver and sockets keeps a lost COMMIT reply uncertain and never attempts rollback or returns that client healthy"
+ },
+ {
+ "suite": "helpers.realPostgresCancellation.test.ts",
+ "fullName": "hides frontend Terminate and FIN as well as replies without closing the backend socket"
+ },
+ {
+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",
+ "fullName": "cancellable transaction server idle backstop with native pool max one bounds an effective idle timeout of 0 before BEGIN and restores it after acknowledged COMMIT"
+ },
+ {
+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",
+ "fullName": "cancellable transaction server idle backstop with native pool max one bounds an effective idle timeout of 45000 before BEGIN and restores it after acknowledged COMMIT"
+ },
+ {
+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",
+ "fullName": "cancellable transaction server idle backstop with native pool max one preserves a smaller positive server idle limit without changing the session"
+ },
+ {
+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",
+ "fullName": "cancellable transaction server idle backstop with native pool max one fails before BEGIN for effective statement timeout 0 and physically evicts the checkout"
+ },
+ {
+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",
+ "fullName": "cancellable transaction server idle backstop with native pool max one fails before BEGIN for effective statement timeout 99 and physically evicts the checkout"
+ },
+ {
+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",
+ "fullName": "cancellable transaction server idle backstop with native pool max one fails before BEGIN for effective statement timeout 30001 and physically evicts the checkout"
+ },
+ {
+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",
+ "fullName": "cancellable transaction server idle backstop with native pool max one fails before BEGIN for effective statement timeout not-a-number and physically evicts the checkout"
+ },
+ {
+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",
+ "fullName": "cancellable transaction server idle backstop with native pool max one fails before BEGIN for effective statement timeout null and physically evicts the checkout"
+ },
+ {
+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",
+ "fullName": "cancellable transaction server idle backstop with native pool max one restores the session after acknowledged ROLLBACK without replacing the primary work error"
+ },
+ {
+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",
+ "fullName": "cancellable transaction server idle backstop with native pool max one poisons and physically ends a failed session setup without running work"
+ },
+ {
+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",
+ "fullName": "cancellable transaction server idle backstop with native pool max one evicts a failed restoration while preserving the acknowledged COMMIT outcome"
+ },
+ {
+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",
+ "fullName": "cancellable transaction server idle backstop with native pool max one evicts a failed restoration after ROLLBACK and keeps the original work failure"
+ },
+ {
+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",
+ "fullName": "cancellable transaction server idle backstop with native pool max one retains the sole pool checkout until restoration finishes before serving the next borrower"
}
]
diff --git a/scripts/tests/test-codex-subscription-t0.sh b/scripts/tests/test-codex-subscription-t0.sh
index 641b4b6a6..1a6b2d4a7 100755
--- a/scripts/tests/test-codex-subscription-t0.sh
+++ b/scripts/tests/test-codex-subscription-t0.sh
@@ -245,6 +245,7 @@ run_group "control-api" "control-api" \
"test/routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts" \
"test/llmProviderAttemptBodyAdmission.test.ts" \
"test/llmProviderAttemptAuthorizer.cancellation.test.ts" \
+ "test/helpers.realPostgresCancellation.test.ts" \
"test/routes.mcp-host.llmProviderAttempts.test.ts" \
"test/routes.adminRecipes.test.ts" \
"test/routes.adminPluginWorkloadSdk.test.ts" \
diff --git a/scripts/tests/test-grok-subscription-t0.sh b/scripts/tests/test-grok-subscription-t0.sh
index a0536876d..2cb258393 100755
--- a/scripts/tests/test-grok-subscription-t0.sh
+++ b/scripts/tests/test-grok-subscription-t0.sh
@@ -277,6 +277,7 @@ run_group "control-api grok" "control-api" \
"test/routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts" \
"test/llmProviderAttemptBodyAdmission.test.ts" \
"test/llmProviderAttemptAuthorizer.cancellation.test.ts" \
+ "test/helpers.realPostgresCancellation.test.ts" \
"test/routes.internal.llmProviderAttempts.grok.test.ts" \
"test/routes.adminPluginWorkloadSdk.test.ts" \
"test/routes.adminRecipes.test.ts" \