Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(control-api): bound abandoned transaction sessions
Alfredo Lopez2842a38018b7
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex a74a93fe0..b87a81497 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.705",+ "version": "0.1.706","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.705",+ "version": "0.1.706","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex 13c95663f..ff1079edf 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.705",+ "version": "0.1.706","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/db.ts b/control-api/src/db.tsindex 3a04e3fb8..ae81db69a 100644--- a/control-api/src/db.ts+++ b/control-api/src/db.ts@@ -6728,6 +6728,8 @@ export async function withTransaction<T>(}let transactionStarted = falselet commitSent = false+ let transactionFinished = false+ let originalIdleTimeout: string | undefinedlet releaseError: Error | boolean | undefined// pg-pool detaches its own idle 'error' listener from a client while it is@@ -6747,9 +6749,9 @@ export async function withTransaction<T>(client.on('error', onClientError)let termination: Promise<void> | undefined- const onAbort = (): void => {+ const terminateClient = (reason: Error | boolean): void => {+ releaseError ??= reasonif (termination) return- releaseError = signal?.reason instanceof Error ? signal.reason : truetermination = client.end().catch(error => {releaseError = error instanceof Error ? error : true})@@ -6761,11 +6763,58 @@ export async function withTransaction<T>(// finish the caller ahead of that event.client.connection.stream.destroy()}+ const onAbort = (): void => {+ terminateClient(signal?.reason instanceof Error ? signal.reason : true)+ }signal?.addEventListener('abort', onAbort, { once: true })if (signal?.aborted) onAbort()try {signal?.throwIfAborted()+ if (signal) {+ // Read normalized milliseconds from this backend, not from client startup+ // options: a role/database default or a reused session can be different.+ const settings = await client.query<{+ statement_timeout_ms: string | null+ idle_timeout_ms: string | null+ }>(`SELECT+ (SELECT setting FROM pg_settings WHERE name = 'statement_timeout') AS statement_timeout_ms,+ (SELECT setting FROM pg_settings WHERE name = 'idle_in_transaction_session_timeout') AS idle_timeout_ms`)+ signal.throwIfAborted()+ const setting = settings.rows[0]+ const statementTimeout = setting?.statement_timeout_ms+ const idleTimeout = setting?.idle_timeout_ms+ const statementMs = Number(statementTimeout)+ const idleMs = Number(idleTimeout)+ // These are the existing createBoundedPgPoolForConnection startup bounds.+ // An unbounded or invalid effective server setting cannot supply a finite+ // remote cleanup backstop, so this cancellable transaction must fail loud.+ if (+ typeof statementTimeout !== 'string' ||+ !/^\d+$/.test(statementTimeout) ||+ !Number.isInteger(statementMs) ||+ statementMs < 100 ||+ statementMs > 30_000 ||+ typeof idleTimeout !== 'string' ||+ !/^\d+$/.test(idleTimeout) ||+ !Number.isSafeInteger(idleMs)+ ) {+ throw new Error('Invalid effective PostgreSQL cancellation timeout bounds')+ }+ if (idleMs === 0 || idleMs > statementMs) {+ // This must be SESSION scope before BEGIN. AbortTransaction restores a+ // SET LOCAL value before entering idle-in-transaction (aborted), which+ // would remove the very backstop needed when a blackhole hides our FIN.+ // Record the original before sending: a missing set_config reply is also+ // uncertain, and that checkout must be ended rather than returned healthy.+ originalIdleTimeout = idleTimeout+ await client.query('SELECT set_config($1, $2, false)', [+ 'idle_in_transaction_session_timeout',+ statementTimeout,+ ])+ signal.throwIfAborted()+ }+ }await client.query('BEGIN')transactionStarted = trueif (signal) {@@ -6795,6 +6844,7 @@ export async function withTransaction<T>(signal?.throwIfAborted()commitSent = trueawait client.query('COMMIT')+ transactionFinished = truesignal?.throwIfAborted()return result} catch (error) {@@ -6803,6 +6853,7 @@ export async function withTransaction<T>(} else {try {await client.query('ROLLBACK')+ transactionFinished = true} catch (rollbackError) {releaseError = rollbackError instanceof Error ? rollbackError : true}@@ -6813,6 +6864,36 @@ export async function withTransaction<T>(throw error} finally {// Work (including its own finally) and any rollback have already unwound.+ // Only an acknowledged, healthy transaction may restore session state and+ // return to the pool. Lost COMMIT/ROLLBACK replies leave the server backstop+ // installed; local transport completion is not remote cleanup observation.+ if (+ originalIdleTimeout !== undefined &&+ transactionFinished &&+ !releaseError &&+ !asyncClientError &&+ !signal?.aborted+ ) {+ try {+ await client.query('SELECT set_config($1, $2, false)', [+ 'idle_in_transaction_session_timeout',+ originalIdleTimeout,+ ])+ } catch (restoreError) {+ // Do not replace the work outcome or relabel an acknowledged COMMIT as+ // a rollback. A session that failed restoration is physically ended.+ releaseError = restoreError instanceof Error ? restoreError : true+ }+ }+ if (signal && (releaseError || asyncClientError || signal.aborted)) {+ terminateClient(+ signal.aborted+ ? signal.reason instanceof Error+ ? signal.reason+ : true+ : (releaseError ?? asyncClientError ?? true)+ )+ }// Keep both listeners and the checkout until physical termination completes.if (termination) await terminationsignal?.removeEventListener('abort', onAbort)diff --git a/control-api/test/helpers.realPostgresCancellation.test.ts b/control-api/test/helpers.realPostgresCancellation.test.tsindex a59539726..fa265f4de 100644--- a/control-api/test/helpers.realPostgresCancellation.test.ts+++ b/control-api/test/helpers.realPostgresCancellation.test.ts@@ -2,7 +2,10 @@ import { expect, it } from 'vitest'import { once } from 'node:events'import { type Socket, createConnection, createServer } from 'node:net'import { setImmediate as nextTurn } from 'node:timers/promises'-import { createPostgresCommitReplyBlackhole } from './helpers/realPostgresCancellation.js'+import {+ createPostgresCommitReplyBlackhole,+ createPostgresDisconnectBlackhole,+} from './helpers/realPostgresCancellation.js'async function observed(check: () => boolean): Promise<void> {const deadline = Date.now() + 1_000@@ -90,3 +93,62 @@ it('refuses an encrypted protocol prerequisite instead of pretending to observe)).rejects.toThrow('plaintext PostgreSQL connection')})++it('hides frontend Terminate and FIN as well as replies without closing the backend socket', async () => {+ const sockets = new Set<Socket>()+ let backendSocket: Socket | undefined+ let received = Buffer.alloc(0)+ let ended = false+ const backend = createServer({ allowHalfOpen: true }, socket => {+ backendSocket = socket+ sockets.add(socket)+ socket.on('data', chunk => {+ received = Buffer.concat([received, chunk])+ socket.write('visible-reply')+ })+ socket.on('end', () => {+ ended = true+ })+ socket.on('error', () => {})+ })+ backend.listen(0, '127.0.0.1')+ await once(backend, 'listening')+ const address = backend.address()+ if (!address || typeof address === 'string') throw new Error('missing fixture backend address')+ const proxy = await createPostgresDisconnectBlackhole(+ ['postgresql:', `//[email protected]:${address.port}/fixture`].join('')+ )+ const target = new URL(proxy.connectionString)+ const client = createConnection({ host: target.hostname, port: Number(target.port) })+ let replies = Buffer.alloc(0)+ client.on('data', chunk => {+ replies = Buffer.concat([replies, chunk])+ })+ client.on('error', () => {})+ try {+ await once(client, 'connect')+ const startup = Buffer.alloc(8)+ startup.writeInt32BE(8)+ startup.writeInt32BE(196608, 4)+ client.write(startup)+ await observed(() => replies.toString() === 'visible-reply')+ proxy.dropRepliesAndDisconnects()+ const terminate = Buffer.from([88, 0, 0, 0, 4])+ client.end(terminate)+ await observed(() => proxy.suppressedDisconnects === 1)+ expect(proxy.discardedFrontendBytes).toBe(terminate.length)+ expect(received).toEqual(startup)+ expect(ended).toBe(false)+ expect(backendSocket?.destroyed).toBe(false)+ backendSocket!.write('hidden-reply')+ await observed(() => proxy.discardedReplyBytes === Buffer.byteLength('hidden-reply'))+ expect(replies.toString()).toBe('visible-reply')+ } finally {+ client.destroy()+ await proxy.close()+ for (const socket of sockets) socket.destroy()+ await new Promise<void>((resolve, reject) =>+ backend.close(error => (error ? reject(error) : resolve()))+ )+ }+})diff --git a/control-api/test/helpers/realPostgresCancellation.ts b/control-api/test/helpers/realPostgresCancellation.tsindex a2aa3c6cf..98bc625be 100644--- a/control-api/test/helpers/realPostgresCancellation.ts+++ b/control-api/test/helpers/realPostgresCancellation.ts@@ -3,6 +3,15 @@ import { type Socket, createConnection, createServer } from 'node:net'/** Observe a real COMMIT while discarding its reply on the owned local PG lane. */export async function createPostgresCommitReplyBlackhole(connectionString: string) {+ return createPostgresProtocolBlackhole(connectionString, true)+}++/** Hide replies and client disconnects without ending the real backend socket. */+export async function createPostgresDisconnectBlackhole(connectionString: string) {+ return createPostgresProtocolBlackhole(connectionString, false)+}++async function createPostgresProtocolBlackhole(connectionString: string, dropCommitReply: boolean) {const target = new URL(connectionString)const sslMode = target.searchParams.get('sslmode')if (sslMode && sslMode !== 'disable') {@@ -14,23 +23,39 @@ export async function createPostgresCommitReplyBlackhole(connectionString: strinconst commitForwarded = new Promise<void>(resolve => {commit = resolve})- let armed = true+ let armed = dropCommitReplylet dropping = falselet discardedReplyBytes = 0- const server = createServer(front => {+ let discardedFrontendBytes = 0+ let suppressDisconnects = false+ let suppressedDisconnects = 0+ const server = createServer({ allowHalfOpen: true }, front => {const host = target.hostname.replace(/^\[|\]$/g, '')- const back = createConnection({ host, port: Number(target.port || 5432) })+ const back = createConnection({ host, port: Number(target.port || 5432), allowHalfOpen: true })for (const socket of [front, back]) {sockets.add(socket)socket.once('close', () => sockets.delete(socket))}- front.on('error', () => back.destroy())+ front.on('error', () => {+ if (suppressDisconnects) suppressedDisconnects+++ else back.destroy()+ })back.on('error', () => front.destroy())- front.on('end', () => back.end())+ front.on('end', () => {+ if (suppressDisconnects) suppressedDisconnects+++ else back.end()+ })back.on('end', () => front.end())let startup = truelet buffered = Buffer.alloc(0)front.on('data', chunk => {+ // pg may send Terminate before FIN on an idle checkout. A network fault+ // that hides disconnects must suppress both; forwarding either would let+ // the test pass because of EOF instead of the server timeout backstop.+ if (suppressDisconnects) {+ discardedFrontendBytes += chunk.length+ return+ }buffered = Buffer.concat([buffered, chunk])if (startup && buffered.length >= 4 && buffered.length >= buffered.readInt32BE(0)) {buffered = buffered.subarray(buffered.readInt32BE(0))@@ -70,8 +95,20 @@ export async function createPostgresCommitReplyBlackhole(connectionString: stringet discardedReplyBytes() {return discardedReplyBytes},+ get discardedFrontendBytes() {+ return discardedFrontendBytes+ },+ get suppressedDisconnects() {+ return suppressedDisconnects+ },+ dropRepliesAndDisconnects: () => {+ armed = false+ dropping = true+ suppressDisconnects = true+ },allowReplies: () => {dropping = false+ suppressDisconnects = false},close: async () => {for (const socket of sockets) socket.destroy()diff --git a/control-api/test/llmProviderAttemptAuthorizer.cancellation.realPostgres.integration.test.ts b/control-api/test/llmProviderAttemptAuthorizer.cancellation.realPostgres.integration.test.tsindex acc57ea8c..1db5d7fd5 100644--- a/control-api/test/llmProviderAttemptAuthorizer.cancellation.realPostgres.integration.test.ts+++ b/control-api/test/llmProviderAttemptAuthorizer.cancellation.realPostgres.integration.test.ts@@ -6,7 +6,10 @@ import { Pool } from 'pg'import { type DbClient, initDb, withTransaction } from '../src/db.js'import { reserveInDangerZone } from '../src/services/budgets/reservations.js'import { insertLlmProviderAttempt } from '../src/services/llmProviderAttemptStore.js'-import { createPostgresCommitReplyBlackhole } from './helpers/realPostgresCancellation.js'+import {+ createPostgresCommitReplyBlackhole,+ createPostgresDisconnectBlackhole,+} from './helpers/realPostgresCancellation.js'import { endPoolAndWaitForClients } from './helpers/realPostgresTeardown.js'const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL@@ -245,4 +248,101 @@ realPostgres('scoped transaction cancellation mechanism and durable outcome', ()await proxy.close()}}, 10_000)+ it.each(['idle', 'active-then-aborted'] as const)(+ 'bounds the remote %s transaction after local cleanup when a protocol blackhole hides both replies and disconnects',+ async mode => {+ const proxy = await createPostgresDisconnectBlackhole(connectionString)+ const isolated = new Pool({+ connectionString: proxy.connectionString,+ ssl: false,+ max: 1,+ connectionTimeoutMillis: 2_000,+ statement_timeout: 2_000,+ options: '-c idle_in_transaction_session_timeout=0',+ })+ const invocationId = randomUUID()+ const lock = mode === 'idle' ? 806303 : 806304+ const entered = deferred<number>()+ const controller = new AbortController()+ const reason = Object.assign(new Error('authorize_timeout'), { code: 'authorize_timeout' })+ try {+ const outcome = withTransaction(+ async db => {+ const identity = await db.query('SELECT pg_backend_pid() AS pid')+ const setting = await db.query(+ "SELECT setting FROM pg_settings WHERE name = 'idle_in_transaction_session_timeout'"+ )+ expect((setting.rows[0] as { setting: string }).setting).toBe('2000')+ await writeAuthorizeRows(db, invocationId)+ await db.query('SELECT pg_advisory_xact_lock($1)', [lock])+ entered.resolve((identity.rows[0] as { pid: number }).pid)+ if (mode === 'active-then-aborted') await db.query('SELECT pg_sleep(30)')+ else await once(controller.signal, 'abort')+ },+ isolated,+ { signal: controller.signal }+ ).then(+ value => ({ value }),+ error => ({ error })+ )+ const pid = await Promise.race([+ entered.promise,+ outcome.then(early => {+ if ('error' in early) throw early.error+ throw new Error('transaction ended before the physical fixture was ready')+ }),+ ])+ const backendState = async () => {+ const result = await observer.query<{ state: string; query: string }>(+ 'SELECT state, query FROM pg_stat_activity WHERE pid = $1 AND datname = $2',+ [pid, database]+ )+ return result.rows[0]+ }+ await observe(async () => {+ const backend = await backendState()+ return mode === 'idle'+ ? backend?.state === 'idle in transaction'+ : backend?.state === 'active' && backend.query === 'SELECT pg_sleep(30)'+ })+ proxy.dropRepliesAndDisconnects()+ controller.abort(reason)+ expect(((await outcome) as { error: unknown }).error).toBe(reason)+ expect(isolated.totalCount).toBe(0)+ await observe(async () => proxy.suppressedDisconnects > 0)+ // Local completion cannot establish remote cleanup. The backend still+ // exists with its socket held by the fault proxy, independently observed.+ expect(await backendState()).toBeDefined()+ if (mode === 'active-then-aborted') {+ // The server statement timeout first enters TRANS_ABORT. A SET LOCAL+ // idle limit would have reverted to zero here and leave this backend+ // alive indefinitely; the pre-BEGIN session backstop must remain.+ await observe(+ async () => (await backendState())?.state === 'idle in transaction (aborted)',+ 4_000+ )+ expect(proxy.discardedReplyBytes).toBeGreaterThan(0)+ }+ await observe(async () => !(await backendState()), 6_000)+ await lockAvailable(lock)+ expect(await rowCounts(invocationId)).toEqual({ attempts: 0, tickets: 0, reservations: 0 })+ proxy.allowReplies()++ // With max one, a fresh physical backend still serves and commits the+ // real writer after both the local checkout and remote backend ended.+ const later = randomUUID()+ await withTransaction(db => writeAuthorizeRows(db, later), isolated, {+ signal: new AbortController().signal,+ })+ expect(await rowCounts(later)).toEqual({ attempts: 1, tickets: 1, reservations: 1 })+ await pool.query('DELETE FROM budget_pending_reservations WHERE task_ref = $1', [later])+ } finally {+ controller.abort(reason)+ proxy.allowReplies()+ await endPoolAndWaitForClients(isolated)+ await proxy.close()+ }+ },+ 20_000+ )})diff --git a/control-api/test/llmProviderAttemptAuthorizer.cancellation.test.ts b/control-api/test/llmProviderAttemptAuthorizer.cancellation.test.tsindex 55b72e154..5d5290759 100644--- a/control-api/test/llmProviderAttemptAuthorizer.cancellation.test.ts+++ b/control-api/test/llmProviderAttemptAuthorizer.cancellation.test.ts@@ -309,12 +309,43 @@ function protocolMessage(type: string, body: Buffer): Buffer {return Buffer.concat([header, body])}-async function postgresPeer(handshake = true) {+function textRows(columns: string[], values: Array<string | null>): Buffer[] {+ const count = Buffer.alloc(2)+ count.writeInt16BE(columns.length)+ const fields = columns.map(name => {+ const description = Buffer.alloc(18)+ description.writeInt32BE(25, 6)+ description.writeInt16BE(-1, 10)+ description.writeInt32BE(-1, 12)+ return Buffer.concat([Buffer.from(`${name}\0`), description])+ })+ const row = values.map(value => {+ const bytes = value === null ? undefined : Buffer.from(value)+ const length = Buffer.alloc(4)+ length.writeInt32BE(bytes?.length ?? -1)+ return bytes ? Buffer.concat([length, bytes]) : length+ })+ return [+ protocolMessage('T', Buffer.concat([count, ...fields])),+ protocolMessage('D', Buffer.concat([count, ...row])),+ ]+}++async function postgresPeer(+ handshake = true,+ settings: { statement?: string | null; idle?: string | null } = {}+) {const connected = deferred()const queries: string[] = []+ const sessionChanges: Array<{ sql: string; values: Array<string | null> }> = []+ const lifecycle: string[] = []const sockets = new Set<Socket>()+ const stalledReplies: Array<() => void> = []let stalledSql = ''+ let stalledOccurrence = 1let failedSql = ''+ let failedOccurrence = 1+ const occurrences = new Map<string, number>()const server = createTcpServer({ allowHalfOpen: true }, socket => {sockets.add(socket)socket.on('error', () => {})@@ -322,6 +353,60 @@ async function postgresPeer(handshake = true) {connected.resolve()let startup = truelet buffered = Buffer.alloc(0)+ let statement = ''+ let parameters: Array<string | null> = []+ let state = 'I'+ let waiting = false+ let pendingSync = false+ let idleTimeout = settings.idle === undefined ? '0' : settings.idle+ const statementTimeout = settings.statement === undefined ? '15000' : settings.statement+ const ready = () => socket.write(protocolMessage('Z', Buffer.from(state)))+ const describe = (sql: string): [string[], Array<string | null>] => {+ if (sql.includes('FROM pg_settings')) {+ return [+ ['statement_timeout_ms', 'idle_timeout_ms'],+ [statementTimeout, idleTimeout],+ ]+ }+ if (sql === 'SELECT set_config($1, $2, false)') return [['set_config'], [parameters[1]]]+ return [[], []]+ }+ const execute = (sql: string, simple: boolean) => {+ queries.push(sql)+ const occurrence = (occurrences.get(sql) ?? 0) + 1+ occurrences.set(sql, occurrence)+ if (sql === 'SELECT set_config($1, $2, false)') {+ sessionChanges.push({ sql, values: [...parameters] })+ }+ const complete = () => {+ if (sql === failedSql && occurrence === failedOccurrence) {+ if (state === 'T') state = 'E'+ socket.write(+ protocolMessage('E', Buffer.from('SERROR\0C42704\0Mfixture statement failure\0\0'))+ )+ } else {+ if (sql === 'BEGIN') state = 'T'+ if (sql === 'ROLLBACK' || sql === 'COMMIT') state = 'I'+ if (sql === 'SELECT set_config($1, $2, false)') idleTimeout = parameters[1]+ const [columns, values] = describe(sql)+ if (columns.length) {+ const [description, row] = textRows(columns, values)+ if (simple) socket.write(description)+ socket.write(row)+ }+ socket.write(protocolMessage('C', Buffer.from(`${sql.split(' ')[0]}\0`)))+ }+ waiting = false+ if (simple || pendingSync) {+ pendingSync = false+ ready()+ }+ }+ if (sql === stalledSql && occurrence === stalledOccurrence) {+ waiting = true+ stalledReplies.push(complete)+ } else complete()+ }socket.on('data', chunk => {buffered = Buffer.concat([buffered, chunk])if (startup) {@@ -341,28 +426,38 @@ async function postgresPeer(handshake = true) {const length = buffered.readInt32BE(1)const body = buffered.subarray(5, length + 1)buffered = buffered.subarray(length + 1)- if (type !== 'Q') continue- const sql = body.toString('utf8', 0, body.length - 1)- queries.push(sql)- if (sql === failedSql) {+ if (type === 'Q') {+ parameters = []+ execute(body.toString('utf8', 0, body.length - 1), true)+ } else if (type === 'P') {+ const start = body.indexOf(0) + 1+ statement = body.toString('utf8', start, body.indexOf(0, start))+ socket.write(protocolMessage('1', Buffer.alloc(0)))+ } else if (type === 'B') {+ let offset = body.indexOf(0) + 1+ offset = body.indexOf(0, offset) + 1+ const formats = body.readInt16BE(offset)+ offset += 2 + formats * 2+ const count = body.readInt16BE(offset)+ offset += 2+ parameters = []+ for (let index = 0; index < count; index++) {+ const size = body.readInt32BE(offset)+ offset += 4+ parameters.push(size < 0 ? null : body.toString('utf8', offset, offset + size))+ if (size >= 0) offset += size+ }+ socket.write(protocolMessage('2', Buffer.alloc(0)))+ } else if (type === 'D') {+ const [columns, values] = describe(statement)socket.write(- Buffer.concat([- protocolMessage(- 'E',- Buffer.from('SERROR\0C42704\0Munsupported connection check parameter\0\0')- ),- protocolMessage('Z', Buffer.from('E')),- ])+ columns.length ? textRows(columns, values)[0] : protocolMessage('n', Buffer.alloc(0)))- continue+ } else if (type === 'E') execute(statement, false)+ else if (type === 'S') {+ if (waiting) pendingSync = true+ else ready()}- if (sql === stalledSql) continue- socket.write(- Buffer.concat([- protocolMessage('C', Buffer.from(`${sql.split(' ')[0]}\0`)),- protocolMessage('Z', Buffer.from(sql === 'BEGIN' ? 'T' : 'I')),- ])- )}})// This deliberate half-open peer never sends EOF in response to Terminate.@@ -382,6 +477,8 @@ async function postgresPeer(handshake = true) {idleTimeoutMillis: 1_000,})pool.on('error', () => {})+ pool.on('connect', client => client.once('end', () => lifecycle.push('end')))+ pool.on('remove', () => lifecycle.push('remove'))cleanup.push(async () => {for (const socket of sockets) socket.destroy()await pool.end()@@ -392,12 +489,19 @@ async function postgresPeer(handshake = true) {return {connected,queries,+ sessionChanges,+ lifecycle,pool,- stall: (sql: string) => {+ stall: (sql: string, occurrence = 1) => {stalledSql = sql+ stalledOccurrence = occurrence+ },+ resume: () => {+ for (const reply of stalledReplies.splice(0)) reply()},- fail: (sql: string) => {+ fail: (sql: string, occurrence = 1) => {failedSql = sql+ failedOccurrence = occurrence},}}@@ -425,9 +529,12 @@ describe('scoped transaction cancellation with the native pg driver and sockets').rejects.toMatchObject({ code: '42704' })expect(work).not.toHaveBeenCalled()expect(fixture.queries).toEqual([+ expect.stringContaining('FROM pg_settings'),+ 'SELECT set_config($1, $2, false)','BEGIN',"SET LOCAL client_connection_check_interval = '100ms'",'ROLLBACK',+ 'SELECT set_config($1, $2, false)',])})@@ -586,6 +693,8 @@ describe('scoped transaction cancellation with the native pg driver and sockets'controller.abort(reason)expect(((await within(outcome)) as { error: unknown }).error).toBe(reason)expect(fixture.queries).toEqual([+ expect.stringContaining('FROM pg_settings'),+ 'SELECT set_config($1, $2, false)','BEGIN',"SET LOCAL client_connection_check_interval = '100ms'",'COMMIT',@@ -593,3 +702,162 @@ describe('scoped transaction cancellation with the native pg driver and sockets'expect(fixture.pool.totalCount).toBe(0)})})++describe('cancellable transaction server idle backstop with native pool max one', () => {+ const settingSql = 'SELECT set_config($1, $2, false)'++ it.each(['0', '45000'])(+ 'bounds an effective idle timeout of %s before BEGIN and restores it after acknowledged COMMIT',+ async idle => {+ const fixture = await postgresPeer(true, { idle })+ await expect(+ withTransaction(async () => 'committed', fixture.pool, {+ signal: new AbortController().signal,+ })+ ).resolves.toBe('committed')+ expect(fixture.sessionChanges).toEqual([+ { sql: settingSql, values: ['idle_in_transaction_session_timeout', '15000'] },+ { sql: settingSql, values: ['idle_in_transaction_session_timeout', idle] },+ ])+ expect(fixture.queries.indexOf(settingSql)).toBeLessThan(fixture.queries.indexOf('BEGIN'))+ expect(fixture.queries.lastIndexOf(settingSql)).toBeGreaterThan(+ fixture.queries.indexOf('COMMIT')+ )+ expect(fixture.pool.totalCount).toBe(1)+ expect(fixture.pool.idleCount).toBe(1)+ const inspected = await fixture.pool.query(+ "SELECT setting FROM pg_settings WHERE name = 'idle_in_transaction_session_timeout'"+ )+ expect(inspected.rows[0].idle_timeout_ms).toBe(idle)+ }+ )++ it('preserves a smaller positive server idle limit without changing the session', async () => {+ const fixture = await postgresPeer(true, { idle: '300' })+ await expect(+ withTransaction(async () => 'committed', fixture.pool, {+ signal: new AbortController().signal,+ })+ ).resolves.toBe('committed')+ expect(fixture.sessionChanges).toEqual([])+ expect(fixture.pool.idleCount).toBe(1)+ })++ it.each(['0', '99', '30001', 'not-a-number', null])(+ 'fails before BEGIN for effective statement timeout %s and physically evicts the checkout',+ async statement => {+ const fixture = await postgresPeer(true, { statement })+ const work = vi.fn()+ const removed = vi.fn()+ fixture.pool.on('remove', removed)+ await expect(+ withTransaction(work, fixture.pool, { signal: new AbortController().signal })+ ).rejects.toThrow('Invalid effective PostgreSQL cancellation timeout bounds')+ expect(work).not.toHaveBeenCalled()+ expect(fixture.queries).toHaveLength(1)+ expect(fixture.queries).not.toContain('BEGIN')+ expect(fixture.pool.totalCount).toBe(0)+ expect(removed).toHaveBeenCalledOnce()+ expect(fixture.lifecycle).toEqual(['end', 'remove'])+ }+ )++ it('restores the session after acknowledged ROLLBACK without replacing the primary work error', async () => {+ const fixture = await postgresPeer()+ const primary = new Error('work failed')+ await expect(+ withTransaction(+ async () => {+ throw primary+ },+ fixture.pool,+ { signal: new AbortController().signal }+ )+ ).rejects.toBe(primary)+ expect(fixture.queries.indexOf('ROLLBACK')).toBeLessThan(+ fixture.queries.lastIndexOf(settingSql)+ )+ expect(fixture.sessionChanges.at(-1)?.values).toEqual([+ 'idle_in_transaction_session_timeout',+ '0',+ ])+ expect(fixture.pool.idleCount).toBe(1)+ })++ it('poisons and physically ends a failed session setup without running work', async () => {+ const fixture = await postgresPeer()+ fixture.fail(settingSql)+ const work = vi.fn()+ const removed = vi.fn()+ fixture.pool.on('remove', removed)+ await expect(+ withTransaction(work, fixture.pool, { signal: new AbortController().signal })+ ).rejects.toMatchObject({ code: '42704' })+ expect(work).not.toHaveBeenCalled()+ expect(fixture.queries).not.toContain('BEGIN')+ expect(fixture.sessionChanges).toHaveLength(1)+ expect(fixture.pool.totalCount).toBe(0)+ expect(removed).toHaveBeenCalledOnce()+ expect(fixture.lifecycle).toEqual(['end', 'remove'])+ })++ it('evicts a failed restoration while preserving the acknowledged COMMIT outcome', async () => {+ const fixture = await postgresPeer()+ fixture.fail(settingSql, 2)+ const removed = vi.fn()+ fixture.pool.on('remove', removed)+ await expect(+ withTransaction(async () => 'committed', fixture.pool, {+ signal: new AbortController().signal,+ })+ ).resolves.toBe('committed')+ expect(fixture.queries).toContain('COMMIT')+ expect(fixture.queries).not.toContain('ROLLBACK')+ expect(fixture.pool.totalCount).toBe(0)+ expect(removed).toHaveBeenCalledOnce()+ expect(fixture.lifecycle).toEqual(['end', 'remove'])+ await expect(withTransaction(async () => 'recovered', fixture.pool)).resolves.toBe('recovered')+ })++ it('evicts a failed restoration after ROLLBACK and keeps the original work failure', async () => {+ const fixture = await postgresPeer()+ fixture.fail(settingSql, 2)+ const primary = new Error('work failed')+ await expect(+ withTransaction(+ async () => {+ throw primary+ },+ fixture.pool,+ { signal: new AbortController().signal }+ )+ ).rejects.toBe(primary)+ expect(fixture.queries).toContain('ROLLBACK')+ expect(fixture.pool.totalCount).toBe(0)+ expect(fixture.lifecycle).toEqual(['end', 'remove'])+ })++ it('retains the sole pool checkout until restoration finishes before serving the next borrower', async () => {+ const fixture = await postgresPeer()+ fixture.stall(settingSql, 2)+ let settled = false+ const first = withTransaction(async () => 'first', fixture.pool, {+ signal: new AbortController().signal,+ }).finally(() => {+ settled = true+ })+ await observed(() => fixture.sessionChanges.length === 2)+ const borrowed = (fixture.pool as unknown as { _clients: PoolClient[] })._clients[0]+ const release = vi.spyOn(borrowed, 'release')+ const next = withTransaction(async () => 'next', fixture.pool)+ await observed(() => fixture.pool.waitingCount === 1)+ expect(settled).toBe(false)+ expect(release).not.toHaveBeenCalled()+ expect(fixture.pool.totalCount).toBe(1)+ fixture.resume()+ await expect(first).resolves.toBe('first')+ await expect(next).resolves.toBe('next')+ expect(fixture.pool.totalCount).toBe(1)+ expect(fixture.pool.idleCount).toBe(1)+ })+})diff --git a/docs/architecture/codex-subscription-transport-contract.md b/docs/architecture/codex-subscription-transport-contract.mdindex 49e2d2284..38ef9303d 100644--- a/docs/architecture/codex-subscription-transport-contract.md+++ b/docs/architecture/codex-subscription-transport-contract.md@@ -495,24 +495,24 @@ behavior changes:visual entries (running or queued) of the gate's 10. A token whose `sub`is missing, empty or not a string is refused 401 before any gate orshare is taken. When the share is full, the proxy answers 503- `provider_unavailable` and logs `codex_proxy_admission_refused` with- `reason: visual_host_share`. The share is released together with the- visual slot: on read-deadline expiry, on client close before hand-off,- and when the handler's finally block unwinds.+ `visual_host_share` and logs `codex_proxy_admission_refused` with+ `reason: visual_host_share`. The share is released with the visual slot+ when the parser completion callback clears retained body references, or+ when the stream handler's finally block unwinds. A deadline or response+ close requests cancellation and does not release a live parser owner.**Visual body read deadline and slot lifetime.** Once the visual gategrants a slot, the body must be read and parsed within`BODY_READ_DEADLINE_MS` (10 s). Otherwise the proxy answers 408- `request_timeout` with `connection: close` (the rest of the body is never- read, so the connection cannot be reused) and frees the slot and the- principal share. After the grant, the slot is also freed when the response- closes for any other reason, which covers every refusal before the stream- starts. That close hook releases only while the slot is still held by the- admission code: a kept visual body hands the release to the handler, which- frees it in its outer `finally`, and a demoted body releases through the- admission path before it joins the ordinary stream gate. A visual-gate- refusal (queue full, wait exceeded) is answered 503 `provider_unavailable`- and logged as `codex_proxy_admission_refused` with `reason: visual_gate`.+ `request_timeout` with `connection: close` and starts the bounded+ `VISUAL_READ_CLOSE_GRACE_MS` destruction backstop. The parser callback+ clears retained body references before releasing the slot and principal+ share; neither the deadline nor response close grants capacity early.+ A kept visual body transfers the release to the handler's outer `finally`;+ a demoted body releases through admission after acquiring its byte budget,+ before joining the ordinary stream gate. A visual-gate capacity refusal+ (queue full, wait exceeded) answers 503 `visual_gate` and logs+ `codex_proxy_admission_refused` with `reason: visual_gate`.A client that leaves while queued is not gate saturation: its place is freedsilently, with no log and no response.diff --git a/scripts/ci/pinned-vitest-cases/control-api.json b/scripts/ci/pinned-vitest-cases/control-api.jsonindex e69bd8f49..3bb4f5d29 100644--- a/scripts/ci/pinned-vitest-cases/control-api.json+++ b/scripts/ci/pinned-vitest-cases/control-api.json@@ -222,5 +222,61 @@{"suite": "llmProviderAttemptAuthorizer.cancellation.test.ts","fullName": "scoped transaction cancellation with the native pg driver and sockets keeps a lost COMMIT reply uncertain and never attempts rollback or returns that client healthy"+ },+ {+ "suite": "helpers.realPostgresCancellation.test.ts",+ "fullName": "hides frontend Terminate and FIN as well as replies without closing the backend socket"+ },+ {+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",+ "fullName": "cancellable transaction server idle backstop with native pool max one bounds an effective idle timeout of 0 before BEGIN and restores it after acknowledged COMMIT"+ },+ {+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",+ "fullName": "cancellable transaction server idle backstop with native pool max one bounds an effective idle timeout of 45000 before BEGIN and restores it after acknowledged COMMIT"+ },+ {+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",+ "fullName": "cancellable transaction server idle backstop with native pool max one preserves a smaller positive server idle limit without changing the session"+ },+ {+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",+ "fullName": "cancellable transaction server idle backstop with native pool max one fails before BEGIN for effective statement timeout 0 and physically evicts the checkout"+ },+ {+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",+ "fullName": "cancellable transaction server idle backstop with native pool max one fails before BEGIN for effective statement timeout 99 and physically evicts the checkout"+ },+ {+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",+ "fullName": "cancellable transaction server idle backstop with native pool max one fails before BEGIN for effective statement timeout 30001 and physically evicts the checkout"+ },+ {+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",+ "fullName": "cancellable transaction server idle backstop with native pool max one fails before BEGIN for effective statement timeout not-a-number and physically evicts the checkout"+ },+ {+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",+ "fullName": "cancellable transaction server idle backstop with native pool max one fails before BEGIN for effective statement timeout null and physically evicts the checkout"+ },+ {+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",+ "fullName": "cancellable transaction server idle backstop with native pool max one restores the session after acknowledged ROLLBACK without replacing the primary work error"+ },+ {+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",+ "fullName": "cancellable transaction server idle backstop with native pool max one poisons and physically ends a failed session setup without running work"+ },+ {+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",+ "fullName": "cancellable transaction server idle backstop with native pool max one evicts a failed restoration while preserving the acknowledged COMMIT outcome"+ },+ {+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",+ "fullName": "cancellable transaction server idle backstop with native pool max one evicts a failed restoration after ROLLBACK and keeps the original work failure"+ },+ {+ "suite": "llmProviderAttemptAuthorizer.cancellation.test.ts",+ "fullName": "cancellable transaction server idle backstop with native pool max one retains the sole pool checkout until restoration finishes before serving the next borrower"}]diff --git a/scripts/tests/test-codex-subscription-t0.sh b/scripts/tests/test-codex-subscription-t0.shindex 641b4b6a6..1a6b2d4a7 100755--- a/scripts/tests/test-codex-subscription-t0.sh+++ b/scripts/tests/test-codex-subscription-t0.sh@@ -245,6 +245,7 @@ run_group "control-api" "control-api" \"test/routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts" \"test/llmProviderAttemptBodyAdmission.test.ts" \"test/llmProviderAttemptAuthorizer.cancellation.test.ts" \+ "test/helpers.realPostgresCancellation.test.ts" \"test/routes.mcp-host.llmProviderAttempts.test.ts" \"test/routes.adminRecipes.test.ts" \"test/routes.adminPluginWorkloadSdk.test.ts" \diff --git a/scripts/tests/test-grok-subscription-t0.sh b/scripts/tests/test-grok-subscription-t0.shindex a0536876d..2cb258393 100755--- a/scripts/tests/test-grok-subscription-t0.sh+++ b/scripts/tests/test-grok-subscription-t0.sh@@ -277,6 +277,7 @@ run_group "control-api grok" "control-api" \"test/routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts" \"test/llmProviderAttemptBodyAdmission.test.ts" \"test/llmProviderAttemptAuthorizer.cancellation.test.ts" \+ "test/helpers.realPostgresCancellation.test.ts" \"test/routes.internal.llmProviderAttempts.grok.test.ts" \"test/routes.adminPluginWorkloadSdk.test.ts" \"test/routes.adminRecipes.test.ts" \