Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
import { Pool, type PoolClient } from 'pg'
import {
type BoundedPoolBudget,
type PoolConstructor,
createBoundedPgPoolForConnection,
} from '../../boundedPgPool.js'
import type { DbClient } from '../../db.js'
import {
governedTracePoolAcquisitionDurationSeconds,
governedTracePoolConnections,
governedTracePoolRejectionsTotal,
governedTracePoolStatementTimeoutsTotal,
recordGovernedTraceOperationalError,
} from '../../observability/metrics.js'
import {
type TracingPoolOperationalLimits,
getTracingMaintenancePoolMax,
getTracingPoolOperationalLimits,
} from './operationalLimits.js'
export type TracingPools = {
traceIngestPool: Pool
traceReadPool: Pool
}
const DEFAULT_MAINTENANCE_STATEMENT_TIMEOUT_MS = 30_000
type TracePoolMetricLabel = 'ingest' | 'read' | 'maintenance'
type TracePoolKind = 'ingest' | 'read' | 'maintenance'
function poolBudget(
max: number,
statementTimeoutMillis: number,
limits: Pick<TracingPoolOperationalLimits, 'idleTimeoutMs' | 'connectionTimeoutMs'>
): BoundedPoolBudget {
return {
max,
idleTimeoutMillis: limits.idleTimeoutMs,
connectionTimeoutMillis: limits.connectionTimeoutMs,
statementTimeoutMillis,
}
}
function fallbackRequestTracingConnectionString(): string {
return (
process.env.CONTROL_API_PG_CONNECTION_STRING ??
(process.env.NODE_ENV === 'test' ? 'postgresql://localhost/tracing_test' : '')
)
}
function tracingConnectionString(kind: TracePoolKind): string {
const envName =
kind === 'ingest'
? 'TRACING_INGEST_PG_CONNECTION_STRING'
: kind === 'read'
? 'TRACING_READ_PG_CONNECTION_STRING'
: 'TRACING_MAINTENANCE_PG_CONNECTION_STRING'
const connectionString =
kind === 'maintenance'
? process.env[envName]
: (process.env[envName] ?? fallbackRequestTracingConnectionString())
if (!connectionString?.trim()) {
const requirement =
kind === 'maintenance' ? envName : `${envName} or CONTROL_API_PG_CONNECTION_STRING`
throw new Error(`${requirement} is required`)
}
return connectionString
}
function createTraceIngestPool(
PoolClass: PoolConstructor = Pool,
limits = getTracingPoolOperationalLimits()
): Pool {
return createBoundedPgPoolForConnection(
tracingConnectionString('ingest'),
poolBudget(limits.ingestPoolMax, limits.ingestStatementTimeoutMs, limits),
PoolClass
)
}
function createTraceReadPool(
PoolClass: PoolConstructor = Pool,
limits = getTracingPoolOperationalLimits()
): Pool {
return createBoundedPgPoolForConnection(
tracingConnectionString('read'),
poolBudget(limits.readPoolMax, limits.readStatementTimeoutMs, limits),
PoolClass
)
}
export function createTraceMaintenancePool(PoolClass: PoolConstructor = Pool): Pool {
const limits = getTracingPoolOperationalLimits()
return createBoundedPgPoolForConnection(
tracingConnectionString('maintenance'),
poolBudget(getTracingMaintenancePoolMax(), DEFAULT_MAINTENANCE_STATEMENT_TIMEOUT_MS, limits),
PoolClass
)
}
export function createTracingPools(PoolClass: PoolConstructor = Pool): TracingPools {
const limits = getTracingPoolOperationalLimits()
return {
traceIngestPool: createTraceIngestPool(PoolClass, limits),
traceReadPool: createTraceReadPool(PoolClass, limits),
}
}
let pools: TracingPools | undefined
let maintenancePool: Pool | undefined
export function getTracingPools(): TracingPools {
pools ??= createTracingPools()
return pools
}
export function getTraceMaintenancePool(): Pool {
maintenancePool ??= createTraceMaintenancePool()
return maintenancePool
}
export async function closeTraceMaintenancePool(): Promise<void> {
const pool = maintenancePool
if (!pool) return
await pool.end()
if (maintenancePool === pool) maintenancePool = undefined
}
let closeTracingPoolsPromise: Promise<void> | null = null
/** Close the lazily-created request tracing pools during Control API shutdown. */
export function closeTracingPools(): Promise<void> {
if (closeTracingPoolsPromise) return closeTracingPoolsPromise
const active = pools
if (!active) return Promise.resolve()
pools = undefined
closeTracingPoolsPromise = Promise.allSettled([
active.traceIngestPool.end(),
active.traceReadPool.end(),
]).then(results => {
closeTracingPoolsPromise = null
const failures = results.filter(result => result.status === 'rejected')
if (failures.length > 0) {
throw new AggregateError(
failures.map(result => (result as PromiseRejectedResult).reason),
'One or more tracing pools failed to close'
)
}
})
return closeTracingPoolsPromise
}
async function releaseAfterRollback(client: PoolClient): Promise<void> {
try {
await client.query('ROLLBACK')
} finally {
client.release()
}
}
function observePoolConnections(pool: Pool, label: TracePoolMetricLabel): void {
const total = Number(pool.totalCount ?? 0)
const idle = Number(pool.idleCount ?? 0)
governedTracePoolConnections.set({ pool: label, state: 'active' }, Math.max(0, total - idle))
governedTracePoolConnections.set({ pool: label, state: 'idle' }, idle)
governedTracePoolConnections.set(
{ pool: label, state: 'waiting' },
Number(pool.waitingCount ?? 0)
)
}
function recordStatementTimeout(error: unknown, label: TracePoolMetricLabel): void {
const candidate = error as { code?: unknown; message?: unknown }
if (
candidate?.code === '57014' ||
(typeof candidate?.message === 'string' && candidate.message.includes('statement timeout'))
) {
governedTracePoolStatementTimeoutsTotal.inc({ pool: label })
if (label !== 'maintenance') {
recordGovernedTraceOperationalError(label === 'read' ? 'read' : 'pool', 'statement_timeout')
}
}
}
async function acquireTraceClient(pool: Pool, label: TracePoolMetricLabel): Promise<PoolClient> {
const startedAt = process.hrtime.bigint()
try {
const pendingConnection = pool.connect()
observePoolConnections(pool, label)
return await pendingConnection
} catch (error) {
governedTracePoolRejectionsTotal.inc({ pool: label })
if (label !== 'maintenance') {
recordGovernedTraceOperationalError('pool', 'pool_rejected')
}
throw error
} finally {
governedTracePoolAcquisitionDurationSeconds.observe(
{ pool: label },
Number(process.hrtime.bigint() - startedAt) / 1e9
)
observePoolConnections(pool, label)
}
}
export async function withTraceIngestTransaction<T>(
work: (db: DbClient) => Promise<T>
): Promise<T> {
const pool = getTracingPools().traceIngestPool
const client = await acquireTraceClient(pool, 'ingest')
let released = false
try {
await client.query('BEGIN')
const result = await work(client)
await client.query('COMMIT')
return result
} catch (error) {
recordStatementTimeout(error, 'ingest')
await releaseAfterRollback(client)
released = true
throw error
} finally {
if (!released) client.release()
observePoolConnections(pool, 'ingest')
}
}
export async function withTraceReadTransaction<T>(work: (db: DbClient) => Promise<T>): Promise<T> {
const pool = getTracingPools().traceReadPool
const client = await acquireTraceClient(pool, 'read')
let released = false
try {
await client.query('BEGIN READ ONLY')
await client.query("SELECT set_config('statement_timeout', $1, true)", [
getTracingPoolOperationalLimits().readStatementTimeoutMs,
])
const result = await work(client)
await client.query('COMMIT')
return result
} catch (error) {
recordStatementTimeout(error, 'read')
await releaseAfterRollback(client)
released = true
throw error
} finally {
if (!released) client.release()
observePoolConnections(pool, 'read')
}
}
export async function withTraceMaintenanceClient<T>(
work: (db: PoolClient) => Promise<T>
): Promise<T> {
const pool = getTraceMaintenancePool()
const client = await acquireTraceClient(pool, 'maintenance')
try {
return await work(client)
} catch (error) {
recordStatementTimeout(error, 'maintenance')
throw error
} finally {
client.release()
observePoolConnections(pool, 'maintenance')
}
}