Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
import { beforeEach, describe, expect, it, vi } from 'vitest'
type FakeClient = {
query: ReturnType<typeof vi.fn>
release: ReturnType<typeof vi.fn>
}
const fakePools = vi.hoisted(
(): Array<{
config: Record<string, unknown>
connect: ReturnType<typeof vi.fn>
end: ReturnType<typeof vi.fn>
totalCount: number
idleCount: number
waitingCount: number
}> => []
)
const metricSpies = vi.hoisted(() => ({
acquisitionObserve: vi.fn(),
connectionsSet: vi.fn(),
rejectionsInc: vi.fn(),
statementTimeoutsInc: vi.fn(),
recordOperationalError: vi.fn(),
}))
vi.mock('pg', () => ({
Pool: class FakePool {
config: Record<string, unknown>
connect = vi.fn()
end = vi.fn().mockResolvedValue(undefined)
totalCount = 0
idleCount = 0
waitingCount = 0
constructor(config: Record<string, unknown>) {
this.config = config
fakePools.push(this)
}
},
}))
vi.mock('../src/observability/metrics.js', () => ({
governedTracePoolAcquisitionDurationSeconds: { observe: metricSpies.acquisitionObserve },
governedTracePoolConnections: { set: metricSpies.connectionsSet },
governedTracePoolRejectionsTotal: { inc: metricSpies.rejectionsInc },
governedTracePoolStatementTimeoutsTotal: { inc: metricSpies.statementTimeoutsInc },
recordGovernedTraceOperationalError: metricSpies.recordOperationalError,
}))
describe('tracing pool isolation', () => {
beforeEach(() => {
vi.resetModules()
fakePools.length = 0
delete process.env.TRACING_INGEST_POOL_MAX
delete process.env.TRACING_READ_POOL_MAX
delete process.env.TRACING_MAINTENANCE_POOL_MAX
delete process.env.TRACING_INGEST_PG_CONNECTION_STRING
delete process.env.TRACING_READ_PG_CONNECTION_STRING
delete process.env.TRACING_MAINTENANCE_PG_CONNECTION_STRING
process.env.CONTROL_API_PG_CONNECTION_STRING = 'postgresql://localhost/tracing_test'
metricSpies.acquisitionObserve.mockReset()
metricSpies.connectionsSet.mockReset()
metricSpies.rejectionsInc.mockReset()
metricSpies.statementTimeoutsInc.mockReset()
metricSpies.recordOperationalError.mockReset()
})
it('creates independent bounded ingest and read request pools without maintenance', async () => {
process.env.TRACING_INGEST_POOL_MAX = '5'
process.env.TRACING_READ_POOL_MAX = '4'
const { createTracingPools } = await import('../src/services/tracing/pools.js')
const PoolClass = (await import('pg')).Pool
const pools = createTracingPools(PoolClass)
expect(pools.traceIngestPool).not.toBe(pools.traceReadPool)
expect(fakePools).toHaveLength(2)
expect(fakePools.at(-2)?.config.max).toBe(5)
expect(fakePools.at(-1)?.config.max).toBe(4)
expect(fakePools.at(-2)?.config.statement_timeout).toBe(5_000)
expect(fakePools.at(-1)?.config.statement_timeout).toBe(2_000)
expect(fakePools.at(-2)?.config.connectionString).toBe('postgresql://localhost/tracing_test')
expect(fakePools.at(-1)?.config.connectionString).toBe('postgresql://localhost/tracing_test')
})
it('uses role-specific tracing connection strings when provided', async () => {
process.env.TRACING_INGEST_PG_CONNECTION_STRING = 'postgresql://runtime/ingest'
process.env.TRACING_READ_PG_CONNECTION_STRING = 'postgresql://runtime/read'
const { createTracingPools } = await import('../src/services/tracing/pools.js')
const PoolClass = (await import('pg')).Pool
createTracingPools(PoolClass)
expect(fakePools).toHaveLength(2)
expect(fakePools.at(-2)?.config.connectionString).toBe('postgresql://runtime/ingest')
expect(fakePools.at(-1)?.config.connectionString).toBe('postgresql://runtime/read')
})
it('can create only the maintenance pool for the trace maintenance worker', async () => {
process.env.TRACING_MAINTENANCE_PG_CONNECTION_STRING = 'postgresql://runtime/maintenance'
const { getTraceMaintenancePool } = await import('../src/services/tracing/pools.js')
const pool = getTraceMaintenancePool()
expect(fakePools).toHaveLength(1)
expect(pool).toBe(fakePools[0])
expect(fakePools[0]?.config.connectionString).toBe('postgresql://runtime/maintenance')
expect(fakePools[0]?.config.max).toBe(1)
})
it('closes and resets the maintenance pool during worker shutdown', async () => {
process.env.TRACING_MAINTENANCE_PG_CONNECTION_STRING = 'postgresql://runtime/maintenance'
const { closeTraceMaintenancePool, getTraceMaintenancePool } =
await import('../src/services/tracing/pools.js')
const firstPool = getTraceMaintenancePool()
await closeTraceMaintenancePool()
expect(firstPool.end).toHaveBeenCalledOnce()
expect(getTraceMaintenancePool()).not.toBe(firstPool)
expect(fakePools).toHaveLength(2)
})
it('closes both lazily-created request pools once during API shutdown', async () => {
const { closeTracingPools, getTracingPools } = await import('../src/services/tracing/pools.js')
const first = getTracingPools()
await Promise.all([closeTracingPools(), closeTracingPools()])
expect(first.traceIngestPool.end).toHaveBeenCalledOnce()
expect(first.traceReadPool.end).toHaveBeenCalledOnce()
expect(getTracingPools()).not.toBe(first)
expect(fakePools).toHaveLength(4)
})
it('does not recreate a request pool when an in-flight trace transaction finishes during shutdown', async () => {
const { closeTracingPools, getTracingPools, withTraceIngestTransaction } =
await import('../src/services/tracing/pools.js')
const { traceIngestPool } = getTracingPools()
const client: FakeClient = { query: vi.fn().mockResolvedValue({}), release: vi.fn() }
vi.mocked(traceIngestPool.connect).mockResolvedValue(client as never)
let finishWork!: () => void
let enteredWork!: () => void
const entered = new Promise<void>(resolve => (enteredWork = resolve))
const transaction = withTraceIngestTransaction(
() =>
new Promise<void>(resolve => {
finishWork = resolve
enteredWork()
})
)
await entered
await closeTracingPools()
finishWork()
await transaction
expect(traceIngestPool.end).toHaveBeenCalledOnce()
expect(client.release).toHaveBeenCalledOnce()
expect(fakePools).toHaveLength(2)
})
it('requires an explicit maintenance connection string', async () => {
const { createTraceMaintenancePool, getTraceMaintenancePool } =
await import('../src/services/tracing/pools.js')
const PoolClass = (await import('pg')).Pool
expect(() => createTraceMaintenancePool(PoolClass)).toThrow(
'TRACING_MAINTENANCE_PG_CONNECTION_STRING is required'
)
expect(() => getTraceMaintenancePool()).toThrow(
'TRACING_MAINTENANCE_PG_CONNECTION_STRING is required'
)
expect(fakePools).toHaveLength(0)
})
it('commits and releases an ingest connection on success', async () => {
const { getTracingPools, withTraceIngestTransaction } =
await import('../src/services/tracing/pools.js')
const { traceIngestPool } = getTracingPools()
const client: FakeClient = { query: vi.fn().mockResolvedValue({}), release: vi.fn() }
vi.mocked(traceIngestPool.connect).mockResolvedValue(client as never)
await expect(withTraceIngestTransaction(async () => 'done')).resolves.toBe('done')
expect(client.query).toHaveBeenNthCalledWith(1, 'BEGIN')
expect(client.query).toHaveBeenNthCalledWith(2, 'COMMIT')
expect(client.release).toHaveBeenCalledOnce()
expect(metricSpies.acquisitionObserve).toHaveBeenCalledWith(
{ pool: 'ingest' },
expect.any(Number)
)
expect(metricSpies.rejectionsInc).not.toHaveBeenCalled()
expect(metricSpies.connectionsSet).toHaveBeenCalledWith({ pool: 'ingest', state: 'waiting' }, 0)
})
it('rolls back and releases an ingest connection on failure', async () => {
const { getTracingPools, withTraceIngestTransaction } =
await import('../src/services/tracing/pools.js')
const { traceIngestPool } = getTracingPools()
const client: FakeClient = { query: vi.fn().mockResolvedValue({}), release: vi.fn() }
vi.mocked(traceIngestPool.connect).mockResolvedValue(client as never)
await expect(
withTraceIngestTransaction(async () => {
throw new Error('append failed')
})
).rejects.toThrow('append failed')
expect(client.query).toHaveBeenNthCalledWith(1, 'BEGIN')
expect(client.query).toHaveBeenNthCalledWith(2, 'ROLLBACK')
expect(client.release).toHaveBeenCalledOnce()
})
it('records a read-pool rejection when acquisition fails', async () => {
const { getTracingPools, withTraceReadTransaction } =
await import('../src/services/tracing/pools.js')
const { traceReadPool } = getTracingPools()
vi.mocked(traceReadPool.connect).mockRejectedValue(new Error('pool exhausted'))
await expect(withTraceReadTransaction(async () => 'unreachable')).rejects.toThrow(
'pool exhausted'
)
expect(metricSpies.acquisitionObserve).toHaveBeenCalledWith(
{ pool: 'read' },
expect.any(Number)
)
expect(metricSpies.rejectionsInc).toHaveBeenCalledWith({ pool: 'read' })
expect(metricSpies.recordOperationalError).toHaveBeenCalledWith('pool', 'pool_rejected')
})
it('publishes a live waiter while a read-pool acquisition is pending', async () => {
const { getTracingPools, withTraceReadTransaction } =
await import('../src/services/tracing/pools.js')
const { traceReadPool } = getTracingPools()
const client: FakeClient = { query: vi.fn().mockResolvedValue({}), release: vi.fn() }
let resolveConnection!: (client: FakeClient) => void
const pendingConnection = new Promise<FakeClient>(resolve => {
resolveConnection = resolve
})
const livePool = traceReadPool as unknown as {
waitingCount: number
connect: ReturnType<typeof vi.fn>
}
livePool.waitingCount = 1
livePool.connect.mockReturnValue(pendingConnection)
const transaction = withTraceReadTransaction(async () => 'done')
expect(metricSpies.connectionsSet).toHaveBeenCalledWith({ pool: 'read', state: 'waiting' }, 1)
livePool.waitingCount = 0
resolveConnection(client)
await expect(transaction).resolves.toBe('done')
expect(metricSpies.connectionsSet).toHaveBeenLastCalledWith(
{ pool: 'read', state: 'waiting' },
0
)
})
it('records statement timeout cancellation separately from pool acquisition rejection', async () => {
const { getTracingPools, withTraceReadTransaction } =
await import('../src/services/tracing/pools.js')
const { traceReadPool } = getTracingPools()
const client: FakeClient = { query: vi.fn().mockResolvedValue({}), release: vi.fn() }
vi.mocked(traceReadPool.connect).mockResolvedValue(client as never)
await expect(
withTraceReadTransaction(async () => {
throw Object.assign(new Error('canceling statement due to statement timeout'), {
code: '57014',
})
})
).rejects.toThrow('statement timeout')
expect(metricSpies.statementTimeoutsInc).toHaveBeenCalledWith({ pool: 'read' })
expect(metricSpies.recordOperationalError).toHaveBeenCalledWith('read', 'statement_timeout')
expect(metricSpies.rejectionsInc).not.toHaveBeenCalled()
})
it('keeps maintenance failures out of the ingest/read operations snapshot signal', async () => {
process.env.TRACING_MAINTENANCE_PG_CONNECTION_STRING = 'postgresql://runtime/maintenance'
const { getTraceMaintenancePool, withTraceMaintenanceClient } =
await import('../src/services/tracing/pools.js')
const maintenancePool = getTraceMaintenancePool()
vi.mocked(maintenancePool.connect).mockRejectedValue(new Error('maintenance pool unavailable'))
await expect(withTraceMaintenanceClient(async () => 'unreachable')).rejects.toThrow(
'maintenance pool unavailable'
)
expect(metricSpies.rejectionsInc).toHaveBeenCalledWith({ pool: 'maintenance' })
expect(metricSpies.recordOperationalError).not.toHaveBeenCalled()
})
})