Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'
import { createHash, randomBytes } from 'node:crypto'
import { performance } from 'node:perf_hooks'
import { Pool, type PoolClient } from 'pg'
import request from 'supertest'
import { endPoolAndWaitForClients } from './helpers/realPostgresTeardown.js'
// The limiter's upserts run on their own pool (db.ts `rateLimitPool`, max 6,
// 5 s acquire), separate from the core pool (max 10, 2 s acquire).
//
// Service-level contract when the LIMITER pool is saturated: every
// checkAndIncrement call waits for the limiter pool's acquire timeout, counts
// rate_limit_backend_errors_total (the rate_limit_db_error line is throttled to
// one per bucket key per window), counts nothing in the bucket, and returns allowed:true with
// backendAvailable:false. `allowed` alone is not an admission decision there:
// each caller reads backendAvailable and decides. The external GFS limiter and
// upload admission fail closed with 503 (routes.externalGfsRateLimitFailClosed
// T3); callers that still admit on backendAvailable:false do so by their own
// choice, not because the service decided it.
//
// Isolation: a saturated CORE pool no longer reaches the limiter; it keeps
// counting.
//
// The production pool bounds are used unchanged: the CORE_POOL_* and
// RATE_LIMIT_POOL_* variables are deleted before db.js is imported, and the
// resulting bounds are asserted before anything else.
//
// Route-level corollary: with the core pool held, an external GFS request
// returns 503 session_backend_unavailable, because session auth queries the
// users table first (externalSessionAuth.ts) and fails closed before the
// limiter runs. The only mock is the session-token verifier (identity). Skipped without
// CONTROL_API_REAL_PG_ADMIN_URL.
const mockVerifyExternalSessionToken = vi.fn()
vi.mock('../src/utils/auth/externalSessionAuthToken.js', () => ({
verifyExternalSessionToken: (...args: unknown[]) => mockVerifyExternalSessionToken(...args),
}))
const PRODUCTION_POOL_MAX = 10
const PRODUCTION_ACQUIRE_TIMEOUT_MS = 2_000
const LIMITER_POOL_MAX = 6
const LIMITER_ACQUIRE_TIMEOUT_MS = 5_000
const CONCURRENT_CHECKS = 25
const LIMIT = 2
function databaseUrl(baseUrl: string, database: string): string {
const url = new URL(baseUrl)
url.pathname = `/${database}`
return url.toString()
}
function quoteIdent(value: string): string {
return `"${value.replace(/"/g, '""')}"`
}
const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL
const describeRealPostgres = adminUrl ? describe : describe.skip
describeRealPostgres('rate limiter under pool saturation', () => {
const database = `rate_limiter_saturation_${randomBytes(6).toString('hex')}`
const poolEnvKeys = [
'CONTROL_API_PG_CONNECTION_STRING',
'CORE_POOL_MAX',
'CORE_POOL_CONNECTION_TIMEOUT_MS',
'RATE_LIMIT_POOL_MAX',
'RATE_LIMIT_POOL_CONNECTION_TIMEOUT_MS',
'RATE_LIMIT_POOL_STATEMENT_TIMEOUT_MS',
] as const
const previousEnv = new Map<string, string | undefined>()
let adminPool: Pool
let corePool: Pool
let limiterPool: Pool
let mod: {
createApp: (gateway: unknown) => import('express').Express
config: typeof import('../src/config.js').config
checkAndIncrement: typeof import('../src/services/rateLimiterService.js').checkAndIncrement
rootLogger: typeof import('../src/observability/logger.js').rootLogger
backendErrorsTotal: typeof import('../src/observability/metrics.js').rateLimitBackendErrorsTotal
MockGateway: typeof import('./mockGateway.js').MockGateway
}
async function backendErrors(): Promise<number> {
const [sample] = (await mod.backendErrorsTotal.get()).values
if (!sample) throw new Error('rate_limit_backend_errors_total has no sample')
return sample.value
}
async function holdEveryClient(target: Pool, max: number): Promise<PoolClient[]> {
const held = await Promise.all(Array.from({ length: max }, () => target.connect()))
expect(target.totalCount).toBe(max)
expect(target.idleCount).toBe(0)
return held
}
beforeAll(async () => {
if (adminUrl === undefined) {
throw new Error('CONTROL_API_REAL_PG_ADMIN_URL is required by this suite')
}
const connectionString = databaseUrl(adminUrl, database)
for (const key of poolEnvKeys) previousEnv.set(key, process.env[key])
for (const key of poolEnvKeys) {
if (key !== 'CONTROL_API_PG_CONNECTION_STRING') delete process.env[key]
}
adminPool = new Pool({ connectionString: adminUrl })
await adminPool.query(`CREATE DATABASE ${quoteIdent(database)}`)
process.env.CONTROL_API_PG_CONNECTION_STRING = connectionString
const dbMod = await import('../src/db.js')
corePool = dbMod.pool as unknown as Pool
corePool.on('error', () => {})
limiterPool = dbMod.rateLimitPool as unknown as Pool
limiterPool.on('error', () => {})
const migratePool = new Pool({ connectionString })
await dbMod.initDb({ connect: () => migratePool.connect() })
await endPoolAndWaitForClients(migratePool)
const appMod = await import('../src/app.js')
const configMod = await import('../src/config.js')
const limiterMod = await import('../src/services/rateLimiterService.js')
const loggerMod = await import('../src/observability/logger.js')
const metricsMod = await import('../src/observability/metrics.js')
const { MockGateway } = await import('./mockGateway.js')
mod = {
createApp: appMod.createApp as never,
config: configMod.config,
checkAndIncrement: limiterMod.checkAndIncrement,
rootLogger: loggerMod.rootLogger,
backendErrorsTotal: metricsMod.rateLimitBackendErrorsTotal,
MockGateway,
}
}, 60_000)
afterAll(async () => {
try {
for (const key of poolEnvKeys) {
const value = previousEnv.get(key)
if (value === undefined) delete process.env[key]
else process.env[key] = value
}
await endPoolAndWaitForClients(corePool).catch(() => {})
await endPoolAndWaitForClients(limiterPool).catch(() => {})
if (adminPool) {
await adminPool.query(
`SELECT pg_terminate_backend(pid) FROM pg_stat_activity
WHERE datname = $1 AND pid <> pg_backend_pid()`,
[database]
)
await adminPool.query(`DROP DATABASE IF EXISTS ${quoteIdent(database)}`)
}
} finally {
await adminPool?.end()
}
})
it('runs against the production pool bounds', () => {
type Options = { options: { max: number; connectionTimeoutMillis: number } }
const core = (corePool as unknown as Options).options
expect(core.max).toBe(PRODUCTION_POOL_MAX)
expect(core.connectionTimeoutMillis).toBe(PRODUCTION_ACQUIRE_TIMEOUT_MS)
const limiter = (limiterPool as unknown as Options).options
expect(limiter.max).toBe(LIMITER_POOL_MAX)
expect(limiter.connectionTimeoutMillis).toBe(LIMITER_ACQUIRE_TIMEOUT_MS)
})
it('keeps counting while every core client is held', async () => {
const bucketKey = `core-saturation:${randomBytes(8).toString('hex')}`
const nowMs = Math.floor(Date.now() / 60_000) * 60_000 + 1_000
const warn = vi.spyOn(mod.rootLogger, 'warn').mockImplementation(() => {})
const errorsBefore = await backendErrors()
const held = await holdEveryClient(corePool, PRODUCTION_POOL_MAX)
try {
const results = []
for (let i = 0; i < 3; i += 1) {
results.push(await mod.checkAndIncrement(bucketKey, LIMIT, nowMs))
}
// Witness: the limiter reached Postgres and counted every call.
expect(results.map(result => result.count)).toEqual([1, 2, 3])
expect(results.map(result => result.allowed)).toEqual([true, true, false])
expect(results.every(result => result.backendAvailable)).toBe(true)
expect(
warn.mock.calls.filter(
call => (call[0] as { event?: string }).event === 'rate_limit_db_error'
)
).toHaveLength(0)
expect(await backendErrors()).toBe(errorsBefore)
} finally {
for (const client of held) client.release()
warn.mockRestore()
}
}, 15_000)
it('reports the backend unavailable for every concurrent check while every limiter client is held, then recovers', async () => {
const bucketKey = `saturation:${randomBytes(8).toString('hex')}`
// One fixed window for both phases, so a minute boundary cannot split them.
const nowMs = Math.floor(Date.now() / 60_000) * 60_000 + 1_000
const warn = vi.spyOn(mod.rootLogger, 'warn').mockImplementation(() => {})
const errorsBefore = await backendErrors()
const held = await holdEveryClient(limiterPool, LIMITER_POOL_MAX)
let released = false
try {
const startedAt = performance.now()
const settledAfterMs: number[] = []
const results = await Promise.all(
Array.from({ length: CONCURRENT_CHECKS }, () =>
mod.checkAndIncrement(bucketKey, LIMIT, nowMs).then(result => {
settledAfterMs.push(performance.now() - startedAt)
return result
})
)
)
expect(results).toHaveLength(CONCURRENT_CHECKS)
// The service reports allowed:true; the admission decision belongs to
// the caller, which must read backendAvailable.
expect(results.every(result => result.allowed === true)).toBe(true)
// The backend was unavailable and nothing was counted.
expect(results.every(result => result.backendAvailable === false)).toBe(true)
expect(results.every(result => result.count === 0)).toBe(true)
// Every failed query is counted; the log line is throttled per bucket
// key, so the 25 failures on one key write one line.
expect(await backendErrors()).toBe(errorsBefore + CONCURRENT_CHECKS)
const dbErrors = warn.mock.calls
.map(
call =>
call[0] as { event?: string; hashedKey?: string; err?: string; suppressed?: number }
)
.filter(payload => payload.event === 'rate_limit_db_error')
expect(dbErrors).toHaveLength(1)
// Witness that the failure is the pool's acquire timeout and nothing else.
expect(dbErrors[0]).toMatchObject({
hashedKey: createHash('sha256').update(bucketKey).digest('hex'),
suppressed: 0,
})
expect(dbErrors[0]!.err).toMatch(/timeout exceeded when trying to connect/)
// Every call waited for the acquire timeout: none settled before it,
// which is a property of the pool, not of machine speed. They waited in
// parallel: in series the second call alone would settle after two
// timeouts, so the bound is structural rather than a latency budget.
expect(settledAfterMs).toHaveLength(CONCURRENT_CHECKS)
expect(Math.min(...settledAfterMs)).toBeGreaterThanOrEqual(LIMITER_ACQUIRE_TIMEOUT_MS - 100)
expect(Math.max(...settledAfterMs)).toBeLessThan(2 * LIMITER_ACQUIRE_TIMEOUT_MS)
// Read through a held client: this also proves the database and the
// table are reachable, so "no row" is not an absent table.
const persisted = await held[0]!.query<{ rows: string }>(
'SELECT count(*)::text AS rows FROM rate_limit_buckets WHERE bucket_key = $1',
[bucketKey]
)
expect(persisted.rows[0]?.rows).toBe('0')
for (const client of held) client.release()
released = true
// Recovery witness, same key and window: counting starts at 1, which also
// proves the 25 unavailable calls persisted nothing.
const recovered = []
for (let i = 0; i < 3; i += 1) {
recovered.push(await mod.checkAndIncrement(bucketKey, LIMIT, nowMs))
}
expect(recovered.map(result => result.count)).toEqual([1, 2, 3])
expect(recovered.map(result => result.allowed)).toEqual([true, true, false])
expect(recovered.every(result => result.backendAvailable)).toBe(true)
expect(
warn.mock.calls.filter(
call => (call[0] as { event?: string }).event === 'rate_limit_db_error'
)
).toHaveLength(1)
expect(await backendErrors()).toBe(errorsBefore + CONCURRENT_CHECKS)
} finally {
if (!released) for (const client of held) client.release()
warn.mockRestore()
}
}, 20_000)
it('answers 503 at the route while the core pool is held, because session auth fails first', async () => {
const internalToken = mod.config.internalServiceTokens['external-rest-api']
if (!internalToken) throw new Error('config has no external-rest-api internal service token')
mockVerifyExternalSessionToken.mockReset()
mockVerifyExternalSessionToken.mockReturnValue({
userId: '11111111-aaaa-4aaa-8aaa-111111111111',
email: '[email protected]',
teamId: '33333333-cccc-4ccc-8ccc-333333333333',
role: 'member',
authGeneration: 1,
exp: Math.floor(Date.now() / 1000) + 3600,
})
const app = mod.createApp(new mod.MockGateway())
const send = () =>
request(app)
.get('/api/v1/external/gfs/resources')
.set('Authorization', `Bearer ${internalToken}`)
.set('x-service-token', 'external-rest-api')
.set('x-user-session-token', 'session')
// Liveness witness with the pool free: the request passes internal auth,
// reaches the session verifier, and is denied by the users-table lookup
// (no such user), so the 503 below comes from that same lookup.
const free = await send()
expect(free.status).toBe(401)
expect(mockVerifyExternalSessionToken).toHaveBeenCalledTimes(1)
const held = await holdEveryClient(corePool, PRODUCTION_POOL_MAX)
try {
const saturated = await send()
expect(saturated.status).toBe(503)
expect(saturated.body).toEqual({
error: 'session_backend_unavailable',
retryAfterSeconds: 2,
})
expect(saturated.headers['retry-after']).toBe('2')
expect(mockVerifyExternalSessionToken).toHaveBeenCalledTimes(2)
} finally {
for (const client of held) client.release()
}
}, 15_000)
})