Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

feat(control-api): give the Postgres rate limiter its own bounded pool with synchronous_commit off

Alfredo Lopez7d138d60400b
diff --git a/.github/workflows/ci-public.yml b/.github/workflows/ci-public.yml
index 77d0f6c62..7720c73c6 100644
--- a/.github/workflows/ci-public.yml
+++ b/.github/workflows/ci-public.yml
@@ -308,6 +308,7 @@ jobs:
db.llmProviderAttemptConnectionIntegrity.realPostgres.integration \
services.llmCatalogSync.realPostgres.integration \
routes.externalGfsRateLimitStress.realPostgres.integration \
+ routes.externalGfsRateLimitFailClosed.realPostgres.integration \
services.rateLimiterPoolSaturation.realPostgres.integration
do
grep -Fq "$suite" "$normalized_log" || {
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index 07501440f..a5035d477 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.606",
+ "version": "0.1.607",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.606",
+ "version": "0.1.607",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index 0f2f7f103..59d3d2ffd 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.606",
+ "version": "0.1.607",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/boundedPgPool.ts b/control-api/src/boundedPgPool.ts
index e8d80d8c8..14ed183dd 100644
--- a/control-api/src/boundedPgPool.ts
+++ b/control-api/src/boundedPgPool.ts
@@ -16,10 +16,16 @@ const LIMITS: Record<keyof BoundedPoolBudget, readonly [number, number]> = {
statementTimeoutMillis: [100, 30_000],
}
+/**
+ * `sessionOptions` is the libpq `options` startup parameter (for example
+ * `-c synchronous_commit=off`). pg sends it in the startup packet, so every
+ * connection of the pool runs with those settings from its first statement.
+ */
export function createBoundedPgPoolForConnection(
connectionString: string,
budget: BoundedPoolBudget,
- PoolClass: PoolConstructor = Pool
+ PoolClass: PoolConstructor = Pool,
+ sessionOptions?: string
): Pool {
for (const [name, value] of Object.entries(budget) as Array<[keyof BoundedPoolBudget, number]>) {
const [min, max] = LIMITS[name]
@@ -33,5 +39,6 @@ export function createBoundedPgPoolForConnection(
idleTimeoutMillis: budget.idleTimeoutMillis,
connectionTimeoutMillis: budget.connectionTimeoutMillis,
statement_timeout: budget.statementTimeoutMillis,
+ ...(sessionOptions === undefined ? {} : { options: sessionOptions }),
})
}
diff --git a/control-api/src/db.ts b/control-api/src/db.ts
index e710d5d98..5ceb8076d 100644
--- a/control-api/src/db.ts
+++ b/control-api/src/db.ts
@@ -121,11 +121,33 @@ const DEFAULT_CORE_POOL_IDLE_TIMEOUT_MS = 30_000
const DEFAULT_CORE_POOL_CONNECTION_TIMEOUT_MS = 2_000
const DEFAULT_CORE_POOL_STATEMENT_TIMEOUT_MS = 15_000
+// The Postgres rate limiter (`rate_limit_buckets` upserts and their prune) has
+// its own pool so a saturated core pool (session auth, SSE LISTEN clients,
+// polling) cannot starve it, and a limiter burst cannot take connections from
+// the rest of the service. synchronous_commit=off: the rows are 60 s counters
+// that the pruner deletes anyway, so losing the last ~200 ms of increments on
+// a Postgres crash is acceptable, and the commit no longer waits on WAL flush
+// while holding the hot bucket row lock.
+const MAX_RATE_LIMIT_POOL_MAX = 16
+const DEFAULT_RATE_LIMIT_POOL_MAX = 6
+const DEFAULT_RATE_LIMIT_POOL_IDLE_TIMEOUT_MS = 30_000
+const DEFAULT_RATE_LIMIT_POOL_CONNECTION_TIMEOUT_MS = 5_000
+const DEFAULT_RATE_LIMIT_POOL_STATEMENT_TIMEOUT_MS = 3_000
+export const RATE_LIMIT_POOL_SESSION_OPTIONS = '-c synchronous_commit=off'
+
+/**
+ * Unset (or empty) means the default. A set value outside `[min, max]`, or one
+ * that is not an integer, stops startup instead of silently becoming the
+ * default.
+ */
function boundedEnvInteger(name: string, fallback: number, min: number, max: number): number {
const raw = process.env[name]
if (!raw) return fallback
const value = Number(raw)
- return Number.isInteger(value) && value >= min && value <= max ? value : fallback
+ if (!Number.isInteger(value) || value < min || value > max) {
+ throw new Error(`${name} must be an integer in [${min}, ${max}], got ${JSON.stringify(raw)}`)
+ }
+ return value
}
export function createBoundedPgPool(
@@ -166,6 +188,41 @@ export function createCorePool(PoolClass: PoolConstructor = Pool): Pool {
export const pool = createCorePool()
export const corePool = pool
+export function rateLimitPoolBudget(): BoundedPoolBudget {
+ return {
+ max: boundedEnvInteger(
+ 'RATE_LIMIT_POOL_MAX',
+ DEFAULT_RATE_LIMIT_POOL_MAX,
+ MIN_POOL_MAX,
+ MAX_RATE_LIMIT_POOL_MAX
+ ),
+ idleTimeoutMillis: DEFAULT_RATE_LIMIT_POOL_IDLE_TIMEOUT_MS,
+ connectionTimeoutMillis: boundedEnvInteger(
+ 'RATE_LIMIT_POOL_CONNECTION_TIMEOUT_MS',
+ DEFAULT_RATE_LIMIT_POOL_CONNECTION_TIMEOUT_MS,
+ MIN_CONNECTION_TIMEOUT_MS,
+ MAX_CONNECTION_TIMEOUT_MS
+ ),
+ statementTimeoutMillis: boundedEnvInteger(
+ 'RATE_LIMIT_POOL_STATEMENT_TIMEOUT_MS',
+ DEFAULT_RATE_LIMIT_POOL_STATEMENT_TIMEOUT_MS,
+ MIN_STATEMENT_TIMEOUT_MS,
+ MAX_STATEMENT_TIMEOUT_MS
+ ),
+ }
+}
+
+export function createRateLimitPool(PoolClass: PoolConstructor = Pool): Pool {
+ return createBoundedPgPoolForConnection(
+ config.pgConnectionString,
+ rateLimitPoolBudget(),
+ PoolClass,
+ RATE_LIMIT_POOL_SESSION_OPTIONS
+ )
+}
+
+export const rateLimitPool = createRateLimitPool()
+
async function applyBaselineSchema(db: DbClient): Promise<void> {
// Baseline includes additive Phase 0 workflow-trigger tables for fresh
// clusters. Existing clusters receive the same tables through migration
diff --git a/control-api/src/main.ts b/control-api/src/main.ts
index d12a0a357..76fcf1bba 100644
--- a/control-api/src/main.ts
+++ b/control-api/src/main.ts
@@ -1,5 +1,5 @@
import { config } from './config.js'
-import { assertDbReady, pool } from './db.js'
+import { assertDbReady, pool, rateLimitPool } from './db.js'
import { K8sGateway } from './k8s.js'
import { reconcileAllowedModelsConfigMapOnBoot } from './llmAllowedModelsBootReconcile.js'
import { logRegistryConnectionState } from './registryBootGuard.js'
@@ -228,5 +228,6 @@ main().catch(error => {
stopSubscriptionCatalogSyncCron()
stopWorkflowApprovalTraceProjector()
void pool.end()
+ void rateLimitPool.end()
process.exit(1)
})
diff --git a/control-api/src/services/rateLimiterService.ts b/control-api/src/services/rateLimiterService.ts
index 948ffc203..3bc69fd51 100644
--- a/control-api/src/services/rateLimiterService.ts
+++ b/control-api/src/services/rateLimiterService.ts
@@ -1,6 +1,6 @@
import { createHash } from 'node:crypto'
import type { PoolClient } from 'pg'
-import { pool } from '../db.js'
+import { pool, rateLimitPool } from '../db.js'
import { rootLogger } from '../observability/logger.js'
/**
@@ -51,7 +51,7 @@ export async function checkAndIncrement(
cost = 1
): Promise<RateLimitCheck> {
return checkAndIncrementWithQuery(
- (text, values) => pool.query(text, values),
+ (text, values) => rateLimitPool.query(text, values),
bucketKey,
maxPerMinute,
nowMs,
@@ -61,7 +61,7 @@ export async function checkAndIncrement(
/**
* Same atomic limiter operation against an explicitly supplied query
- * function. Production callers use the process-wide pool above; the narrow
+ * function. Production callers use the dedicated limiter pool above; the narrow
* seam lets the real-Postgres integration suite exercise this exact SQL
* against an isolated database without changing runtime ownership.
*/
@@ -190,6 +190,10 @@ function advisoryKey(value: string): AdvisoryLockKey {
return { high: digest.readInt32BE(0), low: digest.readInt32BE(4) }
}
+// The advisory-lock client stays on the core pool: advisory locks write no WAL,
+// so synchronous_commit=off buys nothing, and this client is held for the life
+// of the process, which would take one of the limiter pool's few connections
+// permanently.
async function getConcurrencyClient(): Promise<PoolClient> {
if (concurrencyClient) return concurrencyClient
if (!concurrencyClientPromise) {
@@ -348,9 +352,10 @@ export async function acquireRateLimitConcurrencyLease(
*/
export async function cleanupExpiredBuckets(nowMs = Date.now()): Promise<number> {
const cutoff = nowMs - 5 * 60_000
- const result = await pool.query(`DELETE FROM rate_limit_buckets WHERE window_start_ms < $1`, [
- cutoff,
- ])
+ const result = await rateLimitPool.query(
+ `DELETE FROM rate_limit_buckets WHERE window_start_ms < $1`,
+ [cutoff]
+ )
return result.rowCount ?? 0
}
diff --git a/control-api/test/db.poolFactory.test.ts b/control-api/test/db.poolFactory.test.ts
index 913e708df..e22c5e739 100644
--- a/control-api/test/db.poolFactory.test.ts
+++ b/control-api/test/db.poolFactory.test.ts
@@ -21,6 +21,9 @@ describe('core Postgres pool budget', () => {
delete process.env.CORE_POOL_IDLE_TIMEOUT_MS
delete process.env.CORE_POOL_CONNECTION_TIMEOUT_MS
delete process.env.CORE_POOL_STATEMENT_TIMEOUT_MS
+ delete process.env.RATE_LIMIT_POOL_MAX
+ delete process.env.RATE_LIMIT_POOL_CONNECTION_TIMEOUT_MS
+ delete process.env.RATE_LIMIT_POOL_STATEMENT_TIMEOUT_MS
})
it('creates the core pool with explicit bounded connection and statement budgets', async () => {
@@ -58,3 +61,74 @@ describe('core Postgres pool budget', () => {
).toThrow('Invalid bounded Postgres pool budget: max')
})
})
+
+describe('rate limiter Postgres pool', () => {
+ beforeEach(() => {
+ vi.resetModules()
+ fakePools.length = 0
+ delete process.env.CORE_POOL_MAX
+ delete process.env.RATE_LIMIT_POOL_MAX
+ delete process.env.RATE_LIMIT_POOL_CONNECTION_TIMEOUT_MS
+ delete process.env.RATE_LIMIT_POOL_STATEMENT_TIMEOUT_MS
+ })
+
+ it('builds the limiter pool with 6/5000/3000 and synchronous_commit off, and leaves the core pool untouched', async () => {
+ const db = await import('../src/db.js')
+
+ // Module load builds exactly the core pool, then the limiter pool.
+ expect(fakePools).toHaveLength(2)
+ const [core, limiter] = fakePools.map(p => p.config)
+ expect(limiter).toMatchObject({
+ max: 6,
+ idleTimeoutMillis: 30_000,
+ connectionTimeoutMillis: 5_000,
+ statement_timeout: 3_000,
+ options: '-c synchronous_commit=off',
+ })
+ expect(db.RATE_LIMIT_POOL_SESSION_OPTIONS).toBe('-c synchronous_commit=off')
+ expect(core).toMatchObject({
+ max: 10,
+ connectionTimeoutMillis: 2_000,
+ statement_timeout: 15_000,
+ })
+ expect(core).not.toHaveProperty('options')
+ })
+
+ it('reads the limiter pool bounds from the environment', async () => {
+ process.env.RATE_LIMIT_POOL_MAX = '16'
+ process.env.RATE_LIMIT_POOL_CONNECTION_TIMEOUT_MS = '100'
+ process.env.RATE_LIMIT_POOL_STATEMENT_TIMEOUT_MS = '30000'
+
+ const { rateLimitPoolBudget } = await import('../src/db.js')
+
+ expect(rateLimitPoolBudget()).toEqual({
+ max: 16,
+ idleTimeoutMillis: 30_000,
+ connectionTimeoutMillis: 100,
+ statementTimeoutMillis: 30_000,
+ })
+ })
+
+ it.each([
+ // [env, value, range, pools built before the refusal (core is built first)]
+ ['RATE_LIMIT_POOL_MAX', '17', '[1, 16]', 1],
+ ['RATE_LIMIT_POOL_MAX', '0', '[1, 16]', 1],
+ ['RATE_LIMIT_POOL_CONNECTION_TIMEOUT_MS', '99', '[100, 30000]', 1],
+ ['RATE_LIMIT_POOL_STATEMENT_TIMEOUT_MS', '30001', '[100, 30000]', 1],
+ ['RATE_LIMIT_POOL_MAX', '6.5', '[1, 16]', 1],
+ ['CORE_POOL_MAX', 'twelve', '[1, 64]', 0],
+ ['CORE_POOL_MAX', '65', '[1, 64]', 0],
+ ] as const)(
+ 'refuses %s=%s at startup instead of falling back to the default',
+ async (name, value, range, built) => {
+ process.env[name] = value
+
+ await expect(import('../src/db.js')).rejects.toThrow(
+ `${name} must be an integer in ${range}, got "${value}"`
+ )
+ // The refusal stops construction at the pool that reads the bad value.
+ expect(fakePools).toHaveLength(built)
+ delete process.env[name]
+ }
+ )
+})
diff --git a/control-api/test/pluginWorkloadSdkGrantUpdateNotify.realPostgres.integration.test.ts b/control-api/test/pluginWorkloadSdkGrantUpdateNotify.realPostgres.integration.test.ts
index 507405b76..0fbb3eb9e 100644
--- a/control-api/test/pluginWorkloadSdkGrantUpdateNotify.realPostgres.integration.test.ts
+++ b/control-api/test/pluginWorkloadSdkGrantUpdateNotify.realPostgres.integration.test.ts
@@ -113,6 +113,7 @@ describeRealPostgres(
listenClient?.release()
await listenPool?.end()
await db?.pool.end()
+ await db?.rateLimitPool.end()
if (previousPgConnectionString === undefined) {
delete process.env.CONTROL_API_PG_CONNECTION_STRING
} else {
diff --git a/control-api/test/pluginWorkloadSdkSteplessQuota.realPostgres.integration.test.ts b/control-api/test/pluginWorkloadSdkSteplessQuota.realPostgres.integration.test.ts
index 2f55bc2c9..bbcc366de 100644
--- a/control-api/test/pluginWorkloadSdkSteplessQuota.realPostgres.integration.test.ts
+++ b/control-api/test/pluginWorkloadSdkSteplessQuota.realPostgres.integration.test.ts
@@ -110,6 +110,7 @@ describeRealPostgres(
afterAll(async () => {
await db?.pool.end()
+ await db?.rateLimitPool.end()
if (previousPgConnectionString === undefined) {
delete process.env.CONTROL_API_PG_CONNECTION_STRING
} else {
diff --git a/control-api/test/registry.test.ts b/control-api/test/registry.test.ts
index 8263e5106..1d8b9a7e5 100644
--- a/control-api/test/registry.test.ts
+++ b/control-api/test/registry.test.ts
@@ -57,6 +57,9 @@ vi.mock('../src/db.js', () => ({
query: (...args: unknown[]) => mockPoolQuery(args[0], args[1] as unknown[] | undefined),
connect: vi.fn(),
},
+ rateLimitPool: {
+ query: (...args: unknown[]) => mockPoolQuery(args[0], args[1] as unknown[] | undefined),
+ },
withTransaction: vi.fn(),
}))
diff --git a/control-api/test/routes.adminWorkflows.test.ts b/control-api/test/routes.adminWorkflows.test.ts
index e24015975..29437a81f 100644
--- a/control-api/test/routes.adminWorkflows.test.ts
+++ b/control-api/test/routes.adminWorkflows.test.ts
@@ -31,6 +31,9 @@ vi.mock('../src/db.js', () => ({
query: (...args: unknown[]) => mockPoolQuery(...args),
connect: (...args: unknown[]) => mockPoolConnect(...args),
},
+ rateLimitPool: {
+ query: (...args: unknown[]) => mockPoolQuery(...args),
+ },
withTransaction: (work: (db: { query: typeof mockPoolQuery }) => unknown) =>
work({ query: mockPoolQuery }),
}))
diff --git a/control-api/test/routes.auth.mcp-host.issue.security.test.ts b/control-api/test/routes.auth.mcp-host.issue.security.test.ts
index 118c5d653..4b6ea4bcb 100644
--- a/control-api/test/routes.auth.mcp-host.issue.security.test.ts
+++ b/control-api/test/routes.auth.mcp-host.issue.security.test.ts
@@ -55,6 +55,9 @@ vi.mock('../src/db.js', () => ({
query: (...args: unknown[]) => mockPoolQueryDispatch(args[0], args[1] as unknown[] | undefined),
connect: vi.fn(),
},
+ rateLimitPool: {
+ query: (...args: unknown[]) => mockPoolQueryDispatch(args[0], args[1] as unknown[] | undefined),
+ },
withTransaction: vi.fn(),
}))
diff --git a/control-api/test/routes.externalGfs.test.ts b/control-api/test/routes.externalGfs.test.ts
index ae33b5d52..4a47ef33c 100644
--- a/control-api/test/routes.externalGfs.test.ts
+++ b/control-api/test/routes.externalGfs.test.ts
@@ -89,6 +89,7 @@ vi.mock('../src/middleware/gfsUploadAdmission.js', async importOriginal => {
})
vi.mock('../src/db.js', () => ({
pool: { query: (...a: unknown[]) => mockQuery(...a) },
+ rateLimitPool: { query: (...a: unknown[]) => mockQuery(...a) },
withTransaction: (...a: unknown[]) => mockWithTransaction(...a),
}))
vi.mock('../src/services/tracing/controlApiPermissionEvents.js', () => ({
diff --git a/control-api/test/routes.externalGfsRateLimitFailClosed.realPostgres.integration.test.ts b/control-api/test/routes.externalGfsRateLimitFailClosed.realPostgres.integration.test.ts
new file mode 100644
index 000000000..20b4866b2
--- /dev/null
+++ b/control-api/test/routes.externalGfsRateLimitFailClosed.realPostgres.integration.test.ts
@@ -0,0 +1,103 @@
+import { afterAll, beforeAll, describe, expect, it } from 'vitest'
+import { randomBytes } from 'node:crypto'
+import { Pool } from 'pg'
+
+// The external GFS limiter's Postgres backend, against real PostgreSQL at the
+// production pool bounds: the CORE_POOL_* and RATE_LIMIT_POOL_* variables are
+// deleted before db.js is imported. Skipped without
+// CONTROL_API_REAL_PG_ADMIN_URL.
+
+function databaseUrl(baseUrl: string, database: string): string {
+ const url = new URL(baseUrl)
+ url.pathname = `/${database}`
+ return url.toString()
+}
+
+function quoteIdent(value: string): string {
+ return `"${value.replace(/"/g, '""')}"`
+}
+
+const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL
+const describeRealPostgres = adminUrl ? describe : describe.skip
+
+describeRealPostgres('external GFS rate limiter backend (real PostgreSQL)', () => {
+ const database = `control_api_gfs_rl_fail_closed_${randomBytes(6).toString('hex')}`
+ const envKeys = [
+ 'CONTROL_API_PG_CONNECTION_STRING',
+ 'CORE_POOL_MAX',
+ 'CORE_POOL_CONNECTION_TIMEOUT_MS',
+ 'CORE_POOL_STATEMENT_TIMEOUT_MS',
+ 'RATE_LIMIT_POOL_MAX',
+ 'RATE_LIMIT_POOL_CONNECTION_TIMEOUT_MS',
+ 'RATE_LIMIT_POOL_STATEMENT_TIMEOUT_MS',
+ ] as const
+ const previousEnv = new Map<string, string | undefined>()
+
+ let adminPool: Pool
+ let corePool: Pool
+ let limiterPool: Pool
+
+ beforeAll(async () => {
+ if (adminUrl === undefined) {
+ throw new Error('CONTROL_API_REAL_PG_ADMIN_URL is required by this suite')
+ }
+ const connectionString = databaseUrl(adminUrl, database)
+ for (const key of envKeys) {
+ previousEnv.set(key, process.env[key])
+ delete process.env[key]
+ }
+ adminPool = new Pool({ connectionString: adminUrl })
+ await adminPool.query(`CREATE DATABASE ${quoteIdent(database)}`)
+ process.env.CONTROL_API_PG_CONNECTION_STRING = connectionString
+
+ const dbMod = await import('../src/db.js')
+ corePool = dbMod.pool as unknown as Pool
+ corePool.on('error', () => {})
+ limiterPool = dbMod.rateLimitPool as unknown as Pool
+ limiterPool.on('error', () => {})
+ const migratePool = new Pool({ connectionString })
+ await dbMod.initDb({ connect: () => migratePool.connect() })
+ await migratePool.end()
+ }, 60_000)
+
+ afterAll(async () => {
+ for (const key of envKeys) {
+ const value = previousEnv.get(key)
+ if (value === undefined) delete process.env[key]
+ else process.env[key] = value
+ }
+ await corePool?.end().catch(() => {})
+ await limiterPool?.end().catch(() => {})
+ if (!adminPool) return
+ await adminPool.query(
+ `SELECT pg_terminate_backend(pid) FROM pg_stat_activity
+ WHERE datname = $1 AND pid <> pg_backend_pid()`,
+ [database]
+ )
+ await adminPool.query(`DROP DATABASE IF EXISTS ${quoteIdent(database)}`)
+ await adminPool.end()
+ })
+
+ it('U1: the limiter pool runs with synchronous_commit off and a 3 s statement timeout; the core pool keeps the defaults', async () => {
+ type Options = {
+ options: { max: number; connectionTimeoutMillis: number; statement_timeout: number }
+ }
+ expect((limiterPool as unknown as Options).options).toMatchObject({
+ max: 6,
+ connectionTimeoutMillis: 5_000,
+ statement_timeout: 3_000,
+ })
+
+ const settings = `SELECT current_setting('synchronous_commit') AS sync,
+ current_setting('statement_timeout') AS timeout,
+ current_database() AS db`
+ const limiter = await limiterPool.query<{ sync: string; timeout: string; db: string }>(settings)
+ const core = await corePool.query<{ sync: string; timeout: string; db: string }>(settings)
+
+ // Witness: both answers come from this suite's database.
+ expect(limiter.rows[0]?.db).toBe(database)
+ expect(core.rows[0]?.db).toBe(database)
+ expect(limiter.rows[0]).toMatchObject({ sync: 'off', timeout: '3s' })
+ expect(core.rows[0]).toMatchObject({ sync: 'on', timeout: '15s' })
+ })
+})
diff --git a/control-api/test/routes.externalGfsRateLimitStress.realPostgres.integration.test.ts b/control-api/test/routes.externalGfsRateLimitStress.realPostgres.integration.test.ts
index 1ef3a9b07..e9ead490a 100644
--- a/control-api/test/routes.externalGfsRateLimitStress.realPostgres.integration.test.ts
+++ b/control-api/test/routes.externalGfsRateLimitStress.realPostgres.integration.test.ts
@@ -4,7 +4,7 @@ import { Pool } from 'pg'
import request from 'supertest'
// Concurrent users at the external GFS rate limits, through the production
-// Express app against real PostgreSQL. Config, the core pool, both limiters,
+// Express app against real PostgreSQL. Config, the core and limiter pools, both limiters,
// the session lifecycle query, authority resolution, the prom registry and the
// logger are real; the ONLY mock is the session-token verifier (identity),
// which decodes a JSON test token so each scenario controls users and sessions.
@@ -87,6 +87,7 @@ describeRealPostgres('external GFS rate limits under concurrent load (real Postg
let adminPool: Pool
let corePool: Pool
+ let limiterPool: Pool
let mod: {
createApp: (gateway: unknown) => import('express').Express
config: typeof import('../src/config.js').config
@@ -108,14 +109,18 @@ describeRealPostgres('external GFS rate limits under concurrent load (real Postg
adminPool = new Pool({ connectionString: adminUrl })
await adminPool.query(`CREATE DATABASE ${quoteIdent(database)}`)
process.env.CONTROL_API_PG_CONNECTION_STRING = connectionString
- // A 200-request burst queues on 10 connections; the production 2 s acquire
- // timeout would make the limiter fail open by accident (see
+ // A 200-request burst queues its session and authority queries on the 10
+ // core connections; the production 2 s acquire timeout would turn some of
+ // them into 500s by accident (see
// services.rateLimiterPoolSaturation.realPostgres.integration.test.ts).
+ // The limiter's own pool runs at its production bounds.
process.env.CORE_POOL_CONNECTION_TIMEOUT_MS = '10000'
const dbMod = await import('../src/db.js')
corePool = dbMod.pool as unknown as Pool
corePool.on('error', () => {})
+ limiterPool = dbMod.rateLimitPool as unknown as Pool
+ limiterPool.on('error', () => {})
const migratePool = new Pool({ connectionString })
await dbMod.initDb({ connect: () => migratePool.connect() })
await migratePool.end()
@@ -148,6 +153,7 @@ describeRealPostgres('external GFS rate limits under concurrent load (real Postg
else process.env[key] = value
}
await corePool?.end().catch(() => {})
+ await limiterPool?.end().catch(() => {})
if (!adminPool) return
await adminPool.query(
`SELECT pg_terminate_backend(pid) FROM pg_stat_activity
diff --git a/control-api/test/routes.externalWorkflows.test.ts b/control-api/test/routes.externalWorkflows.test.ts
index 2adb9e1e6..de2bf55bb 100644
--- a/control-api/test/routes.externalWorkflows.test.ts
+++ b/control-api/test/routes.externalWorkflows.test.ts
@@ -25,6 +25,7 @@ vi.mock('../src/db.js', () => ({
return mockPoolQuery(...args)
},
},
+ rateLimitPool: { query: (...args: unknown[]) => mockPoolQuery(...args) },
withTransaction: (...args: unknown[]) => mockWithTransaction(...args),
}))
diff --git a/control-api/test/routes.internalPluginWorkloadSdk.test.ts b/control-api/test/routes.internalPluginWorkloadSdk.test.ts
index 1fb2b8d96..1ff7df43a 100644
--- a/control-api/test/routes.internalPluginWorkloadSdk.test.ts
+++ b/control-api/test/routes.internalPluginWorkloadSdk.test.ts
@@ -6,6 +6,14 @@ import { config } from '../src/config.js'
import { createInternalPluginWorkloadSdkRouter } from '../src/routes/internal/pluginWorkloadSdk.js'
const db = vi.hoisted(() => ({ revoke: vi.fn(), finalize: vi.fn() }))
+const limiterQuery = vi.hoisted(() => vi.fn())
+
+// The internal SDK limiter counts in the dedicated limiter pool. Answer its
+// upsert at the boundary so the route never waits on an unreachable database.
+vi.mock('../src/db.js', () => ({
+ pool: { query: vi.fn(), connect: vi.fn() },
+ rateLimitPool: { query: (...args: unknown[]) => limiterQuery(...args) },
+}))
vi.mock('../src/services/pluginWorkloadSdkDb.js', () => ({
revokePluginWorkloadSdkForRecipe: (...args: unknown[]) => db.revoke(...args),
@@ -31,6 +39,8 @@ describe('internal Plugin Workload SDK revocation', () => {
beforeEach(() => {
db.revoke.mockReset()
db.finalize.mockReset()
+ limiterQuery.mockReset()
+ limiterQuery.mockResolvedValue({ rows: [{ count: 1 }], rowCount: 1 })
db.revoke.mockResolvedValue({
state: 'revoking',
revocationId: '11111111-1111-4111-8111-111111111111',
@@ -81,6 +91,11 @@ describe('internal Plugin Workload SDK revocation', () => {
internalPrincipal: expect.objectContaining({ credentialId: 'wrc-revocation-test' }),
})
)
+ // Both requests were counted against the WRC principal's bucket.
+ expect(limiterQuery.mock.calls.map(call => (call[1] as unknown[])[0])).toEqual([
+ 'plugin_workload_sdk_internal:wrc:wrc-provisioner',
+ 'plugin_workload_sdk_internal:wrc:wrc-provisioner',
+ ])
})
it('rejects a different internal issuer before the mutation boundary', async () => {
diff --git a/control-api/test/routes.llmModelGrantRace.realPostgres.integration.test.ts b/control-api/test/routes.llmModelGrantRace.realPostgres.integration.test.ts
index dafa8b0b2..a07bc463c 100644
--- a/control-api/test/routes.llmModelGrantRace.realPostgres.integration.test.ts
+++ b/control-api/test/routes.llmModelGrantRace.realPostgres.integration.test.ts
@@ -101,6 +101,7 @@ describeRealPostgres('llm-model reductor ↔ grant upsert serialization (R1-H3 f
let adminPool: Pool
let corePool: Pool
+ let limiterPool: Pool
// A dedicated pool so the raw reductor transaction holds ONE connection (and its
// session-scoped advisory lock) independent of the app's module pool. Advisory
// locks are database-global, so a lock taken here blocks the app's grant upsert.
@@ -132,6 +133,8 @@ describeRealPostgres('llm-model reductor ↔ grant upsert serialization (R1-H3 f
const dbMod = await import('../src/db.js')
corePool = dbMod.pool as unknown as Pool
corePool.on('error', () => {})
+ limiterPool = dbMod.rateLimitPool as unknown as Pool
+ limiterPool.on('error', () => {})
const migratePool = new Pool({ connectionString })
await dbMod.initDb({ connect: () => migratePool.connect() })
await migratePool.end()
@@ -167,6 +170,7 @@ describeRealPostgres('llm-model reductor ↔ grant upsert serialization (R1-H3 f
else process.env.CONTROL_API_PG_CONNECTION_STRING = previousPgEnv
await racePool?.end().catch(() => {})
await corePool?.end().catch(() => {})
+ await limiterPool?.end().catch(() => {})
if (!adminPool) return
await adminPool.query(
`SELECT pg_terminate_backend(pid) FROM pg_stat_activity
diff --git a/control-api/test/routes.llmModelHostRace.realPostgres.integration.test.ts b/control-api/test/routes.llmModelHostRace.realPostgres.integration.test.ts
index 39d97374d..df7adaaeb 100644
--- a/control-api/test/routes.llmModelHostRace.realPostgres.integration.test.ts
+++ b/control-api/test/routes.llmModelHostRace.realPostgres.integration.test.ts
@@ -81,6 +81,7 @@ describeRealPostgres('llm-model reductor ↔ host referencer serialization (R1-H
let adminPool: Pool
let corePool: Pool
+ let limiterPool: Pool
// Loaded dynamically AFTER the env points the module pool at the test DB.
let mod: {
createApp: (gateway: unknown) => import('express').Express
@@ -115,6 +116,8 @@ describeRealPostgres('llm-model reductor ↔ host referencer serialization (R1-H
// Absorb it so teardown is clean (the pool is being torn down anyway).
corePool = dbMod.pool as unknown as Pool
corePool.on('error', () => {})
+ limiterPool = dbMod.rateLimitPool as unknown as Pool
+ limiterPool.on('error', () => {})
const migratePool = new Pool({ connectionString })
await dbMod.initDb({ connect: () => migratePool.connect() })
await migratePool.end()
@@ -171,6 +174,7 @@ describeRealPostgres('llm-model reductor ↔ host referencer serialization (R1-H
else process.env.CONTROL_API_PG_CONNECTION_STRING = previousPgEnv
// Drain the module pool's connections first so DROP DATABASE has no live users.
await corePool?.end().catch(() => {})
+ await limiterPool?.end().catch(() => {})
if (!adminPool) return
await adminPool.query(
`SELECT pg_terminate_backend(pid) FROM pg_stat_activity
diff --git a/control-api/test/routes.mcp-host.plugin-workload-sdk.test.ts b/control-api/test/routes.mcp-host.plugin-workload-sdk.test.ts
index ac443de71..f50d28cf1 100644
--- a/control-api/test/routes.mcp-host.plugin-workload-sdk.test.ts
+++ b/control-api/test/routes.mcp-host.plugin-workload-sdk.test.ts
@@ -1,7 +1,7 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
import express from 'express'
import request from 'supertest'
-import { pool } from '../src/db.js'
+import { rateLimitPool } from '../src/db.js'
import { createMcpHostPluginWorkloadSdkRoutes } from '../src/routes/mcp-host/plugin-workload-sdk.routes.js'
import * as codexConnection from '../src/services/codexSubscriptionConnection.js'
import * as grokConnection from '../src/services/grokSubscriptionConnection.js'
@@ -18,6 +18,9 @@ vi.mock('../src/db.js', () => ({
query: vi.fn().mockResolvedValue({ rows: [], rowCount: 0 }),
connect: vi.fn(),
},
+ rateLimitPool: {
+ query: vi.fn().mockResolvedValue({ rows: [], rowCount: 0 }),
+ },
withTransaction: vi.fn(),
}))
@@ -1448,7 +1451,7 @@ describe('POST /mcp-host/plugin-workload-sdk/credential-ticket/introspect', () =
// before the tighter credential-ticket bucket. Keep the first query under
// that outer limit and exhaust the credential-specific bucket on the
// second query so this test remains focused on ticket work protection.
- vi.mocked(pool.query)
+ vi.mocked(rateLimitPool.query)
.mockResolvedValueOnce({ rows: [{ count: 1 }], rowCount: 1 } as never)
.mockResolvedValueOnce({ rows: [{ count: 121 }], rowCount: 1 } as never)
const res = await request(buildApp())
diff --git a/control-api/test/routes.mcp-host.renewal.refresh.rate-limit.test.ts b/control-api/test/routes.mcp-host.renewal.refresh.rate-limit.test.ts
index f9f06edab..5fdfcaa0d 100644
--- a/control-api/test/routes.mcp-host.renewal.refresh.rate-limit.test.ts
+++ b/control-api/test/routes.mcp-host.renewal.refresh.rate-limit.test.ts
@@ -54,6 +54,9 @@ vi.mock('../src/db.js', () => ({
query: (...args: unknown[]) => mockPoolQuery(args[0], args[1] as unknown[] | undefined),
connect: vi.fn(),
},
+ rateLimitPool: {
+ query: (...args: unknown[]) => mockPoolQuery(args[0], args[1] as unknown[] | undefined),
+ },
withTransaction: vi.fn(),
}))
diff --git a/control-api/test/routes.mcp-host.renewal.reissue.test.ts b/control-api/test/routes.mcp-host.renewal.reissue.test.ts
index 52737fbfd..8304d998f 100644
--- a/control-api/test/routes.mcp-host.renewal.reissue.test.ts
+++ b/control-api/test/routes.mcp-host.renewal.reissue.test.ts
@@ -79,6 +79,9 @@ vi.mock('../src/db.js', () => ({
query: (...args: unknown[]) => mockPoolQuery(args[0], args[1] as unknown[] | undefined),
connect: vi.fn(),
},
+ rateLimitPool: {
+ query: (...args: unknown[]) => mockPoolQuery(args[0], args[1] as unknown[] | undefined),
+ },
withTransaction: vi.fn(),
}))
diff --git a/control-api/test/services.rateLimiter.test.ts b/control-api/test/services.rateLimiter.test.ts
index 312782d4a..0c01964d1 100644
--- a/control-api/test/services.rateLimiter.test.ts
+++ b/control-api/test/services.rateLimiter.test.ts
@@ -33,7 +33,7 @@ const mockConcurrencyClient = {
}
const mockPoolConnect = vi.fn(async () => mockConcurrencyClient)
-const mockPoolQuery = vi.fn(async (sql: unknown, params?: unknown[]) => {
+const mockRateLimitPoolQuery = vi.fn(async (sql: unknown, params?: unknown[]) => {
const text = typeof sql === 'string' ? sql : ''
if (/INSERT INTO rate_limit_buckets/i.test(text)) {
const bucketKey = String(params?.[0] ?? '')
@@ -58,11 +58,22 @@ const mockPoolQuery = vi.fn(async (sql: unknown, params?: unknown[]) => {
return { rows: [], rowCount: 0 }
})
+// The limiter's upserts and prune go to the dedicated limiter pool; the core
+// pool only lends the long-lived advisory-lock client. A limiter statement on
+// the core pool is a wiring regression, so the core query rejects loudly.
+const mockCorePoolQuery = vi.fn(async (sql: unknown) => {
+ throw new Error(`core pool must not serve the rate limiter: ${String(sql).slice(0, 40)}`)
+})
+
vi.mock('../src/db.js', () => ({
pool: {
- query: (...args: unknown[]) => mockPoolQuery(args[0], args[1] as unknown[] | undefined),
+ query: (...args: unknown[]) => mockCorePoolQuery(args[0]),
connect: (...args: unknown[]) => mockPoolConnect(...args),
},
+ rateLimitPool: {
+ query: (...args: unknown[]) =>
+ mockRateLimitPoolQuery(args[0], args[1] as unknown[] | undefined),
+ },
withTransaction: vi.fn(),
}))
@@ -70,10 +81,23 @@ describe('rateLimiterService', () => {
beforeEach(() => {
buckets.clear()
advisoryLocks.clear()
- mockPoolQuery.mockClear()
+ mockRateLimitPoolQuery.mockClear()
+ mockCorePoolQuery.mockClear()
mockClientQuery.mockClear()
})
+ it('serves the upsert and the prune from the limiter pool, never the core pool', async () => {
+ const r = await checkAndIncrement('test:bucket:pool-wiring', 5)
+ await cleanupExpiredBuckets(1_700_000_000_000)
+
+ // Witness: both statements reached the limiter pool.
+ expect(r).toMatchObject({ allowed: true, count: 1, backendAvailable: true })
+ expect(
+ mockRateLimitPoolQuery.mock.calls.map(([sql]) => String(sql).trim().split(/\s+/)[0])
+ ).toEqual(['INSERT', 'DELETE'])
+ expect(mockCorePoolQuery).not.toHaveBeenCalled()
+ })
+
it('allows requests up to the limit and denies the (limit+1)-th in the same window', async () => {
const key = 'test:bucket:1'
const limit = 5
@@ -129,7 +153,7 @@ describe('rateLimiterService', () => {
const second = await checkAndIncrement('test:bucket:weighted', 16, 1_700_000_000_000, 10)
expect(first).toMatchObject({ allowed: true, count: 7, remaining: 9 })
expect(second).toMatchObject({ allowed: false, count: 17, remaining: 0 })
- expect(mockPoolQuery).toHaveBeenLastCalledWith(expect.any(String), [
+ expect(mockRateLimitPoolQuery).toHaveBeenLastCalledWith(expect.any(String), [
'test:bucket:weighted',
currentWindowStartMs(1_700_000_000_000),
10,
@@ -176,8 +200,8 @@ describe('rateLimiterService', () => {
await replacement.release()
})
- it('fails open when pool.query throws (DB error)', async () => {
- mockPoolQuery.mockRejectedValueOnce(new Error('connection refused'))
+ it('fails open when the limiter pool query throws (DB error)', async () => {
+ mockRateLimitPoolQuery.mockRejectedValueOnce(new Error('connection refused'))
const r = await checkAndIncrement('test:bucket:failopen', 5)
expect(r.allowed).toBe(true)
expect(r.count).toBe(0)
@@ -185,8 +209,8 @@ describe('rateLimiterService', () => {
expect(r.backendAvailable).toBe(false)
})
- it('fails open when pool.query returns empty rows', async () => {
- mockPoolQuery.mockResolvedValueOnce({ rows: [], rowCount: 0 })
+ it('fails open when the limiter pool query returns empty rows', async () => {
+ mockRateLimitPoolQuery.mockResolvedValueOnce({ rows: [], rowCount: 0 })
const r = await checkAndIncrement('test:bucket:emptyrows', 5)
expect(r.allowed).toBe(true)
expect(r.count).toBe(0)
diff --git a/control-api/test/services.rateLimiterPoolSaturation.realPostgres.integration.test.ts b/control-api/test/services.rateLimiterPoolSaturation.realPostgres.integration.test.ts
index 144305483..4577eba44 100644
--- a/control-api/test/services.rateLimiterPoolSaturation.realPostgres.integration.test.ts
+++ b/control-api/test/services.rateLimiterPoolSaturation.realPostgres.integration.test.ts
@@ -4,19 +4,24 @@ import { performance } from 'node:perf_hooks'
import { Pool, type PoolClient } from 'pg'
import request from 'supertest'
+// The limiter's upserts run on their own pool (db.ts `rateLimitPool`, max 6,
+// 5 s acquire), separate from the core pool (max 10, 2 s acquire).
+//
// Characterization of a KNOWN defect, not a specification of desired
-// behaviour: when the production core pool is saturated, checkAndIncrement
-// fails OPEN (rateLimiterService.ts, the catch around the INSERT). Every call
-// waits for the pool's acquire timeout, logs rate_limit_db_error, and returns
-// allowed:true with backendAvailable:false. The fix is tracked separately;
-// when it lands, the `allowed` assertion below is the one line that flips.
+// behaviour: when the LIMITER pool is saturated, checkAndIncrement fails OPEN
+// (rateLimiterService.ts, the catch around the INSERT). Every call waits for
+// the limiter pool's acquire timeout, logs rate_limit_db_error, and returns
+// allowed:true with backendAvailable:false.
+//
+// Isolation: a saturated CORE pool no longer reaches the limiter; it keeps
+// counting.
//
-// The production pool bounds are used unchanged (max 10, 2 s acquire): the
-// two CORE_POOL_* variables are deleted before db.js is imported, and the
+// The production pool bounds are used unchanged: the CORE_POOL_* and
+// RATE_LIMIT_POOL_* variables are deleted before db.js is imported, and the
// resulting bounds are asserted before anything else.
//
-// Route-level corollary: with the pool held, an external GFS request returns
-// 500, because session auth queries the users table first
+// Route-level corollary: with the core pool held, an external GFS request
+// returns 500, because session auth queries the users table first
// (externalSessionAuth.ts) and fails closed before the limiter runs. The only
// mock is the session-token verifier (identity). Skipped without
// CONTROL_API_REAL_PG_ADMIN_URL.
@@ -29,6 +34,8 @@ vi.mock('../src/utils/auth/externalSessionAuthToken.js', () => ({
const PRODUCTION_POOL_MAX = 10
const PRODUCTION_ACQUIRE_TIMEOUT_MS = 2_000
+const LIMITER_POOL_MAX = 6
+const LIMITER_ACQUIRE_TIMEOUT_MS = 5_000
const CONCURRENT_CHECKS = 25
const LIMIT = 2
@@ -45,17 +52,21 @@ function quoteIdent(value: string): string {
const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL
const describeRealPostgres = adminUrl ? describe : describe.skip
-describeRealPostgres('rate limiter under core pool saturation (known fail-open)', () => {
+describeRealPostgres('rate limiter under pool saturation (known fail-open)', () => {
const database = `rate_limiter_saturation_${randomBytes(6).toString('hex')}`
const poolEnvKeys = [
'CONTROL_API_PG_CONNECTION_STRING',
'CORE_POOL_MAX',
'CORE_POOL_CONNECTION_TIMEOUT_MS',
+ 'RATE_LIMIT_POOL_MAX',
+ 'RATE_LIMIT_POOL_CONNECTION_TIMEOUT_MS',
+ 'RATE_LIMIT_POOL_STATEMENT_TIMEOUT_MS',
] as const
const previousEnv = new Map<string, string | undefined>()
let adminPool: Pool
let corePool: Pool
+ let limiterPool: Pool
let mod: {
createApp: (gateway: unknown) => import('express').Express
config: typeof import('../src/config.js').config
@@ -64,12 +75,10 @@ describeRealPostgres('rate limiter under core pool saturation (known fail-open)'
MockGateway: typeof import('./mockGateway.js').MockGateway
}
- async function holdEveryPoolClient(): Promise<PoolClient[]> {
- const held = await Promise.all(
- Array.from({ length: PRODUCTION_POOL_MAX }, () => corePool.connect())
- )
- expect(corePool.totalCount).toBe(PRODUCTION_POOL_MAX)
- expect(corePool.idleCount).toBe(0)
+ async function holdEveryClient(target: Pool, max: number): Promise<PoolClient[]> {
+ const held = await Promise.all(Array.from({ length: max }, () => target.connect()))
+ expect(target.totalCount).toBe(max)
+ expect(target.idleCount).toBe(0)
return held
}
@@ -79,8 +88,9 @@ describeRealPostgres('rate limiter under core pool saturation (known fail-open)'
}
const connectionString = databaseUrl(adminUrl, database)
for (const key of poolEnvKeys) previousEnv.set(key, process.env[key])
- delete process.env.CORE_POOL_MAX
- delete process.env.CORE_POOL_CONNECTION_TIMEOUT_MS
+ for (const key of poolEnvKeys) {
+ if (key !== 'CONTROL_API_PG_CONNECTION_STRING') delete process.env[key]
+ }
adminPool = new Pool({ connectionString: adminUrl })
await adminPool.query(`CREATE DATABASE ${quoteIdent(database)}`)
@@ -89,6 +99,8 @@ describeRealPostgres('rate limiter under core pool saturation (known fail-open)'
const dbMod = await import('../src/db.js')
corePool = dbMod.pool as unknown as Pool
corePool.on('error', () => {})
+ limiterPool = dbMod.rateLimitPool as unknown as Pool
+ limiterPool.on('error', () => {})
const migratePool = new Pool({ connectionString })
await dbMod.initDb({ connect: () => migratePool.connect() })
await migratePool.end()
@@ -114,6 +126,7 @@ describeRealPostgres('rate limiter under core pool saturation (known fail-open)'
else process.env[key] = value
}
await corePool?.end().catch(() => {})
+ await limiterPool?.end().catch(() => {})
if (!adminPool) return
await adminPool.query(
`SELECT pg_terminate_backend(pid) FROM pg_stat_activity
@@ -125,21 +138,46 @@ describeRealPostgres('rate limiter under core pool saturation (known fail-open)'
})
it('runs against the production pool bounds', () => {
- const options = (
- corePool as unknown as {
- options: { max: number; connectionTimeoutMillis: number }
- }
- ).options
- expect(options.max).toBe(PRODUCTION_POOL_MAX)
- expect(options.connectionTimeoutMillis).toBe(PRODUCTION_ACQUIRE_TIMEOUT_MS)
+ type Options = { options: { max: number; connectionTimeoutMillis: number } }
+ const core = (corePool as unknown as Options).options
+ expect(core.max).toBe(PRODUCTION_POOL_MAX)
+ expect(core.connectionTimeoutMillis).toBe(PRODUCTION_ACQUIRE_TIMEOUT_MS)
+ const limiter = (limiterPool as unknown as Options).options
+ expect(limiter.max).toBe(LIMITER_POOL_MAX)
+ expect(limiter.connectionTimeoutMillis).toBe(LIMITER_ACQUIRE_TIMEOUT_MS)
})
- it('fails open for every concurrent check while every client is held, then recovers', async () => {
+ it('keeps counting while every core client is held', async () => {
+ const bucketKey = `core-saturation:${randomBytes(8).toString('hex')}`
+ const nowMs = Math.floor(Date.now() / 60_000) * 60_000 + 1_000
+ const warn = vi.spyOn(mod.rootLogger, 'warn').mockImplementation(() => {})
+ const held = await holdEveryClient(corePool, PRODUCTION_POOL_MAX)
+ try {
+ const results = []
+ for (let i = 0; i < 3; i += 1) {
+ results.push(await mod.checkAndIncrement(bucketKey, LIMIT, nowMs))
+ }
+ // Witness: the limiter reached Postgres and counted every call.
+ expect(results.map(result => result.count)).toEqual([1, 2, 3])
+ expect(results.map(result => result.allowed)).toEqual([true, true, false])
+ expect(results.every(result => result.backendAvailable)).toBe(true)
+ expect(
+ warn.mock.calls.filter(
+ call => (call[0] as { event?: string }).event === 'rate_limit_db_error'
+ )
+ ).toHaveLength(0)
+ } finally {
+ for (const client of held) client.release()
+ warn.mockRestore()
+ }
+ }, 15_000)
+
+ it('fails open for every concurrent check while every limiter client is held, then recovers', async () => {
const bucketKey = `saturation:${randomBytes(8).toString('hex')}`
// One fixed window for both phases, so a minute boundary cannot split them.
const nowMs = Math.floor(Date.now() / 60_000) * 60_000 + 1_000
const warn = vi.spyOn(mod.rootLogger, 'warn').mockImplementation(() => {})
- const held = await holdEveryPoolClient()
+ const held = await holdEveryClient(limiterPool, LIMITER_POOL_MAX)
let released = false
try {
const startedAt = performance.now()
@@ -151,7 +189,7 @@ describeRealPostgres('rate limiter under core pool saturation (known fail-open)'
const elapsedMs = performance.now() - startedAt
expect(results).toHaveLength(CONCURRENT_CHECKS)
- // The known defect: a saturated pool admits every request.
+ // The known defect: a saturated limiter pool admits every request.
expect(results.every(result => result.allowed === true)).toBe(true)
// These two stay true after the fix: the backend was unavailable and
// nothing was counted.
@@ -169,8 +207,8 @@ describeRealPostgres('rate limiter under core pool saturation (known fail-open)'
}
// Every call waited for the acquire timeout, in parallel, not in series.
- expect(elapsedMs).toBeGreaterThanOrEqual(PRODUCTION_ACQUIRE_TIMEOUT_MS - 100)
- expect(elapsedMs).toBeLessThan(5_000)
+ expect(elapsedMs).toBeGreaterThanOrEqual(LIMITER_ACQUIRE_TIMEOUT_MS - 100)
+ expect(elapsedMs).toBeLessThan(LIMITER_ACQUIRE_TIMEOUT_MS + 3_000)
// Read through a held client: this also proves the database and the
// table are reachable, so "no row" is not an absent table.
@@ -201,9 +239,9 @@ describeRealPostgres('rate limiter under core pool saturation (known fail-open)'
if (!released) for (const client of held) client.release()
warn.mockRestore()
}
- }, 15_000)
+ }, 20_000)
- it('answers 500 at the route while the pool is held, because session auth fails first', async () => {
+ it('answers 500 at the route while the core pool is held, because session auth fails first', async () => {
const internalToken = mod.config.internalServiceTokens['external-rest-api']
if (!internalToken) throw new Error('config has no external-rest-api internal service token')
mockVerifyExternalSessionToken.mockReset()
@@ -230,7 +268,7 @@ describeRealPostgres('rate limiter under core pool saturation (known fail-open)'
expect(free.status).toBe(401)
expect(mockVerifyExternalSessionToken).toHaveBeenCalledTimes(1)
- const held = await holdEveryPoolClient()
+ const held = await holdEveryClient(corePool, PRODUCTION_POOL_MAX)
try {
const saturated = await send()
expect(saturated.status).toBe(500)