Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
feat(control-api): give the Postgres rate limiter its own bounded pool with synchronous_commit off
Alfredo Lopez7d138d60400b
diff --git a/.github/workflows/ci-public.yml b/.github/workflows/ci-public.ymlindex 77d0f6c62..7720c73c6 100644--- a/.github/workflows/ci-public.yml+++ b/.github/workflows/ci-public.yml@@ -308,6 +308,7 @@ jobs:db.llmProviderAttemptConnectionIntegrity.realPostgres.integration \services.llmCatalogSync.realPostgres.integration \routes.externalGfsRateLimitStress.realPostgres.integration \+ routes.externalGfsRateLimitFailClosed.realPostgres.integration \services.rateLimiterPoolSaturation.realPostgres.integrationdogrep -Fq "$suite" "$normalized_log" || {diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex 07501440f..a5035d477 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.606",+ "version": "0.1.607","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.606",+ "version": "0.1.607","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex 0f2f7f103..59d3d2ffd 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.606",+ "version": "0.1.607","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/boundedPgPool.ts b/control-api/src/boundedPgPool.tsindex e8d80d8c8..14ed183dd 100644--- a/control-api/src/boundedPgPool.ts+++ b/control-api/src/boundedPgPool.ts@@ -16,10 +16,16 @@ const LIMITS: Record<keyof BoundedPoolBudget, readonly [number, number]> = {statementTimeoutMillis: [100, 30_000],}+/**+ * `sessionOptions` is the libpq `options` startup parameter (for example+ * `-c synchronous_commit=off`). pg sends it in the startup packet, so every+ * connection of the pool runs with those settings from its first statement.+ */export function createBoundedPgPoolForConnection(connectionString: string,budget: BoundedPoolBudget,- PoolClass: PoolConstructor = Pool+ PoolClass: PoolConstructor = Pool,+ sessionOptions?: string): Pool {for (const [name, value] of Object.entries(budget) as Array<[keyof BoundedPoolBudget, number]>) {const [min, max] = LIMITS[name]@@ -33,5 +39,6 @@ export function createBoundedPgPoolForConnection(idleTimeoutMillis: budget.idleTimeoutMillis,connectionTimeoutMillis: budget.connectionTimeoutMillis,statement_timeout: budget.statementTimeoutMillis,+ ...(sessionOptions === undefined ? {} : { options: sessionOptions }),})}diff --git a/control-api/src/db.ts b/control-api/src/db.tsindex e710d5d98..5ceb8076d 100644--- a/control-api/src/db.ts+++ b/control-api/src/db.ts@@ -121,11 +121,33 @@ const DEFAULT_CORE_POOL_IDLE_TIMEOUT_MS = 30_000const DEFAULT_CORE_POOL_CONNECTION_TIMEOUT_MS = 2_000const DEFAULT_CORE_POOL_STATEMENT_TIMEOUT_MS = 15_000+// The Postgres rate limiter (`rate_limit_buckets` upserts and their prune) has+// its own pool so a saturated core pool (session auth, SSE LISTEN clients,+// polling) cannot starve it, and a limiter burst cannot take connections from+// the rest of the service. synchronous_commit=off: the rows are 60 s counters+// that the pruner deletes anyway, so losing the last ~200 ms of increments on+// a Postgres crash is acceptable, and the commit no longer waits on WAL flush+// while holding the hot bucket row lock.+const MAX_RATE_LIMIT_POOL_MAX = 16+const DEFAULT_RATE_LIMIT_POOL_MAX = 6+const DEFAULT_RATE_LIMIT_POOL_IDLE_TIMEOUT_MS = 30_000+const DEFAULT_RATE_LIMIT_POOL_CONNECTION_TIMEOUT_MS = 5_000+const DEFAULT_RATE_LIMIT_POOL_STATEMENT_TIMEOUT_MS = 3_000+export const RATE_LIMIT_POOL_SESSION_OPTIONS = '-c synchronous_commit=off'++/**+ * Unset (or empty) means the default. A set value outside `[min, max]`, or one+ * that is not an integer, stops startup instead of silently becoming the+ * default.+ */function boundedEnvInteger(name: string, fallback: number, min: number, max: number): number {const raw = process.env[name]if (!raw) return fallbackconst value = Number(raw)- return Number.isInteger(value) && value >= min && value <= max ? value : fallback+ if (!Number.isInteger(value) || value < min || value > max) {+ throw new Error(`${name} must be an integer in [${min}, ${max}], got ${JSON.stringify(raw)}`)+ }+ return value}export function createBoundedPgPool(@@ -166,6 +188,41 @@ export function createCorePool(PoolClass: PoolConstructor = Pool): Pool {export const pool = createCorePool()export const corePool = pool+export function rateLimitPoolBudget(): BoundedPoolBudget {+ return {+ max: boundedEnvInteger(+ 'RATE_LIMIT_POOL_MAX',+ DEFAULT_RATE_LIMIT_POOL_MAX,+ MIN_POOL_MAX,+ MAX_RATE_LIMIT_POOL_MAX+ ),+ idleTimeoutMillis: DEFAULT_RATE_LIMIT_POOL_IDLE_TIMEOUT_MS,+ connectionTimeoutMillis: boundedEnvInteger(+ 'RATE_LIMIT_POOL_CONNECTION_TIMEOUT_MS',+ DEFAULT_RATE_LIMIT_POOL_CONNECTION_TIMEOUT_MS,+ MIN_CONNECTION_TIMEOUT_MS,+ MAX_CONNECTION_TIMEOUT_MS+ ),+ statementTimeoutMillis: boundedEnvInteger(+ 'RATE_LIMIT_POOL_STATEMENT_TIMEOUT_MS',+ DEFAULT_RATE_LIMIT_POOL_STATEMENT_TIMEOUT_MS,+ MIN_STATEMENT_TIMEOUT_MS,+ MAX_STATEMENT_TIMEOUT_MS+ ),+ }+}++export function createRateLimitPool(PoolClass: PoolConstructor = Pool): Pool {+ return createBoundedPgPoolForConnection(+ config.pgConnectionString,+ rateLimitPoolBudget(),+ PoolClass,+ RATE_LIMIT_POOL_SESSION_OPTIONS+ )+}++export const rateLimitPool = createRateLimitPool()+async function applyBaselineSchema(db: DbClient): Promise<void> {// Baseline includes additive Phase 0 workflow-trigger tables for fresh// clusters. Existing clusters receive the same tables through migrationdiff --git a/control-api/src/main.ts b/control-api/src/main.tsindex d12a0a357..76fcf1bba 100644--- a/control-api/src/main.ts+++ b/control-api/src/main.ts@@ -1,5 +1,5 @@import { config } from './config.js'-import { assertDbReady, pool } from './db.js'+import { assertDbReady, pool, rateLimitPool } from './db.js'import { K8sGateway } from './k8s.js'import { reconcileAllowedModelsConfigMapOnBoot } from './llmAllowedModelsBootReconcile.js'import { logRegistryConnectionState } from './registryBootGuard.js'@@ -228,5 +228,6 @@ main().catch(error => {stopSubscriptionCatalogSyncCron()stopWorkflowApprovalTraceProjector()void pool.end()+ void rateLimitPool.end()process.exit(1)})diff --git a/control-api/src/services/rateLimiterService.ts b/control-api/src/services/rateLimiterService.tsindex 948ffc203..3bc69fd51 100644--- a/control-api/src/services/rateLimiterService.ts+++ b/control-api/src/services/rateLimiterService.ts@@ -1,6 +1,6 @@import { createHash } from 'node:crypto'import type { PoolClient } from 'pg'-import { pool } from '../db.js'+import { pool, rateLimitPool } from '../db.js'import { rootLogger } from '../observability/logger.js'/**@@ -51,7 +51,7 @@ export async function checkAndIncrement(cost = 1): Promise<RateLimitCheck> {return checkAndIncrementWithQuery(- (text, values) => pool.query(text, values),+ (text, values) => rateLimitPool.query(text, values),bucketKey,maxPerMinute,nowMs,@@ -61,7 +61,7 @@ export async function checkAndIncrement(/*** Same atomic limiter operation against an explicitly supplied query- * function. Production callers use the process-wide pool above; the narrow+ * function. Production callers use the dedicated limiter pool above; the narrow* seam lets the real-Postgres integration suite exercise this exact SQL* against an isolated database without changing runtime ownership.*/@@ -190,6 +190,10 @@ function advisoryKey(value: string): AdvisoryLockKey {return { high: digest.readInt32BE(0), low: digest.readInt32BE(4) }}+// The advisory-lock client stays on the core pool: advisory locks write no WAL,+// so synchronous_commit=off buys nothing, and this client is held for the life+// of the process, which would take one of the limiter pool's few connections+// permanently.async function getConcurrencyClient(): Promise<PoolClient> {if (concurrencyClient) return concurrencyClientif (!concurrencyClientPromise) {@@ -348,9 +352,10 @@ export async function acquireRateLimitConcurrencyLease(*/export async function cleanupExpiredBuckets(nowMs = Date.now()): Promise<number> {const cutoff = nowMs - 5 * 60_000- const result = await pool.query(`DELETE FROM rate_limit_buckets WHERE window_start_ms < $1`, [- cutoff,- ])+ const result = await rateLimitPool.query(+ `DELETE FROM rate_limit_buckets WHERE window_start_ms < $1`,+ [cutoff]+ )return result.rowCount ?? 0}diff --git a/control-api/test/db.poolFactory.test.ts b/control-api/test/db.poolFactory.test.tsindex 913e708df..e22c5e739 100644--- a/control-api/test/db.poolFactory.test.ts+++ b/control-api/test/db.poolFactory.test.ts@@ -21,6 +21,9 @@ describe('core Postgres pool budget', () => {delete process.env.CORE_POOL_IDLE_TIMEOUT_MSdelete process.env.CORE_POOL_CONNECTION_TIMEOUT_MSdelete process.env.CORE_POOL_STATEMENT_TIMEOUT_MS+ delete process.env.RATE_LIMIT_POOL_MAX+ delete process.env.RATE_LIMIT_POOL_CONNECTION_TIMEOUT_MS+ delete process.env.RATE_LIMIT_POOL_STATEMENT_TIMEOUT_MS})it('creates the core pool with explicit bounded connection and statement budgets', async () => {@@ -58,3 +61,74 @@ describe('core Postgres pool budget', () => {).toThrow('Invalid bounded Postgres pool budget: max')})})++describe('rate limiter Postgres pool', () => {+ beforeEach(() => {+ vi.resetModules()+ fakePools.length = 0+ delete process.env.CORE_POOL_MAX+ delete process.env.RATE_LIMIT_POOL_MAX+ delete process.env.RATE_LIMIT_POOL_CONNECTION_TIMEOUT_MS+ delete process.env.RATE_LIMIT_POOL_STATEMENT_TIMEOUT_MS+ })++ it('builds the limiter pool with 6/5000/3000 and synchronous_commit off, and leaves the core pool untouched', async () => {+ const db = await import('../src/db.js')++ // Module load builds exactly the core pool, then the limiter pool.+ expect(fakePools).toHaveLength(2)+ const [core, limiter] = fakePools.map(p => p.config)+ expect(limiter).toMatchObject({+ max: 6,+ idleTimeoutMillis: 30_000,+ connectionTimeoutMillis: 5_000,+ statement_timeout: 3_000,+ options: '-c synchronous_commit=off',+ })+ expect(db.RATE_LIMIT_POOL_SESSION_OPTIONS).toBe('-c synchronous_commit=off')+ expect(core).toMatchObject({+ max: 10,+ connectionTimeoutMillis: 2_000,+ statement_timeout: 15_000,+ })+ expect(core).not.toHaveProperty('options')+ })++ it('reads the limiter pool bounds from the environment', async () => {+ process.env.RATE_LIMIT_POOL_MAX = '16'+ process.env.RATE_LIMIT_POOL_CONNECTION_TIMEOUT_MS = '100'+ process.env.RATE_LIMIT_POOL_STATEMENT_TIMEOUT_MS = '30000'++ const { rateLimitPoolBudget } = await import('../src/db.js')++ expect(rateLimitPoolBudget()).toEqual({+ max: 16,+ idleTimeoutMillis: 30_000,+ connectionTimeoutMillis: 100,+ statementTimeoutMillis: 30_000,+ })+ })++ it.each([+ // [env, value, range, pools built before the refusal (core is built first)]+ ['RATE_LIMIT_POOL_MAX', '17', '[1, 16]', 1],+ ['RATE_LIMIT_POOL_MAX', '0', '[1, 16]', 1],+ ['RATE_LIMIT_POOL_CONNECTION_TIMEOUT_MS', '99', '[100, 30000]', 1],+ ['RATE_LIMIT_POOL_STATEMENT_TIMEOUT_MS', '30001', '[100, 30000]', 1],+ ['RATE_LIMIT_POOL_MAX', '6.5', '[1, 16]', 1],+ ['CORE_POOL_MAX', 'twelve', '[1, 64]', 0],+ ['CORE_POOL_MAX', '65', '[1, 64]', 0],+ ] as const)(+ 'refuses %s=%s at startup instead of falling back to the default',+ async (name, value, range, built) => {+ process.env[name] = value++ await expect(import('../src/db.js')).rejects.toThrow(+ `${name} must be an integer in ${range}, got "${value}"`+ )+ // The refusal stops construction at the pool that reads the bad value.+ expect(fakePools).toHaveLength(built)+ delete process.env[name]+ }+ )+})diff --git a/control-api/test/pluginWorkloadSdkGrantUpdateNotify.realPostgres.integration.test.ts b/control-api/test/pluginWorkloadSdkGrantUpdateNotify.realPostgres.integration.test.tsindex 507405b76..0fbb3eb9e 100644--- a/control-api/test/pluginWorkloadSdkGrantUpdateNotify.realPostgres.integration.test.ts+++ b/control-api/test/pluginWorkloadSdkGrantUpdateNotify.realPostgres.integration.test.ts@@ -113,6 +113,7 @@ describeRealPostgres(listenClient?.release()await listenPool?.end()await db?.pool.end()+ await db?.rateLimitPool.end()if (previousPgConnectionString === undefined) {delete process.env.CONTROL_API_PG_CONNECTION_STRING} else {diff --git a/control-api/test/pluginWorkloadSdkSteplessQuota.realPostgres.integration.test.ts b/control-api/test/pluginWorkloadSdkSteplessQuota.realPostgres.integration.test.tsindex 2f55bc2c9..bbcc366de 100644--- a/control-api/test/pluginWorkloadSdkSteplessQuota.realPostgres.integration.test.ts+++ b/control-api/test/pluginWorkloadSdkSteplessQuota.realPostgres.integration.test.ts@@ -110,6 +110,7 @@ describeRealPostgres(afterAll(async () => {await db?.pool.end()+ await db?.rateLimitPool.end()if (previousPgConnectionString === undefined) {delete process.env.CONTROL_API_PG_CONNECTION_STRING} else {diff --git a/control-api/test/registry.test.ts b/control-api/test/registry.test.tsindex 8263e5106..1d8b9a7e5 100644--- a/control-api/test/registry.test.ts+++ b/control-api/test/registry.test.ts@@ -57,6 +57,9 @@ vi.mock('../src/db.js', () => ({query: (...args: unknown[]) => mockPoolQuery(args[0], args[1] as unknown[] | undefined),connect: vi.fn(),},+ rateLimitPool: {+ query: (...args: unknown[]) => mockPoolQuery(args[0], args[1] as unknown[] | undefined),+ },withTransaction: vi.fn(),}))diff --git a/control-api/test/routes.adminWorkflows.test.ts b/control-api/test/routes.adminWorkflows.test.tsindex e24015975..29437a81f 100644--- a/control-api/test/routes.adminWorkflows.test.ts+++ b/control-api/test/routes.adminWorkflows.test.ts@@ -31,6 +31,9 @@ vi.mock('../src/db.js', () => ({query: (...args: unknown[]) => mockPoolQuery(...args),connect: (...args: unknown[]) => mockPoolConnect(...args),},+ rateLimitPool: {+ query: (...args: unknown[]) => mockPoolQuery(...args),+ },withTransaction: (work: (db: { query: typeof mockPoolQuery }) => unknown) =>work({ query: mockPoolQuery }),}))diff --git a/control-api/test/routes.auth.mcp-host.issue.security.test.ts b/control-api/test/routes.auth.mcp-host.issue.security.test.tsindex 118c5d653..4b6ea4bcb 100644--- a/control-api/test/routes.auth.mcp-host.issue.security.test.ts+++ b/control-api/test/routes.auth.mcp-host.issue.security.test.ts@@ -55,6 +55,9 @@ vi.mock('../src/db.js', () => ({query: (...args: unknown[]) => mockPoolQueryDispatch(args[0], args[1] as unknown[] | undefined),connect: vi.fn(),},+ rateLimitPool: {+ query: (...args: unknown[]) => mockPoolQueryDispatch(args[0], args[1] as unknown[] | undefined),+ },withTransaction: vi.fn(),}))diff --git a/control-api/test/routes.externalGfs.test.ts b/control-api/test/routes.externalGfs.test.tsindex ae33b5d52..4a47ef33c 100644--- a/control-api/test/routes.externalGfs.test.ts+++ b/control-api/test/routes.externalGfs.test.ts@@ -89,6 +89,7 @@ vi.mock('../src/middleware/gfsUploadAdmission.js', async importOriginal => {})vi.mock('../src/db.js', () => ({pool: { query: (...a: unknown[]) => mockQuery(...a) },+ rateLimitPool: { query: (...a: unknown[]) => mockQuery(...a) },withTransaction: (...a: unknown[]) => mockWithTransaction(...a),}))vi.mock('../src/services/tracing/controlApiPermissionEvents.js', () => ({diff --git a/control-api/test/routes.externalGfsRateLimitFailClosed.realPostgres.integration.test.ts b/control-api/test/routes.externalGfsRateLimitFailClosed.realPostgres.integration.test.tsnew file mode 100644index 000000000..20b4866b2--- /dev/null+++ b/control-api/test/routes.externalGfsRateLimitFailClosed.realPostgres.integration.test.ts@@ -0,0 +1,103 @@+import { afterAll, beforeAll, describe, expect, it } from 'vitest'+import { randomBytes } from 'node:crypto'+import { Pool } from 'pg'++// The external GFS limiter's Postgres backend, against real PostgreSQL at the+// production pool bounds: the CORE_POOL_* and RATE_LIMIT_POOL_* variables are+// deleted before db.js is imported. Skipped without+// CONTROL_API_REAL_PG_ADMIN_URL.++function databaseUrl(baseUrl: string, database: string): string {+ const url = new URL(baseUrl)+ url.pathname = `/${database}`+ return url.toString()+}++function quoteIdent(value: string): string {+ return `"${value.replace(/"/g, '""')}"`+}++const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL+const describeRealPostgres = adminUrl ? describe : describe.skip++describeRealPostgres('external GFS rate limiter backend (real PostgreSQL)', () => {+ const database = `control_api_gfs_rl_fail_closed_${randomBytes(6).toString('hex')}`+ const envKeys = [+ 'CONTROL_API_PG_CONNECTION_STRING',+ 'CORE_POOL_MAX',+ 'CORE_POOL_CONNECTION_TIMEOUT_MS',+ 'CORE_POOL_STATEMENT_TIMEOUT_MS',+ 'RATE_LIMIT_POOL_MAX',+ 'RATE_LIMIT_POOL_CONNECTION_TIMEOUT_MS',+ 'RATE_LIMIT_POOL_STATEMENT_TIMEOUT_MS',+ ] as const+ const previousEnv = new Map<string, string | undefined>()++ let adminPool: Pool+ let corePool: Pool+ let limiterPool: Pool++ beforeAll(async () => {+ if (adminUrl === undefined) {+ throw new Error('CONTROL_API_REAL_PG_ADMIN_URL is required by this suite')+ }+ const connectionString = databaseUrl(adminUrl, database)+ for (const key of envKeys) {+ previousEnv.set(key, process.env[key])+ delete process.env[key]+ }+ adminPool = new Pool({ connectionString: adminUrl })+ await adminPool.query(`CREATE DATABASE ${quoteIdent(database)}`)+ process.env.CONTROL_API_PG_CONNECTION_STRING = connectionString++ const dbMod = await import('../src/db.js')+ corePool = dbMod.pool as unknown as Pool+ corePool.on('error', () => {})+ limiterPool = dbMod.rateLimitPool as unknown as Pool+ limiterPool.on('error', () => {})+ const migratePool = new Pool({ connectionString })+ await dbMod.initDb({ connect: () => migratePool.connect() })+ await migratePool.end()+ }, 60_000)++ afterAll(async () => {+ for (const key of envKeys) {+ const value = previousEnv.get(key)+ if (value === undefined) delete process.env[key]+ else process.env[key] = value+ }+ await corePool?.end().catch(() => {})+ await limiterPool?.end().catch(() => {})+ if (!adminPool) return+ await adminPool.query(+ `SELECT pg_terminate_backend(pid) FROM pg_stat_activity+ WHERE datname = $1 AND pid <> pg_backend_pid()`,+ [database]+ )+ await adminPool.query(`DROP DATABASE IF EXISTS ${quoteIdent(database)}`)+ await adminPool.end()+ })++ it('U1: the limiter pool runs with synchronous_commit off and a 3 s statement timeout; the core pool keeps the defaults', async () => {+ type Options = {+ options: { max: number; connectionTimeoutMillis: number; statement_timeout: number }+ }+ expect((limiterPool as unknown as Options).options).toMatchObject({+ max: 6,+ connectionTimeoutMillis: 5_000,+ statement_timeout: 3_000,+ })++ const settings = `SELECT current_setting('synchronous_commit') AS sync,+ current_setting('statement_timeout') AS timeout,+ current_database() AS db`+ const limiter = await limiterPool.query<{ sync: string; timeout: string; db: string }>(settings)+ const core = await corePool.query<{ sync: string; timeout: string; db: string }>(settings)++ // Witness: both answers come from this suite's database.+ expect(limiter.rows[0]?.db).toBe(database)+ expect(core.rows[0]?.db).toBe(database)+ expect(limiter.rows[0]).toMatchObject({ sync: 'off', timeout: '3s' })+ expect(core.rows[0]).toMatchObject({ sync: 'on', timeout: '15s' })+ })+})diff --git a/control-api/test/routes.externalGfsRateLimitStress.realPostgres.integration.test.ts b/control-api/test/routes.externalGfsRateLimitStress.realPostgres.integration.test.tsindex 1ef3a9b07..e9ead490a 100644--- a/control-api/test/routes.externalGfsRateLimitStress.realPostgres.integration.test.ts+++ b/control-api/test/routes.externalGfsRateLimitStress.realPostgres.integration.test.ts@@ -4,7 +4,7 @@ import { Pool } from 'pg'import request from 'supertest'// Concurrent users at the external GFS rate limits, through the production-// Express app against real PostgreSQL. Config, the core pool, both limiters,+// Express app against real PostgreSQL. Config, the core and limiter pools, both limiters,// the session lifecycle query, authority resolution, the prom registry and the// logger are real; the ONLY mock is the session-token verifier (identity),// which decodes a JSON test token so each scenario controls users and sessions.@@ -87,6 +87,7 @@ describeRealPostgres('external GFS rate limits under concurrent load (real Postglet adminPool: Poollet corePool: Pool+ let limiterPool: Poollet mod: {createApp: (gateway: unknown) => import('express').Expressconfig: typeof import('../src/config.js').config@@ -108,14 +109,18 @@ describeRealPostgres('external GFS rate limits under concurrent load (real PostgadminPool = new Pool({ connectionString: adminUrl })await adminPool.query(`CREATE DATABASE ${quoteIdent(database)}`)process.env.CONTROL_API_PG_CONNECTION_STRING = connectionString- // A 200-request burst queues on 10 connections; the production 2 s acquire- // timeout would make the limiter fail open by accident (see+ // A 200-request burst queues its session and authority queries on the 10+ // core connections; the production 2 s acquire timeout would turn some of+ // them into 500s by accident (see// services.rateLimiterPoolSaturation.realPostgres.integration.test.ts).+ // The limiter's own pool runs at its production bounds.process.env.CORE_POOL_CONNECTION_TIMEOUT_MS = '10000'const dbMod = await import('../src/db.js')corePool = dbMod.pool as unknown as PoolcorePool.on('error', () => {})+ limiterPool = dbMod.rateLimitPool as unknown as Pool+ limiterPool.on('error', () => {})const migratePool = new Pool({ connectionString })await dbMod.initDb({ connect: () => migratePool.connect() })await migratePool.end()@@ -148,6 +153,7 @@ describeRealPostgres('external GFS rate limits under concurrent load (real Postgelse process.env[key] = value}await corePool?.end().catch(() => {})+ await limiterPool?.end().catch(() => {})if (!adminPool) returnawait adminPool.query(`SELECT pg_terminate_backend(pid) FROM pg_stat_activitydiff --git a/control-api/test/routes.externalWorkflows.test.ts b/control-api/test/routes.externalWorkflows.test.tsindex 2adb9e1e6..de2bf55bb 100644--- a/control-api/test/routes.externalWorkflows.test.ts+++ b/control-api/test/routes.externalWorkflows.test.ts@@ -25,6 +25,7 @@ vi.mock('../src/db.js', () => ({return mockPoolQuery(...args)},},+ rateLimitPool: { query: (...args: unknown[]) => mockPoolQuery(...args) },withTransaction: (...args: unknown[]) => mockWithTransaction(...args),}))diff --git a/control-api/test/routes.internalPluginWorkloadSdk.test.ts b/control-api/test/routes.internalPluginWorkloadSdk.test.tsindex 1fb2b8d96..1ff7df43a 100644--- a/control-api/test/routes.internalPluginWorkloadSdk.test.ts+++ b/control-api/test/routes.internalPluginWorkloadSdk.test.ts@@ -6,6 +6,14 @@ import { config } from '../src/config.js'import { createInternalPluginWorkloadSdkRouter } from '../src/routes/internal/pluginWorkloadSdk.js'const db = vi.hoisted(() => ({ revoke: vi.fn(), finalize: vi.fn() }))+const limiterQuery = vi.hoisted(() => vi.fn())++// The internal SDK limiter counts in the dedicated limiter pool. Answer its+// upsert at the boundary so the route never waits on an unreachable database.+vi.mock('../src/db.js', () => ({+ pool: { query: vi.fn(), connect: vi.fn() },+ rateLimitPool: { query: (...args: unknown[]) => limiterQuery(...args) },+}))vi.mock('../src/services/pluginWorkloadSdkDb.js', () => ({revokePluginWorkloadSdkForRecipe: (...args: unknown[]) => db.revoke(...args),@@ -31,6 +39,8 @@ describe('internal Plugin Workload SDK revocation', () => {beforeEach(() => {db.revoke.mockReset()db.finalize.mockReset()+ limiterQuery.mockReset()+ limiterQuery.mockResolvedValue({ rows: [{ count: 1 }], rowCount: 1 })db.revoke.mockResolvedValue({state: 'revoking',revocationId: '11111111-1111-4111-8111-111111111111',@@ -81,6 +91,11 @@ describe('internal Plugin Workload SDK revocation', () => {internalPrincipal: expect.objectContaining({ credentialId: 'wrc-revocation-test' }),}))+ // Both requests were counted against the WRC principal's bucket.+ expect(limiterQuery.mock.calls.map(call => (call[1] as unknown[])[0])).toEqual([+ 'plugin_workload_sdk_internal:wrc:wrc-provisioner',+ 'plugin_workload_sdk_internal:wrc:wrc-provisioner',+ ])})it('rejects a different internal issuer before the mutation boundary', async () => {diff --git a/control-api/test/routes.llmModelGrantRace.realPostgres.integration.test.ts b/control-api/test/routes.llmModelGrantRace.realPostgres.integration.test.tsindex dafa8b0b2..a07bc463c 100644--- a/control-api/test/routes.llmModelGrantRace.realPostgres.integration.test.ts+++ b/control-api/test/routes.llmModelGrantRace.realPostgres.integration.test.ts@@ -101,6 +101,7 @@ describeRealPostgres('llm-model reductor ↔ grant upsert serialization (R1-H3 flet adminPool: Poollet corePool: Pool+ let limiterPool: Pool// A dedicated pool so the raw reductor transaction holds ONE connection (and its// session-scoped advisory lock) independent of the app's module pool. Advisory// locks are database-global, so a lock taken here blocks the app's grant upsert.@@ -132,6 +133,8 @@ describeRealPostgres('llm-model reductor ↔ grant upsert serialization (R1-H3 fconst dbMod = await import('../src/db.js')corePool = dbMod.pool as unknown as PoolcorePool.on('error', () => {})+ limiterPool = dbMod.rateLimitPool as unknown as Pool+ limiterPool.on('error', () => {})const migratePool = new Pool({ connectionString })await dbMod.initDb({ connect: () => migratePool.connect() })await migratePool.end()@@ -167,6 +170,7 @@ describeRealPostgres('llm-model reductor ↔ grant upsert serialization (R1-H3 felse process.env.CONTROL_API_PG_CONNECTION_STRING = previousPgEnvawait racePool?.end().catch(() => {})await corePool?.end().catch(() => {})+ await limiterPool?.end().catch(() => {})if (!adminPool) returnawait adminPool.query(`SELECT pg_terminate_backend(pid) FROM pg_stat_activitydiff --git a/control-api/test/routes.llmModelHostRace.realPostgres.integration.test.ts b/control-api/test/routes.llmModelHostRace.realPostgres.integration.test.tsindex 39d97374d..df7adaaeb 100644--- a/control-api/test/routes.llmModelHostRace.realPostgres.integration.test.ts+++ b/control-api/test/routes.llmModelHostRace.realPostgres.integration.test.ts@@ -81,6 +81,7 @@ describeRealPostgres('llm-model reductor ↔ host referencer serialization (R1-Hlet adminPool: Poollet corePool: Pool+ let limiterPool: Pool// Loaded dynamically AFTER the env points the module pool at the test DB.let mod: {createApp: (gateway: unknown) => import('express').Express@@ -115,6 +116,8 @@ describeRealPostgres('llm-model reductor ↔ host referencer serialization (R1-H// Absorb it so teardown is clean (the pool is being torn down anyway).corePool = dbMod.pool as unknown as PoolcorePool.on('error', () => {})+ limiterPool = dbMod.rateLimitPool as unknown as Pool+ limiterPool.on('error', () => {})const migratePool = new Pool({ connectionString })await dbMod.initDb({ connect: () => migratePool.connect() })await migratePool.end()@@ -171,6 +174,7 @@ describeRealPostgres('llm-model reductor ↔ host referencer serialization (R1-Helse process.env.CONTROL_API_PG_CONNECTION_STRING = previousPgEnv// Drain the module pool's connections first so DROP DATABASE has no live users.await corePool?.end().catch(() => {})+ await limiterPool?.end().catch(() => {})if (!adminPool) returnawait adminPool.query(`SELECT pg_terminate_backend(pid) FROM pg_stat_activitydiff --git a/control-api/test/routes.mcp-host.plugin-workload-sdk.test.ts b/control-api/test/routes.mcp-host.plugin-workload-sdk.test.tsindex ac443de71..f50d28cf1 100644--- a/control-api/test/routes.mcp-host.plugin-workload-sdk.test.ts+++ b/control-api/test/routes.mcp-host.plugin-workload-sdk.test.ts@@ -1,7 +1,7 @@import { beforeEach, describe, expect, it, vi } from 'vitest'import express from 'express'import request from 'supertest'-import { pool } from '../src/db.js'+import { rateLimitPool } from '../src/db.js'import { createMcpHostPluginWorkloadSdkRoutes } from '../src/routes/mcp-host/plugin-workload-sdk.routes.js'import * as codexConnection from '../src/services/codexSubscriptionConnection.js'import * as grokConnection from '../src/services/grokSubscriptionConnection.js'@@ -18,6 +18,9 @@ vi.mock('../src/db.js', () => ({query: vi.fn().mockResolvedValue({ rows: [], rowCount: 0 }),connect: vi.fn(),},+ rateLimitPool: {+ query: vi.fn().mockResolvedValue({ rows: [], rowCount: 0 }),+ },withTransaction: vi.fn(),}))@@ -1448,7 +1451,7 @@ describe('POST /mcp-host/plugin-workload-sdk/credential-ticket/introspect', () =// before the tighter credential-ticket bucket. Keep the first query under// that outer limit and exhaust the credential-specific bucket on the// second query so this test remains focused on ticket work protection.- vi.mocked(pool.query)+ vi.mocked(rateLimitPool.query).mockResolvedValueOnce({ rows: [{ count: 1 }], rowCount: 1 } as never).mockResolvedValueOnce({ rows: [{ count: 121 }], rowCount: 1 } as never)const res = await request(buildApp())diff --git a/control-api/test/routes.mcp-host.renewal.refresh.rate-limit.test.ts b/control-api/test/routes.mcp-host.renewal.refresh.rate-limit.test.tsindex f9f06edab..5fdfcaa0d 100644--- a/control-api/test/routes.mcp-host.renewal.refresh.rate-limit.test.ts+++ b/control-api/test/routes.mcp-host.renewal.refresh.rate-limit.test.ts@@ -54,6 +54,9 @@ vi.mock('../src/db.js', () => ({query: (...args: unknown[]) => mockPoolQuery(args[0], args[1] as unknown[] | undefined),connect: vi.fn(),},+ rateLimitPool: {+ query: (...args: unknown[]) => mockPoolQuery(args[0], args[1] as unknown[] | undefined),+ },withTransaction: vi.fn(),}))diff --git a/control-api/test/routes.mcp-host.renewal.reissue.test.ts b/control-api/test/routes.mcp-host.renewal.reissue.test.tsindex 52737fbfd..8304d998f 100644--- a/control-api/test/routes.mcp-host.renewal.reissue.test.ts+++ b/control-api/test/routes.mcp-host.renewal.reissue.test.ts@@ -79,6 +79,9 @@ vi.mock('../src/db.js', () => ({query: (...args: unknown[]) => mockPoolQuery(args[0], args[1] as unknown[] | undefined),connect: vi.fn(),},+ rateLimitPool: {+ query: (...args: unknown[]) => mockPoolQuery(args[0], args[1] as unknown[] | undefined),+ },withTransaction: vi.fn(),}))diff --git a/control-api/test/services.rateLimiter.test.ts b/control-api/test/services.rateLimiter.test.tsindex 312782d4a..0c01964d1 100644--- a/control-api/test/services.rateLimiter.test.ts+++ b/control-api/test/services.rateLimiter.test.ts@@ -33,7 +33,7 @@ const mockConcurrencyClient = {}const mockPoolConnect = vi.fn(async () => mockConcurrencyClient)-const mockPoolQuery = vi.fn(async (sql: unknown, params?: unknown[]) => {+const mockRateLimitPoolQuery = vi.fn(async (sql: unknown, params?: unknown[]) => {const text = typeof sql === 'string' ? sql : ''if (/INSERT INTO rate_limit_buckets/i.test(text)) {const bucketKey = String(params?.[0] ?? '')@@ -58,11 +58,22 @@ const mockPoolQuery = vi.fn(async (sql: unknown, params?: unknown[]) => {return { rows: [], rowCount: 0 }})+// The limiter's upserts and prune go to the dedicated limiter pool; the core+// pool only lends the long-lived advisory-lock client. A limiter statement on+// the core pool is a wiring regression, so the core query rejects loudly.+const mockCorePoolQuery = vi.fn(async (sql: unknown) => {+ throw new Error(`core pool must not serve the rate limiter: ${String(sql).slice(0, 40)}`)+})+vi.mock('../src/db.js', () => ({pool: {- query: (...args: unknown[]) => mockPoolQuery(args[0], args[1] as unknown[] | undefined),+ query: (...args: unknown[]) => mockCorePoolQuery(args[0]),connect: (...args: unknown[]) => mockPoolConnect(...args),},+ rateLimitPool: {+ query: (...args: unknown[]) =>+ mockRateLimitPoolQuery(args[0], args[1] as unknown[] | undefined),+ },withTransaction: vi.fn(),}))@@ -70,10 +81,23 @@ describe('rateLimiterService', () => {beforeEach(() => {buckets.clear()advisoryLocks.clear()- mockPoolQuery.mockClear()+ mockRateLimitPoolQuery.mockClear()+ mockCorePoolQuery.mockClear()mockClientQuery.mockClear()})+ it('serves the upsert and the prune from the limiter pool, never the core pool', async () => {+ const r = await checkAndIncrement('test:bucket:pool-wiring', 5)+ await cleanupExpiredBuckets(1_700_000_000_000)++ // Witness: both statements reached the limiter pool.+ expect(r).toMatchObject({ allowed: true, count: 1, backendAvailable: true })+ expect(+ mockRateLimitPoolQuery.mock.calls.map(([sql]) => String(sql).trim().split(/\s+/)[0])+ ).toEqual(['INSERT', 'DELETE'])+ expect(mockCorePoolQuery).not.toHaveBeenCalled()+ })+it('allows requests up to the limit and denies the (limit+1)-th in the same window', async () => {const key = 'test:bucket:1'const limit = 5@@ -129,7 +153,7 @@ describe('rateLimiterService', () => {const second = await checkAndIncrement('test:bucket:weighted', 16, 1_700_000_000_000, 10)expect(first).toMatchObject({ allowed: true, count: 7, remaining: 9 })expect(second).toMatchObject({ allowed: false, count: 17, remaining: 0 })- expect(mockPoolQuery).toHaveBeenLastCalledWith(expect.any(String), [+ expect(mockRateLimitPoolQuery).toHaveBeenLastCalledWith(expect.any(String), ['test:bucket:weighted',currentWindowStartMs(1_700_000_000_000),10,@@ -176,8 +200,8 @@ describe('rateLimiterService', () => {await replacement.release()})- it('fails open when pool.query throws (DB error)', async () => {- mockPoolQuery.mockRejectedValueOnce(new Error('connection refused'))+ it('fails open when the limiter pool query throws (DB error)', async () => {+ mockRateLimitPoolQuery.mockRejectedValueOnce(new Error('connection refused'))const r = await checkAndIncrement('test:bucket:failopen', 5)expect(r.allowed).toBe(true)expect(r.count).toBe(0)@@ -185,8 +209,8 @@ describe('rateLimiterService', () => {expect(r.backendAvailable).toBe(false)})- it('fails open when pool.query returns empty rows', async () => {- mockPoolQuery.mockResolvedValueOnce({ rows: [], rowCount: 0 })+ it('fails open when the limiter pool query returns empty rows', async () => {+ mockRateLimitPoolQuery.mockResolvedValueOnce({ rows: [], rowCount: 0 })const r = await checkAndIncrement('test:bucket:emptyrows', 5)expect(r.allowed).toBe(true)expect(r.count).toBe(0)diff --git a/control-api/test/services.rateLimiterPoolSaturation.realPostgres.integration.test.ts b/control-api/test/services.rateLimiterPoolSaturation.realPostgres.integration.test.tsindex 144305483..4577eba44 100644--- a/control-api/test/services.rateLimiterPoolSaturation.realPostgres.integration.test.ts+++ b/control-api/test/services.rateLimiterPoolSaturation.realPostgres.integration.test.ts@@ -4,19 +4,24 @@ import { performance } from 'node:perf_hooks'import { Pool, type PoolClient } from 'pg'import request from 'supertest'+// The limiter's upserts run on their own pool (db.ts `rateLimitPool`, max 6,+// 5 s acquire), separate from the core pool (max 10, 2 s acquire).+//// Characterization of a KNOWN defect, not a specification of desired-// behaviour: when the production core pool is saturated, checkAndIncrement-// fails OPEN (rateLimiterService.ts, the catch around the INSERT). Every call-// waits for the pool's acquire timeout, logs rate_limit_db_error, and returns-// allowed:true with backendAvailable:false. The fix is tracked separately;-// when it lands, the `allowed` assertion below is the one line that flips.+// behaviour: when the LIMITER pool is saturated, checkAndIncrement fails OPEN+// (rateLimiterService.ts, the catch around the INSERT). Every call waits for+// the limiter pool's acquire timeout, logs rate_limit_db_error, and returns+// allowed:true with backendAvailable:false.+//+// Isolation: a saturated CORE pool no longer reaches the limiter; it keeps+// counting.//-// The production pool bounds are used unchanged (max 10, 2 s acquire): the-// two CORE_POOL_* variables are deleted before db.js is imported, and the+// The production pool bounds are used unchanged: the CORE_POOL_* and+// RATE_LIMIT_POOL_* variables are deleted before db.js is imported, and the// resulting bounds are asserted before anything else.//-// Route-level corollary: with the pool held, an external GFS request returns-// 500, because session auth queries the users table first+// Route-level corollary: with the core pool held, an external GFS request+// returns 500, because session auth queries the users table first// (externalSessionAuth.ts) and fails closed before the limiter runs. The only// mock is the session-token verifier (identity). Skipped without// CONTROL_API_REAL_PG_ADMIN_URL.@@ -29,6 +34,8 @@ vi.mock('../src/utils/auth/externalSessionAuthToken.js', () => ({const PRODUCTION_POOL_MAX = 10const PRODUCTION_ACQUIRE_TIMEOUT_MS = 2_000+const LIMITER_POOL_MAX = 6+const LIMITER_ACQUIRE_TIMEOUT_MS = 5_000const CONCURRENT_CHECKS = 25const LIMIT = 2@@ -45,17 +52,21 @@ function quoteIdent(value: string): string {const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URLconst describeRealPostgres = adminUrl ? describe : describe.skip-describeRealPostgres('rate limiter under core pool saturation (known fail-open)', () => {+describeRealPostgres('rate limiter under pool saturation (known fail-open)', () => {const database = `rate_limiter_saturation_${randomBytes(6).toString('hex')}`const poolEnvKeys = ['CONTROL_API_PG_CONNECTION_STRING','CORE_POOL_MAX','CORE_POOL_CONNECTION_TIMEOUT_MS',+ 'RATE_LIMIT_POOL_MAX',+ 'RATE_LIMIT_POOL_CONNECTION_TIMEOUT_MS',+ 'RATE_LIMIT_POOL_STATEMENT_TIMEOUT_MS',] as constconst previousEnv = new Map<string, string | undefined>()let adminPool: Poollet corePool: Pool+ let limiterPool: Poollet mod: {createApp: (gateway: unknown) => import('express').Expressconfig: typeof import('../src/config.js').config@@ -64,12 +75,10 @@ describeRealPostgres('rate limiter under core pool saturation (known fail-open)'MockGateway: typeof import('./mockGateway.js').MockGateway}- async function holdEveryPoolClient(): Promise<PoolClient[]> {- const held = await Promise.all(- Array.from({ length: PRODUCTION_POOL_MAX }, () => corePool.connect())- )- expect(corePool.totalCount).toBe(PRODUCTION_POOL_MAX)- expect(corePool.idleCount).toBe(0)+ async function holdEveryClient(target: Pool, max: number): Promise<PoolClient[]> {+ const held = await Promise.all(Array.from({ length: max }, () => target.connect()))+ expect(target.totalCount).toBe(max)+ expect(target.idleCount).toBe(0)return held}@@ -79,8 +88,9 @@ describeRealPostgres('rate limiter under core pool saturation (known fail-open)'}const connectionString = databaseUrl(adminUrl, database)for (const key of poolEnvKeys) previousEnv.set(key, process.env[key])- delete process.env.CORE_POOL_MAX- delete process.env.CORE_POOL_CONNECTION_TIMEOUT_MS+ for (const key of poolEnvKeys) {+ if (key !== 'CONTROL_API_PG_CONNECTION_STRING') delete process.env[key]+ }adminPool = new Pool({ connectionString: adminUrl })await adminPool.query(`CREATE DATABASE ${quoteIdent(database)}`)@@ -89,6 +99,8 @@ describeRealPostgres('rate limiter under core pool saturation (known fail-open)'const dbMod = await import('../src/db.js')corePool = dbMod.pool as unknown as PoolcorePool.on('error', () => {})+ limiterPool = dbMod.rateLimitPool as unknown as Pool+ limiterPool.on('error', () => {})const migratePool = new Pool({ connectionString })await dbMod.initDb({ connect: () => migratePool.connect() })await migratePool.end()@@ -114,6 +126,7 @@ describeRealPostgres('rate limiter under core pool saturation (known fail-open)'else process.env[key] = value}await corePool?.end().catch(() => {})+ await limiterPool?.end().catch(() => {})if (!adminPool) returnawait adminPool.query(`SELECT pg_terminate_backend(pid) FROM pg_stat_activity@@ -125,21 +138,46 @@ describeRealPostgres('rate limiter under core pool saturation (known fail-open)'})it('runs against the production pool bounds', () => {- const options = (- corePool as unknown as {- options: { max: number; connectionTimeoutMillis: number }- }- ).options- expect(options.max).toBe(PRODUCTION_POOL_MAX)- expect(options.connectionTimeoutMillis).toBe(PRODUCTION_ACQUIRE_TIMEOUT_MS)+ type Options = { options: { max: number; connectionTimeoutMillis: number } }+ const core = (corePool as unknown as Options).options+ expect(core.max).toBe(PRODUCTION_POOL_MAX)+ expect(core.connectionTimeoutMillis).toBe(PRODUCTION_ACQUIRE_TIMEOUT_MS)+ const limiter = (limiterPool as unknown as Options).options+ expect(limiter.max).toBe(LIMITER_POOL_MAX)+ expect(limiter.connectionTimeoutMillis).toBe(LIMITER_ACQUIRE_TIMEOUT_MS)})- it('fails open for every concurrent check while every client is held, then recovers', async () => {+ it('keeps counting while every core client is held', async () => {+ const bucketKey = `core-saturation:${randomBytes(8).toString('hex')}`+ const nowMs = Math.floor(Date.now() / 60_000) * 60_000 + 1_000+ const warn = vi.spyOn(mod.rootLogger, 'warn').mockImplementation(() => {})+ const held = await holdEveryClient(corePool, PRODUCTION_POOL_MAX)+ try {+ const results = []+ for (let i = 0; i < 3; i += 1) {+ results.push(await mod.checkAndIncrement(bucketKey, LIMIT, nowMs))+ }+ // Witness: the limiter reached Postgres and counted every call.+ expect(results.map(result => result.count)).toEqual([1, 2, 3])+ expect(results.map(result => result.allowed)).toEqual([true, true, false])+ expect(results.every(result => result.backendAvailable)).toBe(true)+ expect(+ warn.mock.calls.filter(+ call => (call[0] as { event?: string }).event === 'rate_limit_db_error'+ )+ ).toHaveLength(0)+ } finally {+ for (const client of held) client.release()+ warn.mockRestore()+ }+ }, 15_000)++ it('fails open for every concurrent check while every limiter client is held, then recovers', async () => {const bucketKey = `saturation:${randomBytes(8).toString('hex')}`// One fixed window for both phases, so a minute boundary cannot split them.const nowMs = Math.floor(Date.now() / 60_000) * 60_000 + 1_000const warn = vi.spyOn(mod.rootLogger, 'warn').mockImplementation(() => {})- const held = await holdEveryPoolClient()+ const held = await holdEveryClient(limiterPool, LIMITER_POOL_MAX)let released = falsetry {const startedAt = performance.now()@@ -151,7 +189,7 @@ describeRealPostgres('rate limiter under core pool saturation (known fail-open)'const elapsedMs = performance.now() - startedAtexpect(results).toHaveLength(CONCURRENT_CHECKS)- // The known defect: a saturated pool admits every request.+ // The known defect: a saturated limiter pool admits every request.expect(results.every(result => result.allowed === true)).toBe(true)// These two stay true after the fix: the backend was unavailable and// nothing was counted.@@ -169,8 +207,8 @@ describeRealPostgres('rate limiter under core pool saturation (known fail-open)'}// Every call waited for the acquire timeout, in parallel, not in series.- expect(elapsedMs).toBeGreaterThanOrEqual(PRODUCTION_ACQUIRE_TIMEOUT_MS - 100)- expect(elapsedMs).toBeLessThan(5_000)+ expect(elapsedMs).toBeGreaterThanOrEqual(LIMITER_ACQUIRE_TIMEOUT_MS - 100)+ expect(elapsedMs).toBeLessThan(LIMITER_ACQUIRE_TIMEOUT_MS + 3_000)// Read through a held client: this also proves the database and the// table are reachable, so "no row" is not an absent table.@@ -201,9 +239,9 @@ describeRealPostgres('rate limiter under core pool saturation (known fail-open)'if (!released) for (const client of held) client.release()warn.mockRestore()}- }, 15_000)+ }, 20_000)- it('answers 500 at the route while the pool is held, because session auth fails first', async () => {+ it('answers 500 at the route while the core pool is held, because session auth fails first', async () => {const internalToken = mod.config.internalServiceTokens['external-rest-api']if (!internalToken) throw new Error('config has no external-rest-api internal service token')mockVerifyExternalSessionToken.mockReset()@@ -230,7 +268,7 @@ describeRealPostgres('rate limiter under core pool saturation (known fail-open)'expect(free.status).toBe(401)expect(mockVerifyExternalSessionToken).toHaveBeenCalledTimes(1)- const held = await holdEveryPoolClient()+ const held = await holdEveryClient(corePool, PRODUCTION_POOL_MAX)try {const saturated = await send()expect(saturated.status).toBe(500)