Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(control-api): size the external GFS read and operation budgets for concurrent desktop surfaces

Alfredo Lopez1dec3d3cc744
diff --git a/.github/workflows/ci-public.yml b/.github/workflows/ci-public.yml
index 35f19dd1d..8b5a6bb6f 100644
--- a/.github/workflows/ci-public.yml
+++ b/.github/workflows/ci-public.yml
@@ -267,9 +267,10 @@ jobs:
echo 'real-Postgres control-api lane reported skipped tests' >&2
exit 1
fi
- # Codex subscription suites must stay inside the real-Postgres filter.
- # A rename or move that drops them from `*realPostgres*` would otherwise
- # still pass the aggregate counts above.
+ # Pinned suites (Codex/Grok subscription, provider attempts, and the
+ # external GFS rate-limit suites) must stay inside the real-Postgres
+ # filter. A rename or move that drops them from `*realPostgres*` would
+ # otherwise still pass the aggregate counts above.
for suite in \
db.codexSubscriptionConnection.realPostgres.integration \
services.codexSubscriptionOAuth.realPostgres.integration \
@@ -285,10 +286,12 @@ jobs:
services.grokProviderAttemptRedemption.refresh.realPostgres.integration \
services.codexSubscriptionLifecycle.realPostgres.integration \
db.llmProviderAttemptConnectionIntegrity.realPostgres.integration \
- services.llmCatalogSync.realPostgres.integration
+ services.llmCatalogSync.realPostgres.integration \
+ routes.externalGfsRateLimitStress.realPostgres.integration \
+ services.rateLimiterPoolSaturation.realPostgres.integration
do
grep -Fq "$suite" "$normalized_log" || {
- echo "real-Postgres control-api lane omitted Codex suite ${suite}" >&2
+ echo "real-Postgres control-api lane omitted pinned suite ${suite}" >&2
exit 1
}
done
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index 716800e60..026eff3ee 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.601",
+ "version": "0.1.602",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.601",
+ "version": "0.1.602",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index a4c6f4544..8ec051e38 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.601",
+ "version": "0.1.602",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/config.ts b/control-api/src/config.ts
index 41ddda9ca..dba2b2fba 100644
--- a/control-api/src/config.ts
+++ b/control-api/src/config.ts
@@ -466,6 +466,10 @@ const WORKFLOW_STEP_DEPENDS_ON_MAX_ITEMS_CEILING = 100
const WORKFLOW_STEP_ALLOWED_TOOLS_MAX_ITEMS_CEILING = 100
// The step mcpServers env can only lower this value; the CRD hard ceiling remains 20.
const WORKFLOW_STEP_MCP_SERVERS_MAX_ITEMS_CEILING = 20
+// The two actor-keyed external GFS budgets accept an environment override up
+// to twice their default; a larger value refuses to boot.
+const EXTERNAL_GFS_READ_RL_PER_MIN_CEILING = 960
+const EXTERNAL_GFS_OPERATION_RL_PER_MIN_CEILING = 180
function normalizePem(value: string): string {
return value.replace(/\\n/g, '\n').trim()
@@ -996,17 +1000,40 @@ export const config: Config = {
600
),
pluginSdkPreauthRlPerMin: positiveIntegerFromEnv('CONTROL_API_PLUGIN_SDK_PREAUTH_PER_MIN', 600),
- // Approved GFS authority-boundary budgets. Keep them fixed here rather than
- // accepting an unreviewed environment override. The 1800/min ingress guard
- // is only a coarse process-local backstop; the aggregate source-IP ceiling,
- // read buckets, token buckets, and mutation/delegation buckets remain
- // independently bounded below.
+ // External GFS authority-boundary budgets. The four keyed by source IP or
+ // by token minting stay compiled: an IP bucket is shared behind a NAT, so
+ // loosening it widens a blast radius beyond one user, and token minting
+ // does not scale with the number of open surfaces. The 1800/min ingress
+ // guard is only a coarse process-local backstop.
externalGfsIngressRlPerMin: 1_800,
externalGfsTokenUserRlPerMin: 10,
externalGfsTokenIpRlPerMin: 600,
externalGfsIpRlPerMin: 1_200,
- externalGfsReadRlPerMin: 120,
- externalGfsOperationRlPerMin: 30,
+ // The two actor-keyed budgets are overridable within a compiled ceiling.
+ // Each value feeds the pre-resolution session and (class, IP) buckets, the
+ // resolved per-actor bucket and the express backstops in
+ // routes/external/gfs.ts. assertExternalGfsBudgetInvariants below refuses a
+ // combination where operations exceed reads or reads exceed the per-IP
+ // all-class bucket.
+ //
+ // Read: one Desktop surface peaks at 58 resource reads/min after PR #702,
+ // and the 2026-09-18 incident peaked at 133 in a sliding 60 s. 480 covers
+ // eight surfaces of one user and is 3.6x the incident peak.
+ externalGfsReadRlPerMin: boundedIntegerFromEnv(
+ 'CONTROL_API_EXTERNAL_GFS_READ_RL_PER_MIN',
+ 480,
+ EXTERNAL_GFS_READ_RL_PER_MIN_CEILING
+ ),
+ // Operation: one maximum-size Desktop upload is at least 27 mutations
+ // (ceil(200 MiB / 8 MiB) parts + create + complete, desktop-app
+ // src/gfs/upload.ts), and a 429 on a part is retried and counted again.
+ // 90 keeps three concurrent uploads out of that retry band. Derivation is
+ // regression-guarded by test/config.externalGfsBudgets.test.ts.
+ externalGfsOperationRlPerMin: boundedIntegerFromEnv(
+ 'CONTROL_API_EXTERNAL_GFS_OPERATION_RL_PER_MIN',
+ 90,
+ EXTERNAL_GFS_OPERATION_RL_PER_MIN_CEILING
+ ),
// Default derived from the wake mechanism's worst case, not picked ad hoc.
// rpc-proxy's wake-and-hold loop re-triggers POST /rpc/hosts/:hostRef/wake
// every wakeRetriggerMs=15000 for up to wakeMaxHoldMs=90000 (defaults in
@@ -1168,6 +1195,41 @@ export const config: Config = {
defaultHookTrustCap: (process.env.CONTROL_API_DEFAULT_HOOK_TRUST_CAP ?? 'mid').toLowerCase(),
}
+/**
+ * Refuse an incoherent set of external GFS budgets at boot. Never clamps: a
+ * corrected value would leave the operator's setting and the running limit
+ * disagreeing.
+ *
+ * - Operations must not be allowed more freely than reads.
+ * - The per-actor read budget must not exceed the per-IP all-class bucket in
+ * front of it; otherwise that bucket caps it under a different key and the
+ * configured number is never reached.
+ */
+export function assertExternalGfsBudgetInvariants(budgets: {
+ readPerMin: number
+ operationPerMin: number
+ ipPerMin: number
+}): void {
+ if (budgets.operationPerMin > budgets.readPerMin) {
+ throw new Error(
+ `CONTROL_API_EXTERNAL_GFS_OPERATION_RL_PER_MIN (${budgets.operationPerMin}) must not ` +
+ `exceed CONTROL_API_EXTERNAL_GFS_READ_RL_PER_MIN (${budgets.readPerMin})`
+ )
+ }
+ if (budgets.readPerMin > budgets.ipPerMin) {
+ throw new Error(
+ `CONTROL_API_EXTERNAL_GFS_READ_RL_PER_MIN (${budgets.readPerMin}) must not exceed ` +
+ `the per-IP external GFS budget externalGfsIpRlPerMin (${budgets.ipPerMin})`
+ )
+ }
+}
+
+assertExternalGfsBudgetInvariants({
+ readPerMin: config.externalGfsReadRlPerMin,
+ operationPerMin: config.externalGfsOperationRlPerMin,
+ ipPerMin: config.externalGfsIpRlPerMin,
+})
+
// Namespace config validation: fail fast if any namespace is empty.
// Empty namespace would target the K8s default namespace — a silent security gap.
const NAMESPACE_KEYS: (keyof Config)[] = [
diff --git a/control-api/test/config.externalGfsBudgets.test.ts b/control-api/test/config.externalGfsBudgets.test.ts
new file mode 100644
index 000000000..d3d68ca4d
--- /dev/null
+++ b/control-api/test/config.externalGfsBudgets.test.ts
@@ -0,0 +1,141 @@
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import { readFileSync } from 'node:fs'
+
+/**
+ * Regression guard for the two actor-keyed external GFS budgets
+ * (config.externalGfsReadRlPerMin, config.externalGfsOperationRlPerMin).
+ *
+ * The operation budget is derived from the largest Desktop upload: one part
+ * per preferred part size up to the default product maximum, plus one create
+ * and one complete call. The two upload constants below are hard-coded
+ * mirrors of desktop-app/src/gfs/upload.ts, and each mirror is verified
+ * against that file, so changing the Desktop part size without re-deriving
+ * this budget fails HERE, in control-api CI.
+ */
+
+// Hard-coded mirrors of desktop-app/src/gfs/upload.ts, in MiB.
+const UPLOAD_PRODUCT_MAX_MIB = 200
+const UPLOAD_PREFERRED_PART_MIB = 8
+// create + complete around the parts.
+const UPLOAD_LIFECYCLE_CALLS = 2
+const MAX_UPLOAD_MUTATIONS =
+ Math.ceil(UPLOAD_PRODUCT_MAX_MIB / UPLOAD_PREFERRED_PART_MIB) + UPLOAD_LIFECYCLE_CALLS
+
+const ENV_KEYS = [
+ 'CONTROL_API_EXTERNAL_GFS_READ_RL_PER_MIN',
+ 'CONTROL_API_EXTERNAL_GFS_OPERATION_RL_PER_MIN',
+] as const
+
+async function loadConfigModuleWith(overrides: Partial<Record<(typeof ENV_KEYS)[number], string>>) {
+ const originalValues = new Map<string, string | undefined>()
+ for (const key of ENV_KEYS) {
+ originalValues.set(key, process.env[key])
+ // Deleted explicitly: a default test that merely omits the variable would
+ // pass for the wrong reason on any host where it is set.
+ delete process.env[key]
+ }
+ Object.assign(process.env, overrides)
+ vi.resetModules()
+ try {
+ return await import('../src/config.js')
+ } finally {
+ for (const key of ENV_KEYS) {
+ const value = originalValues.get(key)
+ if (value === undefined) delete process.env[key]
+ else process.env[key] = value
+ }
+ }
+}
+
+/** Fail-loud single-match extraction — a miss means the source moved. */
+function extractOne(source: string, pattern: RegExp, label: string): string {
+ const match = source.match(pattern)
+ if (!match || match[1] === undefined) {
+ throw new Error(`Could not extract ${label} with ${pattern} — re-derive the mirrors`)
+ }
+ return match[1]
+}
+
+describe('external GFS actor budgets', () => {
+ afterEach(() => {
+ vi.resetModules()
+ })
+
+ it('derives the operation budget from the live Desktop upload constants', async () => {
+ const upload = readFileSync(
+ new URL('../../desktop-app/src/gfs/upload.ts', import.meta.url),
+ 'utf-8'
+ )
+ const productMaxMib = extractOne(
+ upload,
+ /GFS_UPLOAD_V2_DEFAULT_PRODUCT_MAX_BYTES\s*=\s*(\d+)\s*\*\s*1024\s*\*\s*1024\b/,
+ 'Desktop upload product maximum'
+ )
+ const preferredPartMib = extractOne(
+ upload,
+ /GFS_UPLOAD_V2_PREFERRED_PART_BYTES\s*=\s*(\d+)\s*\*\s*1024\s*\*\s*1024\b/,
+ 'Desktop upload preferred part size'
+ )
+ expect(Number(productMaxMib)).toBe(UPLOAD_PRODUCT_MAX_MIB)
+ expect(Number(preferredPartMib)).toBe(UPLOAD_PREFERRED_PART_MIB)
+ expect(MAX_UPLOAD_MUTATIONS).toBe(27)
+
+ // 27 is a floor: larger parts, retried 429s and reconciliation add more.
+ const { config } = await loadConfigModuleWith({})
+ expect(config.externalGfsOperationRlPerMin).toBeGreaterThanOrEqual(MAX_UPLOAD_MUTATIONS)
+ })
+
+ it('defaults to 480 reads and 90 operations per minute', async () => {
+ const { config } = await loadConfigModuleWith({})
+ expect(config.externalGfsReadRlPerMin).toBe(480)
+ expect(config.externalGfsOperationRlPerMin).toBe(90)
+ })
+
+ it('accepts an environment override within the ceiling', async () => {
+ const { config } = await loadConfigModuleWith({
+ CONTROL_API_EXTERNAL_GFS_READ_RL_PER_MIN: '960',
+ CONTROL_API_EXTERNAL_GFS_OPERATION_RL_PER_MIN: '180',
+ })
+ expect(config.externalGfsReadRlPerMin).toBe(960)
+ expect(config.externalGfsOperationRlPerMin).toBe(180)
+ })
+
+ it('refuses to boot above either compiled ceiling', async () => {
+ await expect(
+ loadConfigModuleWith({ CONTROL_API_EXTERNAL_GFS_READ_RL_PER_MIN: '961' })
+ ).rejects.toThrow(
+ 'CONTROL_API_EXTERNAL_GFS_READ_RL_PER_MIN must be an integer between 1 and 960'
+ )
+ await expect(
+ loadConfigModuleWith({ CONTROL_API_EXTERNAL_GFS_OPERATION_RL_PER_MIN: '181' })
+ ).rejects.toThrow(
+ 'CONTROL_API_EXTERNAL_GFS_OPERATION_RL_PER_MIN must be an integer between 1 and 180'
+ )
+ })
+
+ it('refuses to boot when the operation budget exceeds the read budget', async () => {
+ await expect(
+ loadConfigModuleWith({
+ CONTROL_API_EXTERNAL_GFS_READ_RL_PER_MIN: '100',
+ CONTROL_API_EXTERNAL_GFS_OPERATION_RL_PER_MIN: '180',
+ })
+ ).rejects.toThrow(
+ 'CONTROL_API_EXTERNAL_GFS_OPERATION_RL_PER_MIN (180) must not exceed ' +
+ 'CONTROL_API_EXTERNAL_GFS_READ_RL_PER_MIN (100)'
+ )
+ })
+
+ it('refuses a read budget above the per-IP budget and accepts it at the boundary', async () => {
+ const { assertExternalGfsBudgetInvariants } = await loadConfigModuleWith({})
+ expect(() =>
+ assertExternalGfsBudgetInvariants({ readPerMin: 1300, operationPerMin: 90, ipPerMin: 1200 })
+ ).toThrow(
+ 'CONTROL_API_EXTERNAL_GFS_READ_RL_PER_MIN (1300) must not exceed ' +
+ 'the per-IP external GFS budget externalGfsIpRlPerMin (1200)'
+ )
+ // Liveness witness for the check being `<=`, not `<`: equal values pass.
+ expect(() =>
+ assertExternalGfsBudgetInvariants({ readPerMin: 1200, operationPerMin: 90, ipPerMin: 1200 })
+ ).not.toThrow()
+ })
+})
diff --git a/control-api/test/routes.externalGfsRateLimitStress.realPostgres.integration.test.ts b/control-api/test/routes.externalGfsRateLimitStress.realPostgres.integration.test.ts
new file mode 100644
index 000000000..41f45a283
--- /dev/null
+++ b/control-api/test/routes.externalGfsRateLimitStress.realPostgres.integration.test.ts
@@ -0,0 +1,544 @@
+import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'
+import { randomBytes, randomUUID } from 'node:crypto'
+import { Pool } from 'pg'
+import request from 'supertest'
+
+// Concurrent users at the external GFS rate limits, through the production
+// Express app against real PostgreSQL. Config, the core pool, both limiters,
+// the session lifecycle query, authority resolution, the prom registry and the
+// logger are real; the ONLY mock is the session-token verifier (identity),
+// which decodes a JSON test token so each scenario controls users and sessions.
+//
+// Denial source (plan Addendum A3), for status 429 only:
+// - route backstop: RateLimit-Policy === "<route L>;w=60";
+// - Postgres bucket: RateLimit-Policy === "<ingress L>;w=60" AND
+// X-RateLimit-Remaining === "0" AND X-RateLimit-Limit === "<bucket L>";
+// - anything else fails the test.
+// Headers cannot say WHICH Postgres bucket denied, so every scenario also reads
+// the rate_limit_buckets ledger: each hit increments its row once and a bucket
+// only sees requests that passed the buckets before it, so per (key, window)
+// denials = max(0, count - limit). The key prefix names the phase.
+//
+// Both limiters read Date.now(), so faking only Date pins their windows. Time
+// only moves forward, each scenario deletes the ledger, uses fresh identities,
+// and builds a new app after setting the budgets (the express backstops and
+// the grants read limiter capture their limit when the router is created).
+// Skipped without CONTROL_API_REAL_PG_ADMIN_URL.
+
+vi.mock('../src/utils/auth/externalSessionAuthToken.js', () => ({
+ verifyExternalSessionToken: (token: string) => {
+ const parsed = JSON.parse(token) as { userId: string }
+ return {
+ userId: parsed.userId,
+ email: `${parsed.userId}@example.test`,
+ teamId: null,
+ role: 'member',
+ authGeneration: 1,
+ exp: Math.floor(Date.now() / 1000) + 3600,
+ }
+ },
+}))
+
+const INGRESS_L = 1_800
+const IP_L_DEFAULT = 1_200
+const MINUTE_MS = 60_000
+
+function databaseUrl(baseUrl: string, database: string): string {
+ const url = new URL(baseUrl)
+ url.pathname = `/${database}`
+ return url.toString()
+}
+
+function quoteIdent(value: string): string {
+ return `"${value.replace(/"/g, '""')}"`
+}
+
+type Budgets = { read: number; operation: number; ip: number }
+type Sent = {
+ path: string
+ status: number
+ body: { error?: unknown }
+ policy?: string
+ xLimit?: string
+ xRemaining?: string
+}
+type LedgerRow = { key: string; windowStartMs: number; count: number }
+type Phase = 'pre-resolution' | 'resolved-operation' | 'edge-backstop'
+
+const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL
+const describeRealPostgres = adminUrl ? describe : describe.skip
+
+describeRealPostgres('external GFS rate limits under concurrent load (real PostgreSQL)', () => {
+ const database = `control_api_gfs_rl_stress_${randomBytes(6).toString('hex')}`
+ const connectionString = databaseUrl(
+ adminUrl ?? 'postgresql://[email protected]/postgres',
+ database
+ )
+ const envKeys = ['CONTROL_API_PG_CONNECTION_STRING', 'CORE_POOL_CONNECTION_TIMEOUT_MS'] as const
+ const previousEnv = new Map<string, string | undefined>()
+
+ let adminPool: Pool
+ let corePool: Pool
+ let mod: {
+ createApp: (gateway: unknown) => import('express').Express
+ config: typeof import('../src/config.js').config
+ assertExternalGfsBudgetInvariants: typeof import('../src/config.js').assertExternalGfsBudgetInvariants
+ pool: import('../src/db.js').DbClient
+ rootLogger: typeof import('../src/observability/logger.js').rootLogger
+ requestsTotal: typeof import('../src/observability/metrics.js').externalGfsRateLimitRequestsTotal
+ MockGateway: typeof import('./mockGateway.js').MockGateway
+ }
+ // Forward-only fake clock: every scenario starts on a later minute.
+ let nextScenarioMinuteMs = 0
+
+ beforeAll(async () => {
+ for (const key of envKeys) previousEnv.set(key, process.env[key])
+ adminPool = new Pool({ connectionString: adminUrl })
+ await adminPool.query(`CREATE DATABASE ${quoteIdent(database)}`)
+ process.env.CONTROL_API_PG_CONNECTION_STRING = connectionString
+ // A 200-request burst queues on 10 connections; the production 2 s acquire
+ // timeout would make the limiter fail open by accident (see
+ // services.rateLimiterPoolSaturation.realPostgres.integration.test.ts).
+ process.env.CORE_POOL_CONNECTION_TIMEOUT_MS = '10000'
+
+ const dbMod = await import('../src/db.js')
+ corePool = dbMod.pool as unknown as Pool
+ corePool.on('error', () => {})
+ const migratePool = new Pool({ connectionString })
+ await dbMod.initDb({ connect: () => migratePool.connect() })
+ await migratePool.end()
+
+ const appMod = await import('../src/app.js')
+ const configMod = await import('../src/config.js')
+ const loggerMod = await import('../src/observability/logger.js')
+ const metricsMod = await import('../src/observability/metrics.js')
+ const { MockGateway } = await import('./mockGateway.js')
+ mod = {
+ createApp: appMod.createApp as never,
+ config: configMod.config,
+ assertExternalGfsBudgetInvariants: configMod.assertExternalGfsBudgetInvariants,
+ pool: dbMod.pool,
+ rootLogger: loggerMod.rootLogger,
+ requestsTotal: metricsMod.externalGfsRateLimitRequestsTotal,
+ MockGateway,
+ }
+ expect(mod.config.externalGfsIngressRlPerMin).toBe(INGRESS_L)
+ expect(mod.config.externalGfsIpRlPerMin).toBe(IP_L_DEFAULT)
+ vi.useFakeTimers({ toFake: ['Date'] })
+ nextScenarioMinuteMs = Math.floor(Date.now() / MINUTE_MS) * MINUTE_MS + 10 * MINUTE_MS
+ }, 60_000)
+
+ afterAll(async () => {
+ vi.useRealTimers()
+ for (const key of envKeys) {
+ const value = previousEnv.get(key)
+ if (value === undefined) delete process.env[key]
+ else process.env[key] = value
+ }
+ await corePool?.end().catch(() => {})
+ if (!adminPool) return
+ await adminPool.query(
+ `SELECT pg_terminate_backend(pid) FROM pg_stat_activity
+ WHERE datname = $1 AND pid <> pg_backend_pid()`,
+ [database]
+ )
+ await adminPool.query(`DROP DATABASE IF EXISTS ${quoteIdent(database)}`)
+ await adminPool.end()
+ })
+
+ /** Sets a boot-valid budget, clears the ledger, returns a fresh app and the scenario minute. */
+ async function startScenario(budgets: Budgets) {
+ mod.assertExternalGfsBudgetInvariants({
+ readPerMin: budgets.read,
+ operationPerMin: budgets.operation,
+ ipPerMin: budgets.ip,
+ })
+ mod.config.externalGfsReadRlPerMin = budgets.read
+ mod.config.externalGfsOperationRlPerMin = budgets.operation
+ mod.config.externalGfsIpRlPerMin = budgets.ip
+ await mod.pool.query('DELETE FROM rate_limit_buckets')
+ const minuteMs = nextScenarioMinuteMs
+ nextScenarioMinuteMs += 5 * MINUTE_MS
+ vi.setSystemTime(minuteMs + 10_000)
+ return { app: mod.createApp(new mod.MockGateway()), minuteMs }
+ }
+
+ async function seedUser(): Promise<string> {
+ const id = randomUUID()
+ await mod.pool.query(`INSERT INTO users (id, email, name) VALUES ($1, $2, $3)`, [
+ id,
+ `${id}@example.test`,
+ 'stress',
+ ])
+ return id
+ }
+
+ const sessionToken = (userId: string, session = 0) => JSON.stringify({ userId, session })
+ const affordancesPath = (resourceId: string) =>
+ `/api/v1/external/gfs/resources/${resourceId}/affordances?drive=main`
+ const grantsPath = (resourceId: string) =>
+ `/api/v1/external/gfs/grants?drive=main&resourceId=${resourceId}`
+
+ async function send(
+ app: import('express').Express,
+ path: string,
+ token: string,
+ ip: string
+ ): Promise<Sent> {
+ const internalToken = mod.config.internalServiceTokens['external-rest-api']
+ if (!internalToken) throw new Error('config has no external-rest-api internal service token')
+ const res = await request(app)
+ .get(path)
+ .set('Authorization', `Bearer ${internalToken}`)
+ .set('x-service-token', 'external-rest-api')
+ .set('x-user-session-token', token)
+ .set('x-forwarded-for', ip)
+ return {
+ path,
+ status: res.status,
+ body: res.body as { error?: unknown },
+ policy: res.headers['ratelimit-policy'],
+ xLimit: res.headers['x-ratelimit-limit'],
+ xRemaining: res.headers['x-ratelimit-remaining'],
+ }
+ }
+
+ /** Addendum A3 classifier. Allowed responses keep their status. */
+ function classify(response: Sent, routeL: number): string {
+ if (response.status !== 429) return `allowed:${response.status}`
+ if (response.policy === `${routeL};w=60`) return 'route-backstop'
+ if (response.policy === `${INGRESS_L};w=60` && response.xRemaining === '0')
+ return `postgres:${response.xLimit}`
+ return `unclassified:${JSON.stringify(response)}`
+ }
+
+ function tally(labels: string[]): Record<string, number> {
+ const counts: Record<string, number> = {}
+ for (const label of labels) counts[label] = (counts[label] ?? 0) + 1
+ return counts
+ }
+
+ async function readLedger(): Promise<LedgerRow[]> {
+ const result = await mod.pool.query(
+ `SELECT bucket_key, window_start_ms::text AS window_start_ms, count
+ FROM rate_limit_buckets ORDER BY bucket_key, window_start_ms`
+ )
+ return (
+ result.rows as Array<{ bucket_key: string; window_start_ms: string; count: number }>
+ ).map(row => ({
+ key: row.bucket_key,
+ windowStartMs: Number(row.window_start_ms),
+ count: Number(row.count),
+ }))
+ }
+
+ /** Sorted counts of the rows whose key starts with `prefix`, optionally in one window. */
+ function counts(ledger: LedgerRow[], prefix: string, windowStartMs?: number): number[] {
+ return ledger
+ .filter(
+ row =>
+ row.key.startsWith(prefix) &&
+ (windowStartMs === undefined || row.windowStartMs === windowStartMs)
+ )
+ .map(row => row.count)
+ .sort((a, b) => a - b)
+ }
+
+ const sum = (values: number[]) => values.reduce((total, value) => total + value, 0)
+
+ /** Limit of every bucket key these scenarios can create; an unknown key fails loudly. */
+ function limitOf(key: string, budgets: Budgets): number {
+ if (key.startsWith('gfs-ext:pre:ip:')) return budgets.ip
+ if (/^gfs-ext:(pre|resolved):(resource|grants-read):/.test(key)) return budgets.read
+ if (key.startsWith('gfsgrants-ext-read:user:')) return budgets.read
+ throw new Error(`unexpected rate limit bucket in the ledger: ${key}`)
+ }
+
+ /** Ledger identity: Postgres denials = sum over (key, window) of max(0, count - limit). */
+ function ledgerDenials(ledger: LedgerRow[], budgets: Budgets): number {
+ return sum(ledger.map(row => Math.max(0, row.count - limitOf(row.key, budgets))))
+ }
+
+ async function deniedByPhase(): Promise<Record<Phase, number>> {
+ const { values } = await mod.requestsTotal.get()
+ const totals: Record<Phase, number> = {
+ 'pre-resolution': 0,
+ 'resolved-operation': 0,
+ 'edge-backstop': 0,
+ }
+ for (const sample of values) {
+ if (sample.labels.outcome !== 'denied') continue
+ const phase = sample.labels.phase as Phase
+ if (!(phase in totals)) throw new Error(`unexpected rate limit phase label: ${phase}`)
+ totals[phase] += sample.value
+ }
+ return totals
+ }
+
+ /** Captures denial log lines by phase and every limiter DB error during `run`. */
+ async function observed<T>(run: () => Promise<T>) {
+ const warn = vi.spyOn(mod.rootLogger, 'warn').mockImplementation(() => {})
+ const before = await deniedByPhase()
+ try {
+ const result = await run()
+ const after = await deniedByPhase()
+ const payloads = warn.mock.calls.map(call => call[0] as { event?: string; phase?: string })
+ const logged = (phase: Phase) =>
+ payloads.filter(p => p.event === 'external_gfs_rate_limit' && p.phase === phase).length
+ return {
+ result,
+ metricDelta: {
+ 'pre-resolution': after['pre-resolution'] - before['pre-resolution'],
+ 'resolved-operation': after['resolved-operation'] - before['resolved-operation'],
+ 'edge-backstop': after['edge-backstop'] - before['edge-backstop'],
+ },
+ logged: {
+ 'pre-resolution': logged('pre-resolution'),
+ 'resolved-operation': logged('resolved-operation'),
+ 'edge-backstop': logged('edge-backstop'),
+ },
+ dbErrors: payloads.filter(p => p.event === 'rate_limit_db_error').length,
+ }
+ } finally {
+ warn.mockRestore()
+ }
+ }
+
+ it('S1: 20 users with distinct sessions and IPs each get exactly the read budget', async () => {
+ const budgets = { read: 6, operation: 6, ip: IP_L_DEFAULT }
+ const { app } = await startScenario(budgets)
+ const users = await Promise.all(Array.from({ length: 20 }, () => seedUser()))
+ const resourceId = randomUUID()
+
+ const run = await observed(() =>
+ Promise.all(
+ users.flatMap((userId, index) =>
+ Array.from({ length: 10 }, () =>
+ send(app, affordancesPath(resourceId), sessionToken(userId), `198.51.100.${index + 1}`)
+ )
+ )
+ )
+ )
+ const labels = run.result.map(response => classify(response, budgets.read))
+
+ expect(run.result).toHaveLength(200)
+ for (let user = 0; user < 20; user += 1) {
+ expect(tally(labels.slice(user * 10, user * 10 + 10))).toEqual({
+ 'allowed:200': 6,
+ 'postgres:6': 4,
+ })
+ }
+ const ledger = await readLedger()
+ expect(counts(ledger, 'gfs-ext:pre:resource:session:')).toEqual(Array(20).fill(10))
+ expect(counts(ledger, 'gfs-ext:pre:resource:ip:')).toEqual(Array(20).fill(6))
+ expect(counts(ledger, 'gfs-ext:pre:ip:')).toEqual(Array(20).fill(6))
+ expect(counts(ledger, 'gfs-ext:resolved:resource:actor:user-session:')).toEqual(
+ Array(20).fill(6)
+ )
+ expect(ledgerDenials(ledger, budgets)).toBe(80)
+ // S6: the Postgres denials are logged and counted exactly once each; the
+ // 120 allowed requests passed the backstop, which denied none of them.
+ expect(run.dbErrors).toBe(0)
+ expect(run.metricDelta).toEqual({
+ 'pre-resolution': 80,
+ 'resolved-operation': 0,
+ 'edge-backstop': 0,
+ })
+ expect(run.logged).toEqual(run.metricDelta)
+ }, 120_000)
+
+ it('S2a: 20 users behind one IP share the (class, IP) read bucket', async () => {
+ const budgets = { read: 30, operation: 30, ip: IP_L_DEFAULT }
+ const { app } = await startScenario(budgets)
+ const users = await Promise.all(Array.from({ length: 20 }, () => seedUser()))
+ const resourceId = randomUUID()
+
+ const run = await observed(() =>
+ Promise.all(
+ users.flatMap(userId =>
+ Array.from({ length: 5 }, () =>
+ send(app, affordancesPath(resourceId), sessionToken(userId), '198.51.100.30')
+ )
+ )
+ )
+ )
+
+ expect(tally(run.result.map(response => classify(response, budgets.read)))).toEqual({
+ 'allowed:200': 30,
+ 'postgres:30': 70,
+ })
+ const ledger = await readLedger()
+ expect(counts(ledger, 'gfs-ext:pre:resource:session:')).toEqual(Array(20).fill(5))
+ expect(counts(ledger, 'gfs-ext:pre:resource:ip:')).toEqual([100])
+ expect(counts(ledger, 'gfs-ext:pre:ip:')).toEqual([30])
+ expect(sum(counts(ledger, 'gfs-ext:resolved:resource:actor:user-session:'))).toBe(30)
+ expect(ledgerDenials(ledger, budgets)).toBe(70)
+ expect(run.dbErrors).toBe(0)
+ expect(run.metricDelta).toEqual({
+ 'pre-resolution': 70,
+ 'resolved-operation': 0,
+ 'edge-backstop': 0,
+ })
+ expect(run.logged).toEqual(run.metricDelta)
+ }, 120_000)
+
+ it('S2b: the per-IP all-class bucket caps two read classes behind one IP', async () => {
+ // Boot-valid: read (30) <= ip (40). pre:ip can only deny when several
+ // classes share the IP, because each class alone is capped at read L first.
+ const budgets = { read: 30, operation: 30, ip: 40 }
+ const { app } = await startScenario(budgets)
+ const users = await Promise.all(Array.from({ length: 10 }, () => seedUser()))
+ const resourceId = randomUUID()
+
+ const run = await observed(() =>
+ Promise.all(
+ users.flatMap((userId, index) =>
+ Array.from({ length: 5 }, () =>
+ send(
+ app,
+ index < 5 ? affordancesPath(resourceId) : grantsPath(resourceId),
+ sessionToken(userId),
+ '198.51.100.40'
+ )
+ )
+ )
+ )
+ )
+ const labels = run.result.map(response => classify(response, budgets.read))
+ const tallied = tally(labels)
+
+ expect(tallied['postgres:40']).toBe(10)
+ expect((tallied['allowed:200'] ?? 0) + (tallied['allowed:403'] ?? 0)).toBe(40)
+ expect(Object.keys(tallied).sort()).toEqual(
+ ['allowed:200', 'allowed:403', 'postgres:40'].filter(label => tallied[label] !== undefined)
+ )
+ // "Allowed" is defined per route: affordances answer 200, the grants list
+ // answers 403 manage_acl_required for a caller without manage_acl.
+ for (const response of run.result) {
+ if (response.status === 200) expect(response.path).toContain('/affordances')
+ if (response.status === 403) {
+ expect(response.path).toContain('/grants')
+ expect(response.body.error).toBe('manage_acl_required')
+ }
+ }
+ const ledger = await readLedger()
+ expect(counts(ledger, 'gfs-ext:pre:resource:session:')).toEqual(Array(5).fill(5))
+ expect(counts(ledger, 'gfs-ext:pre:grants-read:session:')).toEqual(Array(5).fill(5))
+ expect(counts(ledger, 'gfs-ext:pre:resource:ip:')).toEqual([25])
+ expect(counts(ledger, 'gfs-ext:pre:grants-read:ip:')).toEqual([25])
+ expect(counts(ledger, 'gfs-ext:pre:ip:')).toEqual([50])
+ expect(sum(counts(ledger, 'gfs-ext:resolved:'))).toBe(40)
+ expect(sum(counts(ledger, 'gfsgrants-ext-read:user:'))).toBe(tallied['allowed:403'] ?? 0)
+ expect(ledgerDenials(ledger, budgets)).toBe(10)
+ expect(run.dbErrors).toBe(0)
+ expect(run.metricDelta).toEqual({
+ 'pre-resolution': 10,
+ 'resolved-operation': 0,
+ 'edge-backstop': 0,
+ })
+ expect(run.logged).toEqual(run.metricDelta)
+ }, 120_000)
+
+ it('S3: two sessions of one user share the resolved actor bucket', async () => {
+ const budgets = { read: 10, operation: 10, ip: IP_L_DEFAULT }
+ const { app } = await startScenario(budgets)
+ const userId = await seedUser()
+ const resourceId = randomUUID()
+
+ const run = await observed(() =>
+ Promise.all([
+ ...Array.from({ length: 8 }, () =>
+ send(app, affordancesPath(resourceId), sessionToken(userId, 1), '198.51.100.51')
+ ),
+ ...Array.from({ length: 8 }, () =>
+ send(app, affordancesPath(resourceId), sessionToken(userId, 2), '198.51.100.52')
+ ),
+ ])
+ )
+
+ expect(tally(run.result.map(response => classify(response, budgets.read)))).toEqual({
+ 'allowed:200': 10,
+ 'postgres:10': 6,
+ })
+ const ledger = await readLedger()
+ // Each pre-resolution bucket saw 8 <= 10 and denied nothing; the resolved
+ // actor bucket saw all 16: the phase witness, independent of telemetry.
+ expect(counts(ledger, 'gfs-ext:pre:resource:session:')).toEqual([8, 8])
+ expect(counts(ledger, 'gfs-ext:pre:resource:ip:')).toEqual([8, 8])
+ expect(counts(ledger, 'gfs-ext:pre:ip:')).toEqual([8, 8])
+ expect(counts(ledger, `gfs-ext:resolved:resource:actor:user-session:${userId}`)).toEqual([16])
+ expect(ledgerDenials(ledger, budgets)).toBe(6)
+ expect(run.dbErrors).toBe(0)
+ expect(run.metricDelta).toEqual({
+ 'pre-resolution': 0,
+ 'resolved-operation': 6,
+ 'edge-backstop': 0,
+ })
+ expect(run.logged).toEqual(run.metricDelta)
+ }, 120_000)
+
+ it('S4: a burst straddling a minute boundary is denied only by the route backstop', async () => {
+ const budgets = { read: 8, operation: 8, ip: IP_L_DEFAULT }
+ const { app, minuteMs } = await startScenario(budgets)
+ const userId = await seedUser()
+ const resourceId = randomUUID()
+ const burst = () =>
+ observed(() =>
+ Promise.all(
+ Array.from({ length: 8 }, () =>
+ send(app, affordancesPath(resourceId), sessionToken(userId), '198.51.100.60')
+ )
+ )
+ )
+ const labelsOf = (responses: Sent[]) =>
+ tally(responses.map(response => classify(response, budgets.read)))
+
+ // A: 50 s into window 1. Opens the backstop window, fills the Postgres buckets.
+ vi.setSystemTime(minuteMs + 50_000)
+ const a = await burst()
+ // B: 15 s later. New Postgres window (counts 1..8, allowed), same backstop window.
+ vi.setSystemTime(minuteMs + 65_000)
+ const b = await burst()
+ // C: 60 s after A. The session bucket of window 2 is already at 8.
+ vi.setSystemTime(minuteMs + 110_000)
+ const c = await burst()
+
+ expect(labelsOf(a.result)).toEqual({ 'allowed:200': 8 })
+ expect(labelsOf(b.result)).toEqual({ 'route-backstop': 8 })
+ expect(labelsOf(c.result)).toEqual({ 'postgres:8': 8 })
+
+ const ledger = await readLedger()
+ const window1 = minuteMs
+ const window2 = minuteMs + MINUTE_MS
+ expect(counts(ledger, 'gfs-ext:pre:resource:session:', window1)).toEqual([8])
+ expect(counts(ledger, 'gfs-ext:pre:resource:session:', window2)).toEqual([16])
+ expect(counts(ledger, 'gfs-ext:pre:resource:ip:', window1)).toEqual([8])
+ expect(counts(ledger, 'gfs-ext:pre:resource:ip:', window2)).toEqual([8])
+ expect(counts(ledger, 'gfs-ext:pre:ip:', window1)).toEqual([8])
+ expect(counts(ledger, 'gfs-ext:pre:ip:', window2)).toEqual([8])
+ expect(counts(ledger, 'gfs-ext:resolved:resource:actor:', window1)).toEqual([8])
+ expect(counts(ledger, 'gfs-ext:resolved:resource:actor:', window2)).toEqual([8])
+ // Only C's 8 denials are Postgres denials; B's 8 are the backstop's.
+ expect(ledgerDenials(ledger, budgets)).toBe(8)
+
+ // S6: A is the liveness witness (allowed, nothing reported); B is reported
+ // only as edge-backstop; C only as pre-resolution.
+ for (const run of [a, b, c]) expect(run.dbErrors).toBe(0)
+ expect(a.metricDelta).toEqual({
+ 'pre-resolution': 0,
+ 'resolved-operation': 0,
+ 'edge-backstop': 0,
+ })
+ expect(b.metricDelta).toEqual({
+ 'pre-resolution': 0,
+ 'resolved-operation': 0,
+ 'edge-backstop': 8,
+ })
+ expect(c.metricDelta).toEqual({
+ 'pre-resolution': 8,
+ 'resolved-operation': 0,
+ 'edge-backstop': 0,
+ })
+ for (const run of [a, b, c]) expect(run.logged).toEqual(run.metricDelta)
+ }, 120_000)
+})
diff --git a/control-api/test/services.rateLimiterPoolSaturation.realPostgres.integration.test.ts b/control-api/test/services.rateLimiterPoolSaturation.realPostgres.integration.test.ts
new file mode 100644
index 000000000..7496fd582
--- /dev/null
+++ b/control-api/test/services.rateLimiterPoolSaturation.realPostgres.integration.test.ts
@@ -0,0 +1,242 @@
+import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'
+import { randomBytes } from 'node:crypto'
+import { performance } from 'node:perf_hooks'
+import { Pool, type PoolClient } from 'pg'
+import request from 'supertest'
+
+// Characterization of a KNOWN defect, not a specification of desired
+// behaviour: when the production core pool is saturated, checkAndIncrement
+// fails OPEN (rateLimiterService.ts, the catch around the INSERT). Every call
+// waits for the pool's acquire timeout, logs rate_limit_db_error, and returns
+// allowed:true with backendAvailable:false. The fix is tracked separately;
+// when it lands, the `allowed` assertion below is the one line that flips.
+//
+// The production pool bounds are used unchanged (max 10, 2 s acquire): the
+// two CORE_POOL_* variables are deleted before db.js is imported, and the
+// resulting bounds are asserted before anything else.
+//
+// Route-level corollary: with the pool held, an external GFS request returns
+// 500, because session auth queries the users table first
+// (externalSessionAuth.ts) and fails closed before the limiter runs. The only
+// mock is the session-token verifier (identity). Skipped without
+// CONTROL_API_REAL_PG_ADMIN_URL.
+
+const mockVerifyExternalSessionToken = vi.fn()
+
+vi.mock('../src/utils/auth/externalSessionAuthToken.js', () => ({
+ verifyExternalSessionToken: (...args: unknown[]) => mockVerifyExternalSessionToken(...args),
+}))
+
+const PRODUCTION_POOL_MAX = 10
+const PRODUCTION_ACQUIRE_TIMEOUT_MS = 2_000
+const CONCURRENT_CHECKS = 25
+const LIMIT = 2
+
+function databaseUrl(baseUrl: string, database: string): string {
+ const url = new URL(baseUrl)
+ url.pathname = `/${database}`
+ return url.toString()
+}
+
+function quoteIdent(value: string): string {
+ return `"${value.replace(/"/g, '""')}"`
+}
+
+const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL
+const describeRealPostgres = adminUrl ? describe : describe.skip
+
+describeRealPostgres('rate limiter under core pool saturation (known fail-open)', () => {
+ const database = `rate_limiter_saturation_${randomBytes(6).toString('hex')}`
+ const connectionString = databaseUrl(
+ adminUrl ?? 'postgresql://[email protected]/postgres',
+ database
+ )
+ const poolEnvKeys = [
+ 'CONTROL_API_PG_CONNECTION_STRING',
+ 'CORE_POOL_MAX',
+ 'CORE_POOL_CONNECTION_TIMEOUT_MS',
+ ] as const
+ const previousEnv = new Map<string, string | undefined>()
+
+ let adminPool: Pool
+ let corePool: Pool
+ let mod: {
+ createApp: (gateway: unknown) => import('express').Express
+ config: typeof import('../src/config.js').config
+ checkAndIncrement: typeof import('../src/services/rateLimiterService.js').checkAndIncrement
+ rootLogger: typeof import('../src/observability/logger.js').rootLogger
+ MockGateway: typeof import('./mockGateway.js').MockGateway
+ }
+
+ async function holdEveryPoolClient(): Promise<PoolClient[]> {
+ const held = await Promise.all(
+ Array.from({ length: PRODUCTION_POOL_MAX }, () => corePool.connect())
+ )
+ expect(corePool.totalCount).toBe(PRODUCTION_POOL_MAX)
+ expect(corePool.idleCount).toBe(0)
+ return held
+ }
+
+ beforeAll(async () => {
+ for (const key of poolEnvKeys) previousEnv.set(key, process.env[key])
+ delete process.env.CORE_POOL_MAX
+ delete process.env.CORE_POOL_CONNECTION_TIMEOUT_MS
+
+ adminPool = new Pool({ connectionString: adminUrl })
+ await adminPool.query(`CREATE DATABASE ${quoteIdent(database)}`)
+ process.env.CONTROL_API_PG_CONNECTION_STRING = connectionString
+
+ const dbMod = await import('../src/db.js')
+ corePool = dbMod.pool as unknown as Pool
+ corePool.on('error', () => {})
+ const migratePool = new Pool({ connectionString })
+ await dbMod.initDb({ connect: () => migratePool.connect() })
+ await migratePool.end()
+
+ const appMod = await import('../src/app.js')
+ const configMod = await import('../src/config.js')
+ const limiterMod = await import('../src/services/rateLimiterService.js')
+ const loggerMod = await import('../src/observability/logger.js')
+ const { MockGateway } = await import('./mockGateway.js')
+ mod = {
+ createApp: appMod.createApp as never,
+ config: configMod.config,
+ checkAndIncrement: limiterMod.checkAndIncrement,
+ rootLogger: loggerMod.rootLogger,
+ MockGateway,
+ }
+ }, 60_000)
+
+ afterAll(async () => {
+ for (const key of poolEnvKeys) {
+ const value = previousEnv.get(key)
+ if (value === undefined) delete process.env[key]
+ else process.env[key] = value
+ }
+ await corePool?.end().catch(() => {})
+ if (!adminPool) return
+ await adminPool.query(
+ `SELECT pg_terminate_backend(pid) FROM pg_stat_activity
+ WHERE datname = $1 AND pid <> pg_backend_pid()`,
+ [database]
+ )
+ await adminPool.query(`DROP DATABASE IF EXISTS ${quoteIdent(database)}`)
+ await adminPool.end()
+ })
+
+ it('runs against the production pool bounds', () => {
+ const options = (
+ corePool as unknown as {
+ options: { max: number; connectionTimeoutMillis: number }
+ }
+ ).options
+ expect(options.max).toBe(PRODUCTION_POOL_MAX)
+ expect(options.connectionTimeoutMillis).toBe(PRODUCTION_ACQUIRE_TIMEOUT_MS)
+ })
+
+ it('fails open for every concurrent check while every client is held, then recovers', async () => {
+ const bucketKey = `saturation:${randomBytes(8).toString('hex')}`
+ // One fixed window for both phases, so a minute boundary cannot split them.
+ const nowMs = Math.floor(Date.now() / 60_000) * 60_000 + 1_000
+ const warn = vi.spyOn(mod.rootLogger, 'warn').mockImplementation(() => {})
+ const held = await holdEveryPoolClient()
+ let released = false
+ try {
+ const startedAt = performance.now()
+ const results = await Promise.all(
+ Array.from({ length: CONCURRENT_CHECKS }, () =>
+ mod.checkAndIncrement(bucketKey, LIMIT, nowMs)
+ )
+ )
+ const elapsedMs = performance.now() - startedAt
+
+ expect(results).toHaveLength(CONCURRENT_CHECKS)
+ // The known defect: a saturated pool admits every request.
+ expect(results.every(result => result.allowed === true)).toBe(true)
+ // These two stay true after the fix: the backend was unavailable and
+ // nothing was counted.
+ expect(results.every(result => result.backendAvailable === false)).toBe(true)
+ expect(results.every(result => result.count === 0)).toBe(true)
+
+ const dbErrors = warn.mock.calls
+ .map(call => call[0] as { event?: string; bucketKey?: string; err?: string })
+ .filter(payload => payload.event === 'rate_limit_db_error')
+ expect(dbErrors).toHaveLength(CONCURRENT_CHECKS)
+ // Witness that the failure is the pool's acquire timeout and nothing else.
+ for (const payload of dbErrors) {
+ expect(payload.bucketKey).toBe(bucketKey)
+ expect(payload.err).toMatch(/timeout exceeded when trying to connect/)
+ }
+
+ // Every call waited for the acquire timeout, in parallel, not in series.
+ expect(elapsedMs).toBeGreaterThanOrEqual(PRODUCTION_ACQUIRE_TIMEOUT_MS - 100)
+ expect(elapsedMs).toBeLessThan(5_000)
+
+ // Read through a held client: this also proves the database and the
+ // table are reachable, so "no row" is not an absent table.
+ const persisted = await held[0]!.query<{ rows: string }>(
+ 'SELECT count(*)::text AS rows FROM rate_limit_buckets WHERE bucket_key = $1',
+ [bucketKey]
+ )
+ expect(persisted.rows[0]?.rows).toBe('0')
+
+ for (const client of held) client.release()
+ released = true
+
+ // Recovery witness, same key and window: counting starts at 1, which also
+ // proves the 25 fail-open calls persisted nothing.
+ const recovered = []
+ for (let i = 0; i < 3; i += 1) {
+ recovered.push(await mod.checkAndIncrement(bucketKey, LIMIT, nowMs))
+ }
+ expect(recovered.map(result => result.count)).toEqual([1, 2, 3])
+ expect(recovered.map(result => result.allowed)).toEqual([true, true, false])
+ expect(recovered.every(result => result.backendAvailable)).toBe(true)
+ expect(
+ warn.mock.calls.filter(
+ call => (call[0] as { event?: string }).event === 'rate_limit_db_error'
+ )
+ ).toHaveLength(CONCURRENT_CHECKS)
+ } finally {
+ if (!released) for (const client of held) client.release()
+ warn.mockRestore()
+ }
+ }, 15_000)
+
+ it('answers 500 at the route while the pool is held, because session auth fails first', async () => {
+ const internalToken = mod.config.internalServiceTokens['external-rest-api']
+ if (!internalToken) throw new Error('config has no external-rest-api internal service token')
+ mockVerifyExternalSessionToken.mockReset()
+ mockVerifyExternalSessionToken.mockReturnValue({
+ userId: '11111111-aaaa-4aaa-8aaa-111111111111',
+ email: '[email protected]',
+ teamId: '33333333-cccc-4ccc-8ccc-333333333333',
+ role: 'member',
+ authGeneration: 1,
+ exp: Math.floor(Date.now() / 1000) + 3600,
+ })
+ const app = mod.createApp(new mod.MockGateway())
+ const send = () =>
+ request(app)
+ .get('/api/v1/external/gfs/resources')
+ .set('Authorization', `Bearer ${internalToken}`)
+ .set('x-service-token', 'external-rest-api')
+ .set('x-user-session-token', 'session')
+
+ // Liveness witness with the pool free: the request passes internal auth,
+ // reaches the session verifier, and is denied by the users-table lookup
+ // (no such user), so the 500 below comes from that same lookup.
+ const free = await send()
+ expect(free.status).toBe(401)
+ expect(mockVerifyExternalSessionToken).toHaveBeenCalledTimes(1)
+
+ const held = await holdEveryPoolClient()
+ try {
+ const saturated = await send()
+ expect(saturated.status).toBe(500)
+ expect(mockVerifyExternalSessionToken).toHaveBeenCalledTimes(2)
+ } finally {
+ for (const client of held) client.release()
+ }
+ }, 15_000)
+})