Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(control-api): size the external GFS read and operation budgets for concurrent desktop surfaces
Alfredo Lopez1dec3d3cc744
diff --git a/.github/workflows/ci-public.yml b/.github/workflows/ci-public.ymlindex 35f19dd1d..8b5a6bb6f 100644--- a/.github/workflows/ci-public.yml+++ b/.github/workflows/ci-public.yml@@ -267,9 +267,10 @@ jobs:echo 'real-Postgres control-api lane reported skipped tests' >&2exit 1fi- # Codex subscription suites must stay inside the real-Postgres filter.- # A rename or move that drops them from `*realPostgres*` would otherwise- # still pass the aggregate counts above.+ # Pinned suites (Codex/Grok subscription, provider attempts, and the+ # external GFS rate-limit suites) must stay inside the real-Postgres+ # filter. A rename or move that drops them from `*realPostgres*` would+ # otherwise still pass the aggregate counts above.for suite in \db.codexSubscriptionConnection.realPostgres.integration \services.codexSubscriptionOAuth.realPostgres.integration \@@ -285,10 +286,12 @@ jobs:services.grokProviderAttemptRedemption.refresh.realPostgres.integration \services.codexSubscriptionLifecycle.realPostgres.integration \db.llmProviderAttemptConnectionIntegrity.realPostgres.integration \- services.llmCatalogSync.realPostgres.integration+ services.llmCatalogSync.realPostgres.integration \+ routes.externalGfsRateLimitStress.realPostgres.integration \+ services.rateLimiterPoolSaturation.realPostgres.integrationdogrep -Fq "$suite" "$normalized_log" || {- echo "real-Postgres control-api lane omitted Codex suite ${suite}" >&2+ echo "real-Postgres control-api lane omitted pinned suite ${suite}" >&2exit 1}donediff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex 716800e60..026eff3ee 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.601",+ "version": "0.1.602","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.601",+ "version": "0.1.602","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex a4c6f4544..8ec051e38 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.601",+ "version": "0.1.602","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/config.ts b/control-api/src/config.tsindex 41ddda9ca..dba2b2fba 100644--- a/control-api/src/config.ts+++ b/control-api/src/config.ts@@ -466,6 +466,10 @@ const WORKFLOW_STEP_DEPENDS_ON_MAX_ITEMS_CEILING = 100const WORKFLOW_STEP_ALLOWED_TOOLS_MAX_ITEMS_CEILING = 100// The step mcpServers env can only lower this value; the CRD hard ceiling remains 20.const WORKFLOW_STEP_MCP_SERVERS_MAX_ITEMS_CEILING = 20+// The two actor-keyed external GFS budgets accept an environment override up+// to twice their default; a larger value refuses to boot.+const EXTERNAL_GFS_READ_RL_PER_MIN_CEILING = 960+const EXTERNAL_GFS_OPERATION_RL_PER_MIN_CEILING = 180function normalizePem(value: string): string {return value.replace(/\\n/g, '\n').trim()@@ -996,17 +1000,40 @@ export const config: Config = {600),pluginSdkPreauthRlPerMin: positiveIntegerFromEnv('CONTROL_API_PLUGIN_SDK_PREAUTH_PER_MIN', 600),- // Approved GFS authority-boundary budgets. Keep them fixed here rather than- // accepting an unreviewed environment override. The 1800/min ingress guard- // is only a coarse process-local backstop; the aggregate source-IP ceiling,- // read buckets, token buckets, and mutation/delegation buckets remain- // independently bounded below.+ // External GFS authority-boundary budgets. The four keyed by source IP or+ // by token minting stay compiled: an IP bucket is shared behind a NAT, so+ // loosening it widens a blast radius beyond one user, and token minting+ // does not scale with the number of open surfaces. The 1800/min ingress+ // guard is only a coarse process-local backstop.externalGfsIngressRlPerMin: 1_800,externalGfsTokenUserRlPerMin: 10,externalGfsTokenIpRlPerMin: 600,externalGfsIpRlPerMin: 1_200,- externalGfsReadRlPerMin: 120,- externalGfsOperationRlPerMin: 30,+ // The two actor-keyed budgets are overridable within a compiled ceiling.+ // Each value feeds the pre-resolution session and (class, IP) buckets, the+ // resolved per-actor bucket and the express backstops in+ // routes/external/gfs.ts. assertExternalGfsBudgetInvariants below refuses a+ // combination where operations exceed reads or reads exceed the per-IP+ // all-class bucket.+ //+ // Read: one Desktop surface peaks at 58 resource reads/min after PR #702,+ // and the 2026-09-18 incident peaked at 133 in a sliding 60 s. 480 covers+ // eight surfaces of one user and is 3.6x the incident peak.+ externalGfsReadRlPerMin: boundedIntegerFromEnv(+ 'CONTROL_API_EXTERNAL_GFS_READ_RL_PER_MIN',+ 480,+ EXTERNAL_GFS_READ_RL_PER_MIN_CEILING+ ),+ // Operation: one maximum-size Desktop upload is at least 27 mutations+ // (ceil(200 MiB / 8 MiB) parts + create + complete, desktop-app+ // src/gfs/upload.ts), and a 429 on a part is retried and counted again.+ // 90 keeps three concurrent uploads out of that retry band. Derivation is+ // regression-guarded by test/config.externalGfsBudgets.test.ts.+ externalGfsOperationRlPerMin: boundedIntegerFromEnv(+ 'CONTROL_API_EXTERNAL_GFS_OPERATION_RL_PER_MIN',+ 90,+ EXTERNAL_GFS_OPERATION_RL_PER_MIN_CEILING+ ),// Default derived from the wake mechanism's worst case, not picked ad hoc.// rpc-proxy's wake-and-hold loop re-triggers POST /rpc/hosts/:hostRef/wake// every wakeRetriggerMs=15000 for up to wakeMaxHoldMs=90000 (defaults in@@ -1168,6 +1195,41 @@ export const config: Config = {defaultHookTrustCap: (process.env.CONTROL_API_DEFAULT_HOOK_TRUST_CAP ?? 'mid').toLowerCase(),}+/**+ * Refuse an incoherent set of external GFS budgets at boot. Never clamps: a+ * corrected value would leave the operator's setting and the running limit+ * disagreeing.+ *+ * - Operations must not be allowed more freely than reads.+ * - The per-actor read budget must not exceed the per-IP all-class bucket in+ * front of it; otherwise that bucket caps it under a different key and the+ * configured number is never reached.+ */+export function assertExternalGfsBudgetInvariants(budgets: {+ readPerMin: number+ operationPerMin: number+ ipPerMin: number+}): void {+ if (budgets.operationPerMin > budgets.readPerMin) {+ throw new Error(+ `CONTROL_API_EXTERNAL_GFS_OPERATION_RL_PER_MIN (${budgets.operationPerMin}) must not ` ++ `exceed CONTROL_API_EXTERNAL_GFS_READ_RL_PER_MIN (${budgets.readPerMin})`+ )+ }+ if (budgets.readPerMin > budgets.ipPerMin) {+ throw new Error(+ `CONTROL_API_EXTERNAL_GFS_READ_RL_PER_MIN (${budgets.readPerMin}) must not exceed ` ++ `the per-IP external GFS budget externalGfsIpRlPerMin (${budgets.ipPerMin})`+ )+ }+}++assertExternalGfsBudgetInvariants({+ readPerMin: config.externalGfsReadRlPerMin,+ operationPerMin: config.externalGfsOperationRlPerMin,+ ipPerMin: config.externalGfsIpRlPerMin,+})+// Namespace config validation: fail fast if any namespace is empty.// Empty namespace would target the K8s default namespace — a silent security gap.const NAMESPACE_KEYS: (keyof Config)[] = [diff --git a/control-api/test/config.externalGfsBudgets.test.ts b/control-api/test/config.externalGfsBudgets.test.tsnew file mode 100644index 000000000..d3d68ca4d--- /dev/null+++ b/control-api/test/config.externalGfsBudgets.test.ts@@ -0,0 +1,141 @@+import { afterEach, describe, expect, it, vi } from 'vitest'+import { readFileSync } from 'node:fs'++/**+ * Regression guard for the two actor-keyed external GFS budgets+ * (config.externalGfsReadRlPerMin, config.externalGfsOperationRlPerMin).+ *+ * The operation budget is derived from the largest Desktop upload: one part+ * per preferred part size up to the default product maximum, plus one create+ * and one complete call. The two upload constants below are hard-coded+ * mirrors of desktop-app/src/gfs/upload.ts, and each mirror is verified+ * against that file, so changing the Desktop part size without re-deriving+ * this budget fails HERE, in control-api CI.+ */++// Hard-coded mirrors of desktop-app/src/gfs/upload.ts, in MiB.+const UPLOAD_PRODUCT_MAX_MIB = 200+const UPLOAD_PREFERRED_PART_MIB = 8+// create + complete around the parts.+const UPLOAD_LIFECYCLE_CALLS = 2+const MAX_UPLOAD_MUTATIONS =+ Math.ceil(UPLOAD_PRODUCT_MAX_MIB / UPLOAD_PREFERRED_PART_MIB) + UPLOAD_LIFECYCLE_CALLS++const ENV_KEYS = [+ 'CONTROL_API_EXTERNAL_GFS_READ_RL_PER_MIN',+ 'CONTROL_API_EXTERNAL_GFS_OPERATION_RL_PER_MIN',+] as const++async function loadConfigModuleWith(overrides: Partial<Record<(typeof ENV_KEYS)[number], string>>) {+ const originalValues = new Map<string, string | undefined>()+ for (const key of ENV_KEYS) {+ originalValues.set(key, process.env[key])+ // Deleted explicitly: a default test that merely omits the variable would+ // pass for the wrong reason on any host where it is set.+ delete process.env[key]+ }+ Object.assign(process.env, overrides)+ vi.resetModules()+ try {+ return await import('../src/config.js')+ } finally {+ for (const key of ENV_KEYS) {+ const value = originalValues.get(key)+ if (value === undefined) delete process.env[key]+ else process.env[key] = value+ }+ }+}++/** Fail-loud single-match extraction — a miss means the source moved. */+function extractOne(source: string, pattern: RegExp, label: string): string {+ const match = source.match(pattern)+ if (!match || match[1] === undefined) {+ throw new Error(`Could not extract ${label} with ${pattern} — re-derive the mirrors`)+ }+ return match[1]+}++describe('external GFS actor budgets', () => {+ afterEach(() => {+ vi.resetModules()+ })++ it('derives the operation budget from the live Desktop upload constants', async () => {+ const upload = readFileSync(+ new URL('../../desktop-app/src/gfs/upload.ts', import.meta.url),+ 'utf-8'+ )+ const productMaxMib = extractOne(+ upload,+ /GFS_UPLOAD_V2_DEFAULT_PRODUCT_MAX_BYTES\s*=\s*(\d+)\s*\*\s*1024\s*\*\s*1024\b/,+ 'Desktop upload product maximum'+ )+ const preferredPartMib = extractOne(+ upload,+ /GFS_UPLOAD_V2_PREFERRED_PART_BYTES\s*=\s*(\d+)\s*\*\s*1024\s*\*\s*1024\b/,+ 'Desktop upload preferred part size'+ )+ expect(Number(productMaxMib)).toBe(UPLOAD_PRODUCT_MAX_MIB)+ expect(Number(preferredPartMib)).toBe(UPLOAD_PREFERRED_PART_MIB)+ expect(MAX_UPLOAD_MUTATIONS).toBe(27)++ // 27 is a floor: larger parts, retried 429s and reconciliation add more.+ const { config } = await loadConfigModuleWith({})+ expect(config.externalGfsOperationRlPerMin).toBeGreaterThanOrEqual(MAX_UPLOAD_MUTATIONS)+ })++ it('defaults to 480 reads and 90 operations per minute', async () => {+ const { config } = await loadConfigModuleWith({})+ expect(config.externalGfsReadRlPerMin).toBe(480)+ expect(config.externalGfsOperationRlPerMin).toBe(90)+ })++ it('accepts an environment override within the ceiling', async () => {+ const { config } = await loadConfigModuleWith({+ CONTROL_API_EXTERNAL_GFS_READ_RL_PER_MIN: '960',+ CONTROL_API_EXTERNAL_GFS_OPERATION_RL_PER_MIN: '180',+ })+ expect(config.externalGfsReadRlPerMin).toBe(960)+ expect(config.externalGfsOperationRlPerMin).toBe(180)+ })++ it('refuses to boot above either compiled ceiling', async () => {+ await expect(+ loadConfigModuleWith({ CONTROL_API_EXTERNAL_GFS_READ_RL_PER_MIN: '961' })+ ).rejects.toThrow(+ 'CONTROL_API_EXTERNAL_GFS_READ_RL_PER_MIN must be an integer between 1 and 960'+ )+ await expect(+ loadConfigModuleWith({ CONTROL_API_EXTERNAL_GFS_OPERATION_RL_PER_MIN: '181' })+ ).rejects.toThrow(+ 'CONTROL_API_EXTERNAL_GFS_OPERATION_RL_PER_MIN must be an integer between 1 and 180'+ )+ })++ it('refuses to boot when the operation budget exceeds the read budget', async () => {+ await expect(+ loadConfigModuleWith({+ CONTROL_API_EXTERNAL_GFS_READ_RL_PER_MIN: '100',+ CONTROL_API_EXTERNAL_GFS_OPERATION_RL_PER_MIN: '180',+ })+ ).rejects.toThrow(+ 'CONTROL_API_EXTERNAL_GFS_OPERATION_RL_PER_MIN (180) must not exceed ' ++ 'CONTROL_API_EXTERNAL_GFS_READ_RL_PER_MIN (100)'+ )+ })++ it('refuses a read budget above the per-IP budget and accepts it at the boundary', async () => {+ const { assertExternalGfsBudgetInvariants } = await loadConfigModuleWith({})+ expect(() =>+ assertExternalGfsBudgetInvariants({ readPerMin: 1300, operationPerMin: 90, ipPerMin: 1200 })+ ).toThrow(+ 'CONTROL_API_EXTERNAL_GFS_READ_RL_PER_MIN (1300) must not exceed ' ++ 'the per-IP external GFS budget externalGfsIpRlPerMin (1200)'+ )+ // Liveness witness for the check being `<=`, not `<`: equal values pass.+ expect(() =>+ assertExternalGfsBudgetInvariants({ readPerMin: 1200, operationPerMin: 90, ipPerMin: 1200 })+ ).not.toThrow()+ })+})diff --git a/control-api/test/routes.externalGfsRateLimitStress.realPostgres.integration.test.ts b/control-api/test/routes.externalGfsRateLimitStress.realPostgres.integration.test.tsnew file mode 100644index 000000000..41f45a283--- /dev/null+++ b/control-api/test/routes.externalGfsRateLimitStress.realPostgres.integration.test.ts@@ -0,0 +1,544 @@+import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'+import { randomBytes, randomUUID } from 'node:crypto'+import { Pool } from 'pg'+import request from 'supertest'++// Concurrent users at the external GFS rate limits, through the production+// Express app against real PostgreSQL. Config, the core pool, both limiters,+// the session lifecycle query, authority resolution, the prom registry and the+// logger are real; the ONLY mock is the session-token verifier (identity),+// which decodes a JSON test token so each scenario controls users and sessions.+//+// Denial source (plan Addendum A3), for status 429 only:+// - route backstop: RateLimit-Policy === "<route L>;w=60";+// - Postgres bucket: RateLimit-Policy === "<ingress L>;w=60" AND+// X-RateLimit-Remaining === "0" AND X-RateLimit-Limit === "<bucket L>";+// - anything else fails the test.+// Headers cannot say WHICH Postgres bucket denied, so every scenario also reads+// the rate_limit_buckets ledger: each hit increments its row once and a bucket+// only sees requests that passed the buckets before it, so per (key, window)+// denials = max(0, count - limit). The key prefix names the phase.+//+// Both limiters read Date.now(), so faking only Date pins their windows. Time+// only moves forward, each scenario deletes the ledger, uses fresh identities,+// and builds a new app after setting the budgets (the express backstops and+// the grants read limiter capture their limit when the router is created).+// Skipped without CONTROL_API_REAL_PG_ADMIN_URL.++vi.mock('../src/utils/auth/externalSessionAuthToken.js', () => ({+ verifyExternalSessionToken: (token: string) => {+ const parsed = JSON.parse(token) as { userId: string }+ return {+ userId: parsed.userId,+ email: `${parsed.userId}@example.test`,+ teamId: null,+ role: 'member',+ authGeneration: 1,+ exp: Math.floor(Date.now() / 1000) + 3600,+ }+ },+}))++const INGRESS_L = 1_800+const IP_L_DEFAULT = 1_200+const MINUTE_MS = 60_000++function databaseUrl(baseUrl: string, database: string): string {+ const url = new URL(baseUrl)+ url.pathname = `/${database}`+ return url.toString()+}++function quoteIdent(value: string): string {+ return `"${value.replace(/"/g, '""')}"`+}++type Budgets = { read: number; operation: number; ip: number }+type Sent = {+ path: string+ status: number+ body: { error?: unknown }+ policy?: string+ xLimit?: string+ xRemaining?: string+}+type LedgerRow = { key: string; windowStartMs: number; count: number }+type Phase = 'pre-resolution' | 'resolved-operation' | 'edge-backstop'++const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL+const describeRealPostgres = adminUrl ? describe : describe.skip++describeRealPostgres('external GFS rate limits under concurrent load (real PostgreSQL)', () => {+ const database = `control_api_gfs_rl_stress_${randomBytes(6).toString('hex')}`+ const connectionString = databaseUrl(+ adminUrl ?? 'postgresql://[email protected]/postgres',+ database+ )+ const envKeys = ['CONTROL_API_PG_CONNECTION_STRING', 'CORE_POOL_CONNECTION_TIMEOUT_MS'] as const+ const previousEnv = new Map<string, string | undefined>()++ let adminPool: Pool+ let corePool: Pool+ let mod: {+ createApp: (gateway: unknown) => import('express').Express+ config: typeof import('../src/config.js').config+ assertExternalGfsBudgetInvariants: typeof import('../src/config.js').assertExternalGfsBudgetInvariants+ pool: import('../src/db.js').DbClient+ rootLogger: typeof import('../src/observability/logger.js').rootLogger+ requestsTotal: typeof import('../src/observability/metrics.js').externalGfsRateLimitRequestsTotal+ MockGateway: typeof import('./mockGateway.js').MockGateway+ }+ // Forward-only fake clock: every scenario starts on a later minute.+ let nextScenarioMinuteMs = 0++ beforeAll(async () => {+ for (const key of envKeys) previousEnv.set(key, process.env[key])+ adminPool = new Pool({ connectionString: adminUrl })+ await adminPool.query(`CREATE DATABASE ${quoteIdent(database)}`)+ process.env.CONTROL_API_PG_CONNECTION_STRING = connectionString+ // A 200-request burst queues on 10 connections; the production 2 s acquire+ // timeout would make the limiter fail open by accident (see+ // services.rateLimiterPoolSaturation.realPostgres.integration.test.ts).+ process.env.CORE_POOL_CONNECTION_TIMEOUT_MS = '10000'++ const dbMod = await import('../src/db.js')+ corePool = dbMod.pool as unknown as Pool+ corePool.on('error', () => {})+ const migratePool = new Pool({ connectionString })+ await dbMod.initDb({ connect: () => migratePool.connect() })+ await migratePool.end()++ const appMod = await import('../src/app.js')+ const configMod = await import('../src/config.js')+ const loggerMod = await import('../src/observability/logger.js')+ const metricsMod = await import('../src/observability/metrics.js')+ const { MockGateway } = await import('./mockGateway.js')+ mod = {+ createApp: appMod.createApp as never,+ config: configMod.config,+ assertExternalGfsBudgetInvariants: configMod.assertExternalGfsBudgetInvariants,+ pool: dbMod.pool,+ rootLogger: loggerMod.rootLogger,+ requestsTotal: metricsMod.externalGfsRateLimitRequestsTotal,+ MockGateway,+ }+ expect(mod.config.externalGfsIngressRlPerMin).toBe(INGRESS_L)+ expect(mod.config.externalGfsIpRlPerMin).toBe(IP_L_DEFAULT)+ vi.useFakeTimers({ toFake: ['Date'] })+ nextScenarioMinuteMs = Math.floor(Date.now() / MINUTE_MS) * MINUTE_MS + 10 * MINUTE_MS+ }, 60_000)++ afterAll(async () => {+ vi.useRealTimers()+ for (const key of envKeys) {+ const value = previousEnv.get(key)+ if (value === undefined) delete process.env[key]+ else process.env[key] = value+ }+ await corePool?.end().catch(() => {})+ if (!adminPool) return+ await adminPool.query(+ `SELECT pg_terminate_backend(pid) FROM pg_stat_activity+ WHERE datname = $1 AND pid <> pg_backend_pid()`,+ [database]+ )+ await adminPool.query(`DROP DATABASE IF EXISTS ${quoteIdent(database)}`)+ await adminPool.end()+ })++ /** Sets a boot-valid budget, clears the ledger, returns a fresh app and the scenario minute. */+ async function startScenario(budgets: Budgets) {+ mod.assertExternalGfsBudgetInvariants({+ readPerMin: budgets.read,+ operationPerMin: budgets.operation,+ ipPerMin: budgets.ip,+ })+ mod.config.externalGfsReadRlPerMin = budgets.read+ mod.config.externalGfsOperationRlPerMin = budgets.operation+ mod.config.externalGfsIpRlPerMin = budgets.ip+ await mod.pool.query('DELETE FROM rate_limit_buckets')+ const minuteMs = nextScenarioMinuteMs+ nextScenarioMinuteMs += 5 * MINUTE_MS+ vi.setSystemTime(minuteMs + 10_000)+ return { app: mod.createApp(new mod.MockGateway()), minuteMs }+ }++ async function seedUser(): Promise<string> {+ const id = randomUUID()+ await mod.pool.query(`INSERT INTO users (id, email, name) VALUES ($1, $2, $3)`, [+ id,+ `${id}@example.test`,+ 'stress',+ ])+ return id+ }++ const sessionToken = (userId: string, session = 0) => JSON.stringify({ userId, session })+ const affordancesPath = (resourceId: string) =>+ `/api/v1/external/gfs/resources/${resourceId}/affordances?drive=main`+ const grantsPath = (resourceId: string) =>+ `/api/v1/external/gfs/grants?drive=main&resourceId=${resourceId}`++ async function send(+ app: import('express').Express,+ path: string,+ token: string,+ ip: string+ ): Promise<Sent> {+ const internalToken = mod.config.internalServiceTokens['external-rest-api']+ if (!internalToken) throw new Error('config has no external-rest-api internal service token')+ const res = await request(app)+ .get(path)+ .set('Authorization', `Bearer ${internalToken}`)+ .set('x-service-token', 'external-rest-api')+ .set('x-user-session-token', token)+ .set('x-forwarded-for', ip)+ return {+ path,+ status: res.status,+ body: res.body as { error?: unknown },+ policy: res.headers['ratelimit-policy'],+ xLimit: res.headers['x-ratelimit-limit'],+ xRemaining: res.headers['x-ratelimit-remaining'],+ }+ }++ /** Addendum A3 classifier. Allowed responses keep their status. */+ function classify(response: Sent, routeL: number): string {+ if (response.status !== 429) return `allowed:${response.status}`+ if (response.policy === `${routeL};w=60`) return 'route-backstop'+ if (response.policy === `${INGRESS_L};w=60` && response.xRemaining === '0')+ return `postgres:${response.xLimit}`+ return `unclassified:${JSON.stringify(response)}`+ }++ function tally(labels: string[]): Record<string, number> {+ const counts: Record<string, number> = {}+ for (const label of labels) counts[label] = (counts[label] ?? 0) + 1+ return counts+ }++ async function readLedger(): Promise<LedgerRow[]> {+ const result = await mod.pool.query(+ `SELECT bucket_key, window_start_ms::text AS window_start_ms, count+ FROM rate_limit_buckets ORDER BY bucket_key, window_start_ms`+ )+ return (+ result.rows as Array<{ bucket_key: string; window_start_ms: string; count: number }>+ ).map(row => ({+ key: row.bucket_key,+ windowStartMs: Number(row.window_start_ms),+ count: Number(row.count),+ }))+ }++ /** Sorted counts of the rows whose key starts with `prefix`, optionally in one window. */+ function counts(ledger: LedgerRow[], prefix: string, windowStartMs?: number): number[] {+ return ledger+ .filter(+ row =>+ row.key.startsWith(prefix) &&+ (windowStartMs === undefined || row.windowStartMs === windowStartMs)+ )+ .map(row => row.count)+ .sort((a, b) => a - b)+ }++ const sum = (values: number[]) => values.reduce((total, value) => total + value, 0)++ /** Limit of every bucket key these scenarios can create; an unknown key fails loudly. */+ function limitOf(key: string, budgets: Budgets): number {+ if (key.startsWith('gfs-ext:pre:ip:')) return budgets.ip+ if (/^gfs-ext:(pre|resolved):(resource|grants-read):/.test(key)) return budgets.read+ if (key.startsWith('gfsgrants-ext-read:user:')) return budgets.read+ throw new Error(`unexpected rate limit bucket in the ledger: ${key}`)+ }++ /** Ledger identity: Postgres denials = sum over (key, window) of max(0, count - limit). */+ function ledgerDenials(ledger: LedgerRow[], budgets: Budgets): number {+ return sum(ledger.map(row => Math.max(0, row.count - limitOf(row.key, budgets))))+ }++ async function deniedByPhase(): Promise<Record<Phase, number>> {+ const { values } = await mod.requestsTotal.get()+ const totals: Record<Phase, number> = {+ 'pre-resolution': 0,+ 'resolved-operation': 0,+ 'edge-backstop': 0,+ }+ for (const sample of values) {+ if (sample.labels.outcome !== 'denied') continue+ const phase = sample.labels.phase as Phase+ if (!(phase in totals)) throw new Error(`unexpected rate limit phase label: ${phase}`)+ totals[phase] += sample.value+ }+ return totals+ }++ /** Captures denial log lines by phase and every limiter DB error during `run`. */+ async function observed<T>(run: () => Promise<T>) {+ const warn = vi.spyOn(mod.rootLogger, 'warn').mockImplementation(() => {})+ const before = await deniedByPhase()+ try {+ const result = await run()+ const after = await deniedByPhase()+ const payloads = warn.mock.calls.map(call => call[0] as { event?: string; phase?: string })+ const logged = (phase: Phase) =>+ payloads.filter(p => p.event === 'external_gfs_rate_limit' && p.phase === phase).length+ return {+ result,+ metricDelta: {+ 'pre-resolution': after['pre-resolution'] - before['pre-resolution'],+ 'resolved-operation': after['resolved-operation'] - before['resolved-operation'],+ 'edge-backstop': after['edge-backstop'] - before['edge-backstop'],+ },+ logged: {+ 'pre-resolution': logged('pre-resolution'),+ 'resolved-operation': logged('resolved-operation'),+ 'edge-backstop': logged('edge-backstop'),+ },+ dbErrors: payloads.filter(p => p.event === 'rate_limit_db_error').length,+ }+ } finally {+ warn.mockRestore()+ }+ }++ it('S1: 20 users with distinct sessions and IPs each get exactly the read budget', async () => {+ const budgets = { read: 6, operation: 6, ip: IP_L_DEFAULT }+ const { app } = await startScenario(budgets)+ const users = await Promise.all(Array.from({ length: 20 }, () => seedUser()))+ const resourceId = randomUUID()++ const run = await observed(() =>+ Promise.all(+ users.flatMap((userId, index) =>+ Array.from({ length: 10 }, () =>+ send(app, affordancesPath(resourceId), sessionToken(userId), `198.51.100.${index + 1}`)+ )+ )+ )+ )+ const labels = run.result.map(response => classify(response, budgets.read))++ expect(run.result).toHaveLength(200)+ for (let user = 0; user < 20; user += 1) {+ expect(tally(labels.slice(user * 10, user * 10 + 10))).toEqual({+ 'allowed:200': 6,+ 'postgres:6': 4,+ })+ }+ const ledger = await readLedger()+ expect(counts(ledger, 'gfs-ext:pre:resource:session:')).toEqual(Array(20).fill(10))+ expect(counts(ledger, 'gfs-ext:pre:resource:ip:')).toEqual(Array(20).fill(6))+ expect(counts(ledger, 'gfs-ext:pre:ip:')).toEqual(Array(20).fill(6))+ expect(counts(ledger, 'gfs-ext:resolved:resource:actor:user-session:')).toEqual(+ Array(20).fill(6)+ )+ expect(ledgerDenials(ledger, budgets)).toBe(80)+ // S6: the Postgres denials are logged and counted exactly once each; the+ // 120 allowed requests passed the backstop, which denied none of them.+ expect(run.dbErrors).toBe(0)+ expect(run.metricDelta).toEqual({+ 'pre-resolution': 80,+ 'resolved-operation': 0,+ 'edge-backstop': 0,+ })+ expect(run.logged).toEqual(run.metricDelta)+ }, 120_000)++ it('S2a: 20 users behind one IP share the (class, IP) read bucket', async () => {+ const budgets = { read: 30, operation: 30, ip: IP_L_DEFAULT }+ const { app } = await startScenario(budgets)+ const users = await Promise.all(Array.from({ length: 20 }, () => seedUser()))+ const resourceId = randomUUID()++ const run = await observed(() =>+ Promise.all(+ users.flatMap(userId =>+ Array.from({ length: 5 }, () =>+ send(app, affordancesPath(resourceId), sessionToken(userId), '198.51.100.30')+ )+ )+ )+ )++ expect(tally(run.result.map(response => classify(response, budgets.read)))).toEqual({+ 'allowed:200': 30,+ 'postgres:30': 70,+ })+ const ledger = await readLedger()+ expect(counts(ledger, 'gfs-ext:pre:resource:session:')).toEqual(Array(20).fill(5))+ expect(counts(ledger, 'gfs-ext:pre:resource:ip:')).toEqual([100])+ expect(counts(ledger, 'gfs-ext:pre:ip:')).toEqual([30])+ expect(sum(counts(ledger, 'gfs-ext:resolved:resource:actor:user-session:'))).toBe(30)+ expect(ledgerDenials(ledger, budgets)).toBe(70)+ expect(run.dbErrors).toBe(0)+ expect(run.metricDelta).toEqual({+ 'pre-resolution': 70,+ 'resolved-operation': 0,+ 'edge-backstop': 0,+ })+ expect(run.logged).toEqual(run.metricDelta)+ }, 120_000)++ it('S2b: the per-IP all-class bucket caps two read classes behind one IP', async () => {+ // Boot-valid: read (30) <= ip (40). pre:ip can only deny when several+ // classes share the IP, because each class alone is capped at read L first.+ const budgets = { read: 30, operation: 30, ip: 40 }+ const { app } = await startScenario(budgets)+ const users = await Promise.all(Array.from({ length: 10 }, () => seedUser()))+ const resourceId = randomUUID()++ const run = await observed(() =>+ Promise.all(+ users.flatMap((userId, index) =>+ Array.from({ length: 5 }, () =>+ send(+ app,+ index < 5 ? affordancesPath(resourceId) : grantsPath(resourceId),+ sessionToken(userId),+ '198.51.100.40'+ )+ )+ )+ )+ )+ const labels = run.result.map(response => classify(response, budgets.read))+ const tallied = tally(labels)++ expect(tallied['postgres:40']).toBe(10)+ expect((tallied['allowed:200'] ?? 0) + (tallied['allowed:403'] ?? 0)).toBe(40)+ expect(Object.keys(tallied).sort()).toEqual(+ ['allowed:200', 'allowed:403', 'postgres:40'].filter(label => tallied[label] !== undefined)+ )+ // "Allowed" is defined per route: affordances answer 200, the grants list+ // answers 403 manage_acl_required for a caller without manage_acl.+ for (const response of run.result) {+ if (response.status === 200) expect(response.path).toContain('/affordances')+ if (response.status === 403) {+ expect(response.path).toContain('/grants')+ expect(response.body.error).toBe('manage_acl_required')+ }+ }+ const ledger = await readLedger()+ expect(counts(ledger, 'gfs-ext:pre:resource:session:')).toEqual(Array(5).fill(5))+ expect(counts(ledger, 'gfs-ext:pre:grants-read:session:')).toEqual(Array(5).fill(5))+ expect(counts(ledger, 'gfs-ext:pre:resource:ip:')).toEqual([25])+ expect(counts(ledger, 'gfs-ext:pre:grants-read:ip:')).toEqual([25])+ expect(counts(ledger, 'gfs-ext:pre:ip:')).toEqual([50])+ expect(sum(counts(ledger, 'gfs-ext:resolved:'))).toBe(40)+ expect(sum(counts(ledger, 'gfsgrants-ext-read:user:'))).toBe(tallied['allowed:403'] ?? 0)+ expect(ledgerDenials(ledger, budgets)).toBe(10)+ expect(run.dbErrors).toBe(0)+ expect(run.metricDelta).toEqual({+ 'pre-resolution': 10,+ 'resolved-operation': 0,+ 'edge-backstop': 0,+ })+ expect(run.logged).toEqual(run.metricDelta)+ }, 120_000)++ it('S3: two sessions of one user share the resolved actor bucket', async () => {+ const budgets = { read: 10, operation: 10, ip: IP_L_DEFAULT }+ const { app } = await startScenario(budgets)+ const userId = await seedUser()+ const resourceId = randomUUID()++ const run = await observed(() =>+ Promise.all([+ ...Array.from({ length: 8 }, () =>+ send(app, affordancesPath(resourceId), sessionToken(userId, 1), '198.51.100.51')+ ),+ ...Array.from({ length: 8 }, () =>+ send(app, affordancesPath(resourceId), sessionToken(userId, 2), '198.51.100.52')+ ),+ ])+ )++ expect(tally(run.result.map(response => classify(response, budgets.read)))).toEqual({+ 'allowed:200': 10,+ 'postgres:10': 6,+ })+ const ledger = await readLedger()+ // Each pre-resolution bucket saw 8 <= 10 and denied nothing; the resolved+ // actor bucket saw all 16: the phase witness, independent of telemetry.+ expect(counts(ledger, 'gfs-ext:pre:resource:session:')).toEqual([8, 8])+ expect(counts(ledger, 'gfs-ext:pre:resource:ip:')).toEqual([8, 8])+ expect(counts(ledger, 'gfs-ext:pre:ip:')).toEqual([8, 8])+ expect(counts(ledger, `gfs-ext:resolved:resource:actor:user-session:${userId}`)).toEqual([16])+ expect(ledgerDenials(ledger, budgets)).toBe(6)+ expect(run.dbErrors).toBe(0)+ expect(run.metricDelta).toEqual({+ 'pre-resolution': 0,+ 'resolved-operation': 6,+ 'edge-backstop': 0,+ })+ expect(run.logged).toEqual(run.metricDelta)+ }, 120_000)++ it('S4: a burst straddling a minute boundary is denied only by the route backstop', async () => {+ const budgets = { read: 8, operation: 8, ip: IP_L_DEFAULT }+ const { app, minuteMs } = await startScenario(budgets)+ const userId = await seedUser()+ const resourceId = randomUUID()+ const burst = () =>+ observed(() =>+ Promise.all(+ Array.from({ length: 8 }, () =>+ send(app, affordancesPath(resourceId), sessionToken(userId), '198.51.100.60')+ )+ )+ )+ const labelsOf = (responses: Sent[]) =>+ tally(responses.map(response => classify(response, budgets.read)))++ // A: 50 s into window 1. Opens the backstop window, fills the Postgres buckets.+ vi.setSystemTime(minuteMs + 50_000)+ const a = await burst()+ // B: 15 s later. New Postgres window (counts 1..8, allowed), same backstop window.+ vi.setSystemTime(minuteMs + 65_000)+ const b = await burst()+ // C: 60 s after A. The session bucket of window 2 is already at 8.+ vi.setSystemTime(minuteMs + 110_000)+ const c = await burst()++ expect(labelsOf(a.result)).toEqual({ 'allowed:200': 8 })+ expect(labelsOf(b.result)).toEqual({ 'route-backstop': 8 })+ expect(labelsOf(c.result)).toEqual({ 'postgres:8': 8 })++ const ledger = await readLedger()+ const window1 = minuteMs+ const window2 = minuteMs + MINUTE_MS+ expect(counts(ledger, 'gfs-ext:pre:resource:session:', window1)).toEqual([8])+ expect(counts(ledger, 'gfs-ext:pre:resource:session:', window2)).toEqual([16])+ expect(counts(ledger, 'gfs-ext:pre:resource:ip:', window1)).toEqual([8])+ expect(counts(ledger, 'gfs-ext:pre:resource:ip:', window2)).toEqual([8])+ expect(counts(ledger, 'gfs-ext:pre:ip:', window1)).toEqual([8])+ expect(counts(ledger, 'gfs-ext:pre:ip:', window2)).toEqual([8])+ expect(counts(ledger, 'gfs-ext:resolved:resource:actor:', window1)).toEqual([8])+ expect(counts(ledger, 'gfs-ext:resolved:resource:actor:', window2)).toEqual([8])+ // Only C's 8 denials are Postgres denials; B's 8 are the backstop's.+ expect(ledgerDenials(ledger, budgets)).toBe(8)++ // S6: A is the liveness witness (allowed, nothing reported); B is reported+ // only as edge-backstop; C only as pre-resolution.+ for (const run of [a, b, c]) expect(run.dbErrors).toBe(0)+ expect(a.metricDelta).toEqual({+ 'pre-resolution': 0,+ 'resolved-operation': 0,+ 'edge-backstop': 0,+ })+ expect(b.metricDelta).toEqual({+ 'pre-resolution': 0,+ 'resolved-operation': 0,+ 'edge-backstop': 8,+ })+ expect(c.metricDelta).toEqual({+ 'pre-resolution': 8,+ 'resolved-operation': 0,+ 'edge-backstop': 0,+ })+ for (const run of [a, b, c]) expect(run.logged).toEqual(run.metricDelta)+ }, 120_000)+})diff --git a/control-api/test/services.rateLimiterPoolSaturation.realPostgres.integration.test.ts b/control-api/test/services.rateLimiterPoolSaturation.realPostgres.integration.test.tsnew file mode 100644index 000000000..7496fd582--- /dev/null+++ b/control-api/test/services.rateLimiterPoolSaturation.realPostgres.integration.test.ts@@ -0,0 +1,242 @@+import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'+import { randomBytes } from 'node:crypto'+import { performance } from 'node:perf_hooks'+import { Pool, type PoolClient } from 'pg'+import request from 'supertest'++// Characterization of a KNOWN defect, not a specification of desired+// behaviour: when the production core pool is saturated, checkAndIncrement+// fails OPEN (rateLimiterService.ts, the catch around the INSERT). Every call+// waits for the pool's acquire timeout, logs rate_limit_db_error, and returns+// allowed:true with backendAvailable:false. The fix is tracked separately;+// when it lands, the `allowed` assertion below is the one line that flips.+//+// The production pool bounds are used unchanged (max 10, 2 s acquire): the+// two CORE_POOL_* variables are deleted before db.js is imported, and the+// resulting bounds are asserted before anything else.+//+// Route-level corollary: with the pool held, an external GFS request returns+// 500, because session auth queries the users table first+// (externalSessionAuth.ts) and fails closed before the limiter runs. The only+// mock is the session-token verifier (identity). Skipped without+// CONTROL_API_REAL_PG_ADMIN_URL.++const mockVerifyExternalSessionToken = vi.fn()++vi.mock('../src/utils/auth/externalSessionAuthToken.js', () => ({+ verifyExternalSessionToken: (...args: unknown[]) => mockVerifyExternalSessionToken(...args),+}))++const PRODUCTION_POOL_MAX = 10+const PRODUCTION_ACQUIRE_TIMEOUT_MS = 2_000+const CONCURRENT_CHECKS = 25+const LIMIT = 2++function databaseUrl(baseUrl: string, database: string): string {+ const url = new URL(baseUrl)+ url.pathname = `/${database}`+ return url.toString()+}++function quoteIdent(value: string): string {+ return `"${value.replace(/"/g, '""')}"`+}++const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL+const describeRealPostgres = adminUrl ? describe : describe.skip++describeRealPostgres('rate limiter under core pool saturation (known fail-open)', () => {+ const database = `rate_limiter_saturation_${randomBytes(6).toString('hex')}`+ const connectionString = databaseUrl(+ adminUrl ?? 'postgresql://[email protected]/postgres',+ database+ )+ const poolEnvKeys = [+ 'CONTROL_API_PG_CONNECTION_STRING',+ 'CORE_POOL_MAX',+ 'CORE_POOL_CONNECTION_TIMEOUT_MS',+ ] as const+ const previousEnv = new Map<string, string | undefined>()++ let adminPool: Pool+ let corePool: Pool+ let mod: {+ createApp: (gateway: unknown) => import('express').Express+ config: typeof import('../src/config.js').config+ checkAndIncrement: typeof import('../src/services/rateLimiterService.js').checkAndIncrement+ rootLogger: typeof import('../src/observability/logger.js').rootLogger+ MockGateway: typeof import('./mockGateway.js').MockGateway+ }++ async function holdEveryPoolClient(): Promise<PoolClient[]> {+ const held = await Promise.all(+ Array.from({ length: PRODUCTION_POOL_MAX }, () => corePool.connect())+ )+ expect(corePool.totalCount).toBe(PRODUCTION_POOL_MAX)+ expect(corePool.idleCount).toBe(0)+ return held+ }++ beforeAll(async () => {+ for (const key of poolEnvKeys) previousEnv.set(key, process.env[key])+ delete process.env.CORE_POOL_MAX+ delete process.env.CORE_POOL_CONNECTION_TIMEOUT_MS++ adminPool = new Pool({ connectionString: adminUrl })+ await adminPool.query(`CREATE DATABASE ${quoteIdent(database)}`)+ process.env.CONTROL_API_PG_CONNECTION_STRING = connectionString++ const dbMod = await import('../src/db.js')+ corePool = dbMod.pool as unknown as Pool+ corePool.on('error', () => {})+ const migratePool = new Pool({ connectionString })+ await dbMod.initDb({ connect: () => migratePool.connect() })+ await migratePool.end()++ const appMod = await import('../src/app.js')+ const configMod = await import('../src/config.js')+ const limiterMod = await import('../src/services/rateLimiterService.js')+ const loggerMod = await import('../src/observability/logger.js')+ const { MockGateway } = await import('./mockGateway.js')+ mod = {+ createApp: appMod.createApp as never,+ config: configMod.config,+ checkAndIncrement: limiterMod.checkAndIncrement,+ rootLogger: loggerMod.rootLogger,+ MockGateway,+ }+ }, 60_000)++ afterAll(async () => {+ for (const key of poolEnvKeys) {+ const value = previousEnv.get(key)+ if (value === undefined) delete process.env[key]+ else process.env[key] = value+ }+ await corePool?.end().catch(() => {})+ if (!adminPool) return+ await adminPool.query(+ `SELECT pg_terminate_backend(pid) FROM pg_stat_activity+ WHERE datname = $1 AND pid <> pg_backend_pid()`,+ [database]+ )+ await adminPool.query(`DROP DATABASE IF EXISTS ${quoteIdent(database)}`)+ await adminPool.end()+ })++ it('runs against the production pool bounds', () => {+ const options = (+ corePool as unknown as {+ options: { max: number; connectionTimeoutMillis: number }+ }+ ).options+ expect(options.max).toBe(PRODUCTION_POOL_MAX)+ expect(options.connectionTimeoutMillis).toBe(PRODUCTION_ACQUIRE_TIMEOUT_MS)+ })++ it('fails open for every concurrent check while every client is held, then recovers', async () => {+ const bucketKey = `saturation:${randomBytes(8).toString('hex')}`+ // One fixed window for both phases, so a minute boundary cannot split them.+ const nowMs = Math.floor(Date.now() / 60_000) * 60_000 + 1_000+ const warn = vi.spyOn(mod.rootLogger, 'warn').mockImplementation(() => {})+ const held = await holdEveryPoolClient()+ let released = false+ try {+ const startedAt = performance.now()+ const results = await Promise.all(+ Array.from({ length: CONCURRENT_CHECKS }, () =>+ mod.checkAndIncrement(bucketKey, LIMIT, nowMs)+ )+ )+ const elapsedMs = performance.now() - startedAt++ expect(results).toHaveLength(CONCURRENT_CHECKS)+ // The known defect: a saturated pool admits every request.+ expect(results.every(result => result.allowed === true)).toBe(true)+ // These two stay true after the fix: the backend was unavailable and+ // nothing was counted.+ expect(results.every(result => result.backendAvailable === false)).toBe(true)+ expect(results.every(result => result.count === 0)).toBe(true)++ const dbErrors = warn.mock.calls+ .map(call => call[0] as { event?: string; bucketKey?: string; err?: string })+ .filter(payload => payload.event === 'rate_limit_db_error')+ expect(dbErrors).toHaveLength(CONCURRENT_CHECKS)+ // Witness that the failure is the pool's acquire timeout and nothing else.+ for (const payload of dbErrors) {+ expect(payload.bucketKey).toBe(bucketKey)+ expect(payload.err).toMatch(/timeout exceeded when trying to connect/)+ }++ // Every call waited for the acquire timeout, in parallel, not in series.+ expect(elapsedMs).toBeGreaterThanOrEqual(PRODUCTION_ACQUIRE_TIMEOUT_MS - 100)+ expect(elapsedMs).toBeLessThan(5_000)++ // Read through a held client: this also proves the database and the+ // table are reachable, so "no row" is not an absent table.+ const persisted = await held[0]!.query<{ rows: string }>(+ 'SELECT count(*)::text AS rows FROM rate_limit_buckets WHERE bucket_key = $1',+ [bucketKey]+ )+ expect(persisted.rows[0]?.rows).toBe('0')++ for (const client of held) client.release()+ released = true++ // Recovery witness, same key and window: counting starts at 1, which also+ // proves the 25 fail-open calls persisted nothing.+ const recovered = []+ for (let i = 0; i < 3; i += 1) {+ recovered.push(await mod.checkAndIncrement(bucketKey, LIMIT, nowMs))+ }+ expect(recovered.map(result => result.count)).toEqual([1, 2, 3])+ expect(recovered.map(result => result.allowed)).toEqual([true, true, false])+ expect(recovered.every(result => result.backendAvailable)).toBe(true)+ expect(+ warn.mock.calls.filter(+ call => (call[0] as { event?: string }).event === 'rate_limit_db_error'+ )+ ).toHaveLength(CONCURRENT_CHECKS)+ } finally {+ if (!released) for (const client of held) client.release()+ warn.mockRestore()+ }+ }, 15_000)++ it('answers 500 at the route while the pool is held, because session auth fails first', async () => {+ const internalToken = mod.config.internalServiceTokens['external-rest-api']+ if (!internalToken) throw new Error('config has no external-rest-api internal service token')+ mockVerifyExternalSessionToken.mockReset()+ mockVerifyExternalSessionToken.mockReturnValue({+ userId: '11111111-aaaa-4aaa-8aaa-111111111111',+ email: '[email protected]',+ teamId: '33333333-cccc-4ccc-8ccc-333333333333',+ role: 'member',+ authGeneration: 1,+ exp: Math.floor(Date.now() / 1000) + 3600,+ })+ const app = mod.createApp(new mod.MockGateway())+ const send = () =>+ request(app)+ .get('/api/v1/external/gfs/resources')+ .set('Authorization', `Bearer ${internalToken}`)+ .set('x-service-token', 'external-rest-api')+ .set('x-user-session-token', 'session')++ // Liveness witness with the pool free: the request passes internal auth,+ // reaches the session verifier, and is denied by the users-table lookup+ // (no such user), so the 500 below comes from that same lookup.+ const free = await send()+ expect(free.status).toBe(401)+ expect(mockVerifyExternalSessionToken).toHaveBeenCalledTimes(1)++ const held = await holdEveryPoolClient()+ try {+ const saturated = await send()+ expect(saturated.status).toBe(500)+ expect(mockVerifyExternalSessionToken).toHaveBeenCalledTimes(2)+ } finally {+ for (const client of held) client.release()+ }+ }, 15_000)+})