Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(control-api): fail closed with 503 when the external GFS rate limiter backend is unavailable
Alfredo Lopezbf4733183aee
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex a5035d477..beaecc0fd 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.607",+ "version": "0.1.608","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.607",+ "version": "0.1.608","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex 59d3d2ffd..9919d7f2e 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.607",+ "version": "0.1.608","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/middleware/externalGfsRateLimit.ts b/control-api/src/middleware/externalGfsRateLimit.tsindex fa9d5c8c2..83e61514f 100644--- a/control-api/src/middleware/externalGfsRateLimit.ts+++ b/control-api/src/middleware/externalGfsRateLimit.ts@@ -8,7 +8,11 @@ import {externalGfsRateLimitDurationSeconds,externalGfsRateLimitRequestsTotal,} from '../observability/metrics.js'-import { type RateLimitCheck, checkAndIncrement } from '../services/rateLimiterService.js'+import {+ RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS,+ type RateLimitCheck,+ checkAndIncrement,+} from '../services/rateLimiterService.js'/*** The external Desktop GFS surface has a small, explicit operation matrix.@@ -247,7 +251,7 @@ function reportDecision(input: {bucket: Bucketoperation: ExternalGfsOperationphase: ExternalGfsRateLimitPhase- outcome: 'allowed' | 'denied'+ outcome: 'allowed' | 'denied' | 'unavailable'latencyMs: numberauthorityResolutionAvoided: boolean}): void {@@ -273,7 +277,12 @@ function reportDecision(input: {latencyMs: input.latencyMs,authorityResolutionAvoided: input.authorityResolutionAvoided,}- if (input.outcome === 'denied') {+ if (input.outcome === 'unavailable') {+ rootLogger.warn(+ { ...fields, event: 'external_gfs_rate_limit_unavailable' },+ 'external GFS rate limit backend unavailable'+ )+ } else if (input.outcome === 'denied') {rootLogger.warn(fields, 'external GFS rate limit denied')} else {rootLogger.debug(fields, 'external GFS rate limit checked')@@ -343,6 +352,27 @@ async function enforceBuckets(input: {const startedAt = performance.now()const result = await checkAndIncrement(bucket.key, bucket.maxPerMinute)const latencyMs = performance.now() - startedAt+ if (!result.backendAvailable) {+ // The limiter could not count this request (pool exhausted, query error+ // or no row), so it cannot tell whether the actor is over budget. The+ // external GFS surface fails closed instead of letting an unmetered+ // request reach authority resolution and gfsc (#764).+ reportDecision({+ bucket,+ operation: input.operation,+ phase: input.phase,+ outcome: 'unavailable',+ latencyMs,+ authorityResolutionAvoided: input.phase === 'pre-resolution',+ })+ input.res.setHeader('Retry-After', String(RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS))+ input.res.setHeader('Cache-Control', 'no-store')+ input.res.status(503).json({+ error: 'gfs_rate_limit_unavailable',+ retryAfterSeconds: RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS,+ })+ return false+ }const allowed = result.allowedreportDecision({bucket,diff --git a/control-api/src/middleware/gfsUploadAdmission.ts b/control-api/src/middleware/gfsUploadAdmission.tsindex 23a688c6c..943e36659 100644--- a/control-api/src/middleware/gfsUploadAdmission.ts+++ b/control-api/src/middleware/gfsUploadAdmission.ts@@ -8,6 +8,7 @@ import {gfsUploadAdmissionRequestsTotal,} from '../observability/metrics.js'import {+ RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS,type RateLimitCheck,acquireRateLimitConcurrencyLease,checkAndIncrement,@@ -106,6 +107,7 @@ function retryAfterSeconds(check: RateLimitCheck): number {}function sendBackendUnavailable(res: Response): void {+ res.setHeader('Retry-After', String(RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS))res.status(503).json({ error: 'gfs_upload_admission_unavailable' })}diff --git a/control-api/src/services/rateLimiterService.ts b/control-api/src/services/rateLimiterService.tsindex 3bc69fd51..b6eff5557 100644--- a/control-api/src/services/rateLimiterService.ts+++ b/control-api/src/services/rateLimiterService.ts@@ -29,6 +29,13 @@ export type RateLimitCheck = {backendAvailable: boolean}+/**+ * Retry-After sent by the callers that fail closed with 503 when+ * `backendAvailable` is false. The count is unknown in that case, so no+ * window reset can be computed; a short fixed hint spreads the retries.+ */+export const RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS = 2+export function currentWindowStartMs(nowMs = Date.now()): number {return Math.floor(nowMs / WINDOW_MS) * WINDOW_MS}diff --git a/control-api/test/gfsUploadAdmission.test.ts b/control-api/test/gfsUploadAdmission.test.tsindex d073cc6ee..adc8669fd 100644--- a/control-api/test/gfsUploadAdmission.test.ts+++ b/control-api/test/gfsUploadAdmission.test.ts@@ -49,6 +49,7 @@ vi.mock('../src/config.js', () => ({}))vi.mock('../src/services/rateLimiterService.js', () => ({+ RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS: 2,acquireRateLimitConcurrencyLease: acquireMock,checkAndIncrement: checkMock,}))@@ -280,6 +281,8 @@ describe('GFS upload admission', () => {const response = await request(buildLifecycleApp()).get('/external/gfs/capabilities')expect(response.status).toBe(503)expect(response.body).toEqual({ error: 'gfs_upload_admission_unavailable' })+ expect(response.headers['retry-after']).toBe('2')+ expect(checkMock).toHaveBeenCalled()expect(acquireMock).not.toHaveBeenCalled()})})diff --git a/control-api/test/middleware.externalGfsRateLimit.test.ts b/control-api/test/middleware.externalGfsRateLimit.test.tsindex 5aea5dfc2..59ab47b40 100644--- a/control-api/test/middleware.externalGfsRateLimit.test.ts+++ b/control-api/test/middleware.externalGfsRateLimit.test.ts@@ -35,6 +35,7 @@ vi.mock('../src/config.js', () => ({},}))vi.mock('../src/services/rateLimiterService.js', () => ({+ RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS: 2,checkAndIncrement: (...args: unknown[]) => checkAndIncrement(...args),}))vi.mock('../src/observability/metrics.js', () => metrics)@@ -51,6 +52,7 @@ function allowed(limit = 30, remaining = limit - 1) {resetMs: Date.now() + 60_000,windowStartMs: Date.now(),count: limit - remaining,+ backendAvailable: true,}}@@ -61,6 +63,19 @@ function denied(limit = 30) {resetMs: Date.now() + 30_000,windowStartMs: Date.now(),count: limit + 1,+ backendAvailable: true,+ }+}++/** What checkAndIncrement returns when its query failed or produced no row. */+function unavailable(limit = 30) {+ return {+ allowed: true,+ remaining: limit,+ resetMs: Date.now() + 60_000,+ windowStartMs: Date.now(),+ count: 0,+ backendAvailable: false,}}@@ -195,6 +210,79 @@ describe('external GFS rate boundary', () => {expect(handler).not.toHaveBeenCalled()})+ it('fails closed with 503 before resolution when the limiter backend cannot count the request', async () => {+ checkAndIncrement.mockResolvedValueOnce(unavailable(120))+ const { app, resolver, handler } = buildApp()++ const response = await request(app)+ .get('/external/gfs/resources')+ .set('x-user-session-token', 'session-one')++ expect(response.status).toBe(503)+ expect(response.body).toEqual({ error: 'gfs_rate_limit_unavailable', retryAfterSeconds: 2 })+ expect(response.headers['retry-after']).toBe('2')+ expect(response.headers['cache-control']).toBe('no-store')+ // The first bucket was consulted and nothing after it: no further bucket,+ // no authority resolution, no handler.+ expect(checkAndIncrement).toHaveBeenCalledTimes(1)+ const sessionKey = String(checkAndIncrement.mock.calls[0]?.[0])+ expect(sessionKey).toMatch(/^gfs-ext:pre:resource:session:[0-9a-f]{64}$/)+ expect(resolver).not.toHaveBeenCalled()+ expect(handler).not.toHaveBeenCalled()+ expect(metrics.externalGfsRateLimitRequestsTotal.inc).toHaveBeenCalledTimes(1)+ expect(metrics.externalGfsRateLimitRequestsTotal.inc).toHaveBeenCalledWith({+ operation_class: 'resource',+ route: '/external/gfs/resources',+ outcome: 'unavailable',+ phase: 'pre-resolution',+ authority_resolution_avoided: 'true',+ })+ expect(metrics.externalGfsRateLimitDurationSeconds.observe).toHaveBeenCalledWith(+ expect.objectContaining({ phase: 'pre-resolution', outcome: 'unavailable' }),+ expect.any(Number)+ )+ expect(logger.rootLogger.warn).toHaveBeenCalledTimes(1)+ expect(logger.rootLogger.warn).toHaveBeenCalledWith(+ expect.objectContaining({+ event: 'external_gfs_rate_limit_unavailable',+ outcome: 'unavailable',+ hashedKey: createHash('sha256').update(sessionKey).digest('hex'),+ }),+ 'external GFS rate limit backend unavailable'+ )+ })++ it('fails closed with 503 after resolution when the resolved actor bucket cannot be counted', async () => {+ checkAndIncrement+ .mockResolvedValueOnce(allowed(120))+ .mockResolvedValueOnce(allowed(120))+ .mockResolvedValueOnce(allowed(1200))+ .mockResolvedValueOnce(unavailable(120))+ const { app, resolver, handler } = buildApp()++ const response = await request(app)+ .get('/external/gfs/resources')+ .set('x-user-session-token', 'session-one')++ expect(response.status).toBe(503)+ expect(response.body).toEqual({ error: 'gfs_rate_limit_unavailable', retryAfterSeconds: 2 })+ expect(response.headers['retry-after']).toBe('2')+ expect(checkAndIncrement).toHaveBeenCalledTimes(4)+ expect(checkAndIncrement).toHaveBeenLastCalledWith(+ `gfs-ext:resolved:resource:actor:linked-admin:${CONTROL_ADMIN_ID}`,+ 120+ )+ expect(resolver).toHaveBeenCalledTimes(1)+ expect(handler).not.toHaveBeenCalled()+ expect(metrics.externalGfsRateLimitRequestsTotal.inc).toHaveBeenLastCalledWith({+ operation_class: 'resource',+ route: '/external/gfs/resources',+ outcome: 'unavailable',+ phase: 'resolved-operation',+ authority_resolution_avoided: 'false',+ })+ })+it('rejects before resolution and handler work when the pre-resolution session bucket is exhausted', async () => {checkAndIncrement.mockResolvedValueOnce(denied(120))const { app, resolver, handler } = buildApp()diff --git a/control-api/test/routes.externalGfs.test.ts b/control-api/test/routes.externalGfs.test.tsindex 4a47ef33c..90dfd7ebe 100644--- a/control-api/test/routes.externalGfs.test.ts+++ b/control-api/test/routes.externalGfs.test.ts@@ -193,7 +193,7 @@ beforeEach(() => {if (text.includes('SELECT lifecycle_state, lifecycle_version')) {return { rows: [{ lifecycle_state: 'active', lifecycle_version: 1 }] }}- return { rows: [] }+ return limiterRow(text) ?? { rows: [] }})mockWithTransaction.mockImplementation(async (work: (db: { query: typeof mockQuery }) => Promise<unknown>) =>@@ -217,6 +217,15 @@ function activeSessionLifecycleResult(: null}+/**+ * The limiter upsert as a reachable Postgres answers it: the first request of+ * the window. The external GFS limiter fails closed on a missing row, so every+ * mock that does not script the limiter must still answer it.+ */+function limiterRow(text: string): { rows: [{ count: number }] } | null {+ return text.includes('rate_limit_buckets') ? { rows: [{ count: 1 }] } : null+}+function combinedAuthorityRow(grants: Record<string, unknown>[],values?: unknown[]@@ -1078,6 +1087,8 @@ describe('linked Desktop operator authority contract', () => {linked()mockQuery.mockImplementation(async (text: string) => {const lifecycle = activeSessionLifecycleResult(text)+ const limiter = limiterRow(text)+ if (limiter) return limiterif (lifecycle) return lifecycleif (text.includes('parent_resource_id IS NULL')) {return {@@ -1285,7 +1296,7 @@ describe('linked Desktop operator authority contract', () => {auth()mockResolveActiveLink.mockResolvedValue(ACTIVE_LINK)mockQuery.mockImplementation(- async (text: string) => activeSessionLifecycleResult(text) ?? { rows: [] }+ async (text: string) => activeSessionLifecycleResult(text) ?? limiterRow(text) ?? { rows: [] })const res = await request(await buildApp())@@ -1487,6 +1498,8 @@ describe('PUT /external/gfs/grants (user delegation via existing engine)', () =>]mockQuery.mockImplementation(async (text: string, values?: unknown[]) => {const lifecycle = activeSessionLifecycleResult(text)+ const limiter = limiterRow(text)+ if (limiter) return limiterif (lifecycle) return lifecycleif (text.includes('FROM team_members')) return { rows: [{ team_id: T2 }] }if (text.includes('authority_grants AS') && text.includes('authority_shares AS')) {@@ -1959,7 +1972,8 @@ describe('authenticated bulk grant/share transport', () => {async (_label, method, path, subjectFields) => {auth()mockQuery.mockImplementation(- async (text: string) => activeSessionLifecycleResult(text) ?? { rows: [] }+ async (text: string) =>+ activeSessionLifecycleResult(text) ?? limiterRow(text) ?? { rows: [] })const app = await buildApp()@@ -1985,7 +1999,8 @@ describe('authenticated bulk grant/share transport', () => {async (_label, method, path) => {auth()mockQuery.mockImplementation(- async (text: string) => activeSessionLifecycleResult(text) ?? { rows: [] }+ async (text: string) =>+ activeSessionLifecycleResult(text) ?? limiterRow(text) ?? { rows: [] })const app = await buildApp()@@ -2193,6 +2208,8 @@ describe('GET /external/gfs/resources/:id/affordances', () => {auth()mockQuery.mockImplementation(async (text: string, values?: unknown[]) => {const lifecycle = activeSessionLifecycleResult(text)+ const limiter = limiterRow(text)+ if (limiter) return limiterif (lifecycle) return lifecycleif (text.includes('FROM team_members')) return { rows: [{ team_id: T2 }] }if (text.includes('WITH RECURSIVE chain'))@@ -2397,6 +2414,8 @@ describe('GET /external/gfs/resources', () => {auth()mockQuery.mockImplementation(async (text: string, values?: unknown[]) => {const lifecycle = activeSessionLifecycleResult(text)+ const limiter = limiterRow(text)+ if (limiter) return limiterif (lifecycle) return lifecycleif (text.includes('FROM team_members')) return { rows: [{ team_id: T1 }] }if (text.includes('FROM gfs_grants') && text.includes('JOIN requested_subjects')) {diff --git a/control-api/test/routes.externalGfsRateLimitFailClosed.realPostgres.integration.test.ts b/control-api/test/routes.externalGfsRateLimitFailClosed.realPostgres.integration.test.tsindex 20b4866b2..82e18e0b1 100644--- a/control-api/test/routes.externalGfsRateLimitFailClosed.realPostgres.integration.test.ts+++ b/control-api/test/routes.externalGfsRateLimitFailClosed.realPostgres.integration.test.ts@@ -1,12 +1,38 @@-import { afterAll, beforeAll, describe, expect, it } from 'vitest'-import { randomBytes } from 'node:crypto'-import { Pool } from 'pg'+import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'+import { randomBytes, randomUUID } from 'node:crypto'+import { Pool, type PoolClient } from 'pg'+import request from 'supertest'// The external GFS limiter's Postgres backend, against real PostgreSQL at the// production pool bounds: the CORE_POOL_* and RATE_LIMIT_POOL_* variables are-// deleted before db.js is imported. Skipped without+// deleted before db.js is imported. Requests go through the production+// Express app; the ONLY mock is the session-token verifier (identity), as in+// routes.externalGfsRateLimitStress. Skipped without// CONTROL_API_REAL_PG_ADMIN_URL.+vi.mock('../src/utils/auth/externalSessionAuthToken.js', () => ({+ verifyExternalSessionToken: (token: string) => {+ const parsed = JSON.parse(token) as { userId: string }+ return {+ userId: parsed.userId,+ email: `${parsed.userId}@example.test`,+ teamId: null,+ role: 'member',+ authGeneration: 1,+ exp: Math.floor(Date.now() / 1000) + 3600,+ }+ },+}))++const LIMITER_POOL_MAX = 6+const LIMITER_ACQUIRE_TIMEOUT_MS = 5_000+// An affordances read walks four Postgres buckets in series (three+// pre-resolution, one resolved). Were the limiter to fail open again, each+// would wait out the acquire timeout before the request reached the handler;+// the budget covers that walk so the regression fails on the status assertion,+// not on the test timeout.+const T3_TIMEOUT_MS = (4 + 2) * LIMITER_ACQUIRE_TIMEOUT_MS+function databaseUrl(baseUrl: string, database: string): string {const url = new URL(baseUrl)url.pathname = `/${database}`@@ -36,6 +62,13 @@ describeRealPostgres('external GFS rate limiter backend (real PostgreSQL)', () =let adminPool: Poollet corePool: Poollet limiterPool: Pool+ let mod: {+ createApp: (gateway: unknown) => import('express').Express+ config: typeof import('../src/config.js').config+ rootLogger: typeof import('../src/observability/logger.js').rootLogger+ requestsTotal: typeof import('../src/observability/metrics.js').externalGfsRateLimitRequestsTotal+ MockGateway: typeof import('./mockGateway.js').MockGateway+ }beforeAll(async () => {if (adminUrl === undefined) {@@ -58,6 +91,19 @@ describeRealPostgres('external GFS rate limiter backend (real PostgreSQL)', () =const migratePool = new Pool({ connectionString })await dbMod.initDb({ connect: () => migratePool.connect() })await migratePool.end()++ const appMod = await import('../src/app.js')+ const configMod = await import('../src/config.js')+ const loggerMod = await import('../src/observability/logger.js')+ const metricsMod = await import('../src/observability/metrics.js')+ const { MockGateway } = await import('./mockGateway.js')+ mod = {+ createApp: appMod.createApp as never,+ config: configMod.config,+ rootLogger: loggerMod.rootLogger,+ requestsTotal: metricsMod.externalGfsRateLimitRequestsTotal,+ MockGateway,+ }}, 60_000)afterAll(async () => {@@ -100,4 +146,87 @@ describeRealPostgres('external GFS rate limiter backend (real PostgreSQL)', () =expect(limiter.rows[0]).toMatchObject({ sync: 'off', timeout: '3s' })expect(core.rows[0]).toMatchObject({ sync: 'on', timeout: '15s' })})++ async function seedUser(): Promise<string> {+ const id = randomUUID()+ await corePool.query(`INSERT INTO users (id, email, name) VALUES ($1, $2, $3)`, [+ id,+ `${id}@example.test`,+ 'fail-closed',+ ])+ return id+ }++ async function getAffordances(app: import('express').Express, userId: string) {+ const internalToken = mod.config.internalServiceTokens['external-rest-api']+ if (!internalToken) throw new Error('config has no external-rest-api internal service token')+ return request(app)+ .get(`/api/v1/external/gfs/resources/${randomUUID()}/affordances?drive=main`)+ .set('Authorization', `Bearer ${internalToken}`)+ .set('x-service-token', 'external-rest-api')+ .set('x-user-session-token', JSON.stringify({ userId }))+ .set('x-forwarded-for', '203.0.113.7')+ }++ async function unavailableDecisions(): Promise<number> {+ const { values } = await mod.requestsTotal.get()+ return values+ .filter(sample => sample.labels.outcome === 'unavailable')+ .reduce((total, sample) => total + sample.value, 0)+ }++ /** Read through the core pool: the limiter pool is the one being held. */+ async function ledgerKeys(): Promise<string[]> {+ const result = await corePool.query<{ bucket_key: string }>(+ 'SELECT bucket_key FROM rate_limit_buckets ORDER BY bucket_key'+ )+ return result.rows.map(row => row.bucket_key)+ }++ it(+ 'T3: with every limiter connection held, an external GFS read fails closed with 503 and is not counted; after release it passes',+ async () => {+ const app = mod.createApp(new mod.MockGateway())+ const userId = await seedUser()+ await corePool.query('DELETE FROM rate_limit_buckets')+ const unavailableBefore = await unavailableDecisions()++ const held: PoolClient[] = []+ let denied: Awaited<ReturnType<typeof getAffordances>>+ let elapsedMs: number+ try {+ for (let index = 0; index < LIMITER_POOL_MAX; index += 1) {+ held.push(await limiterPool.connect())+ }+ // Witness: the pool is exhausted, not merely busy.+ expect(limiterPool.totalCount).toBe(LIMITER_POOL_MAX)+ expect(limiterPool.idleCount).toBe(0)++ const startedAt = Date.now()+ denied = await getAffordances(app, userId)+ elapsedMs = Date.now() - startedAt+ } finally {+ for (const client of held) client.release()+ }++ expect(denied.status).toBe(503)+ expect(denied.body).toEqual({ error: 'gfs_rate_limit_unavailable', retryAfterSeconds: 2 })+ expect(denied.headers['retry-after']).toBe('2')+ expect(denied.headers['cache-control']).toBe('no-store')+ // Witness: the 503 came from waiting out the limiter's acquire timeout.+ expect(elapsedMs).toBeGreaterThanOrEqual(LIMITER_ACQUIRE_TIMEOUT_MS - 250)+ expect((await unavailableDecisions()) - unavailableBefore).toBe(1)+ expect(await ledgerKeys()).toEqual([])++ const allowed = await getAffordances(app, userId)+ expect(allowed.status).toBe(200)+ // Liveness witness for the empty ledger above: the same request, once the+ // pool is free, is counted in rate_limit_buckets.+ const keys = await ledgerKeys()+ expect(keys.length).toBeGreaterThan(0)+ expect(keys.every(key => key.startsWith('gfs-ext:'))).toBe(true)+ expect((await unavailableDecisions()) - unavailableBefore).toBe(1)+ },+ T3_TIMEOUT_MS+ )})diff --git a/control-api/test/services.rateLimiterPoolSaturation.realPostgres.integration.test.ts b/control-api/test/services.rateLimiterPoolSaturation.realPostgres.integration.test.tsindex 4577eba44..8aa618cd7 100644--- a/control-api/test/services.rateLimiterPoolSaturation.realPostgres.integration.test.ts+++ b/control-api/test/services.rateLimiterPoolSaturation.realPostgres.integration.test.ts@@ -7,11 +7,14 @@ import request from 'supertest'// The limiter's upserts run on their own pool (db.ts `rateLimitPool`, max 6,// 5 s acquire), separate from the core pool (max 10, 2 s acquire).//-// Characterization of a KNOWN defect, not a specification of desired-// behaviour: when the LIMITER pool is saturated, checkAndIncrement fails OPEN-// (rateLimiterService.ts, the catch around the INSERT). Every call waits for-// the limiter pool's acquire timeout, logs rate_limit_db_error, and returns-// allowed:true with backendAvailable:false.+// Service-level contract when the LIMITER pool is saturated: every+// checkAndIncrement call waits for the limiter pool's acquire timeout, logs+// rate_limit_db_error, counts nothing, and returns allowed:true with+// backendAvailable:false. `allowed` alone is not an admission decision there:+// each caller reads backendAvailable and decides. The external GFS limiter and+// upload admission fail closed with 503 (routes.externalGfsRateLimitFailClosed+// T3); callers that still admit on backendAvailable:false do so by their own+// choice, not because the service decided it.//// Isolation: a saturated CORE pool no longer reaches the limiter; it keeps// counting.@@ -52,7 +55,7 @@ function quoteIdent(value: string): string {const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URLconst describeRealPostgres = adminUrl ? describe : describe.skip-describeRealPostgres('rate limiter under pool saturation (known fail-open)', () => {+describeRealPostgres('rate limiter under pool saturation', () => {const database = `rate_limiter_saturation_${randomBytes(6).toString('hex')}`const poolEnvKeys = ['CONTROL_API_PG_CONNECTION_STRING',@@ -172,7 +175,7 @@ describeRealPostgres('rate limiter under pool saturation (known fail-open)', ()}}, 15_000)- it('fails open for every concurrent check while every limiter client is held, then recovers', async () => {+ it('reports the backend unavailable for every concurrent check while every limiter client is held, then recovers', async () => {const bucketKey = `saturation:${randomBytes(8).toString('hex')}`// One fixed window for both phases, so a minute boundary cannot split them.const nowMs = Math.floor(Date.now() / 60_000) * 60_000 + 1_000@@ -189,10 +192,10 @@ describeRealPostgres('rate limiter under pool saturation (known fail-open)', ()const elapsedMs = performance.now() - startedAtexpect(results).toHaveLength(CONCURRENT_CHECKS)- // The known defect: a saturated limiter pool admits every request.+ // The service reports allowed:true; the admission decision belongs to+ // the caller, which must read backendAvailable.expect(results.every(result => result.allowed === true)).toBe(true)- // These two stay true after the fix: the backend was unavailable and- // nothing was counted.+ // The backend was unavailable and nothing was counted.expect(results.every(result => result.backendAvailable === false)).toBe(true)expect(results.every(result => result.count === 0)).toBe(true)@@ -222,7 +225,7 @@ describeRealPostgres('rate limiter under pool saturation (known fail-open)', ()released = true// Recovery witness, same key and window: counting starts at 1, which also- // proves the 25 fail-open calls persisted nothing.+ // proves the 25 unavailable calls persisted nothing.const recovered = []for (let i = 0; i < 3; i += 1) {recovered.push(await mod.checkAndIncrement(bucketKey, LIMIT, nowMs))