Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(control-api): fail closed with 503 when the external GFS rate limiter backend is unavailable

Alfredo Lopezbf4733183aee
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index a5035d477..beaecc0fd 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.607",
+ "version": "0.1.608",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.607",
+ "version": "0.1.608",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index 59d3d2ffd..9919d7f2e 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.607",
+ "version": "0.1.608",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/middleware/externalGfsRateLimit.ts b/control-api/src/middleware/externalGfsRateLimit.ts
index fa9d5c8c2..83e61514f 100644
--- a/control-api/src/middleware/externalGfsRateLimit.ts
+++ b/control-api/src/middleware/externalGfsRateLimit.ts
@@ -8,7 +8,11 @@ import {
externalGfsRateLimitDurationSeconds,
externalGfsRateLimitRequestsTotal,
} from '../observability/metrics.js'
-import { type RateLimitCheck, checkAndIncrement } from '../services/rateLimiterService.js'
+import {
+ RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS,
+ type RateLimitCheck,
+ checkAndIncrement,
+} from '../services/rateLimiterService.js'
/**
* The external Desktop GFS surface has a small, explicit operation matrix.
@@ -247,7 +251,7 @@ function reportDecision(input: {
bucket: Bucket
operation: ExternalGfsOperation
phase: ExternalGfsRateLimitPhase
- outcome: 'allowed' | 'denied'
+ outcome: 'allowed' | 'denied' | 'unavailable'
latencyMs: number
authorityResolutionAvoided: boolean
}): void {
@@ -273,7 +277,12 @@ function reportDecision(input: {
latencyMs: input.latencyMs,
authorityResolutionAvoided: input.authorityResolutionAvoided,
}
- if (input.outcome === 'denied') {
+ if (input.outcome === 'unavailable') {
+ rootLogger.warn(
+ { ...fields, event: 'external_gfs_rate_limit_unavailable' },
+ 'external GFS rate limit backend unavailable'
+ )
+ } else if (input.outcome === 'denied') {
rootLogger.warn(fields, 'external GFS rate limit denied')
} else {
rootLogger.debug(fields, 'external GFS rate limit checked')
@@ -343,6 +352,27 @@ async function enforceBuckets(input: {
const startedAt = performance.now()
const result = await checkAndIncrement(bucket.key, bucket.maxPerMinute)
const latencyMs = performance.now() - startedAt
+ if (!result.backendAvailable) {
+ // The limiter could not count this request (pool exhausted, query error
+ // or no row), so it cannot tell whether the actor is over budget. The
+ // external GFS surface fails closed instead of letting an unmetered
+ // request reach authority resolution and gfsc (#764).
+ reportDecision({
+ bucket,
+ operation: input.operation,
+ phase: input.phase,
+ outcome: 'unavailable',
+ latencyMs,
+ authorityResolutionAvoided: input.phase === 'pre-resolution',
+ })
+ input.res.setHeader('Retry-After', String(RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS))
+ input.res.setHeader('Cache-Control', 'no-store')
+ input.res.status(503).json({
+ error: 'gfs_rate_limit_unavailable',
+ retryAfterSeconds: RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS,
+ })
+ return false
+ }
const allowed = result.allowed
reportDecision({
bucket,
diff --git a/control-api/src/middleware/gfsUploadAdmission.ts b/control-api/src/middleware/gfsUploadAdmission.ts
index 23a688c6c..943e36659 100644
--- a/control-api/src/middleware/gfsUploadAdmission.ts
+++ b/control-api/src/middleware/gfsUploadAdmission.ts
@@ -8,6 +8,7 @@ import {
gfsUploadAdmissionRequestsTotal,
} from '../observability/metrics.js'
import {
+ RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS,
type RateLimitCheck,
acquireRateLimitConcurrencyLease,
checkAndIncrement,
@@ -106,6 +107,7 @@ function retryAfterSeconds(check: RateLimitCheck): number {
}
function sendBackendUnavailable(res: Response): void {
+ res.setHeader('Retry-After', String(RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS))
res.status(503).json({ error: 'gfs_upload_admission_unavailable' })
}
diff --git a/control-api/src/services/rateLimiterService.ts b/control-api/src/services/rateLimiterService.ts
index 3bc69fd51..b6eff5557 100644
--- a/control-api/src/services/rateLimiterService.ts
+++ b/control-api/src/services/rateLimiterService.ts
@@ -29,6 +29,13 @@ export type RateLimitCheck = {
backendAvailable: boolean
}
+/**
+ * Retry-After sent by the callers that fail closed with 503 when
+ * `backendAvailable` is false. The count is unknown in that case, so no
+ * window reset can be computed; a short fixed hint spreads the retries.
+ */
+export const RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS = 2
+
export function currentWindowStartMs(nowMs = Date.now()): number {
return Math.floor(nowMs / WINDOW_MS) * WINDOW_MS
}
diff --git a/control-api/test/gfsUploadAdmission.test.ts b/control-api/test/gfsUploadAdmission.test.ts
index d073cc6ee..adc8669fd 100644
--- a/control-api/test/gfsUploadAdmission.test.ts
+++ b/control-api/test/gfsUploadAdmission.test.ts
@@ -49,6 +49,7 @@ vi.mock('../src/config.js', () => ({
}))
vi.mock('../src/services/rateLimiterService.js', () => ({
+ RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS: 2,
acquireRateLimitConcurrencyLease: acquireMock,
checkAndIncrement: checkMock,
}))
@@ -280,6 +281,8 @@ describe('GFS upload admission', () => {
const response = await request(buildLifecycleApp()).get('/external/gfs/capabilities')
expect(response.status).toBe(503)
expect(response.body).toEqual({ error: 'gfs_upload_admission_unavailable' })
+ expect(response.headers['retry-after']).toBe('2')
+ expect(checkMock).toHaveBeenCalled()
expect(acquireMock).not.toHaveBeenCalled()
})
})
diff --git a/control-api/test/middleware.externalGfsRateLimit.test.ts b/control-api/test/middleware.externalGfsRateLimit.test.ts
index 5aea5dfc2..59ab47b40 100644
--- a/control-api/test/middleware.externalGfsRateLimit.test.ts
+++ b/control-api/test/middleware.externalGfsRateLimit.test.ts
@@ -35,6 +35,7 @@ vi.mock('../src/config.js', () => ({
},
}))
vi.mock('../src/services/rateLimiterService.js', () => ({
+ RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS: 2,
checkAndIncrement: (...args: unknown[]) => checkAndIncrement(...args),
}))
vi.mock('../src/observability/metrics.js', () => metrics)
@@ -51,6 +52,7 @@ function allowed(limit = 30, remaining = limit - 1) {
resetMs: Date.now() + 60_000,
windowStartMs: Date.now(),
count: limit - remaining,
+ backendAvailable: true,
}
}
@@ -61,6 +63,19 @@ function denied(limit = 30) {
resetMs: Date.now() + 30_000,
windowStartMs: Date.now(),
count: limit + 1,
+ backendAvailable: true,
+ }
+}
+
+/** What checkAndIncrement returns when its query failed or produced no row. */
+function unavailable(limit = 30) {
+ return {
+ allowed: true,
+ remaining: limit,
+ resetMs: Date.now() + 60_000,
+ windowStartMs: Date.now(),
+ count: 0,
+ backendAvailable: false,
}
}
@@ -195,6 +210,79 @@ describe('external GFS rate boundary', () => {
expect(handler).not.toHaveBeenCalled()
})
+ it('fails closed with 503 before resolution when the limiter backend cannot count the request', async () => {
+ checkAndIncrement.mockResolvedValueOnce(unavailable(120))
+ const { app, resolver, handler } = buildApp()
+
+ const response = await request(app)
+ .get('/external/gfs/resources')
+ .set('x-user-session-token', 'session-one')
+
+ expect(response.status).toBe(503)
+ expect(response.body).toEqual({ error: 'gfs_rate_limit_unavailable', retryAfterSeconds: 2 })
+ expect(response.headers['retry-after']).toBe('2')
+ expect(response.headers['cache-control']).toBe('no-store')
+ // The first bucket was consulted and nothing after it: no further bucket,
+ // no authority resolution, no handler.
+ expect(checkAndIncrement).toHaveBeenCalledTimes(1)
+ const sessionKey = String(checkAndIncrement.mock.calls[0]?.[0])
+ expect(sessionKey).toMatch(/^gfs-ext:pre:resource:session:[0-9a-f]{64}$/)
+ expect(resolver).not.toHaveBeenCalled()
+ expect(handler).not.toHaveBeenCalled()
+ expect(metrics.externalGfsRateLimitRequestsTotal.inc).toHaveBeenCalledTimes(1)
+ expect(metrics.externalGfsRateLimitRequestsTotal.inc).toHaveBeenCalledWith({
+ operation_class: 'resource',
+ route: '/external/gfs/resources',
+ outcome: 'unavailable',
+ phase: 'pre-resolution',
+ authority_resolution_avoided: 'true',
+ })
+ expect(metrics.externalGfsRateLimitDurationSeconds.observe).toHaveBeenCalledWith(
+ expect.objectContaining({ phase: 'pre-resolution', outcome: 'unavailable' }),
+ expect.any(Number)
+ )
+ expect(logger.rootLogger.warn).toHaveBeenCalledTimes(1)
+ expect(logger.rootLogger.warn).toHaveBeenCalledWith(
+ expect.objectContaining({
+ event: 'external_gfs_rate_limit_unavailable',
+ outcome: 'unavailable',
+ hashedKey: createHash('sha256').update(sessionKey).digest('hex'),
+ }),
+ 'external GFS rate limit backend unavailable'
+ )
+ })
+
+ it('fails closed with 503 after resolution when the resolved actor bucket cannot be counted', async () => {
+ checkAndIncrement
+ .mockResolvedValueOnce(allowed(120))
+ .mockResolvedValueOnce(allowed(120))
+ .mockResolvedValueOnce(allowed(1200))
+ .mockResolvedValueOnce(unavailable(120))
+ const { app, resolver, handler } = buildApp()
+
+ const response = await request(app)
+ .get('/external/gfs/resources')
+ .set('x-user-session-token', 'session-one')
+
+ expect(response.status).toBe(503)
+ expect(response.body).toEqual({ error: 'gfs_rate_limit_unavailable', retryAfterSeconds: 2 })
+ expect(response.headers['retry-after']).toBe('2')
+ expect(checkAndIncrement).toHaveBeenCalledTimes(4)
+ expect(checkAndIncrement).toHaveBeenLastCalledWith(
+ `gfs-ext:resolved:resource:actor:linked-admin:${CONTROL_ADMIN_ID}`,
+ 120
+ )
+ expect(resolver).toHaveBeenCalledTimes(1)
+ expect(handler).not.toHaveBeenCalled()
+ expect(metrics.externalGfsRateLimitRequestsTotal.inc).toHaveBeenLastCalledWith({
+ operation_class: 'resource',
+ route: '/external/gfs/resources',
+ outcome: 'unavailable',
+ phase: 'resolved-operation',
+ authority_resolution_avoided: 'false',
+ })
+ })
+
it('rejects before resolution and handler work when the pre-resolution session bucket is exhausted', async () => {
checkAndIncrement.mockResolvedValueOnce(denied(120))
const { app, resolver, handler } = buildApp()
diff --git a/control-api/test/routes.externalGfs.test.ts b/control-api/test/routes.externalGfs.test.ts
index 4a47ef33c..90dfd7ebe 100644
--- a/control-api/test/routes.externalGfs.test.ts
+++ b/control-api/test/routes.externalGfs.test.ts
@@ -193,7 +193,7 @@ beforeEach(() => {
if (text.includes('SELECT lifecycle_state, lifecycle_version')) {
return { rows: [{ lifecycle_state: 'active', lifecycle_version: 1 }] }
}
- return { rows: [] }
+ return limiterRow(text) ?? { rows: [] }
})
mockWithTransaction.mockImplementation(
async (work: (db: { query: typeof mockQuery }) => Promise<unknown>) =>
@@ -217,6 +217,15 @@ function activeSessionLifecycleResult(
: null
}
+/**
+ * The limiter upsert as a reachable Postgres answers it: the first request of
+ * the window. The external GFS limiter fails closed on a missing row, so every
+ * mock that does not script the limiter must still answer it.
+ */
+function limiterRow(text: string): { rows: [{ count: number }] } | null {
+ return text.includes('rate_limit_buckets') ? { rows: [{ count: 1 }] } : null
+}
+
function combinedAuthorityRow(
grants: Record<string, unknown>[],
values?: unknown[]
@@ -1078,6 +1087,8 @@ describe('linked Desktop operator authority contract', () => {
linked()
mockQuery.mockImplementation(async (text: string) => {
const lifecycle = activeSessionLifecycleResult(text)
+ const limiter = limiterRow(text)
+ if (limiter) return limiter
if (lifecycle) return lifecycle
if (text.includes('parent_resource_id IS NULL')) {
return {
@@ -1285,7 +1296,7 @@ describe('linked Desktop operator authority contract', () => {
auth()
mockResolveActiveLink.mockResolvedValue(ACTIVE_LINK)
mockQuery.mockImplementation(
- async (text: string) => activeSessionLifecycleResult(text) ?? { rows: [] }
+ async (text: string) => activeSessionLifecycleResult(text) ?? limiterRow(text) ?? { rows: [] }
)
const res = await request(await buildApp())
@@ -1487,6 +1498,8 @@ describe('PUT /external/gfs/grants (user delegation via existing engine)', () =>
]
mockQuery.mockImplementation(async (text: string, values?: unknown[]) => {
const lifecycle = activeSessionLifecycleResult(text)
+ const limiter = limiterRow(text)
+ if (limiter) return limiter
if (lifecycle) return lifecycle
if (text.includes('FROM team_members')) return { rows: [{ team_id: T2 }] }
if (text.includes('authority_grants AS') && text.includes('authority_shares AS')) {
@@ -1959,7 +1972,8 @@ describe('authenticated bulk grant/share transport', () => {
async (_label, method, path, subjectFields) => {
auth()
mockQuery.mockImplementation(
- async (text: string) => activeSessionLifecycleResult(text) ?? { rows: [] }
+ async (text: string) =>
+ activeSessionLifecycleResult(text) ?? limiterRow(text) ?? { rows: [] }
)
const app = await buildApp()
@@ -1985,7 +1999,8 @@ describe('authenticated bulk grant/share transport', () => {
async (_label, method, path) => {
auth()
mockQuery.mockImplementation(
- async (text: string) => activeSessionLifecycleResult(text) ?? { rows: [] }
+ async (text: string) =>
+ activeSessionLifecycleResult(text) ?? limiterRow(text) ?? { rows: [] }
)
const app = await buildApp()
@@ -2193,6 +2208,8 @@ describe('GET /external/gfs/resources/:id/affordances', () => {
auth()
mockQuery.mockImplementation(async (text: string, values?: unknown[]) => {
const lifecycle = activeSessionLifecycleResult(text)
+ const limiter = limiterRow(text)
+ if (limiter) return limiter
if (lifecycle) return lifecycle
if (text.includes('FROM team_members')) return { rows: [{ team_id: T2 }] }
if (text.includes('WITH RECURSIVE chain'))
@@ -2397,6 +2414,8 @@ describe('GET /external/gfs/resources', () => {
auth()
mockQuery.mockImplementation(async (text: string, values?: unknown[]) => {
const lifecycle = activeSessionLifecycleResult(text)
+ const limiter = limiterRow(text)
+ if (limiter) return limiter
if (lifecycle) return lifecycle
if (text.includes('FROM team_members')) return { rows: [{ team_id: T1 }] }
if (text.includes('FROM gfs_grants') && text.includes('JOIN requested_subjects')) {
diff --git a/control-api/test/routes.externalGfsRateLimitFailClosed.realPostgres.integration.test.ts b/control-api/test/routes.externalGfsRateLimitFailClosed.realPostgres.integration.test.ts
index 20b4866b2..82e18e0b1 100644
--- a/control-api/test/routes.externalGfsRateLimitFailClosed.realPostgres.integration.test.ts
+++ b/control-api/test/routes.externalGfsRateLimitFailClosed.realPostgres.integration.test.ts
@@ -1,12 +1,38 @@
-import { afterAll, beforeAll, describe, expect, it } from 'vitest'
-import { randomBytes } from 'node:crypto'
-import { Pool } from 'pg'
+import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'
+import { randomBytes, randomUUID } from 'node:crypto'
+import { Pool, type PoolClient } from 'pg'
+import request from 'supertest'
// The external GFS limiter's Postgres backend, against real PostgreSQL at the
// production pool bounds: the CORE_POOL_* and RATE_LIMIT_POOL_* variables are
-// deleted before db.js is imported. Skipped without
+// deleted before db.js is imported. Requests go through the production
+// Express app; the ONLY mock is the session-token verifier (identity), as in
+// routes.externalGfsRateLimitStress. Skipped without
// CONTROL_API_REAL_PG_ADMIN_URL.
+vi.mock('../src/utils/auth/externalSessionAuthToken.js', () => ({
+ verifyExternalSessionToken: (token: string) => {
+ const parsed = JSON.parse(token) as { userId: string }
+ return {
+ userId: parsed.userId,
+ email: `${parsed.userId}@example.test`,
+ teamId: null,
+ role: 'member',
+ authGeneration: 1,
+ exp: Math.floor(Date.now() / 1000) + 3600,
+ }
+ },
+}))
+
+const LIMITER_POOL_MAX = 6
+const LIMITER_ACQUIRE_TIMEOUT_MS = 5_000
+// An affordances read walks four Postgres buckets in series (three
+// pre-resolution, one resolved). Were the limiter to fail open again, each
+// would wait out the acquire timeout before the request reached the handler;
+// the budget covers that walk so the regression fails on the status assertion,
+// not on the test timeout.
+const T3_TIMEOUT_MS = (4 + 2) * LIMITER_ACQUIRE_TIMEOUT_MS
+
function databaseUrl(baseUrl: string, database: string): string {
const url = new URL(baseUrl)
url.pathname = `/${database}`
@@ -36,6 +62,13 @@ describeRealPostgres('external GFS rate limiter backend (real PostgreSQL)', () =
let adminPool: Pool
let corePool: Pool
let limiterPool: Pool
+ let mod: {
+ createApp: (gateway: unknown) => import('express').Express
+ config: typeof import('../src/config.js').config
+ rootLogger: typeof import('../src/observability/logger.js').rootLogger
+ requestsTotal: typeof import('../src/observability/metrics.js').externalGfsRateLimitRequestsTotal
+ MockGateway: typeof import('./mockGateway.js').MockGateway
+ }
beforeAll(async () => {
if (adminUrl === undefined) {
@@ -58,6 +91,19 @@ describeRealPostgres('external GFS rate limiter backend (real PostgreSQL)', () =
const migratePool = new Pool({ connectionString })
await dbMod.initDb({ connect: () => migratePool.connect() })
await migratePool.end()
+
+ const appMod = await import('../src/app.js')
+ const configMod = await import('../src/config.js')
+ const loggerMod = await import('../src/observability/logger.js')
+ const metricsMod = await import('../src/observability/metrics.js')
+ const { MockGateway } = await import('./mockGateway.js')
+ mod = {
+ createApp: appMod.createApp as never,
+ config: configMod.config,
+ rootLogger: loggerMod.rootLogger,
+ requestsTotal: metricsMod.externalGfsRateLimitRequestsTotal,
+ MockGateway,
+ }
}, 60_000)
afterAll(async () => {
@@ -100,4 +146,87 @@ describeRealPostgres('external GFS rate limiter backend (real PostgreSQL)', () =
expect(limiter.rows[0]).toMatchObject({ sync: 'off', timeout: '3s' })
expect(core.rows[0]).toMatchObject({ sync: 'on', timeout: '15s' })
})
+
+ async function seedUser(): Promise<string> {
+ const id = randomUUID()
+ await corePool.query(`INSERT INTO users (id, email, name) VALUES ($1, $2, $3)`, [
+ id,
+ `${id}@example.test`,
+ 'fail-closed',
+ ])
+ return id
+ }
+
+ async function getAffordances(app: import('express').Express, userId: string) {
+ const internalToken = mod.config.internalServiceTokens['external-rest-api']
+ if (!internalToken) throw new Error('config has no external-rest-api internal service token')
+ return request(app)
+ .get(`/api/v1/external/gfs/resources/${randomUUID()}/affordances?drive=main`)
+ .set('Authorization', `Bearer ${internalToken}`)
+ .set('x-service-token', 'external-rest-api')
+ .set('x-user-session-token', JSON.stringify({ userId }))
+ .set('x-forwarded-for', '203.0.113.7')
+ }
+
+ async function unavailableDecisions(): Promise<number> {
+ const { values } = await mod.requestsTotal.get()
+ return values
+ .filter(sample => sample.labels.outcome === 'unavailable')
+ .reduce((total, sample) => total + sample.value, 0)
+ }
+
+ /** Read through the core pool: the limiter pool is the one being held. */
+ async function ledgerKeys(): Promise<string[]> {
+ const result = await corePool.query<{ bucket_key: string }>(
+ 'SELECT bucket_key FROM rate_limit_buckets ORDER BY bucket_key'
+ )
+ return result.rows.map(row => row.bucket_key)
+ }
+
+ it(
+ 'T3: with every limiter connection held, an external GFS read fails closed with 503 and is not counted; after release it passes',
+ async () => {
+ const app = mod.createApp(new mod.MockGateway())
+ const userId = await seedUser()
+ await corePool.query('DELETE FROM rate_limit_buckets')
+ const unavailableBefore = await unavailableDecisions()
+
+ const held: PoolClient[] = []
+ let denied: Awaited<ReturnType<typeof getAffordances>>
+ let elapsedMs: number
+ try {
+ for (let index = 0; index < LIMITER_POOL_MAX; index += 1) {
+ held.push(await limiterPool.connect())
+ }
+ // Witness: the pool is exhausted, not merely busy.
+ expect(limiterPool.totalCount).toBe(LIMITER_POOL_MAX)
+ expect(limiterPool.idleCount).toBe(0)
+
+ const startedAt = Date.now()
+ denied = await getAffordances(app, userId)
+ elapsedMs = Date.now() - startedAt
+ } finally {
+ for (const client of held) client.release()
+ }
+
+ expect(denied.status).toBe(503)
+ expect(denied.body).toEqual({ error: 'gfs_rate_limit_unavailable', retryAfterSeconds: 2 })
+ expect(denied.headers['retry-after']).toBe('2')
+ expect(denied.headers['cache-control']).toBe('no-store')
+ // Witness: the 503 came from waiting out the limiter's acquire timeout.
+ expect(elapsedMs).toBeGreaterThanOrEqual(LIMITER_ACQUIRE_TIMEOUT_MS - 250)
+ expect((await unavailableDecisions()) - unavailableBefore).toBe(1)
+ expect(await ledgerKeys()).toEqual([])
+
+ const allowed = await getAffordances(app, userId)
+ expect(allowed.status).toBe(200)
+ // Liveness witness for the empty ledger above: the same request, once the
+ // pool is free, is counted in rate_limit_buckets.
+ const keys = await ledgerKeys()
+ expect(keys.length).toBeGreaterThan(0)
+ expect(keys.every(key => key.startsWith('gfs-ext:'))).toBe(true)
+ expect((await unavailableDecisions()) - unavailableBefore).toBe(1)
+ },
+ T3_TIMEOUT_MS
+ )
})
diff --git a/control-api/test/services.rateLimiterPoolSaturation.realPostgres.integration.test.ts b/control-api/test/services.rateLimiterPoolSaturation.realPostgres.integration.test.ts
index 4577eba44..8aa618cd7 100644
--- a/control-api/test/services.rateLimiterPoolSaturation.realPostgres.integration.test.ts
+++ b/control-api/test/services.rateLimiterPoolSaturation.realPostgres.integration.test.ts
@@ -7,11 +7,14 @@ import request from 'supertest'
// The limiter's upserts run on their own pool (db.ts `rateLimitPool`, max 6,
// 5 s acquire), separate from the core pool (max 10, 2 s acquire).
//
-// Characterization of a KNOWN defect, not a specification of desired
-// behaviour: when the LIMITER pool is saturated, checkAndIncrement fails OPEN
-// (rateLimiterService.ts, the catch around the INSERT). Every call waits for
-// the limiter pool's acquire timeout, logs rate_limit_db_error, and returns
-// allowed:true with backendAvailable:false.
+// Service-level contract when the LIMITER pool is saturated: every
+// checkAndIncrement call waits for the limiter pool's acquire timeout, logs
+// rate_limit_db_error, counts nothing, and returns allowed:true with
+// backendAvailable:false. `allowed` alone is not an admission decision there:
+// each caller reads backendAvailable and decides. The external GFS limiter and
+// upload admission fail closed with 503 (routes.externalGfsRateLimitFailClosed
+// T3); callers that still admit on backendAvailable:false do so by their own
+// choice, not because the service decided it.
//
// Isolation: a saturated CORE pool no longer reaches the limiter; it keeps
// counting.
@@ -52,7 +55,7 @@ function quoteIdent(value: string): string {
const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL
const describeRealPostgres = adminUrl ? describe : describe.skip
-describeRealPostgres('rate limiter under pool saturation (known fail-open)', () => {
+describeRealPostgres('rate limiter under pool saturation', () => {
const database = `rate_limiter_saturation_${randomBytes(6).toString('hex')}`
const poolEnvKeys = [
'CONTROL_API_PG_CONNECTION_STRING',
@@ -172,7 +175,7 @@ describeRealPostgres('rate limiter under pool saturation (known fail-open)', ()
}
}, 15_000)
- it('fails open for every concurrent check while every limiter client is held, then recovers', async () => {
+ it('reports the backend unavailable for every concurrent check while every limiter client is held, then recovers', async () => {
const bucketKey = `saturation:${randomBytes(8).toString('hex')}`
// One fixed window for both phases, so a minute boundary cannot split them.
const nowMs = Math.floor(Date.now() / 60_000) * 60_000 + 1_000
@@ -189,10 +192,10 @@ describeRealPostgres('rate limiter under pool saturation (known fail-open)', ()
const elapsedMs = performance.now() - startedAt
expect(results).toHaveLength(CONCURRENT_CHECKS)
- // The known defect: a saturated limiter pool admits every request.
+ // The service reports allowed:true; the admission decision belongs to
+ // the caller, which must read backendAvailable.
expect(results.every(result => result.allowed === true)).toBe(true)
- // These two stay true after the fix: the backend was unavailable and
- // nothing was counted.
+ // The backend was unavailable and nothing was counted.
expect(results.every(result => result.backendAvailable === false)).toBe(true)
expect(results.every(result => result.count === 0)).toBe(true)
@@ -222,7 +225,7 @@ describeRealPostgres('rate limiter under pool saturation (known fail-open)', ()
released = true
// Recovery witness, same key and window: counting starts at 1, which also
- // proves the 25 fail-open calls persisted nothing.
+ // proves the 25 unavailable calls persisted nothing.
const recovered = []
for (let i = 0; i < 3; i += 1) {
recovered.push(await mod.checkAndIncrement(bucketKey, LIMIT, nowMs))