Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(control-api): throttle rate-limit backend log lines and count every backend error

Alfredo Lopezdd8bd86a13f0
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index dc8f73ce9..e09311723 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.615",
+ "version": "0.1.616",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.615",
+ "version": "0.1.616",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index e3dc19c99..9e41085c9 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.615",
+ "version": "0.1.616",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/middleware/externalGfsRateLimit.ts b/control-api/src/middleware/externalGfsRateLimit.ts
index 83e61514f..b836a1575 100644
--- a/control-api/src/middleware/externalGfsRateLimit.ts
+++ b/control-api/src/middleware/externalGfsRateLimit.ts
@@ -3,6 +3,7 @@ import { createHash } from 'node:crypto'
import { isIP } from 'node:net'
import { config } from '../config.js'
import type { ExternalGfsAuthority } from '../gfs/externalAuthority.js'
+import { LogThrottle } from '../observability/logThrottle.js'
import { rootLogger } from '../observability/logger.js'
import {
externalGfsRateLimitDurationSeconds,
@@ -145,6 +146,11 @@ export function externalGfsOperationFor(
return null
}
+/**
+ * The hashedKey logged for a bucket: a correlation id that joins the Postgres
+ * and backstop lines, not anonymization. It is unsalted, so a low-entropy key
+ * (an IP, an email) can be recovered by guessing.
+ */
function digest(value: string): string {
return createHash('sha256').update(value).digest('hex')
}
@@ -247,6 +253,8 @@ function applyAllowedRateHeaders(
res.setHeader('X-RateLimit-Reset', String(Math.floor(result.resetMs / 1000)))
}
+const unavailableLogThrottle = new LogThrottle(60_000)
+
function reportDecision(input: {
bucket: Bucket
operation: ExternalGfsOperation
@@ -265,8 +273,9 @@ function reportDecision(input: {
externalGfsRateLimitRequestsTotal.inc(labels)
externalGfsRateLimitDurationSeconds.observe(labels, input.latencyMs / 1_000)
- // The hash gives operators a join key for a single limiter identity without
- // exposing session tokens, source IPs, or stable internal IDs in logs.
+ // The hash is a join key for one limiter identity, so the raw key (session
+ // token, source IP, internal id) is not written. It is not anonymization: the
+ // SHA-256 is unsalted, and a low-entropy key such as an IP can be guessed.
const fields = {
event: 'external_gfs_rate_limit',
operationClass: input.operation.operationClass,
@@ -278,10 +287,15 @@ function reportDecision(input: {
authorityResolutionAvoided: input.authorityResolutionAvoided,
}
if (input.outcome === 'unavailable') {
- rootLogger.warn(
- { ...fields, event: 'external_gfs_rate_limit_unavailable' },
- 'external GFS rate limit backend unavailable'
- )
+ // Every request fails while the backend is down, so the line is written
+ // once per key per minute; the counter above still records each one.
+ const suppressed = unavailableLogThrottle.admit(fields.hashedKey)
+ if (suppressed !== undefined) {
+ rootLogger.warn(
+ { ...fields, event: 'external_gfs_rate_limit_unavailable', suppressed },
+ 'external GFS rate limit backend unavailable'
+ )
+ }
} else if (input.outcome === 'denied') {
rootLogger.warn(fields, 'external GFS rate limit denied')
} else {
diff --git a/control-api/src/observability/logThrottle.ts b/control-api/src/observability/logThrottle.ts
new file mode 100644
index 000000000..d497c19c5
--- /dev/null
+++ b/control-api/src/observability/logThrottle.ts
@@ -0,0 +1,54 @@
+/**
+ * Bounds how many lines one event writes: at most one per key per interval.
+ * A suppressed event is only counted, and the next admitted line for that key
+ * carries the count, so no occurrence disappears from the log without a trace.
+ * Callers keep their metrics unthrottled; this only sheds log volume, such as
+ * one warn line per request while the rate-limit backend is down.
+ *
+ * Keys idle for two intervals are dropped by a sweep that runs at most once per
+ * interval, so the map is bounded by the keys active in the last two intervals.
+ * A count still pending when its key is swept is lost from the log, not from
+ * the metrics.
+ */
+export class LogThrottle {
+ private readonly entries = new Map<string, { emittedAt: number; suppressed: number }>()
+ private lastSweep: number
+
+ constructor(
+ private readonly intervalMs: number,
+ private readonly now: () => number = () => Date.now()
+ ) {
+ if (!(intervalMs > 0)) throw new Error('LogThrottle: intervalMs must be > 0')
+ this.lastSweep = now()
+ }
+
+ /** Number of keys currently held. Read by tests to observe the sweep. */
+ get trackedKeyCount(): number {
+ return this.entries.size
+ }
+
+ /**
+ * Record one event for `key`. Returns the number of events suppressed since
+ * the key's previous line when a line is due now, or `undefined` when this
+ * event is suppressed.
+ */
+ admit(key: string): number | undefined {
+ const now = this.now()
+ this.sweep(now)
+ const entry = this.entries.get(key)
+ if (entry && now - entry.emittedAt < this.intervalMs) {
+ entry.suppressed += 1
+ return undefined
+ }
+ this.entries.set(key, { emittedAt: now, suppressed: 0 })
+ return entry?.suppressed ?? 0
+ }
+
+ private sweep(now: number): void {
+ if (now - this.lastSweep < this.intervalMs) return
+ for (const [key, entry] of this.entries) {
+ if (now - entry.emittedAt >= 2 * this.intervalMs) this.entries.delete(key)
+ }
+ this.lastSweep = now
+ }
+}
diff --git a/control-api/src/observability/metrics.ts b/control-api/src/observability/metrics.ts
index 1d2737f93..d8e8e58ba 100644
--- a/control-api/src/observability/metrics.ts
+++ b/control-api/src/observability/metrics.ts
@@ -250,10 +250,18 @@ export const approvalsArchiveDurationSeconds = getOrCreateHistogram({
// ─── Rate limiter metrics ─────────────────────────────────────────────────
export const rateLimitHitsTotal = getOrCreateCounter({
name: 'rate_limit_hits_total',
- help: 'Rate-limiter hits, labelled by bucket_type and result (allowed|denied|unavailable).',
+ help: 'Rate-limiter hits, labelled by bucket_type and result (allowed|denied|unavailable|fallback_allowed|fallback_denied).',
labelNames: ['bucket_type', 'result'] as const as Array<'bucket_type' | 'result'>,
})
+// Every limiter query that failed, counted even when its warn line is
+// throttled. No label: the bucket key is unbounded (see below).
+export const rateLimitBackendErrorsTotal = getOrCreateCounter({
+ name: 'rate_limit_backend_errors_total',
+ help: 'Rate-limiter queries that failed, so the request was not counted.',
+ labelNames: [] as string[],
+})
+
export const gfsUploadAdmissionRequestsTotal = getOrCreateCounter({
name: 'gfs_upload_admission_requests_total',
help: 'Upload-v2 edge admission decisions by bounded limit family and result.',
diff --git a/control-api/src/services/rateLimiterService.ts b/control-api/src/services/rateLimiterService.ts
index 850799bd9..ba2adc912 100644
--- a/control-api/src/services/rateLimiterService.ts
+++ b/control-api/src/services/rateLimiterService.ts
@@ -1,7 +1,9 @@
import { createHash } from 'node:crypto'
import type { PoolClient } from 'pg'
import { pool, rateLimitPool } from '../db.js'
+import { LogThrottle } from '../observability/logThrottle.js'
import { rootLogger } from '../observability/logger.js'
+import { rateLimitBackendErrorsTotal } from '../observability/metrics.js'
/**
* PG-backed sliding-window-ish rate limiter.
@@ -20,6 +22,9 @@ import { rootLogger } from '../observability/logger.js'
const WINDOW_MS = 60_000
+// One rate_limit_db_error line per bucket per window.
+const dbErrorLogThrottle = new LogThrottle(WINDOW_MS)
+
export type RateLimitCheck = {
allowed: boolean
remaining: number
@@ -94,19 +99,30 @@ export async function checkAndIncrementWithQuery(
[bucketKey, windowStartMs, cost]
)
} catch (err) {
- // Fail-open on DB errors — the limiter is an abuse gate, not a security
- // boundary. Log and let the request through so a DB blip cannot take
- // down all traffic.
+ // The service does not decide admission. It reports backendAvailable:false
+ // and each caller decides: rateLimitMiddleware answers 503 or counts the
+ // request in its process-memory fallback, and the external GFS limiter
+ // answers 503.
// Bucket keys carry user and workload ids; the log gets the same SHA-256
- // the external GFS limiter logs as hashedKey, so the lines still join.
- rootLogger.warn(
- {
- event: 'rate_limit_db_error',
- hashedKey: createHash('sha256').update(bucketKey).digest('hex'),
- err: err instanceof Error ? err.message : String(err),
- },
- 'rate limiter DB error, failing open'
- )
+ // the external GFS limiter logs as hashedKey, so the lines still join. It
+ // is a correlation id, not anonymization: unsalted, so a low-entropy key
+ // (an IP, an email) can be recovered by guessing. While the backend is
+ // down every request lands here, so the line is throttled per bucket and
+ // the counter keeps the full count.
+ rateLimitBackendErrorsTotal.inc()
+ const hashedKey = createHash('sha256').update(bucketKey).digest('hex')
+ const suppressed = dbErrorLogThrottle.admit(hashedKey)
+ if (suppressed !== undefined) {
+ rootLogger.warn(
+ {
+ event: 'rate_limit_db_error',
+ hashedKey,
+ err: err instanceof Error ? err.message : String(err),
+ suppressed,
+ },
+ 'rate limiter DB error, backend unavailable'
+ )
+ }
return {
allowed: true,
remaining: maxPerMinute,
@@ -116,8 +132,8 @@ export async function checkAndIncrementWithQuery(
backendAvailable: false,
}
}
- // Fail-open if the DB returns no row (e.g. test harness mocking pool.query
- // with empty rows). Same rationale as the DB-error branch.
+ // No row (e.g. a test harness mocking pool.query with empty rows) is also
+ // reported as backendAvailable:false; the caller decides, as above.
const rows = result && Array.isArray(result.rows) ? result.rows : []
const row = (rows[0] ?? null) as { count: number | string } | null
if (!row) {
diff --git a/control-api/test/middleware.externalGfsRateLimit.test.ts b/control-api/test/middleware.externalGfsRateLimit.test.ts
index 59ab47b40..9406e3cbf 100644
--- a/control-api/test/middleware.externalGfsRateLimit.test.ts
+++ b/control-api/test/middleware.externalGfsRateLimit.test.ts
@@ -247,11 +247,47 @@ describe('external GFS rate boundary', () => {
event: 'external_gfs_rate_limit_unavailable',
outcome: 'unavailable',
hashedKey: createHash('sha256').update(sessionKey).digest('hex'),
+ suppressed: 0,
}),
'external GFS rate limit backend unavailable'
)
})
+ it('writes one unavailable line per key per minute and counts every 503', async () => {
+ vi.useFakeTimers({ toFake: ['Date'] })
+ try {
+ vi.setSystemTime(1_800_000_000_000)
+ checkAndIncrement.mockResolvedValue(unavailable(120))
+ const { app, handler } = buildApp()
+ const send = () =>
+ request(app).get('/external/gfs/resources').set('x-user-session-token', 'session-throttled')
+
+ const statuses: number[] = []
+ for (let n = 0; n < 50; n += 1) statuses.push((await send()).status)
+
+ // Witness: all 50 reached the limiter and were refused with 503.
+ expect(statuses.filter(status => status === 503)).toHaveLength(50)
+ expect(checkAndIncrement).toHaveBeenCalledTimes(50)
+ expect(metrics.externalGfsRateLimitRequestsTotal.inc).toHaveBeenCalledTimes(50)
+ expect(logger.rootLogger.warn).toHaveBeenCalledTimes(1)
+ expect(logger.rootLogger.warn.mock.calls[0]![0]).toMatchObject({
+ event: 'external_gfs_rate_limit_unavailable',
+ suppressed: 0,
+ })
+
+ vi.setSystemTime(1_800_000_060_000)
+ expect((await send()).status).toBe(503)
+ expect(logger.rootLogger.warn).toHaveBeenCalledTimes(2)
+ expect(logger.rootLogger.warn.mock.calls[1]![0]).toMatchObject({
+ event: 'external_gfs_rate_limit_unavailable',
+ suppressed: 49,
+ })
+ expect(handler).not.toHaveBeenCalled()
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+
it('fails closed with 503 after resolution when the resolved actor bucket cannot be counted', async () => {
checkAndIncrement
.mockResolvedValueOnce(allowed(120))
@@ -267,6 +303,7 @@ describe('external GFS rate boundary', () => {
expect(response.status).toBe(503)
expect(response.body).toEqual({ error: 'gfs_rate_limit_unavailable', retryAfterSeconds: 2 })
expect(response.headers['retry-after']).toBe('2')
+ expect(response.headers['cache-control']).toBe('no-store')
expect(checkAndIncrement).toHaveBeenCalledTimes(4)
expect(checkAndIncrement).toHaveBeenLastCalledWith(
`gfs-ext:resolved:resource:actor:linked-admin:${CONTROL_ADMIN_ID}`,
diff --git a/control-api/test/observability.logThrottle.test.ts b/control-api/test/observability.logThrottle.test.ts
new file mode 100644
index 000000000..ab8054cf6
--- /dev/null
+++ b/control-api/test/observability.logThrottle.test.ts
@@ -0,0 +1,58 @@
+import { describe, expect, it } from 'vitest'
+import { LogThrottle } from '../src/observability/logThrottle.js'
+
+describe('LogThrottle', () => {
+ it('admits the first event per key, suppresses the rest of the interval, and reports how many it dropped', () => {
+ let t = 1_000
+ const throttle = new LogThrottle(60_000, () => t)
+
+ expect(throttle.admit('a')).toBe(0)
+ const inInterval = Array.from({ length: 49 }, () => throttle.admit('a'))
+ // Witness: 49 events were offered and every one was suppressed.
+ expect(inInterval).toHaveLength(49)
+ expect(inInterval.every(r => r === undefined)).toBe(true)
+
+ t += 60_000
+ expect(throttle.admit('a')).toBe(49)
+ expect(throttle.admit('a')).toBeUndefined()
+ })
+
+ it('throttles each key on its own', () => {
+ const t = 1_000
+ const throttle = new LogThrottle(60_000, () => t)
+
+ expect(throttle.admit('a')).toBe(0)
+ expect(throttle.admit('b')).toBe(0)
+ expect(throttle.admit('a')).toBeUndefined()
+ expect(throttle.admit('b')).toBeUndefined()
+ })
+
+ it('keeps a pending suppressed count through a sweep until the second interval ends', () => {
+ let t = 1_000
+ const throttle = new LogThrottle(60_000, () => t)
+ throttle.admit('a')
+ throttle.admit('a')
+ throttle.admit('a')
+
+ // Another key's event at 1.5 intervals runs the sweep first.
+ t += 90_000
+ throttle.admit('b')
+ expect(throttle.admit('a')).toBe(2)
+ })
+
+ it('sweeps keys idle for two intervals, so the map is bounded by the keys active recently', () => {
+ let t = 1_000
+ const throttle = new LogThrottle(60_000, () => t)
+ for (let i = 0; i < 1_000; i += 1) throttle.admit(`k${i}`)
+ // Witness: every key is tracked before the sweep.
+ expect(throttle.trackedKeyCount).toBe(1_000)
+
+ t += 120_000
+ throttle.admit('live')
+ expect(throttle.trackedKeyCount).toBe(1)
+ })
+
+ it('rejects a non-positive interval', () => {
+ expect(() => new LogThrottle(0)).toThrow(/intervalMs must be > 0/)
+ })
+})
diff --git a/control-api/test/routes.externalGfsRateLimitStress.realPostgres.integration.test.ts b/control-api/test/routes.externalGfsRateLimitStress.realPostgres.integration.test.ts
index fc7fef64d..068d716ab 100644
--- a/control-api/test/routes.externalGfsRateLimitStress.realPostgres.integration.test.ts
+++ b/control-api/test/routes.externalGfsRateLimitStress.realPostgres.integration.test.ts
@@ -144,6 +144,7 @@ describeRealPostgres('external GFS rate limits under concurrent load (real Postg
pool: import('../src/db.js').DbClient
rootLogger: typeof import('../src/observability/logger.js').rootLogger
requestsTotal: typeof import('../src/observability/metrics.js').externalGfsRateLimitRequestsTotal
+ backendErrorsTotal: typeof import('../src/observability/metrics.js').rateLimitBackendErrorsTotal
MockGateway: typeof import('./mockGateway.js').MockGateway
}
// Forward-only fake clock: every scenario starts on a later minute.
@@ -193,6 +194,7 @@ describeRealPostgres('external GFS rate limits under concurrent load (real Postg
pool: dbMod.pool,
rootLogger: loggerMod.rootLogger,
requestsTotal: metricsMod.externalGfsRateLimitRequestsTotal,
+ backendErrorsTotal: metricsMod.rateLimitBackendErrorsTotal,
MockGateway,
}
expect(mod.config.externalGfsIngressRlPerMin).toBe(INGRESS_L)
@@ -450,13 +452,23 @@ describeRealPostgres('external GFS rate limits under concurrent load (real Postg
return totals
}
- /** Captures denial log lines by phase and every limiter DB error during `run`. */
+ async function backendErrors(): Promise<number> {
+ return (await mod.backendErrorsTotal.get()).values[0]?.value ?? 0
+ }
+
+ /**
+ * Captures denial log lines by phase and every limiter DB error during `run`.
+ * DB errors are read from the counter: the rate_limit_db_error line is
+ * throttled, so counting lines would undercount.
+ */
async function observed<T>(run: () => Promise<T>) {
const warn = vi.spyOn(mod.rootLogger, 'warn').mockImplementation(() => {})
const before = await deniedByPhase()
+ const errorsBefore = await backendErrors()
try {
const result = await run()
const after = await deniedByPhase()
+ const errorsAfter = await backendErrors()
const payloads = warn.mock.calls.map(call => call[0] as { event?: string; phase?: string })
const logged = (phase: Phase) =>
payloads.filter(p => p.event === 'external_gfs_rate_limit' && p.phase === phase).length
@@ -472,7 +484,7 @@ describeRealPostgres('external GFS rate limits under concurrent load (real Postg
'resolved-operation': logged('resolved-operation'),
'edge-backstop': logged('edge-backstop'),
},
- dbErrors: payloads.filter(p => p.event === 'rate_limit_db_error').length,
+ dbErrors: errorsAfter - errorsBefore,
}
} finally {
warn.mockRestore()
diff --git a/control-api/test/services.rateLimiter.test.ts b/control-api/test/services.rateLimiter.test.ts
index 2739d6e24..6a3928140 100644
--- a/control-api/test/services.rateLimiter.test.ts
+++ b/control-api/test/services.rateLimiter.test.ts
@@ -1,6 +1,7 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
import { createHash } from 'node:crypto'
import { rootLogger } from '../src/observability/logger.js'
+import { rateLimitBackendErrorsTotal } from '../src/observability/metrics.js'
import {
acquireRateLimitConcurrencyLease,
checkAndIncrement,
@@ -10,6 +11,11 @@ import {
stopRateLimiterCleanup,
} from '../src/services/rateLimiterService.js'
+async function errorCount(): Promise<number> {
+ const { values } = await rateLimitBackendErrorsTotal.get()
+ return values[0]?.value ?? 0
+}
+
// In-memory simulation of (bucket_key, window_start_ms) → count, matching the
// real rate_limit_buckets unique index semantics.
const buckets = new Map<string, number>()
@@ -225,6 +231,7 @@ describe('rateLimiterService', () => {
event: 'rate_limit_db_error',
hashedKey: createHash('sha256').update(bucketKey).digest('hex'),
err: 'connection refused',
+ suppressed: 0,
})
expect(JSON.stringify(warn.mock.calls)).not.toContain(desktopUserId)
} finally {
@@ -232,6 +239,38 @@ describe('rateLimiterService', () => {
}
})
+ it('writes one DB-error line per bucket per minute, counts every error, and reports the suppressed count', async () => {
+ const bucketKey = 'test:bucket:dberror-throttle'
+ const warn = vi.spyOn(rootLogger, 'warn').mockImplementation(() => undefined)
+ vi.useFakeTimers({ toFake: ['Date'] })
+ try {
+ vi.setSystemTime(1_800_000_000_000)
+ const before = await errorCount()
+ for (let i = 0; i < 50; i += 1) {
+ mockRateLimitPoolQuery.mockRejectedValueOnce(new Error('connection refused'))
+ await checkAndIncrement(bucketKey, 5)
+ }
+
+ // Witness: the one line exists and is this bucket's DB error.
+ expect(warn).toHaveBeenCalledTimes(1)
+ expect(warn.mock.calls[0]?.[0]).toMatchObject({ event: 'rate_limit_db_error', suppressed: 0 })
+ expect((await errorCount()) - before).toBe(50)
+
+ vi.setSystemTime(1_800_000_060_000)
+ mockRateLimitPoolQuery.mockRejectedValueOnce(new Error('connection refused'))
+ await checkAndIncrement(bucketKey, 5)
+ expect(warn).toHaveBeenCalledTimes(2)
+ expect(warn.mock.calls[1]?.[0]).toMatchObject({
+ event: 'rate_limit_db_error',
+ suppressed: 49,
+ })
+ expect((await errorCount()) - before).toBe(51)
+ } finally {
+ vi.useRealTimers()
+ warn.mockRestore()
+ }
+ })
+
it('fails open when the limiter pool query returns empty rows', async () => {
mockRateLimitPoolQuery.mockResolvedValueOnce({ rows: [], rowCount: 0 })
const r = await checkAndIncrement('test:bucket:emptyrows', 5)
diff --git a/control-api/test/services.rateLimiterPoolSaturation.realPostgres.integration.test.ts b/control-api/test/services.rateLimiterPoolSaturation.realPostgres.integration.test.ts
index 518903300..b9e2c325c 100644
--- a/control-api/test/services.rateLimiterPoolSaturation.realPostgres.integration.test.ts
+++ b/control-api/test/services.rateLimiterPoolSaturation.realPostgres.integration.test.ts
@@ -8,8 +8,9 @@ import request from 'supertest'
// 5 s acquire), separate from the core pool (max 10, 2 s acquire).
//
// Service-level contract when the LIMITER pool is saturated: every
-// checkAndIncrement call waits for the limiter pool's acquire timeout, logs
-// rate_limit_db_error, counts nothing, and returns allowed:true with
+// checkAndIncrement call waits for the limiter pool's acquire timeout, counts
+// rate_limit_backend_errors_total (the rate_limit_db_error line is throttled to
+// one per bucket key per window), counts nothing in the bucket, and returns allowed:true with
// backendAvailable:false. `allowed` alone is not an admission decision there:
// each caller reads backendAvailable and decides. The external GFS limiter and
// upload admission fail closed with 503 (routes.externalGfsRateLimitFailClosed
@@ -75,9 +76,16 @@ describeRealPostgres('rate limiter under pool saturation', () => {
config: typeof import('../src/config.js').config
checkAndIncrement: typeof import('../src/services/rateLimiterService.js').checkAndIncrement
rootLogger: typeof import('../src/observability/logger.js').rootLogger
+ backendErrorsTotal: typeof import('../src/observability/metrics.js').rateLimitBackendErrorsTotal
MockGateway: typeof import('./mockGateway.js').MockGateway
}
+ async function backendErrors(): Promise<number> {
+ const [sample] = (await mod.backendErrorsTotal.get()).values
+ if (!sample) throw new Error('rate_limit_backend_errors_total has no sample')
+ return sample.value
+ }
+
async function holdEveryClient(target: Pool, max: number): Promise<PoolClient[]> {
const held = await Promise.all(Array.from({ length: max }, () => target.connect()))
expect(target.totalCount).toBe(max)
@@ -112,12 +120,14 @@ describeRealPostgres('rate limiter under pool saturation', () => {
const configMod = await import('../src/config.js')
const limiterMod = await import('../src/services/rateLimiterService.js')
const loggerMod = await import('../src/observability/logger.js')
+ const metricsMod = await import('../src/observability/metrics.js')
const { MockGateway } = await import('./mockGateway.js')
mod = {
createApp: appMod.createApp as never,
config: configMod.config,
checkAndIncrement: limiterMod.checkAndIncrement,
rootLogger: loggerMod.rootLogger,
+ backendErrorsTotal: metricsMod.rateLimitBackendErrorsTotal,
MockGateway,
}
}, 60_000)
@@ -154,6 +164,7 @@ describeRealPostgres('rate limiter under pool saturation', () => {
const bucketKey = `core-saturation:${randomBytes(8).toString('hex')}`
const nowMs = Math.floor(Date.now() / 60_000) * 60_000 + 1_000
const warn = vi.spyOn(mod.rootLogger, 'warn').mockImplementation(() => {})
+ const errorsBefore = await backendErrors()
const held = await holdEveryClient(corePool, PRODUCTION_POOL_MAX)
try {
const results = []
@@ -169,6 +180,7 @@ describeRealPostgres('rate limiter under pool saturation', () => {
call => (call[0] as { event?: string }).event === 'rate_limit_db_error'
)
).toHaveLength(0)
+ expect(await backendErrors()).toBe(errorsBefore)
} finally {
for (const client of held) client.release()
warn.mockRestore()
@@ -180,6 +192,7 @@ describeRealPostgres('rate limiter under pool saturation', () => {
// One fixed window for both phases, so a minute boundary cannot split them.
const nowMs = Math.floor(Date.now() / 60_000) * 60_000 + 1_000
const warn = vi.spyOn(mod.rootLogger, 'warn').mockImplementation(() => {})
+ const errorsBefore = await backendErrors()
const held = await holdEveryClient(limiterPool, LIMITER_POOL_MAX)
let released = false
try {
@@ -202,15 +215,22 @@ describeRealPostgres('rate limiter under pool saturation', () => {
expect(results.every(result => result.backendAvailable === false)).toBe(true)
expect(results.every(result => result.count === 0)).toBe(true)
+ // Every failed query is counted; the log line is throttled per bucket
+ // key, so the 25 failures on one key write one line.
+ expect(await backendErrors()).toBe(errorsBefore + CONCURRENT_CHECKS)
const dbErrors = warn.mock.calls
- .map(call => call[0] as { event?: string; hashedKey?: string; err?: string })
+ .map(
+ call =>
+ call[0] as { event?: string; hashedKey?: string; err?: string; suppressed?: number }
+ )
.filter(payload => payload.event === 'rate_limit_db_error')
- expect(dbErrors).toHaveLength(CONCURRENT_CHECKS)
+ expect(dbErrors).toHaveLength(1)
// Witness that the failure is the pool's acquire timeout and nothing else.
- for (const payload of dbErrors) {
- expect(payload.hashedKey).toBe(createHash('sha256').update(bucketKey).digest('hex'))
- expect(payload.err).toMatch(/timeout exceeded when trying to connect/)
- }
+ expect(dbErrors[0]).toMatchObject({
+ hashedKey: createHash('sha256').update(bucketKey).digest('hex'),
+ suppressed: 0,
+ })
+ expect(dbErrors[0]!.err).toMatch(/timeout exceeded when trying to connect/)
// Every call waited for the acquire timeout: none settled before it,
// which is a property of the pool, not of machine speed. They waited in
@@ -244,7 +264,8 @@ describeRealPostgres('rate limiter under pool saturation', () => {
warn.mock.calls.filter(
call => (call[0] as { event?: string }).event === 'rate_limit_db_error'
)
- ).toHaveLength(CONCURRENT_CHECKS)
+ ).toHaveLength(1)
+ expect(await backendErrors()).toBe(errorsBefore + CONCURRENT_CHECKS)
} finally {
if (!released) for (const client of held) client.release()
warn.mockRestore()