Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(control-api): throttle rate-limit backend log lines and count every backend error
Alfredo Lopezdd8bd86a13f0
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex dc8f73ce9..e09311723 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.615",+ "version": "0.1.616","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.615",+ "version": "0.1.616","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex e3dc19c99..9e41085c9 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.615",+ "version": "0.1.616","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/middleware/externalGfsRateLimit.ts b/control-api/src/middleware/externalGfsRateLimit.tsindex 83e61514f..b836a1575 100644--- a/control-api/src/middleware/externalGfsRateLimit.ts+++ b/control-api/src/middleware/externalGfsRateLimit.ts@@ -3,6 +3,7 @@ import { createHash } from 'node:crypto'import { isIP } from 'node:net'import { config } from '../config.js'import type { ExternalGfsAuthority } from '../gfs/externalAuthority.js'+import { LogThrottle } from '../observability/logThrottle.js'import { rootLogger } from '../observability/logger.js'import {externalGfsRateLimitDurationSeconds,@@ -145,6 +146,11 @@ export function externalGfsOperationFor(return null}+/**+ * The hashedKey logged for a bucket: a correlation id that joins the Postgres+ * and backstop lines, not anonymization. It is unsalted, so a low-entropy key+ * (an IP, an email) can be recovered by guessing.+ */function digest(value: string): string {return createHash('sha256').update(value).digest('hex')}@@ -247,6 +253,8 @@ function applyAllowedRateHeaders(res.setHeader('X-RateLimit-Reset', String(Math.floor(result.resetMs / 1000)))}+const unavailableLogThrottle = new LogThrottle(60_000)+function reportDecision(input: {bucket: Bucketoperation: ExternalGfsOperation@@ -265,8 +273,9 @@ function reportDecision(input: {externalGfsRateLimitRequestsTotal.inc(labels)externalGfsRateLimitDurationSeconds.observe(labels, input.latencyMs / 1_000)- // The hash gives operators a join key for a single limiter identity without- // exposing session tokens, source IPs, or stable internal IDs in logs.+ // The hash is a join key for one limiter identity, so the raw key (session+ // token, source IP, internal id) is not written. It is not anonymization: the+ // SHA-256 is unsalted, and a low-entropy key such as an IP can be guessed.const fields = {event: 'external_gfs_rate_limit',operationClass: input.operation.operationClass,@@ -278,10 +287,15 @@ function reportDecision(input: {authorityResolutionAvoided: input.authorityResolutionAvoided,}if (input.outcome === 'unavailable') {- rootLogger.warn(- { ...fields, event: 'external_gfs_rate_limit_unavailable' },- 'external GFS rate limit backend unavailable'- )+ // Every request fails while the backend is down, so the line is written+ // once per key per minute; the counter above still records each one.+ const suppressed = unavailableLogThrottle.admit(fields.hashedKey)+ if (suppressed !== undefined) {+ rootLogger.warn(+ { ...fields, event: 'external_gfs_rate_limit_unavailable', suppressed },+ 'external GFS rate limit backend unavailable'+ )+ }} else if (input.outcome === 'denied') {rootLogger.warn(fields, 'external GFS rate limit denied')} else {diff --git a/control-api/src/observability/logThrottle.ts b/control-api/src/observability/logThrottle.tsnew file mode 100644index 000000000..d497c19c5--- /dev/null+++ b/control-api/src/observability/logThrottle.ts@@ -0,0 +1,54 @@+/**+ * Bounds how many lines one event writes: at most one per key per interval.+ * A suppressed event is only counted, and the next admitted line for that key+ * carries the count, so no occurrence disappears from the log without a trace.+ * Callers keep their metrics unthrottled; this only sheds log volume, such as+ * one warn line per request while the rate-limit backend is down.+ *+ * Keys idle for two intervals are dropped by a sweep that runs at most once per+ * interval, so the map is bounded by the keys active in the last two intervals.+ * A count still pending when its key is swept is lost from the log, not from+ * the metrics.+ */+export class LogThrottle {+ private readonly entries = new Map<string, { emittedAt: number; suppressed: number }>()+ private lastSweep: number++ constructor(+ private readonly intervalMs: number,+ private readonly now: () => number = () => Date.now()+ ) {+ if (!(intervalMs > 0)) throw new Error('LogThrottle: intervalMs must be > 0')+ this.lastSweep = now()+ }++ /** Number of keys currently held. Read by tests to observe the sweep. */+ get trackedKeyCount(): number {+ return this.entries.size+ }++ /**+ * Record one event for `key`. Returns the number of events suppressed since+ * the key's previous line when a line is due now, or `undefined` when this+ * event is suppressed.+ */+ admit(key: string): number | undefined {+ const now = this.now()+ this.sweep(now)+ const entry = this.entries.get(key)+ if (entry && now - entry.emittedAt < this.intervalMs) {+ entry.suppressed += 1+ return undefined+ }+ this.entries.set(key, { emittedAt: now, suppressed: 0 })+ return entry?.suppressed ?? 0+ }++ private sweep(now: number): void {+ if (now - this.lastSweep < this.intervalMs) return+ for (const [key, entry] of this.entries) {+ if (now - entry.emittedAt >= 2 * this.intervalMs) this.entries.delete(key)+ }+ this.lastSweep = now+ }+}diff --git a/control-api/src/observability/metrics.ts b/control-api/src/observability/metrics.tsindex 1d2737f93..d8e8e58ba 100644--- a/control-api/src/observability/metrics.ts+++ b/control-api/src/observability/metrics.ts@@ -250,10 +250,18 @@ export const approvalsArchiveDurationSeconds = getOrCreateHistogram({// ─── Rate limiter metrics ─────────────────────────────────────────────────export const rateLimitHitsTotal = getOrCreateCounter({name: 'rate_limit_hits_total',- help: 'Rate-limiter hits, labelled by bucket_type and result (allowed|denied|unavailable).',+ help: 'Rate-limiter hits, labelled by bucket_type and result (allowed|denied|unavailable|fallback_allowed|fallback_denied).',labelNames: ['bucket_type', 'result'] as const as Array<'bucket_type' | 'result'>,})+// Every limiter query that failed, counted even when its warn line is+// throttled. No label: the bucket key is unbounded (see below).+export const rateLimitBackendErrorsTotal = getOrCreateCounter({+ name: 'rate_limit_backend_errors_total',+ help: 'Rate-limiter queries that failed, so the request was not counted.',+ labelNames: [] as string[],+})+export const gfsUploadAdmissionRequestsTotal = getOrCreateCounter({name: 'gfs_upload_admission_requests_total',help: 'Upload-v2 edge admission decisions by bounded limit family and result.',diff --git a/control-api/src/services/rateLimiterService.ts b/control-api/src/services/rateLimiterService.tsindex 850799bd9..ba2adc912 100644--- a/control-api/src/services/rateLimiterService.ts+++ b/control-api/src/services/rateLimiterService.ts@@ -1,7 +1,9 @@import { createHash } from 'node:crypto'import type { PoolClient } from 'pg'import { pool, rateLimitPool } from '../db.js'+import { LogThrottle } from '../observability/logThrottle.js'import { rootLogger } from '../observability/logger.js'+import { rateLimitBackendErrorsTotal } from '../observability/metrics.js'/*** PG-backed sliding-window-ish rate limiter.@@ -20,6 +22,9 @@ import { rootLogger } from '../observability/logger.js'const WINDOW_MS = 60_000+// One rate_limit_db_error line per bucket per window.+const dbErrorLogThrottle = new LogThrottle(WINDOW_MS)+export type RateLimitCheck = {allowed: booleanremaining: number@@ -94,19 +99,30 @@ export async function checkAndIncrementWithQuery([bucketKey, windowStartMs, cost])} catch (err) {- // Fail-open on DB errors — the limiter is an abuse gate, not a security- // boundary. Log and let the request through so a DB blip cannot take- // down all traffic.+ // The service does not decide admission. It reports backendAvailable:false+ // and each caller decides: rateLimitMiddleware answers 503 or counts the+ // request in its process-memory fallback, and the external GFS limiter+ // answers 503.// Bucket keys carry user and workload ids; the log gets the same SHA-256- // the external GFS limiter logs as hashedKey, so the lines still join.- rootLogger.warn(- {- event: 'rate_limit_db_error',- hashedKey: createHash('sha256').update(bucketKey).digest('hex'),- err: err instanceof Error ? err.message : String(err),- },- 'rate limiter DB error, failing open'- )+ // the external GFS limiter logs as hashedKey, so the lines still join. It+ // is a correlation id, not anonymization: unsalted, so a low-entropy key+ // (an IP, an email) can be recovered by guessing. While the backend is+ // down every request lands here, so the line is throttled per bucket and+ // the counter keeps the full count.+ rateLimitBackendErrorsTotal.inc()+ const hashedKey = createHash('sha256').update(bucketKey).digest('hex')+ const suppressed = dbErrorLogThrottle.admit(hashedKey)+ if (suppressed !== undefined) {+ rootLogger.warn(+ {+ event: 'rate_limit_db_error',+ hashedKey,+ err: err instanceof Error ? err.message : String(err),+ suppressed,+ },+ 'rate limiter DB error, backend unavailable'+ )+ }return {allowed: true,remaining: maxPerMinute,@@ -116,8 +132,8 @@ export async function checkAndIncrementWithQuery(backendAvailable: false,}}- // Fail-open if the DB returns no row (e.g. test harness mocking pool.query- // with empty rows). Same rationale as the DB-error branch.+ // No row (e.g. a test harness mocking pool.query with empty rows) is also+ // reported as backendAvailable:false; the caller decides, as above.const rows = result && Array.isArray(result.rows) ? result.rows : []const row = (rows[0] ?? null) as { count: number | string } | nullif (!row) {diff --git a/control-api/test/middleware.externalGfsRateLimit.test.ts b/control-api/test/middleware.externalGfsRateLimit.test.tsindex 59ab47b40..9406e3cbf 100644--- a/control-api/test/middleware.externalGfsRateLimit.test.ts+++ b/control-api/test/middleware.externalGfsRateLimit.test.ts@@ -247,11 +247,47 @@ describe('external GFS rate boundary', () => {event: 'external_gfs_rate_limit_unavailable',outcome: 'unavailable',hashedKey: createHash('sha256').update(sessionKey).digest('hex'),+ suppressed: 0,}),'external GFS rate limit backend unavailable')})+ it('writes one unavailable line per key per minute and counts every 503', async () => {+ vi.useFakeTimers({ toFake: ['Date'] })+ try {+ vi.setSystemTime(1_800_000_000_000)+ checkAndIncrement.mockResolvedValue(unavailable(120))+ const { app, handler } = buildApp()+ const send = () =>+ request(app).get('/external/gfs/resources').set('x-user-session-token', 'session-throttled')++ const statuses: number[] = []+ for (let n = 0; n < 50; n += 1) statuses.push((await send()).status)++ // Witness: all 50 reached the limiter and were refused with 503.+ expect(statuses.filter(status => status === 503)).toHaveLength(50)+ expect(checkAndIncrement).toHaveBeenCalledTimes(50)+ expect(metrics.externalGfsRateLimitRequestsTotal.inc).toHaveBeenCalledTimes(50)+ expect(logger.rootLogger.warn).toHaveBeenCalledTimes(1)+ expect(logger.rootLogger.warn.mock.calls[0]![0]).toMatchObject({+ event: 'external_gfs_rate_limit_unavailable',+ suppressed: 0,+ })++ vi.setSystemTime(1_800_000_060_000)+ expect((await send()).status).toBe(503)+ expect(logger.rootLogger.warn).toHaveBeenCalledTimes(2)+ expect(logger.rootLogger.warn.mock.calls[1]![0]).toMatchObject({+ event: 'external_gfs_rate_limit_unavailable',+ suppressed: 49,+ })+ expect(handler).not.toHaveBeenCalled()+ } finally {+ vi.useRealTimers()+ }+ })+it('fails closed with 503 after resolution when the resolved actor bucket cannot be counted', async () => {checkAndIncrement.mockResolvedValueOnce(allowed(120))@@ -267,6 +303,7 @@ describe('external GFS rate boundary', () => {expect(response.status).toBe(503)expect(response.body).toEqual({ error: 'gfs_rate_limit_unavailable', retryAfterSeconds: 2 })expect(response.headers['retry-after']).toBe('2')+ expect(response.headers['cache-control']).toBe('no-store')expect(checkAndIncrement).toHaveBeenCalledTimes(4)expect(checkAndIncrement).toHaveBeenLastCalledWith(`gfs-ext:resolved:resource:actor:linked-admin:${CONTROL_ADMIN_ID}`,diff --git a/control-api/test/observability.logThrottle.test.ts b/control-api/test/observability.logThrottle.test.tsnew file mode 100644index 000000000..ab8054cf6--- /dev/null+++ b/control-api/test/observability.logThrottle.test.ts@@ -0,0 +1,58 @@+import { describe, expect, it } from 'vitest'+import { LogThrottle } from '../src/observability/logThrottle.js'++describe('LogThrottle', () => {+ it('admits the first event per key, suppresses the rest of the interval, and reports how many it dropped', () => {+ let t = 1_000+ const throttle = new LogThrottle(60_000, () => t)++ expect(throttle.admit('a')).toBe(0)+ const inInterval = Array.from({ length: 49 }, () => throttle.admit('a'))+ // Witness: 49 events were offered and every one was suppressed.+ expect(inInterval).toHaveLength(49)+ expect(inInterval.every(r => r === undefined)).toBe(true)++ t += 60_000+ expect(throttle.admit('a')).toBe(49)+ expect(throttle.admit('a')).toBeUndefined()+ })++ it('throttles each key on its own', () => {+ const t = 1_000+ const throttle = new LogThrottle(60_000, () => t)++ expect(throttle.admit('a')).toBe(0)+ expect(throttle.admit('b')).toBe(0)+ expect(throttle.admit('a')).toBeUndefined()+ expect(throttle.admit('b')).toBeUndefined()+ })++ it('keeps a pending suppressed count through a sweep until the second interval ends', () => {+ let t = 1_000+ const throttle = new LogThrottle(60_000, () => t)+ throttle.admit('a')+ throttle.admit('a')+ throttle.admit('a')++ // Another key's event at 1.5 intervals runs the sweep first.+ t += 90_000+ throttle.admit('b')+ expect(throttle.admit('a')).toBe(2)+ })++ it('sweeps keys idle for two intervals, so the map is bounded by the keys active recently', () => {+ let t = 1_000+ const throttle = new LogThrottle(60_000, () => t)+ for (let i = 0; i < 1_000; i += 1) throttle.admit(`k${i}`)+ // Witness: every key is tracked before the sweep.+ expect(throttle.trackedKeyCount).toBe(1_000)++ t += 120_000+ throttle.admit('live')+ expect(throttle.trackedKeyCount).toBe(1)+ })++ it('rejects a non-positive interval', () => {+ expect(() => new LogThrottle(0)).toThrow(/intervalMs must be > 0/)+ })+})diff --git a/control-api/test/routes.externalGfsRateLimitStress.realPostgres.integration.test.ts b/control-api/test/routes.externalGfsRateLimitStress.realPostgres.integration.test.tsindex fc7fef64d..068d716ab 100644--- a/control-api/test/routes.externalGfsRateLimitStress.realPostgres.integration.test.ts+++ b/control-api/test/routes.externalGfsRateLimitStress.realPostgres.integration.test.ts@@ -144,6 +144,7 @@ describeRealPostgres('external GFS rate limits under concurrent load (real Postgpool: import('../src/db.js').DbClientrootLogger: typeof import('../src/observability/logger.js').rootLoggerrequestsTotal: typeof import('../src/observability/metrics.js').externalGfsRateLimitRequestsTotal+ backendErrorsTotal: typeof import('../src/observability/metrics.js').rateLimitBackendErrorsTotalMockGateway: typeof import('./mockGateway.js').MockGateway}// Forward-only fake clock: every scenario starts on a later minute.@@ -193,6 +194,7 @@ describeRealPostgres('external GFS rate limits under concurrent load (real Postgpool: dbMod.pool,rootLogger: loggerMod.rootLogger,requestsTotal: metricsMod.externalGfsRateLimitRequestsTotal,+ backendErrorsTotal: metricsMod.rateLimitBackendErrorsTotal,MockGateway,}expect(mod.config.externalGfsIngressRlPerMin).toBe(INGRESS_L)@@ -450,13 +452,23 @@ describeRealPostgres('external GFS rate limits under concurrent load (real Postgreturn totals}- /** Captures denial log lines by phase and every limiter DB error during `run`. */+ async function backendErrors(): Promise<number> {+ return (await mod.backendErrorsTotal.get()).values[0]?.value ?? 0+ }++ /**+ * Captures denial log lines by phase and every limiter DB error during `run`.+ * DB errors are read from the counter: the rate_limit_db_error line is+ * throttled, so counting lines would undercount.+ */async function observed<T>(run: () => Promise<T>) {const warn = vi.spyOn(mod.rootLogger, 'warn').mockImplementation(() => {})const before = await deniedByPhase()+ const errorsBefore = await backendErrors()try {const result = await run()const after = await deniedByPhase()+ const errorsAfter = await backendErrors()const payloads = warn.mock.calls.map(call => call[0] as { event?: string; phase?: string })const logged = (phase: Phase) =>payloads.filter(p => p.event === 'external_gfs_rate_limit' && p.phase === phase).length@@ -472,7 +484,7 @@ describeRealPostgres('external GFS rate limits under concurrent load (real Postg'resolved-operation': logged('resolved-operation'),'edge-backstop': logged('edge-backstop'),},- dbErrors: payloads.filter(p => p.event === 'rate_limit_db_error').length,+ dbErrors: errorsAfter - errorsBefore,}} finally {warn.mockRestore()diff --git a/control-api/test/services.rateLimiter.test.ts b/control-api/test/services.rateLimiter.test.tsindex 2739d6e24..6a3928140 100644--- a/control-api/test/services.rateLimiter.test.ts+++ b/control-api/test/services.rateLimiter.test.ts@@ -1,6 +1,7 @@import { beforeEach, describe, expect, it, vi } from 'vitest'import { createHash } from 'node:crypto'import { rootLogger } from '../src/observability/logger.js'+import { rateLimitBackendErrorsTotal } from '../src/observability/metrics.js'import {acquireRateLimitConcurrencyLease,checkAndIncrement,@@ -10,6 +11,11 @@ import {stopRateLimiterCleanup,} from '../src/services/rateLimiterService.js'+async function errorCount(): Promise<number> {+ const { values } = await rateLimitBackendErrorsTotal.get()+ return values[0]?.value ?? 0+}+// In-memory simulation of (bucket_key, window_start_ms) → count, matching the// real rate_limit_buckets unique index semantics.const buckets = new Map<string, number>()@@ -225,6 +231,7 @@ describe('rateLimiterService', () => {event: 'rate_limit_db_error',hashedKey: createHash('sha256').update(bucketKey).digest('hex'),err: 'connection refused',+ suppressed: 0,})expect(JSON.stringify(warn.mock.calls)).not.toContain(desktopUserId)} finally {@@ -232,6 +239,38 @@ describe('rateLimiterService', () => {}})+ it('writes one DB-error line per bucket per minute, counts every error, and reports the suppressed count', async () => {+ const bucketKey = 'test:bucket:dberror-throttle'+ const warn = vi.spyOn(rootLogger, 'warn').mockImplementation(() => undefined)+ vi.useFakeTimers({ toFake: ['Date'] })+ try {+ vi.setSystemTime(1_800_000_000_000)+ const before = await errorCount()+ for (let i = 0; i < 50; i += 1) {+ mockRateLimitPoolQuery.mockRejectedValueOnce(new Error('connection refused'))+ await checkAndIncrement(bucketKey, 5)+ }++ // Witness: the one line exists and is this bucket's DB error.+ expect(warn).toHaveBeenCalledTimes(1)+ expect(warn.mock.calls[0]?.[0]).toMatchObject({ event: 'rate_limit_db_error', suppressed: 0 })+ expect((await errorCount()) - before).toBe(50)++ vi.setSystemTime(1_800_000_060_000)+ mockRateLimitPoolQuery.mockRejectedValueOnce(new Error('connection refused'))+ await checkAndIncrement(bucketKey, 5)+ expect(warn).toHaveBeenCalledTimes(2)+ expect(warn.mock.calls[1]?.[0]).toMatchObject({+ event: 'rate_limit_db_error',+ suppressed: 49,+ })+ expect((await errorCount()) - before).toBe(51)+ } finally {+ vi.useRealTimers()+ warn.mockRestore()+ }+ })+it('fails open when the limiter pool query returns empty rows', async () => {mockRateLimitPoolQuery.mockResolvedValueOnce({ rows: [], rowCount: 0 })const r = await checkAndIncrement('test:bucket:emptyrows', 5)diff --git a/control-api/test/services.rateLimiterPoolSaturation.realPostgres.integration.test.ts b/control-api/test/services.rateLimiterPoolSaturation.realPostgres.integration.test.tsindex 518903300..b9e2c325c 100644--- a/control-api/test/services.rateLimiterPoolSaturation.realPostgres.integration.test.ts+++ b/control-api/test/services.rateLimiterPoolSaturation.realPostgres.integration.test.ts@@ -8,8 +8,9 @@ import request from 'supertest'// 5 s acquire), separate from the core pool (max 10, 2 s acquire).//// Service-level contract when the LIMITER pool is saturated: every-// checkAndIncrement call waits for the limiter pool's acquire timeout, logs-// rate_limit_db_error, counts nothing, and returns allowed:true with+// checkAndIncrement call waits for the limiter pool's acquire timeout, counts+// rate_limit_backend_errors_total (the rate_limit_db_error line is throttled to+// one per bucket key per window), counts nothing in the bucket, and returns allowed:true with// backendAvailable:false. `allowed` alone is not an admission decision there:// each caller reads backendAvailable and decides. The external GFS limiter and// upload admission fail closed with 503 (routes.externalGfsRateLimitFailClosed@@ -75,9 +76,16 @@ describeRealPostgres('rate limiter under pool saturation', () => {config: typeof import('../src/config.js').configcheckAndIncrement: typeof import('../src/services/rateLimiterService.js').checkAndIncrementrootLogger: typeof import('../src/observability/logger.js').rootLogger+ backendErrorsTotal: typeof import('../src/observability/metrics.js').rateLimitBackendErrorsTotalMockGateway: typeof import('./mockGateway.js').MockGateway}+ async function backendErrors(): Promise<number> {+ const [sample] = (await mod.backendErrorsTotal.get()).values+ if (!sample) throw new Error('rate_limit_backend_errors_total has no sample')+ return sample.value+ }+async function holdEveryClient(target: Pool, max: number): Promise<PoolClient[]> {const held = await Promise.all(Array.from({ length: max }, () => target.connect()))expect(target.totalCount).toBe(max)@@ -112,12 +120,14 @@ describeRealPostgres('rate limiter under pool saturation', () => {const configMod = await import('../src/config.js')const limiterMod = await import('../src/services/rateLimiterService.js')const loggerMod = await import('../src/observability/logger.js')+ const metricsMod = await import('../src/observability/metrics.js')const { MockGateway } = await import('./mockGateway.js')mod = {createApp: appMod.createApp as never,config: configMod.config,checkAndIncrement: limiterMod.checkAndIncrement,rootLogger: loggerMod.rootLogger,+ backendErrorsTotal: metricsMod.rateLimitBackendErrorsTotal,MockGateway,}}, 60_000)@@ -154,6 +164,7 @@ describeRealPostgres('rate limiter under pool saturation', () => {const bucketKey = `core-saturation:${randomBytes(8).toString('hex')}`const nowMs = Math.floor(Date.now() / 60_000) * 60_000 + 1_000const warn = vi.spyOn(mod.rootLogger, 'warn').mockImplementation(() => {})+ const errorsBefore = await backendErrors()const held = await holdEveryClient(corePool, PRODUCTION_POOL_MAX)try {const results = []@@ -169,6 +180,7 @@ describeRealPostgres('rate limiter under pool saturation', () => {call => (call[0] as { event?: string }).event === 'rate_limit_db_error')).toHaveLength(0)+ expect(await backendErrors()).toBe(errorsBefore)} finally {for (const client of held) client.release()warn.mockRestore()@@ -180,6 +192,7 @@ describeRealPostgres('rate limiter under pool saturation', () => {// One fixed window for both phases, so a minute boundary cannot split them.const nowMs = Math.floor(Date.now() / 60_000) * 60_000 + 1_000const warn = vi.spyOn(mod.rootLogger, 'warn').mockImplementation(() => {})+ const errorsBefore = await backendErrors()const held = await holdEveryClient(limiterPool, LIMITER_POOL_MAX)let released = falsetry {@@ -202,15 +215,22 @@ describeRealPostgres('rate limiter under pool saturation', () => {expect(results.every(result => result.backendAvailable === false)).toBe(true)expect(results.every(result => result.count === 0)).toBe(true)+ // Every failed query is counted; the log line is throttled per bucket+ // key, so the 25 failures on one key write one line.+ expect(await backendErrors()).toBe(errorsBefore + CONCURRENT_CHECKS)const dbErrors = warn.mock.calls- .map(call => call[0] as { event?: string; hashedKey?: string; err?: string })+ .map(+ call =>+ call[0] as { event?: string; hashedKey?: string; err?: string; suppressed?: number }+ ).filter(payload => payload.event === 'rate_limit_db_error')- expect(dbErrors).toHaveLength(CONCURRENT_CHECKS)+ expect(dbErrors).toHaveLength(1)// Witness that the failure is the pool's acquire timeout and nothing else.- for (const payload of dbErrors) {- expect(payload.hashedKey).toBe(createHash('sha256').update(bucketKey).digest('hex'))- expect(payload.err).toMatch(/timeout exceeded when trying to connect/)- }+ expect(dbErrors[0]).toMatchObject({+ hashedKey: createHash('sha256').update(bucketKey).digest('hex'),+ suppressed: 0,+ })+ expect(dbErrors[0]!.err).toMatch(/timeout exceeded when trying to connect/)// Every call waited for the acquire timeout: none settled before it,// which is a property of the pool, not of machine speed. They waited in@@ -244,7 +264,8 @@ describeRealPostgres('rate limiter under pool saturation', () => {warn.mock.calls.filter(call => (call[0] as { event?: string }).event === 'rate_limit_db_error')- ).toHaveLength(CONCURRENT_CHECKS)+ ).toHaveLength(1)+ expect(await backendErrors()).toBe(errorsBefore + CONCURRENT_CHECKS)} finally {if (!released) for (const client of held) client.release()warn.mockRestore()