Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
import { beforeEach, describe, expect, it, vi } from 'vitest'
import { createHash } from 'node:crypto'
import { rootLogger } from '../src/observability/logger.js'
import { rateLimitBackendErrorsTotal } from '../src/observability/metrics.js'
import {
acquireRateLimitConcurrencyLease,
checkAndIncrement,
cleanupExpiredBuckets,
currentWindowStartMs,
startRateLimiterCleanup,
stopRateLimiterCleanup,
} from '../src/services/rateLimiterService.js'
async function errorCount(): Promise<number> {
const { values } = await rateLimitBackendErrorsTotal.get()
return values[0]?.value ?? 0
}
// In-memory simulation of (bucket_key, window_start_ms) → count, matching the
// real rate_limit_buckets unique index semantics.
const buckets = new Map<string, number>()
const advisoryLocks = new Set<string>()
const mockClientQuery = vi.fn(async (sql: unknown, params?: unknown[]) => {
const text = String(sql)
const identity = `${String(params?.[0])}:${String(params?.[1])}`
if (text.includes('pg_try_advisory_lock')) {
if (advisoryLocks.has(identity)) return { rows: [{ acquired: false }] }
advisoryLocks.add(identity)
return { rows: [{ acquired: true }] }
}
if (text.includes('pg_advisory_unlock')) {
const unlocked = advisoryLocks.delete(identity)
return { rows: [{ pg_advisory_unlock: unlocked }] }
}
return { rows: [] }
})
const mockConcurrencyClient = {
query: mockClientQuery,
once: vi.fn(),
release: vi.fn(),
}
const mockPoolConnect = vi.fn(async () => mockConcurrencyClient)
const mockRateLimitPoolQuery = vi.fn(async (sql: unknown, params?: unknown[]) => {
const text = typeof sql === 'string' ? sql : ''
if (/INSERT INTO rate_limit_buckets/i.test(text)) {
const bucketKey = String(params?.[0] ?? '')
const windowStart = Number(params?.[1] ?? 0)
const mapKey = `${bucketKey}|${windowStart}`
const next = (buckets.get(mapKey) ?? 0) + Number(params?.[2] ?? 1)
buckets.set(mapKey, next)
return { rows: [{ count: next }], rowCount: 1 }
}
if (/DELETE FROM rate_limit_buckets/i.test(text)) {
const cutoff = Number(params?.[0] ?? 0)
let removed = 0
for (const key of Array.from(buckets.keys())) {
const windowStart = Number(key.split('|')[1])
if (windowStart < cutoff) {
buckets.delete(key)
removed++
}
}
return { rows: [], rowCount: removed }
}
return { rows: [], rowCount: 0 }
})
// The limiter's upserts and prune go to the dedicated limiter pool; the core
// pool only lends the long-lived advisory-lock client. A limiter statement on
// the core pool is a wiring regression, so the core query rejects loudly.
const mockCorePoolQuery = vi.fn(async (sql: unknown) => {
if (String(sql).includes('DELETE FROM password_identifier_state'))
return { rows: [], rowCount: 0 }
throw new Error(`core pool must not serve the rate limiter: ${String(sql).slice(0, 40)}`)
})
vi.mock('../src/db.js', () => ({
pool: {
query: (...args: unknown[]) => mockCorePoolQuery(args[0]),
connect: (...args: unknown[]) => mockPoolConnect(...args),
},
rateLimitPool: {
query: (...args: unknown[]) =>
mockRateLimitPoolQuery(args[0], args[1] as unknown[] | undefined),
},
withTransaction: vi.fn(),
}))
describe('rateLimiterService', () => {
beforeEach(() => {
buckets.clear()
advisoryLocks.clear()
mockRateLimitPoolQuery.mockClear()
mockCorePoolQuery.mockClear()
mockClientQuery.mockClear()
})
it('keeps legacy upserts/prune on the limiter pool and credential cleanup on the core pool', async () => {
const r = await checkAndIncrement('test:bucket:pool-wiring', 5)
await cleanupExpiredBuckets(1_700_000_000_000)
// Witness: both statements reached the limiter pool.
expect(r).toMatchObject({ allowed: true, count: 1, backendAvailable: true })
expect(
mockRateLimitPoolQuery.mock.calls.map(([sql]) => String(sql).trim().split(/\s+/)[0])
).toEqual(['INSERT', 'DELETE'])
expect(mockCorePoolQuery).toHaveBeenCalledExactlyOnceWith(
expect.stringContaining('DELETE FROM password_identifier_state')
)
})
it('allows requests up to the limit and denies the (limit+1)-th in the same window', async () => {
const key = 'test:bucket:1'
const limit = 5
const nowMs = 1_700_000_000_000
for (let i = 0; i < limit; i++) {
const r = await checkAndIncrement(key, limit, nowMs)
expect(r.allowed).toBe(true)
expect(r.count).toBe(i + 1)
expect(r.remaining).toBe(limit - (i + 1))
}
const over = await checkAndIncrement(key, limit, nowMs)
expect(over.allowed).toBe(false)
expect(over.count).toBe(limit + 1)
expect(over.remaining).toBe(0)
})
it('exposes resetMs aligned to the next 60s window boundary', async () => {
const nowMs = 1_700_000_000_000
const r = await checkAndIncrement('test:bucket:reset', 10, nowMs)
expect(r.windowStartMs).toBe(Math.floor(nowMs / 60_000) * 60_000)
expect(r.resetMs).toBe(r.windowStartMs + 60_000)
expect(r.resetMs - nowMs).toBeLessThanOrEqual(60_000)
})
it('rolls over to a fresh slot when the 60s window advances', async () => {
const key = 'test:bucket:roll'
const limit = 2
const t0 = 1_700_000_000_000 // aligned to minute start
// Fill the window at t0.
await checkAndIncrement(key, limit, t0)
await checkAndIncrement(key, limit, t0)
const denied = await checkAndIncrement(key, limit, t0)
expect(denied.allowed).toBe(false)
// New window (+60_001 ms) — fresh counter.
const fresh = await checkAndIncrement(key, limit, t0 + 60_001)
expect(fresh.allowed).toBe(true)
expect(fresh.count).toBe(1)
})
it('concurrent increments observe distinct counter values (no race)', async () => {
const key = 'test:bucket:concurrent'
const limit = 100
const nowMs = 1_700_000_000_000
const results = await Promise.all(
Array.from({ length: 10 }, () => checkAndIncrement(key, limit, nowMs))
)
const counts = results.map(r => r.count).sort((a, b) => a - b)
expect(counts).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
expect(results.every(r => r.allowed)).toBe(true)
})
it('atomically charges a weighted cost in the existing PostgreSQL bucket', async () => {
const first = await checkAndIncrement('test:bucket:weighted', 16, 1_700_000_000_000, 7)
const second = await checkAndIncrement('test:bucket:weighted', 16, 1_700_000_000_000, 10)
expect(first).toMatchObject({ allowed: true, count: 7, remaining: 9 })
expect(second).toMatchObject({ allowed: false, count: 17, remaining: 0 })
expect(mockRateLimitPoolQuery).toHaveBeenLastCalledWith(expect.any(String), [
'test:bucket:weighted',
currentWindowStartMs(1_700_000_000_000),
10,
])
})
it('holds replica-safe advisory slots until release and then admits the next request', async () => {
const requirements = [{ bucketKey: 'gfs-upload:test-active', maxConcurrent: 2 }]
const [first, second, denied] = await Promise.all([
acquireRateLimitConcurrencyLease(requirements),
acquireRateLimitConcurrencyLease(requirements),
acquireRateLimitConcurrencyLease(requirements),
])
expect(first).toMatchObject({ allowed: true, backendAvailable: true })
expect(second).toMatchObject({ allowed: true, backendAvailable: true })
expect(denied).toMatchObject({ allowed: false, backendAvailable: true })
await first.release()
const admittedAfterRelease = await acquireRateLimitConcurrencyLease(requirements)
expect(admittedAfterRelease).toMatchObject({ allowed: true, backendAvailable: true })
await second.release()
await admittedAfterRelease.release()
expect(advisoryLocks.size).toBe(0)
expect(mockPoolConnect).toHaveBeenCalledTimes(1)
})
it('destroys the dedicated connection when an advisory unlock outcome is unknown', async () => {
const lease = await acquireRateLimitConcurrencyLease([
{ bucketKey: 'gfs-upload:unlock-failure', maxConcurrent: 1 },
])
expect(lease.allowed).toBe(true)
mockClientQuery.mockRejectedValueOnce(new Error('connection lost during unlock'))
await lease.release()
expect(mockConcurrencyClient.release).toHaveBeenCalledWith(true)
// The fake client does not emulate PostgreSQL's session teardown; model
// the advisory lock release that the destroyed real connection provides.
advisoryLocks.clear()
const replacement = await acquireRateLimitConcurrencyLease([
{ bucketKey: 'gfs-upload:unlock-failure', maxConcurrent: 1 },
])
expect(replacement).toMatchObject({ allowed: true, backendAvailable: true })
await replacement.release()
})
it('reports the backend unavailable when the limiter pool query throws (DB error); the caller decides the policy', async () => {
mockRateLimitPoolQuery.mockRejectedValueOnce(new Error('connection refused'))
const r = await checkAndIncrement('test:bucket:failopen', 5)
expect(r.allowed).toBe(true)
expect(r.count).toBe(0)
expect(r.remaining).toBe(5)
expect(r.backendAvailable).toBe(false)
})
it('logs the SHA-256 of the bucket key on a DB error, never the key or the user id it carries', async () => {
const desktopUserId = '5f0c2d1e-7a4b-4c3d-9e8f-0a1b2c3d4e5f'
const bucketKey = `gfs-ext:pre:token:user:${desktopUserId}`
const warn = vi.spyOn(rootLogger, 'warn').mockImplementation(() => undefined)
try {
mockRateLimitPoolQuery.mockRejectedValueOnce(new Error('connection refused'))
const r = await checkAndIncrement(bucketKey, 5)
expect(r.backendAvailable).toBe(false)
// Witness: the DB error wrote exactly one line, and it is that event.
expect(warn).toHaveBeenCalledTimes(1)
expect(warn.mock.calls[0]?.[0]).toEqual({
event: 'rate_limit_db_error',
hashedKey: createHash('sha256').update(bucketKey).digest('hex'),
err: 'connection refused',
suppressed: 0,
})
expect(JSON.stringify(warn.mock.calls)).not.toContain(desktopUserId)
} finally {
warn.mockRestore()
}
})
it('writes one DB-error line per bucket per minute, counts every error, and reports the suppressed count', async () => {
const bucketKey = 'test:bucket:dberror-throttle'
const warn = vi.spyOn(rootLogger, 'warn').mockImplementation(() => undefined)
vi.useFakeTimers({ toFake: ['Date'] })
try {
vi.setSystemTime(1_800_000_000_000)
const before = await errorCount()
for (let i = 0; i < 50; i += 1) {
mockRateLimitPoolQuery.mockRejectedValueOnce(new Error('connection refused'))
await checkAndIncrement(bucketKey, 5)
}
// Witness: the one line exists and is this bucket's DB error.
expect(warn).toHaveBeenCalledTimes(1)
expect(warn.mock.calls[0]?.[0]).toMatchObject({ event: 'rate_limit_db_error', suppressed: 0 })
expect((await errorCount()) - before).toBe(50)
vi.setSystemTime(1_800_000_060_000)
mockRateLimitPoolQuery.mockRejectedValueOnce(new Error('connection refused'))
await checkAndIncrement(bucketKey, 5)
expect(warn).toHaveBeenCalledTimes(2)
expect(warn.mock.calls[1]?.[0]).toMatchObject({
event: 'rate_limit_db_error',
suppressed: 49,
})
expect((await errorCount()) - before).toBe(51)
} finally {
vi.useRealTimers()
warn.mockRestore()
}
})
it('reports the backend unavailable when the limiter pool query returns empty rows', async () => {
mockRateLimitPoolQuery.mockResolvedValueOnce({ rows: [], rowCount: 0 })
const r = await checkAndIncrement('test:bucket:emptyrows', 5)
expect(r.allowed).toBe(true)
expect(r.count).toBe(0)
expect(r.backendAvailable).toBe(false)
})
it('cleanupExpiredBuckets removes rows older than 5 minutes', async () => {
const now = 1_700_000_000_000
// Seed two buckets: one old (>5 min ago), one fresh.
const oldKey = `old-bucket|${now - 10 * 60_000}` // 10 min ago
const freshKey = `fresh-bucket|${now - 60_000}` // 1 min ago
buckets.set(oldKey, 1)
buckets.set(freshKey, 1)
const removed = await cleanupExpiredBuckets(now)
expect(removed).toBe(1)
expect(buckets.has(oldKey)).toBe(false)
expect(buckets.has(freshKey)).toBe(true)
})
it('currentWindowStartMs floors now to the minute boundary', () => {
const minuteStart = 1_700_000_040_000 // divisible by 60_000
expect(minuteStart % 60_000).toBe(0)
const t = minuteStart + 37_000 // 37s past a minute
expect(currentWindowStartMs(t)).toBe(minuteStart)
expect(currentWindowStartMs(minuteStart)).toBe(minuteStart)
})
it('startRateLimiterCleanup + stopRateLimiterCleanup are idempotent', () => {
// No assertion beyond "does not throw" — this is a lifecycle smoke test.
startRateLimiterCleanup(60_000)
startRateLimiterCleanup(60_000) // second call is no-op
stopRateLimiterCleanup()
stopRateLimiterCleanup() // double stop is safe
})
it('releases the idle dedicated session after finite credential work', async () => {
mockConcurrencyClient.release.mockClear()
const lease = await acquireRateLimitConcurrencyLease(
[{ bucketKey: 'finite-password-work', maxConcurrent: 1 }],
{ releaseIdleClient: true }
)
expect(lease.allowed).toBe(true)
await lease.release()
expect(mockConcurrencyClient.release).toHaveBeenCalledExactlyOnceWith(true)
})
it('releases an idle dedicated session after busy denial', async () => {
mockConcurrencyClient.release.mockClear()
mockClientQuery.mockResolvedValueOnce({ rows: [{ acquired: false }] } as never)
const lease = await acquireRateLimitConcurrencyLease(
[{ bucketKey: 'finite-busy-work', maxConcurrent: 1 }],
{ releaseIdleClient: true }
)
expect(lease.allowed).toBe(false)
expect(lease.backendAvailable).toBe(true)
expect(mockConcurrencyClient.release).toHaveBeenCalledExactlyOnceWith(true)
})
it('releases an idle dedicated session after an admission query failure', async () => {
mockConcurrencyClient.release.mockClear()
mockClientQuery.mockRejectedValueOnce(new Error('synthetic admission failure'))
const lease = await acquireRateLimitConcurrencyLease(
[{ bucketKey: 'finite-failed-work', maxConcurrent: 1 }],
{ releaseIdleClient: true }
)
expect(lease.allowed).toBe(false)
expect(lease.backendAvailable).toBe(false)
expect(mockConcurrencyClient.release).toHaveBeenCalledExactlyOnceWith(true)
})
it('retains the dedicated session when another lease remains active', async () => {
const active = await acquireRateLimitConcurrencyLease([
{ bucketKey: 'retained-active-work', maxConcurrent: 1 },
])
mockConcurrencyClient.release.mockClear()
mockClientQuery.mockResolvedValueOnce({ rows: [{ acquired: false }] } as never)
const denied = await acquireRateLimitConcurrencyLease(
[{ bucketKey: 'finite-denied-work', maxConcurrent: 1 }],
{ releaseIdleClient: true }
)
expect(denied.allowed).toBe(false)
expect(mockConcurrencyClient.release).not.toHaveBeenCalled()
await active.release()
})
})