Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
14.8 KB
Raw Blame
import { beforeEach, describe, expect, it, vi } from 'vitest'
import { createHash } from 'node:crypto'
import { rootLogger } from '../src/observability/logger.js'
import { rateLimitBackendErrorsTotal } from '../src/observability/metrics.js'
import {
  acquireRateLimitConcurrencyLease,
  checkAndIncrement,
  cleanupExpiredBuckets,
  currentWindowStartMs,
  startRateLimiterCleanup,
  stopRateLimiterCleanup,
} from '../src/services/rateLimiterService.js'

async function errorCount(): Promise<number> {
  const { values } = await rateLimitBackendErrorsTotal.get()
  return values[0]?.value ?? 0
}

// In-memory simulation of (bucket_key, window_start_ms) → count, matching the
// real rate_limit_buckets unique index semantics.
const buckets = new Map<string, number>()
const advisoryLocks = new Set<string>()
const mockClientQuery = vi.fn(async (sql: unknown, params?: unknown[]) => {
  const text = String(sql)
  const identity = `${String(params?.[0])}:${String(params?.[1])}`
  if (text.includes('pg_try_advisory_lock')) {
    if (advisoryLocks.has(identity)) return { rows: [{ acquired: false }] }
    advisoryLocks.add(identity)
    return { rows: [{ acquired: true }] }
  }
  if (text.includes('pg_advisory_unlock')) {
    const unlocked = advisoryLocks.delete(identity)
    return { rows: [{ pg_advisory_unlock: unlocked }] }
  }
  return { rows: [] }
})
const mockConcurrencyClient = {
  query: mockClientQuery,
  once: vi.fn(),
  release: vi.fn(),
}
const mockPoolConnect = vi.fn(async () => mockConcurrencyClient)

const mockRateLimitPoolQuery = vi.fn(async (sql: unknown, params?: unknown[]) => {
  const text = typeof sql === 'string' ? sql : ''
  if (/INSERT INTO rate_limit_buckets/i.test(text)) {
    const bucketKey = String(params?.[0] ?? '')
    const windowStart = Number(params?.[1] ?? 0)
    const mapKey = `${bucketKey}|${windowStart}`
    const next = (buckets.get(mapKey) ?? 0) + Number(params?.[2] ?? 1)
    buckets.set(mapKey, next)
    return { rows: [{ count: next }], rowCount: 1 }
  }
  if (/DELETE FROM rate_limit_buckets/i.test(text)) {
    const cutoff = Number(params?.[0] ?? 0)
    let removed = 0
    for (const key of Array.from(buckets.keys())) {
      const windowStart = Number(key.split('|')[1])
      if (windowStart < cutoff) {
        buckets.delete(key)
        removed++
      }
    }
    return { rows: [], rowCount: removed }
  }
  return { rows: [], rowCount: 0 }
})

// The limiter's upserts and prune go to the dedicated limiter pool; the core
// pool only lends the long-lived advisory-lock client. A limiter statement on
// the core pool is a wiring regression, so the core query rejects loudly.
const mockCorePoolQuery = vi.fn(async (sql: unknown) => {
  if (String(sql).includes('DELETE FROM password_identifier_state'))
    return { rows: [], rowCount: 0 }
  throw new Error(`core pool must not serve the rate limiter: ${String(sql).slice(0, 40)}`)
})

vi.mock('../src/db.js', () => ({
  pool: {
    query: (...args: unknown[]) => mockCorePoolQuery(args[0]),
    connect: (...args: unknown[]) => mockPoolConnect(...args),
  },
  rateLimitPool: {
    query: (...args: unknown[]) =>
      mockRateLimitPoolQuery(args[0], args[1] as unknown[] | undefined),
  },
  withTransaction: vi.fn(),
}))

describe('rateLimiterService', () => {
  beforeEach(() => {
    buckets.clear()
    advisoryLocks.clear()
    mockRateLimitPoolQuery.mockClear()
    mockCorePoolQuery.mockClear()
    mockClientQuery.mockClear()
  })

  it('keeps legacy upserts/prune on the limiter pool and credential cleanup on the core pool', async () => {
    const r = await checkAndIncrement('test:bucket:pool-wiring', 5)
    await cleanupExpiredBuckets(1_700_000_000_000)

    // Witness: both statements reached the limiter pool.
    expect(r).toMatchObject({ allowed: true, count: 1, backendAvailable: true })
    expect(
      mockRateLimitPoolQuery.mock.calls.map(([sql]) => String(sql).trim().split(/\s+/)[0])
    ).toEqual(['INSERT', 'DELETE'])
    expect(mockCorePoolQuery).toHaveBeenCalledExactlyOnceWith(
      expect.stringContaining('DELETE FROM password_identifier_state')
    )
  })

  it('allows requests up to the limit and denies the (limit+1)-th in the same window', async () => {
    const key = 'test:bucket:1'
    const limit = 5
    const nowMs = 1_700_000_000_000
    for (let i = 0; i < limit; i++) {
      const r = await checkAndIncrement(key, limit, nowMs)
      expect(r.allowed).toBe(true)
      expect(r.count).toBe(i + 1)
      expect(r.remaining).toBe(limit - (i + 1))
    }
    const over = await checkAndIncrement(key, limit, nowMs)
    expect(over.allowed).toBe(false)
    expect(over.count).toBe(limit + 1)
    expect(over.remaining).toBe(0)
  })

  it('exposes resetMs aligned to the next 60s window boundary', async () => {
    const nowMs = 1_700_000_000_000
    const r = await checkAndIncrement('test:bucket:reset', 10, nowMs)
    expect(r.windowStartMs).toBe(Math.floor(nowMs / 60_000) * 60_000)
    expect(r.resetMs).toBe(r.windowStartMs + 60_000)
    expect(r.resetMs - nowMs).toBeLessThanOrEqual(60_000)
  })

  it('rolls over to a fresh slot when the 60s window advances', async () => {
    const key = 'test:bucket:roll'
    const limit = 2
    const t0 = 1_700_000_000_000 // aligned to minute start
    // Fill the window at t0.
    await checkAndIncrement(key, limit, t0)
    await checkAndIncrement(key, limit, t0)
    const denied = await checkAndIncrement(key, limit, t0)
    expect(denied.allowed).toBe(false)

    // New window (+60_001 ms) — fresh counter.
    const fresh = await checkAndIncrement(key, limit, t0 + 60_001)
    expect(fresh.allowed).toBe(true)
    expect(fresh.count).toBe(1)
  })

  it('concurrent increments observe distinct counter values (no race)', async () => {
    const key = 'test:bucket:concurrent'
    const limit = 100
    const nowMs = 1_700_000_000_000
    const results = await Promise.all(
      Array.from({ length: 10 }, () => checkAndIncrement(key, limit, nowMs))
    )
    const counts = results.map(r => r.count).sort((a, b) => a - b)
    expect(counts).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
    expect(results.every(r => r.allowed)).toBe(true)
  })

  it('atomically charges a weighted cost in the existing PostgreSQL bucket', async () => {
    const first = await checkAndIncrement('test:bucket:weighted', 16, 1_700_000_000_000, 7)
    const second = await checkAndIncrement('test:bucket:weighted', 16, 1_700_000_000_000, 10)
    expect(first).toMatchObject({ allowed: true, count: 7, remaining: 9 })
    expect(second).toMatchObject({ allowed: false, count: 17, remaining: 0 })
    expect(mockRateLimitPoolQuery).toHaveBeenLastCalledWith(expect.any(String), [
      'test:bucket:weighted',
      currentWindowStartMs(1_700_000_000_000),
      10,
    ])
  })

  it('holds replica-safe advisory slots until release and then admits the next request', async () => {
    const requirements = [{ bucketKey: 'gfs-upload:test-active', maxConcurrent: 2 }]
    const [first, second, denied] = await Promise.all([
      acquireRateLimitConcurrencyLease(requirements),
      acquireRateLimitConcurrencyLease(requirements),
      acquireRateLimitConcurrencyLease(requirements),
    ])
    expect(first).toMatchObject({ allowed: true, backendAvailable: true })
    expect(second).toMatchObject({ allowed: true, backendAvailable: true })
    expect(denied).toMatchObject({ allowed: false, backendAvailable: true })

    await first.release()
    const admittedAfterRelease = await acquireRateLimitConcurrencyLease(requirements)
    expect(admittedAfterRelease).toMatchObject({ allowed: true, backendAvailable: true })
    await second.release()
    await admittedAfterRelease.release()
    expect(advisoryLocks.size).toBe(0)
    expect(mockPoolConnect).toHaveBeenCalledTimes(1)
  })

  it('destroys the dedicated connection when an advisory unlock outcome is unknown', async () => {
    const lease = await acquireRateLimitConcurrencyLease([
      { bucketKey: 'gfs-upload:unlock-failure', maxConcurrent: 1 },
    ])
    expect(lease.allowed).toBe(true)
    mockClientQuery.mockRejectedValueOnce(new Error('connection lost during unlock'))

    await lease.release()

    expect(mockConcurrencyClient.release).toHaveBeenCalledWith(true)
    // The fake client does not emulate PostgreSQL's session teardown; model
    // the advisory lock release that the destroyed real connection provides.
    advisoryLocks.clear()
    const replacement = await acquireRateLimitConcurrencyLease([
      { bucketKey: 'gfs-upload:unlock-failure', maxConcurrent: 1 },
    ])
    expect(replacement).toMatchObject({ allowed: true, backendAvailable: true })
    await replacement.release()
  })

  it('reports the backend unavailable when the limiter pool query throws (DB error); the caller decides the policy', async () => {
    mockRateLimitPoolQuery.mockRejectedValueOnce(new Error('connection refused'))
    const r = await checkAndIncrement('test:bucket:failopen', 5)
    expect(r.allowed).toBe(true)
    expect(r.count).toBe(0)
    expect(r.remaining).toBe(5)
    expect(r.backendAvailable).toBe(false)
  })

  it('logs the SHA-256 of the bucket key on a DB error, never the key or the user id it carries', async () => {
    const desktopUserId = '5f0c2d1e-7a4b-4c3d-9e8f-0a1b2c3d4e5f'
    const bucketKey = `gfs-ext:pre:token:user:${desktopUserId}`
    const warn = vi.spyOn(rootLogger, 'warn').mockImplementation(() => undefined)
    try {
      mockRateLimitPoolQuery.mockRejectedValueOnce(new Error('connection refused'))
      const r = await checkAndIncrement(bucketKey, 5)
      expect(r.backendAvailable).toBe(false)
      // Witness: the DB error wrote exactly one line, and it is that event.
      expect(warn).toHaveBeenCalledTimes(1)
      expect(warn.mock.calls[0]?.[0]).toEqual({
        event: 'rate_limit_db_error',
        hashedKey: createHash('sha256').update(bucketKey).digest('hex'),
        err: 'connection refused',
        suppressed: 0,
      })
      expect(JSON.stringify(warn.mock.calls)).not.toContain(desktopUserId)
    } finally {
      warn.mockRestore()
    }
  })

  it('writes one DB-error line per bucket per minute, counts every error, and reports the suppressed count', async () => {
    const bucketKey = 'test:bucket:dberror-throttle'
    const warn = vi.spyOn(rootLogger, 'warn').mockImplementation(() => undefined)
    vi.useFakeTimers({ toFake: ['Date'] })
    try {
      vi.setSystemTime(1_800_000_000_000)
      const before = await errorCount()
      for (let i = 0; i < 50; i += 1) {
        mockRateLimitPoolQuery.mockRejectedValueOnce(new Error('connection refused'))
        await checkAndIncrement(bucketKey, 5)
      }

      // Witness: the one line exists and is this bucket's DB error.
      expect(warn).toHaveBeenCalledTimes(1)
      expect(warn.mock.calls[0]?.[0]).toMatchObject({ event: 'rate_limit_db_error', suppressed: 0 })
      expect((await errorCount()) - before).toBe(50)

      vi.setSystemTime(1_800_000_060_000)
      mockRateLimitPoolQuery.mockRejectedValueOnce(new Error('connection refused'))
      await checkAndIncrement(bucketKey, 5)
      expect(warn).toHaveBeenCalledTimes(2)
      expect(warn.mock.calls[1]?.[0]).toMatchObject({
        event: 'rate_limit_db_error',
        suppressed: 49,
      })
      expect((await errorCount()) - before).toBe(51)
    } finally {
      vi.useRealTimers()
      warn.mockRestore()
    }
  })

  it('reports the backend unavailable when the limiter pool query returns empty rows', async () => {
    mockRateLimitPoolQuery.mockResolvedValueOnce({ rows: [], rowCount: 0 })
    const r = await checkAndIncrement('test:bucket:emptyrows', 5)
    expect(r.allowed).toBe(true)
    expect(r.count).toBe(0)
    expect(r.backendAvailable).toBe(false)
  })

  it('cleanupExpiredBuckets removes rows older than 5 minutes', async () => {
    const now = 1_700_000_000_000
    // Seed two buckets: one old (>5 min ago), one fresh.
    const oldKey = `old-bucket|${now - 10 * 60_000}` // 10 min ago
    const freshKey = `fresh-bucket|${now - 60_000}` // 1 min ago
    buckets.set(oldKey, 1)
    buckets.set(freshKey, 1)

    const removed = await cleanupExpiredBuckets(now)
    expect(removed).toBe(1)
    expect(buckets.has(oldKey)).toBe(false)
    expect(buckets.has(freshKey)).toBe(true)
  })

  it('currentWindowStartMs floors now to the minute boundary', () => {
    const minuteStart = 1_700_000_040_000 // divisible by 60_000
    expect(minuteStart % 60_000).toBe(0)
    const t = minuteStart + 37_000 // 37s past a minute
    expect(currentWindowStartMs(t)).toBe(minuteStart)
    expect(currentWindowStartMs(minuteStart)).toBe(minuteStart)
  })

  it('startRateLimiterCleanup + stopRateLimiterCleanup are idempotent', () => {
    // No assertion beyond "does not throw" — this is a lifecycle smoke test.
    startRateLimiterCleanup(60_000)
    startRateLimiterCleanup(60_000) // second call is no-op
    stopRateLimiterCleanup()
    stopRateLimiterCleanup() // double stop is safe
  })
  it('releases the idle dedicated session after finite credential work', async () => {
    mockConcurrencyClient.release.mockClear()
    const lease = await acquireRateLimitConcurrencyLease(
      [{ bucketKey: 'finite-password-work', maxConcurrent: 1 }],
      { releaseIdleClient: true }
    )
    expect(lease.allowed).toBe(true)
    await lease.release()
    expect(mockConcurrencyClient.release).toHaveBeenCalledExactlyOnceWith(true)
  })
  it('releases an idle dedicated session after busy denial', async () => {
    mockConcurrencyClient.release.mockClear()
    mockClientQuery.mockResolvedValueOnce({ rows: [{ acquired: false }] } as never)
    const lease = await acquireRateLimitConcurrencyLease(
      [{ bucketKey: 'finite-busy-work', maxConcurrent: 1 }],
      { releaseIdleClient: true }
    )
    expect(lease.allowed).toBe(false)
    expect(lease.backendAvailable).toBe(true)
    expect(mockConcurrencyClient.release).toHaveBeenCalledExactlyOnceWith(true)
  })
  it('releases an idle dedicated session after an admission query failure', async () => {
    mockConcurrencyClient.release.mockClear()
    mockClientQuery.mockRejectedValueOnce(new Error('synthetic admission failure'))
    const lease = await acquireRateLimitConcurrencyLease(
      [{ bucketKey: 'finite-failed-work', maxConcurrent: 1 }],
      { releaseIdleClient: true }
    )
    expect(lease.allowed).toBe(false)
    expect(lease.backendAvailable).toBe(false)
    expect(mockConcurrencyClient.release).toHaveBeenCalledExactlyOnceWith(true)
  })
  it('retains the dedicated session when another lease remains active', async () => {
    const active = await acquireRateLimitConcurrencyLease([
      { bucketKey: 'retained-active-work', maxConcurrent: 1 },
    ])
    mockConcurrencyClient.release.mockClear()
    mockClientQuery.mockResolvedValueOnce({ rows: [{ acquired: false }] } as never)
    const denied = await acquireRateLimitConcurrencyLease(
      [{ bucketKey: 'finite-denied-work', maxConcurrent: 1 }],
      { releaseIdleClient: true }
    )
    expect(denied.allowed).toBe(false)
    expect(mockConcurrencyClient.release).not.toHaveBeenCalled()
    await active.release()
  })
})