Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
10.3 KB
Raw Blame
import { NextFunction, Request, Response } from 'express'
import { createHash } from 'node:crypto'
import { LogThrottle } from '../observability/logThrottle.js'
import { rateLimitHitsTotal } from '../observability/metrics.js'
import { boundedBucketKey } from '../services/rateLimitBucketKey.js'
import {
  RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS,
  checkAndIncrement,
} from '../services/rateLimiterService.js'
import { ProcessMemoryRateLimiter } from './processMemoryRateLimiter.js'

/** What a limiter does when its Postgres backend cannot count a request. */
export type RateLimitBackendUnavailableMode = 'process-memory' | 'closed'

// Compile-time guard, checked by the CI typecheck of src/. The `'open'` mode
// (let the request through uncounted) was removed; adding it back to the union
// makes this alias fail to compile (TS2344).
type AssertTrue<T extends true> = T
export type OpenIsNotABackendUnavailableMode = AssertTrue<
  'open' extends RateLimitBackendUnavailableMode ? false : true
>

export type RateLimitEnforcerOptions = {
  bucketType: string
  maxPerMinute: number
  onBackendUnavailable: RateLimitBackendUnavailableMode
  onLimited?: (req: Request, res: Response, retryAfterSeconds: number) => void
}

/**
 * Charges one request to `key` and answers it when it must not proceed.
 * Resolves `true` when the caller may continue (the `X-RateLimit-*` headers are
 * set), or `false` after a 429 or 503 has been sent.
 */
export type RateLimitEnforcer = (req: Request, res: Response, key: string) => Promise<boolean>

/** The last enforcing counter owns every advertised quota/header family. */
function writeRateLimitHeaders(
  res: Response,
  limit: number,
  remaining: number,
  resetMs: number
): void {
  const resetSeconds = Math.max(1, Math.ceil((resetMs - Date.now()) / 1000))
  res.setHeader('X-RateLimit-Limit', String(limit))
  res.setHeader('X-RateLimit-Remaining', String(remaining))
  res.setHeader('X-RateLimit-Reset', String(Math.floor(resetMs / 1000)))
  res.setHeader('RateLimit', `limit=${limit}, remaining=${remaining}, reset=${resetSeconds}`)
  res.setHeader('RateLimit-Policy', `${limit};w=60`)
  // Keep existing draft-6 clients consistent with the ledger as well.
  res.setHeader('RateLimit-Limit', String(limit))
  res.setHeader('RateLimit-Remaining', String(remaining))
  res.setHeader('RateLimit-Reset', String(resetSeconds))
}

/**
 * Factory for a per-request rate limit middleware backed by a PG token bucket.
 *
 * Usage:
 *   router.post("/foo",
 *     rateLimitMiddleware({
 *       bucketType: "recipe",
 *       maxPerMinute: 10,
 *       getBucketKey: (req) => mcpHostRateLimitBucketKey('recipe', req.mcpHostJwt),
 *       onBackendUnavailable: 'process-memory',
 *     }),
 *     handler,
 *   );
 *
 * The middleware:
 *   - Returns 429 with `Retry-After` (seconds until window end) when the limit
 *     is exceeded.
 *   - Emits `rate_limit_hits_total{bucket_type, result}` on every call, with
 *     `result` one of `allowed`, `denied`, `unavailable`, `fallback_allowed`,
 *     `fallback_denied`.
 *   - If `getBucketKey` returns `null` (e.g. unable to derive key), the
 *     request is *allowed* but not counted — fail-open. Callers that need
 *     strict enforcement should return a sentinel key instead.
 *   - A key longer than 512 UTF-8 bytes is counted under its SHA-256
 *     (`boundedBucketKey`), so a client-supplied value cannot grow either
 *     backend's storage per request.
 *   - When the limiter backend cannot count the request
 *     (`backendAvailable: false`: the limiter pool is saturated, Postgres
 *     returned an error, or the upsert returned no row), `onBackendUnavailable`
 *     decides. The option is required so every caller states that choice.
 *       - `'closed'` answers 503 `rate_limit_unavailable` with `Retry-After`.
 *       - `'process-memory'` counts the request in this middleware instance's
 *         own in-memory counter: the same `maxPerMinute`, in a 60 s window per
 *         key. Over the limit it answers 429 like a Postgres denial, and the
 *         warn line carries `source: 'process-memory'`. The in-memory counter
 *         is used only for requests Postgres could not count; the two counts
 *         are never added together. So when an outage ends inside a window, a
 *         key can get up to 2 × `maxPerMinute` in that window: up to the limit
 *         counted in memory during the outage, then up to the limit counted in
 *         Postgres after it. An upsert that commits but whose reply is lost is
 *         counted by both, which only denies sooner. It tracks at most 100 000
 *         keys per 60 s;
 *         a new key beyond that answers 503 as `'closed'` does. The counter
 *         lives in one process: control-api runs one replica today, so the
 *         in-memory limit holds for the whole service only while that is
 *         true. With N replicas a caller can get N × `maxPerMinute` requests
 *         per minute while Postgres is unavailable.
 *   - A 503 carries no `X-RateLimit-*` headers, including ones an earlier
 *     limiter on the same route set, because no count backs them.
 */
export function rateLimitMiddleware(
  opts: RateLimitEnforcerOptions & { getBucketKey: (req: Request) => string | null }
) {
  const enforce = createRateLimitEnforcer(opts)
  return function rateLimitMw(req: Request, res: Response, next: NextFunction): void {
    void (async () => {
      try {
        const key = opts.getBucketKey(req)
        if (!key) {
          // Fail-open: no key means we cannot attribute the call (usually a
          // pre-auth path); let it through rather than 500.
          next()
          return
        }
        if (await enforce(req, res, key)) next()
      } catch (err) {
        next(err)
      }
    })()
  }
}

/** Remove every quota family when no authoritative count may be advertised. */
export function clearRateLimitHeaders(res: Response): void {
  res.removeHeader('X-RateLimit-Limit')
  res.removeHeader('X-RateLimit-Remaining')
  res.removeHeader('X-RateLimit-Reset')
  res.removeHeader('RateLimit')
  res.removeHeader('RateLimit-Policy')
  res.removeHeader('RateLimit-Limit')
  res.removeHeader('RateLimit-Remaining')
  res.removeHeader('RateLimit-Reset')
}

/**
 * The limiter behind `rateLimitMiddleware`, for a route that can only derive
 * its key inside the handler (after authenticating the caller). Create it once
 * per route set, not per request: with `'process-memory'` it owns the
 * in-memory counter. Same semantics as `rateLimitMiddleware`.
 */
export function createRateLimitEnforcer(opts: RateLimitEnforcerOptions): RateLimitEnforcer {
  // While the backend is down every request on this bucket fails the same way,
  // so the warn line is written once per minute; the counter records each one.
  const unavailableLogThrottle = new LogThrottle(60_000)
  const processMemory =
    opts.onBackendUnavailable === 'process-memory'
      ? new ProcessMemoryRateLimiter(opts.maxPerMinute)
      : null

  function answerUnavailable(req: Request, res: Response): void {
    rateLimitHitsTotal.inc({ bucket_type: opts.bucketType, result: 'unavailable' }, 1)
    const suppressed = unavailableLogThrottle.admit(opts.bucketType)
    if (req.log && suppressed !== undefined) {
      req.log.warn(
        { event: 'rate_limit_unavailable', bucketType: opts.bucketType, suppressed },
        'rate limit backend unavailable'
      )
    }
    clearRateLimitHeaders(res)
    res.setHeader('Retry-After', String(RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS))
    res.setHeader('Cache-Control', 'no-store')
    res.status(503).json({
      error: 'rate_limit_unavailable',
      retryAfterSeconds: RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS,
    })
  }

  function answerDenied(
    req: Request,
    res: Response,
    key: string,
    denial: { count: number; resetMs: number; source?: 'process-memory' }
  ): void {
    const retryAfterSec = Math.max(1, Math.ceil((denial.resetMs - Date.now()) / 1000))
    res.setHeader('Retry-After', String(retryAfterSec))
    writeRateLimitHeaders(res, opts.maxPerMinute, 0, denial.resetMs)
    if (req.log) {
      req.log.warn(
        {
          event: 'rate_limit_denied',
          bucketType: opts.bucketType,
          // Keys carry user ids (e.g. desktopUserId). The SHA-256 is a
          // correlation id, not anonymization: it is unsalted, so a
          // low-entropy key (an IP, an email) can be recovered by guessing.
          hashedKey: createHash('sha256').update(key).digest('hex'),
          count: denial.count,
          maxPerMinute: opts.maxPerMinute,
          ...(denial.source === undefined ? {} : { source: denial.source }),
        },
        'rate limit exceeded'
      )
    }
    if (opts.onLimited) opts.onLimited(req, res, retryAfterSec)
    else
      res.status(429).json({
        error: 'Too Many Requests',
        code: 'rate_limited',
        message: `This request limit has been reached. Try again in ${retryAfterSec} seconds.`,
        retryAfterSeconds: retryAfterSec,
      })
  }

  return async function enforce(req: Request, res: Response, rawKey: string): Promise<boolean> {
    const key = boundedBucketKey(rawKey)
    const result = await checkAndIncrement(key, opts.maxPerMinute)
    if (!result.backendAvailable) {
      if (processMemory === null) {
        answerUnavailable(req, res)
        return false
      }
      const decision = await processMemory.hit(key)
      if (decision.outcome === 'key_cap_reached') {
        answerUnavailable(req, res)
        return false
      }
      if (decision.outcome === 'denied') {
        rateLimitHitsTotal.inc({ bucket_type: opts.bucketType, result: 'fallback_denied' }, 1)
        answerDenied(req, res, key, {
          count: decision.totalHits,
          resetMs: decision.resetMs,
          source: 'process-memory',
        })
        return false
      }
      rateLimitHitsTotal.inc({ bucket_type: opts.bucketType, result: 'fallback_allowed' }, 1)
      writeRateLimitHeaders(
        res,
        opts.maxPerMinute,
        opts.maxPerMinute - decision.totalHits,
        decision.resetMs
      )
      return true
    }
    if (!result.allowed) {
      rateLimitHitsTotal.inc({ bucket_type: opts.bucketType, result: 'denied' }, 1)
      answerDenied(req, res, key, { count: result.count, resetMs: result.resetMs })
      return false
    }

    rateLimitHitsTotal.inc({ bucket_type: opts.bucketType, result: 'allowed' }, 1)
    writeRateLimitHeaders(res, opts.maxPerMinute, result.remaining, result.resetMs)
    return true
  }
}