Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
17.1 KB
Raw Blame
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'
import { randomBytes, randomUUID } from 'node:crypto'
import { Pool, type PoolClient } from 'pg'
import request from 'supertest'
import { endPoolAndWaitForClients } from './helpers/realPostgresTeardown.js'

// The external GFS limiter's Postgres backend, against real PostgreSQL at the
// production pool bounds: the CORE_POOL_* and RATE_LIMIT_POOL_* variables are
// deleted before db.js is imported. Requests go through the production
// Express app; the ONLY mock is the session-token verifier (identity), as in
// routes.externalGfsRateLimitStress. Skipped without
// CONTROL_API_REAL_PG_ADMIN_URL.

vi.mock('../src/utils/auth/externalSessionAuthToken.js', () => ({
  // A token that is not a JSON identity fails verification, as a bad
  // signature does in production.
  verifyExternalSessionToken: (token: string) => {
    if (!token.startsWith('{')) return null
    const parsed = JSON.parse(token) as { userId: string }
    return {
      userId: parsed.userId,
      email: `${parsed.userId}@example.test`,
      teamId: null,
      role: 'member',
      authGeneration: 1,
      exp: Math.floor(Date.now() / 1000) + 3600,
    }
  },
}))

const LIMITER_POOL_MAX = 6
const LIMITER_ACQUIRE_TIMEOUT_MS = 5_000
// An affordances read walks four Postgres buckets in series (three
// pre-resolution, one resolved). Were the limiter to fail open again, each
// would wait out the acquire timeout before the request reached the handler;
// the budget covers that walk so the regression fails on the status assertion,
// not on the test timeout.
const T3_TIMEOUT_MS = (4 + 2) * LIMITER_ACQUIRE_TIMEOUT_MS
const CORE_POOL_MAX = 10
const CORE_ACQUIRE_TIMEOUT_MS = 2_000
// T5: serial same-row upserts per pool. 10 ms per upsert is the regression
// line; the plan's reversal rule (report and stop) sits at 2.5 ms with sync off.
const T5_UPSERTS = 200
const T5_MAX_TOTAL_MS = T5_UPSERTS * 10

function databaseUrl(baseUrl: string, database: string): string {
  const url = new URL(baseUrl)
  url.pathname = `/${database}`
  return url.toString()
}

function quoteIdent(value: string): string {
  return `"${value.replace(/"/g, '""')}"`
}

const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL
const describeRealPostgres = adminUrl ? describe : describe.skip

describeRealPostgres('external GFS rate limiter backend (real PostgreSQL)', () => {
  const database = `control_api_gfs_rl_fail_closed_${randomBytes(6).toString('hex')}`
  const envKeys = [
    'CONTROL_API_PG_CONNECTION_STRING',
    'CORE_POOL_MAX',
    'CORE_POOL_CONNECTION_TIMEOUT_MS',
    'CORE_POOL_STATEMENT_TIMEOUT_MS',
    'RATE_LIMIT_POOL_MAX',
    'RATE_LIMIT_POOL_CONNECTION_TIMEOUT_MS',
    'RATE_LIMIT_POOL_STATEMENT_TIMEOUT_MS',
  ] as const
  const previousEnv = new Map<string, string | undefined>()

  let adminPool: Pool
  let corePool: Pool
  let limiterPool: Pool
  let mod: {
    createApp: (gateway: unknown) => import('express').Express
    config: typeof import('../src/config.js').config
    requestsTotal: typeof import('../src/observability/metrics.js').externalGfsRateLimitRequestsTotal
    MockGateway: typeof import('./mockGateway.js').MockGateway
  }

  beforeAll(async () => {
    if (adminUrl === undefined) {
      throw new Error('CONTROL_API_REAL_PG_ADMIN_URL is required by this suite')
    }
    const connectionString = databaseUrl(adminUrl, database)
    for (const key of envKeys) {
      previousEnv.set(key, process.env[key])
      delete process.env[key]
    }
    adminPool = new Pool({ connectionString: adminUrl })
    await adminPool.query(`CREATE DATABASE ${quoteIdent(database)}`)
    process.env.CONTROL_API_PG_CONNECTION_STRING = connectionString

    const dbMod = await import('../src/db.js')
    corePool = dbMod.pool as unknown as Pool
    corePool.on('error', () => {})
    limiterPool = dbMod.rateLimitPool as unknown as Pool
    limiterPool.on('error', () => {})
    const migratePool = new Pool({ connectionString })
    await dbMod.initDb({ connect: () => migratePool.connect() })
    await endPoolAndWaitForClients(migratePool)

    const appMod = await import('../src/app.js')
    const configMod = await import('../src/config.js')
    const metricsMod = await import('../src/observability/metrics.js')
    const { MockGateway } = await import('./mockGateway.js')
    mod = {
      createApp: appMod.createApp as never,
      config: configMod.config,
      requestsTotal: metricsMod.externalGfsRateLimitRequestsTotal,
      MockGateway,
    }
  }, 60_000)

  afterAll(async () => {
    try {
      for (const key of envKeys) {
        const value = previousEnv.get(key)
        if (value === undefined) delete process.env[key]
        else process.env[key] = value
      }
      await endPoolAndWaitForClients(corePool).catch(() => {})
      await endPoolAndWaitForClients(limiterPool).catch(() => {})
      if (adminPool) {
        await adminPool.query(
          `SELECT pg_terminate_backend(pid) FROM pg_stat_activity
        WHERE datname = $1 AND pid <> pg_backend_pid()`,
          [database]
        )
        await adminPool.query(`DROP DATABASE IF EXISTS ${quoteIdent(database)}`)
      }
    } finally {
      await adminPool?.end()
    }
  })

  it('U1: the limiter pool runs with synchronous_commit off and a 3 s statement timeout; the core pool keeps the defaults', async () => {
    type Options = {
      options: { max: number; connectionTimeoutMillis: number; statement_timeout: number }
    }
    expect((limiterPool as unknown as Options).options).toMatchObject({
      max: 6,
      connectionTimeoutMillis: 5_000,
      statement_timeout: 3_000,
    })

    const settings = `SELECT current_setting('synchronous_commit') AS sync,
                             current_setting('statement_timeout') AS timeout,
                             current_database() AS db`
    const limiter = await limiterPool.query<{ sync: string; timeout: string; db: string }>(settings)
    const core = await corePool.query<{ sync: string; timeout: string; db: string }>(settings)

    // Witness: both answers come from this suite's database.
    expect(limiter.rows[0]?.db).toBe(database)
    expect(core.rows[0]?.db).toBe(database)
    expect(limiter.rows[0]).toMatchObject({ sync: 'off', timeout: '3s' })
    expect(core.rows[0]).toMatchObject({ sync: 'on', timeout: '15s' })
  })

  async function seedUser(): Promise<string> {
    const id = randomUUID()
    await corePool.query(`INSERT INTO users (id, email, name) VALUES ($1, $2, $3)`, [
      id,
      `${id}@example.test`,
      'fail-closed',
    ])
    return id
  }

  async function getAffordances(app: import('express').Express, userId: string) {
    const internalToken = mod.config.internalServiceTokens['external-rest-api']
    if (!internalToken) throw new Error('config has no external-rest-api internal service token')
    return request(app)
      .get(`/api/v1/external/gfs/resources/${randomUUID()}/affordances?drive=main`)
      .set('Authorization', `Bearer ${internalToken}`)
      .set('x-service-token', 'external-rest-api')
      .set('x-user-session-token', JSON.stringify({ userId }))
      .set('x-forwarded-for', '203.0.113.7')
  }

  async function unavailableDecisions(): Promise<number> {
    const { values } = await mod.requestsTotal.get()
    return values
      .filter(sample => sample.labels.outcome === 'unavailable')
      .reduce((total, sample) => total + sample.value, 0)
  }

  /** Read through the core pool: the limiter pool is the one being held. */
  async function ledgerKeys(): Promise<string[]> {
    const result = await corePool.query<{ bucket_key: string }>(
      'SELECT bucket_key FROM rate_limit_buckets ORDER BY bucket_key'
    )
    return result.rows.map(row => row.bucket_key)
  }

  it(
    'T3: with every limiter connection held, an external GFS read fails closed with 503 and is not counted; after release it passes',
    async () => {
      const app = mod.createApp(new mod.MockGateway())
      const userId = await seedUser()
      await corePool.query('DELETE FROM rate_limit_buckets')
      const unavailableBefore = await unavailableDecisions()

      const held: PoolClient[] = []
      let denied: Awaited<ReturnType<typeof getAffordances>>
      let elapsedMs: number
      try {
        for (let index = 0; index < LIMITER_POOL_MAX; index += 1) {
          held.push(await limiterPool.connect())
        }
        // Witness: the pool is exhausted, not merely busy.
        expect(limiterPool.totalCount).toBe(LIMITER_POOL_MAX)
        expect(limiterPool.idleCount).toBe(0)

        const startedAt = Date.now()
        denied = await getAffordances(app, userId)
        elapsedMs = Date.now() - startedAt
      } finally {
        for (const client of held) client.release()
      }

      expect(denied.status).toBe(503)
      expect(denied.body).toEqual({ error: 'gfs_rate_limit_unavailable', retryAfterSeconds: 2 })
      expect(denied.headers['retry-after']).toBe('2')
      expect(denied.headers['cache-control']).toBe('no-store')
      // Witness: the 503 came from waiting out the limiter's acquire timeout.
      expect(elapsedMs).toBeGreaterThanOrEqual(LIMITER_ACQUIRE_TIMEOUT_MS - 250)
      expect((await unavailableDecisions()) - unavailableBefore).toBe(1)
      expect(await ledgerKeys()).toEqual([])

      const allowed = await getAffordances(app, userId)
      expect(allowed.status).toBe(200)
      // Liveness witness for the empty ledger above: the same request, once the
      // pool is free, is counted in rate_limit_buckets.
      const keys = await ledgerKeys()
      expect(keys.length).toBeGreaterThan(0)
      expect(keys.every(key => key.startsWith('gfs-ext:'))).toBe(true)
      expect((await unavailableDecisions()) - unavailableBefore).toBe(1)
    },
    T3_TIMEOUT_MS
  )

  // Holding the pool (T3) cannot reach the grants rateLimitMiddleware: the
  // pre-resolution Postgres bucket fails closed first. A trigger instead
  // fails every limiter upsert whose key is not an external GFS Postgres
  // bucket, so those keep counting and the grants bucket gets a real query
  // error (plan addendum A2).
  async function injectLimiterFault(): Promise<void> {
    await corePool.query(`
      CREATE FUNCTION t3b_limiter_fault() RETURNS trigger LANGUAGE plpgsql AS $$
      BEGIN
        IF NEW.bucket_key NOT LIKE 'gfs-ext:%' THEN
          RAISE EXCEPTION 't3b injected limiter fault';
        END IF;
        RETURN NEW;
      END
      $$`)
    await corePool.query(`
      CREATE TRIGGER t3b_limiter_fault BEFORE INSERT ON rate_limit_buckets
        FOR EACH ROW EXECUTE FUNCTION t3b_limiter_fault()`)
  }

  async function removeLimiterFault(): Promise<void> {
    await corePool.query('DROP TRIGGER IF EXISTS t3b_limiter_fault ON rate_limit_buckets')
    await corePool.query('DROP FUNCTION IF EXISTS t3b_limiter_fault()')
  }

  it('T3b: the external grants limiter fails closed on a backend error while a process-memory route still passes', async () => {
    const app = mod.createApp(new mod.MockGateway())
    const userId = await seedUser()
    const internalToken = mod.config.internalServiceTokens['external-rest-api']
    if (!internalToken) throw new Error('config has no external-rest-api internal service token')
    const external = (req: request.Test) =>
      req
        .set('Authorization', `Bearer ${internalToken}`)
        .set('x-service-token', 'external-rest-api')
        .set('x-user-session-token', JSON.stringify({ userId }))
        .set('x-forwarded-for', '203.0.113.7')
    await corePool.query('DELETE FROM rate_limit_buckets')

    let grants: request.Response
    let entityChanges: request.Response
    let ack: request.Response
    await injectLimiterFault()
    try {
      grants = await external(
        request(app).get(`/api/v1/external/gfs/grants?drive=main&resourceId=${randomUUID()}`)
      )
      entityChanges = await external(request(app).get('/api/v1/external/entity-changes/stream'))
      ack = await external(request(app).post(`/api/v1/external/notifications/${randomUUID()}/ack`))
    } finally {
      await removeLimiterFault()
    }

    // The 503 comes from rateLimitMiddleware, not from the external GFS
    // Postgres buckets (whose body is gfs_rate_limit_unavailable).
    expect(grants.status).toBe(503)
    expect(grants.body).toEqual({ error: 'rate_limit_unavailable', retryAfterSeconds: 2 })
    expect(grants.headers['retry-after']).toBe('2')
    expect(grants.headers['cache-control']).toBe('no-store')
    // The entity-change stream uses the same shared rate-limit middleware but
    // chooses fail-closed because its principal budget must remain distributed.
    expect(entityChanges.status).toBe(503)
    expect(entityChanges.body).toEqual({ error: 'rate_limit_unavailable', retryAfterSeconds: 2 })
    expect(entityChanges.headers['retry-after']).toBe('2')
    expect(entityChanges.headers['cache-control']).toBe('no-store')
    // Witness for the process-memory policy under the same fault: the ack
    // route's Postgres limiter could not count either, its in-memory counter
    // admitted the first request, and its handler answered.
    expect(ack.status).toBe(404)
    expect(ack.body).toEqual({ error: 'notification_not_found' })

    // Witness that the fault is scoped as intended: the external GFS Postgres
    // buckets counted, and nothing else reached the ledger.
    const keys = await ledgerKeys()
    expect(keys.length).toBeGreaterThan(0)
    expect(keys.every(key => key.startsWith('gfs-ext:'))).toBe(true)
  }, 30_000)

  it('T4: with every core connection held, session validation answers 503 while an invalid token still answers 401', async () => {
    type Options = { options: { max: number; connectionTimeoutMillis: number } }
    expect((corePool as unknown as Options).options).toMatchObject({
      max: CORE_POOL_MAX,
      connectionTimeoutMillis: CORE_ACQUIRE_TIMEOUT_MS,
    })
    const app = mod.createApp(new mod.MockGateway())
    const userId = await seedUser()
    const internalToken = mod.config.internalServiceTokens['external-rest-api']
    if (!internalToken) throw new Error('config has no external-rest-api internal service token')
    const external = (req: request.Test, sessionToken: string) =>
      req
        .set('Authorization', `Bearer ${internalToken}`)
        .set('x-service-token', 'external-rest-api')
        .set('x-user-session-token', sessionToken)
        .set('x-forwarded-for', '203.0.113.7')
    const affordancesPath = () =>
      `/api/v1/external/gfs/resources/${randomUUID()}/affordances?drive=main`

    const held: PoolClient[] = []
    let unavailable: request.Response
    let invalid: request.Response
    let elapsedMs: number
    try {
      for (let index = 0; index < CORE_POOL_MAX; index += 1) {
        held.push(await corePool.connect())
      }
      // Witness: the core pool is exhausted, not merely busy.
      expect(corePool.totalCount).toBe(CORE_POOL_MAX)
      expect(corePool.idleCount).toBe(0)

      const startedAt = Date.now()
      unavailable = await external(request(app).get(affordancesPath()), JSON.stringify({ userId }))
      elapsedMs = Date.now() - startedAt
      invalid = await external(request(app).get(affordancesPath()), 'not-a-session-token')
    } finally {
      for (const client of held) client.release()
    }

    expect(unavailable.status).toBe(503)
    expect(unavailable.body).toEqual({
      error: 'session_backend_unavailable',
      retryAfterSeconds: 2,
    })
    expect(unavailable.headers['retry-after']).toBe('2')
    expect(unavailable.headers['cache-control']).toBe('no-store')
    // Witness: the 503 came from waiting out the core pool's acquire timeout.
    expect(elapsedMs).toBeGreaterThanOrEqual(CORE_ACQUIRE_TIMEOUT_MS - 250)

    expect(invalid.status).toBe(401)
    expect(invalid.body).toEqual({ error: 'Unauthorized' })

    // Liveness witness: the same valid request, once the pool is free, passes
    // session validation and reaches the handler.
    const allowed = await external(request(app).get(affordancesPath()), JSON.stringify({ userId }))
    expect(allowed.status).toBe(200)
  }, 30_000)

  it('T5 (informational): same-row limiter upserts with synchronous_commit off vs on', async () => {
    const { checkAndIncrementWithQuery } = await import('../src/services/rateLimiterService.js')
    const measure = async (pool: Pool, label: string): Promise<number> => {
      const key = `t5:${label}:${randomUUID()}`
      const nowMs = Date.now()
      const startedAt = process.hrtime.bigint()
      let last = 0
      for (let index = 0; index < T5_UPSERTS; index += 1) {
        const result = await checkAndIncrementWithQuery(
          (text, values) => pool.query(text, values),
          key,
          T5_UPSERTS,
          nowMs
        )
        expect(result.backendAvailable).toBe(true)
        last = result.count
      }
      const totalMs = Number(process.hrtime.bigint() - startedAt) / 1e6
      // Witness: every upsert landed on the same row.
      expect(last).toBe(T5_UPSERTS)
      return totalMs
    }

    const offMs = await measure(limiterPool, 'sync-off')
    const onMs = await measure(corePool, 'sync-on')
    console.info(
      `[T5] ${T5_UPSERTS} serial same-row upserts: synchronous_commit=off ` +
        `${(offMs / T5_UPSERTS).toFixed(3)} ms/upsert, on ${(onMs / T5_UPSERTS).toFixed(3)} ms/upsert`
    )
    expect(offMs).toBeLessThan(T5_MAX_TOTAL_MS)
  }, 30_000)
})