Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
11.6 KB
Raw Blame
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'
import { randomBytes } from 'node:crypto'
import { Pool } from 'pg'
import request from 'supertest'
import { endPoolAndWaitForClients } from './helpers/realPostgresTeardown.js'

// R1-H3 fase 1 (host↔model) — regression test for the TOCTOU race the advisory
// lock closes (mini-spec §7 / T3 / T4). It drives the race through the REAL route
// handlers over HTTP against a REAL Postgres, so the SAME file, run against the
// parent commit (bb51c5eb, no lock), fails BY ASSERTION: without serialization the
// reductor enumerates an empty impact (the host is parked before its CR lands),
// deletes the model, and the host CR is then created referencing a deleted model —
// a dangling reference, and the reductor answers 204 instead of 409.
//
// Determinism WITHOUT a production test hook (adenda A4): a `MockGateway` subclass
// defers `createResource`/`listResource` for Hosts on a barrier the test controls,
// parking one writer WHILE it holds (child) / would hold (parent) the lock, then
// firing the other. Fixtures come from the REAL producers (T1): the allowlist row
// from `createAllowedModel`, the Host CR from the same `gateway.createResource` the
// impact enumeration later LISTs. Skipped without CONTROL_API_REAL_PG_ADMIN_URL.

const mockVerifyAdminToken = vi.fn()
const mockIsAdminTokenRevoked = vi.fn()
const mockFindAdminById = vi.fn()

vi.mock('../src/utils/auth/adminAuthToken.js', () => ({
  verifyAdminToken: (...args: unknown[]) => mockVerifyAdminToken(...args),
}))

vi.mock('../src/services/adminAuthService.js', () => ({
  findAdminById: (...args: unknown[]) => mockFindAdminById(...args),
  isAdminTokenRevoked: (...args: unknown[]) => mockIsAdminTokenRevoked(...args),
}))

const ADMIN_CLAIMS = {
  sub: '00000000-0000-4000-8000-000000000001',
  typ: 'user' as const,
  role: 'admin' as const,
  jti: 'admin-jti',
  exp: Math.floor(Date.now() / 1000) + 3600,
}
const ACTIVE_ADMIN = {
  id: ADMIN_CLAIMS.sub,
  username: 'admin',
  email: '[email protected]',
  passwordHash: 'hash',
  sessionVersion: 0,
  role: 'admin' as const,
  status: 'active' as const,
  failedAttempts: 0,
  lockedUntil: null,
}
const COOKIE = 'control_ui_admin_session=admin-token'

const PROVIDER = 'claude'
const MODEL = 'meta-llama/Llama-3.1-8B' // a name with a slash — never a valid lock/key separator

function deferred<T = void>(): { promise: Promise<T>; resolve: (v: T | PromiseLike<T>) => void } {
  let resolve!: (v: T | PromiseLike<T>) => void
  const promise = new Promise<T>(r => {
    resolve = r
  })
  return { promise, resolve }
}

function databaseUrl(baseUrl: string, database: string): string {
  const url = new URL(baseUrl)
  url.pathname = `/${database}`
  return url.toString()
}

const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL
const describeRealPostgres = adminUrl ? describe : describe.skip

describeRealPostgres('llm-model reductor ↔ host referencer serialization (R1-H3 fase 1)', () => {
  const database = `control_api_host_model_race_${randomBytes(6).toString('hex')}`
  const connectionString = databaseUrl(
    adminUrl ?? 'postgresql://[email protected]/postgres',
    database
  )
  const previousPgEnv = process.env.CONTROL_API_PG_CONNECTION_STRING

  let adminPool: Pool
  let corePool: Pool
  let limiterPool: Pool
  // Loaded dynamically AFTER the env points the module pool at the test DB.
  let mod: {
    createApp: (gateway: unknown) => import('express').Express
    config: typeof import('../src/config.js').config
    pool: import('../src/db.js').DbClient
    createAllowedModel: typeof import('../src/services/llmAllowedModels.js').createAllowedModel
    MockGateway: typeof import('./mockGateway.js').MockGateway
  }
  let BarrierGatewayClass: new (ns?: string) => BarrierGatewayShape

  interface BarrierGatewayShape {
    blockHostCreate: boolean
    blockHostList: boolean
    hostCreateEntered: ReturnType<typeof deferred>
    releaseHostCreate: ReturnType<typeof deferred>
    hostListEntered: ReturnType<typeof deferred>
    releaseHostList: ReturnType<typeof deferred>
    listResource: (plural: string, ns?: string) => Promise<unknown[]>
  }

  beforeAll(async () => {
    adminPool = new Pool({ connectionString: adminUrl })
    await adminPool.query(`CREATE DATABASE "${database.replace(/"/g, '""')}"`)

    // Point the process at the fresh DB BEFORE importing db/config/app so the
    // module pool (and everything the app wires) connects there.
    process.env.CONTROL_API_PG_CONNECTION_STRING = connectionString

    const dbMod = await import('../src/db.js')
    // The module core pool holds idle connections; when afterAll terminates
    // backends before DROP DATABASE, those idle clients surface a 57P01 error.
    // Absorb it so teardown is clean (the pool is being torn down anyway).
    corePool = dbMod.pool as unknown as Pool
    corePool.on('error', () => {})
    limiterPool = dbMod.rateLimitPool as unknown as Pool
    limiterPool.on('error', () => {})
    const migratePool = new Pool({ connectionString })
    await dbMod.initDb({ connect: () => migratePool.connect() })
    await endPoolAndWaitForClients(migratePool)

    const appMod = await import('../src/app.js')
    const configMod = await import('../src/config.js')
    const allowlistMod = await import('../src/services/llmAllowedModels.js')
    const { MockGateway } = await import('./mockGateway.js')
    mod = {
      createApp: appMod.createApp as never,
      config: configMod.config,
      pool: dbMod.pool,
      createAllowedModel: allowlistMod.createAllowedModel,
      MockGateway,
    }

    // A gateway that can park Host create / Host list on a barrier the test drives.
    class BarrierGateway extends MockGateway {
      blockHostCreate = false
      blockHostList = false
      hostCreateEntered = deferred()
      releaseHostCreate = deferred()
      hostListEntered = deferred()
      releaseHostList = deferred()

      override async createResource(
        plural: Parameters<InstanceType<typeof MockGateway>['createResource']>[0],
        body: Parameters<InstanceType<typeof MockGateway>['createResource']>[1],
        namespace?: string
      ): Promise<unknown> {
        if (plural === 'hosts' && this.blockHostCreate) {
          this.hostCreateEntered.resolve()
          await this.releaseHostCreate.promise
        }
        return super.createResource(plural, body, namespace)
      }

      override async listResource(
        plural: Parameters<InstanceType<typeof MockGateway>['listResource']>[0],
        namespace?: string
      ): Promise<unknown[]> {
        if (plural === 'hosts' && this.blockHostList) {
          this.hostListEntered.resolve()
          await this.releaseHostList.promise
        }
        return super.listResource(plural, namespace)
      }
    }
    BarrierGatewayClass = BarrierGateway as never
  }, 60_000)

  afterAll(async () => {
    try {
      if (previousPgEnv === undefined) delete process.env.CONTROL_API_PG_CONNECTION_STRING
      else process.env.CONTROL_API_PG_CONNECTION_STRING = previousPgEnv
      // Drain the module pool's connections first so DROP DATABASE has no live users.
      await endPoolAndWaitForClients(corePool).catch(() => {})
      await endPoolAndWaitForClients(limiterPool).catch(() => {})
      if (adminPool) {
        await adminPool.query(
          `SELECT pg_terminate_backend(pid) FROM pg_stat_activity
        WHERE datname = $1 AND pid <> pg_backend_pid()`,
          [database]
        )
        await adminPool.query(`DROP DATABASE IF EXISTS "${database.replace(/"/g, '""')}"`)
      }
    } finally {
      await adminPool?.end()
    }
  })

  beforeEach(async () => {
    mockVerifyAdminToken.mockReset().mockReturnValue(ADMIN_CLAIMS)
    mockIsAdminTokenRevoked.mockReset().mockResolvedValue(false)
    mockFindAdminById.mockReset().mockResolvedValue(ACTIVE_ADMIN)
    // Clean slate per test: no rows, no hosts leaking between orderings.
    await mod.pool.query('DELETE FROM llm_allowed_models')
  })

  afterEach(async () => {
    await mod.pool.query('DELETE FROM llm_allowed_models')
  })

  async function seedEnabledModel(): Promise<string> {
    const row = await mod.createAllowedModel(
      { provider: PROVIDER, model: MODEL, enabled: true },
      'test-actor'
    )
    return row.id
  }

  const hostBody = (name: string) => ({
    metadata: { name },
    spec: { model: { provider: PROVIDER, name: MODEL } },
  })

  async function modelRow(id: string): Promise<{ enabled: boolean } | undefined> {
    const res = await mod.pool.query('SELECT enabled FROM llm_allowed_models WHERE id = $1', [id])
    return res.rows[0] as { enabled: boolean } | undefined
  }

  it('host commits first → reductor sees the CR under the lock and 409s (no dangling reference)', async () => {
    const id = await seedEnabledModel()
    const gateway = new BarrierGatewayClass()
    const app = mod.createApp(gateway)

    // 1. Park the host-create INSIDE createResource — it holds the model lock.
    gateway.blockHostCreate = true
    const hostDone = (async () =>
      request(app).post('/api/v1/admin/hosts').set('Cookie', COOKIE).send(hostBody('race-host')))()
    await gateway.hostCreateEntered.promise

    // 2. Fire the non-forced DELETE reductor; give it time to block on the lock
    //    (child) or to read+delete an empty impact (parent) before we release.
    const reductorDone = (async () =>
      request(app).delete(`/api/v1/admin/llm-models/${id}`).set('Cookie', COOKIE))()
    await new Promise(r => setTimeout(r, 400))

    // 3. Let the host CR land + commit, releasing the lock.
    gateway.releaseHostCreate.resolve()
    const [hostRes, reductorRes] = await Promise.all([hostDone, reductorDone])

    // Observable final state (T4): the host was created, the reductor was refused
    // with the impact naming the host, and the model row still exists ENABLED — no
    // Host CR points at a removed pair.
    expect(hostRes.status).toBe(201)
    expect(reductorRes.status).toBe(409)
    expect(reductorRes.body.error).toBe('model_in_use')
    expect(
      (reductorRes.body.impact.hostsAffected as Array<{ name: string }>).map(h => h.name)
    ).toContain('race-host')
    expect((await modelRow(id))?.enabled).toBe(true)
    const hosts = await gateway.listResource('hosts', mod.config.hostsNamespace)
    expect(hosts).toHaveLength(1)
  }, 30_000)

  it('reductor commits first → host revalidates under the lock and 422s (no dangling reference)', async () => {
    const id = await seedEnabledModel()
    const gateway = new BarrierGatewayClass()
    const app = mod.createApp(gateway)

    // 1. Park the reductor INSIDE its impact Host LIST — it holds the model lock.
    gateway.blockHostList = true
    const reductorDone = (async () =>
      request(app).delete(`/api/v1/admin/llm-models/${id}`).set('Cookie', COOKIE))()
    await gateway.hostListEntered.promise

    // 2. Fire the host-create; give it time to block on the lock (child) before we
    //    release the reductor's LIST.
    const hostDone = (async () =>
      request(app)
        .post('/api/v1/admin/hosts')
        .set('Cookie', COOKIE)
        .send(hostBody('race-host-2')))()
    await new Promise(r => setTimeout(r, 400))

    // 3. Let the reductor finish its (empty) impact, delete, and commit — releasing
    //    the lock so the host revalidates against the now-deleted model.
    gateway.releaseHostList.resolve()
    const [reductorRes, hostRes] = await Promise.all([reductorDone, hostDone])

    expect(reductorRes.status).toBe(204)
    expect(hostRes.status).toBe(422)
    expect(await modelRow(id)).toBeUndefined()
    const hosts = await gateway.listResource('hosts', mod.config.hostsNamespace)
    expect(hosts).toHaveLength(0)
  }, 30_000)
})