Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
13.6 KB
Raw Blame
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
import { randomBytes, randomUUID } from 'node:crypto'
import { once } from 'node:events'
import { setImmediate as nextTurn } from 'node:timers/promises'
import { Pool } from 'pg'
import { type DbClient, initDb, withTransaction } from '../src/db.js'
import { reserveInDangerZone } from '../src/services/budgets/reservations.js'
import { insertLlmProviderAttempt } from '../src/services/llmProviderAttemptStore.js'
import {
  createPostgresCommitReplyBlackhole,
  createPostgresDisconnectBlackhole,
} from './helpers/realPostgresCancellation.js'
import { endPoolAndWaitForClients } from './helpers/realPostgresTeardown.js'

const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL
if (process.env.CONTROL_API_REAL_PG_REQUIRED === '1' && !adminUrl) {
  throw new Error('physical cancellation tests require CONTROL_API_REAL_PG_ADMIN_URL')
}
const realPostgres = adminUrl ? describe : describe.skip

function deferred<T = void>() {
  let resolve!: (value: T) => void
  const promise = new Promise<T>(res => {
    resolve = res
  })
  return { promise, resolve }
}

function quoteIdent(value: string): string {
  return `"${value.replace(/"/g, '""')}"`
}

async function observe(check: () => Promise<boolean>, timeoutMs = 2_000): Promise<void> {
  const deadline = Date.now() + timeoutMs
  while (!(await check())) {
    if (Date.now() >= deadline) throw new Error('physical PostgreSQL cleanup did not complete')
    await nextTurn()
  }
}

// Mechanism proof uses real schema/helpers but bypasses authorize and signing.
// Full authorization cancellation lives in services.llmProviderAttemptAuthorization.
realPostgres('scoped transaction cancellation mechanism and durable outcome', () => {
  const database = `authorize_cancel_${randomBytes(6).toString('hex')}`
  let admin: Pool
  let observer: Pool
  let pool: Pool
  let connectionString: string
  let budgetId: string

  beforeAll(async () => {
    admin = new Pool({
      connectionString: adminUrl,
      connectionTimeoutMillis: 2_000,
      statement_timeout: 15_000,
    })
    await admin.query(`CREATE DATABASE ${quoteIdent(database)}`)
    const url = new URL(adminUrl!)
    url.pathname = `/${database}`
    connectionString = url.toString()
    pool = new Pool({
      connectionString,
      max: 1,
      connectionTimeoutMillis: 2_000,
      statement_timeout: 15_000,
      idleTimeoutMillis: 1_000,
    })
    observer = new Pool({
      connectionString,
      max: 1,
      connectionTimeoutMillis: 2_000,
      statement_timeout: 2_000,
      idleTimeoutMillis: 1_000,
    })
    const runtime = await pool.query<{ version: string; version_num: string }>(
      "SELECT version(), current_setting('server_version_num') AS version_num"
    )
    expect(Number(runtime.rows[0].version_num)).toBeGreaterThanOrEqual(160000)
    expect(Number(runtime.rows[0].version_num)).toBeLessThan(170000)
    expect(runtime.rows[0].version).toMatch(/linux|darwin|bsd|illumos/i)
    await initDb({ connect: () => pool.connect() })
    const budget = await pool.query<{ id: string }>(
      `INSERT INTO token_budgets (name, unit, limit_amount, period, max_task_amount)
       VALUES ('cancellation fixture', 'tokens', 100, 'daily', 4) RETURNING id`
    )
    budgetId = budget.rows[0].id
  }, 60_000)

  afterAll(async () => {
    try {
      await endPoolAndWaitForClients(pool)
      await endPoolAndWaitForClients(observer)
      if (admin) await admin.query(`DROP DATABASE IF EXISTS ${quoteIdent(database)}`)
    } finally {
      await admin?.end()
    }
  })

  async function writeAuthorizeRows(db: DbClient, invocationId: string) {
    const reservation = await reserveInDangerZone(
      {
        budgetId,
        limit: 100,
        spent: 95,
        minStart: 1,
        estAmount: 4,
        taskRef: invocationId,
        hostRef: 'cancel-host',
      },
      undefined,
      db
    )
    expect(reservation.decision).toBe('allow')
    if (reservation.decision !== 'allow') throw new Error('fixture reservation denied')
    const attempt = await insertLlmProviderAttempt(db, {
      callerKind: 'host',
      hostRef: 'cancel-host',
      recipeNamespace: null,
      recipeName: null,
      invocationId,
      attemptGeneration: 1,
      providerAttemptIndex: 1,
      model: 'gpt-5.1',
      requestHash: 'a'.repeat(64),
      policyRevision: 1,
      policyHash: 'b'.repeat(64),
      budgetReservationId: reservation.reservationId,
      connectionRevision: 1,
      connectionId: null,
    })
    await db.query(
      `INSERT INTO llm_provider_attempt_tickets (jti, provider_attempt_id, status, expires_at)
       VALUES ($1, $2, 'issued', NOW() + INTERVAL '1 minute')`,
      [randomUUID(), attempt.id]
    )
  }

  async function rowCounts(invocationId: string) {
    const result = await observer.query<{
      attempts: number
      tickets: number
      reservations: number
    }>(
      `SELECT
        (SELECT COUNT(*)::int FROM llm_provider_attempts WHERE invocation_id = $1) AS attempts,
        (SELECT COUNT(*)::int FROM llm_provider_attempt_tickets t JOIN llm_provider_attempts a
          ON t.provider_attempt_id = a.id WHERE a.invocation_id = $1) AS tickets,
        (SELECT COUNT(*)::int FROM budget_pending_reservations WHERE task_ref = $1) AS reservations`,
      [invocationId]
    )
    return result.rows[0]
  }

  async function backendGone(pid: number) {
    await observe(async () => {
      const result = await observer.query<{ count: number }>(
        'SELECT COUNT(*)::int AS count FROM pg_stat_activity WHERE pid = $1 AND datname = $2',
        [pid, database]
      )
      return result.rows[0].count === 0
    })
  }

  async function lockAvailable(lock: number) {
    await withTransaction(async db => {
      const result = await db.query('SELECT pg_try_advisory_xact_lock($1) AS available', [lock])
      expect((result.rows[0] as { available: boolean }).available).toBe(true)
    }, observer)
  }

  it.each(['active', 'idle'] as const)(
    'ends the %s backend, releases writer locks and rolls attempt/ticket/reservation back before COMMIT',
    async mode => {
      const invocationId = randomUUID()
      const lock = mode === 'active' ? 806301 : 806302
      const entered = deferred<number>()
      const controller = new AbortController()
      const reason = Object.assign(new Error('authorize_aborted'), { code: 'authorize_aborted' })
      const outcome = withTransaction(
        async db => {
          const identity = await db.query('SELECT pg_backend_pid() AS pid')
          await writeAuthorizeRows(db, invocationId)
          await db.query('SELECT pg_advisory_xact_lock($1)', [lock])
          entered.resolve((identity.rows[0] as { pid: number }).pid)
          if (mode === 'active') await db.query('SELECT pg_sleep(30)')
          else await once(controller.signal, 'abort')
        },
        pool,
        { signal: controller.signal }
      ).then(
        value => ({ value }),
        error => ({ error })
      )
      const pid = await entered.promise
      await observe(async () => {
        const result = await observer.query<{ state: string; query: string }>(
          'SELECT state, query FROM pg_stat_activity WHERE pid = $1',
          [pid]
        )
        return mode === 'active'
          ? result.rows[0]?.state === 'active' && result.rows[0]?.query === 'SELECT pg_sleep(30)'
          : result.rows[0]?.state === 'idle in transaction'
      })
      controller.abort(reason)
      expect(((await outcome) as { error: unknown }).error).toBe(reason)
      expect(pool.totalCount).toBe(0)
      await backendGone(pid)
      await lockAvailable(lock)
      expect(await rowCounts(invocationId)).toEqual({ attempts: 0, tickets: 0, reservations: 0 })

      // The same physical writes still commit normally after cancellation.
      const later = randomUUID()
      await withTransaction(db => writeAuthorizeRows(db, later), pool)
      expect(await rowCounts(later)).toEqual({ attempts: 1, tickets: 1, reservations: 1 })
      await pool.query('DELETE FROM budget_pending_reservations WHERE task_ref = $1', [later])
    },
    10_000
  )

  it('retains a committed attempt/reservation when a real COMMIT reply is lost; cancellation is not rollback proof', async () => {
    const proxy = await createPostgresCommitReplyBlackhole(connectionString)
    const uncertainPool = new Pool({
      connectionString: proxy.connectionString,
      ssl: false,
      max: 1,
      connectionTimeoutMillis: 2_000,
      statement_timeout: 15_000,
    })
    const invocationId = randomUUID()
    const controller = new AbortController()
    const reason = Object.assign(new Error('authorize_timeout'), { code: 'authorize_timeout' })
    try {
      const outcome = withTransaction(db => writeAuthorizeRows(db, invocationId), uncertainPool, {
        signal: controller.signal,
      }).then(
        value => ({ value }),
        error => ({ error })
      )
      await proxy.commitForwarded
      await observe(async () => (await rowCounts(invocationId)).attempts === 1)
      controller.abort(reason)
      expect(((await outcome) as { error: unknown }).error).toBe(reason)
      expect(await rowCounts(invocationId)).toEqual({ attempts: 1, tickets: 1, reservations: 1 })
      expect(uncertainPool.totalCount).toBe(0)
    } finally {
      controller.abort(reason)
      proxy.allowReplies()
      await endPoolAndWaitForClients(uncertainPool)
      await proxy.close()
    }
  }, 10_000)
  it.each(['idle', 'active-then-aborted'] as const)(
    'bounds the remote %s transaction after local cleanup when a protocol blackhole hides both replies and disconnects',
    async mode => {
      const proxy = await createPostgresDisconnectBlackhole(connectionString)
      const isolated = new Pool({
        connectionString: proxy.connectionString,
        ssl: false,
        max: 1,
        connectionTimeoutMillis: 2_000,
        statement_timeout: 2_000,
        options: '-c idle_in_transaction_session_timeout=0',
      })
      const invocationId = randomUUID()
      const lock = mode === 'idle' ? 806303 : 806304
      const entered = deferred<number>()
      const controller = new AbortController()
      const reason = Object.assign(new Error('authorize_timeout'), { code: 'authorize_timeout' })
      try {
        const outcome = withTransaction(
          async db => {
            const identity = await db.query('SELECT pg_backend_pid() AS pid')
            const setting = await db.query(
              "SELECT setting FROM pg_settings WHERE name = 'idle_in_transaction_session_timeout'"
            )
            expect((setting.rows[0] as { setting: string }).setting).toBe('2000')
            await writeAuthorizeRows(db, invocationId)
            await db.query('SELECT pg_advisory_xact_lock($1)', [lock])
            entered.resolve((identity.rows[0] as { pid: number }).pid)
            if (mode === 'active-then-aborted') await db.query('SELECT pg_sleep(30)')
            else await once(controller.signal, 'abort')
          },
          isolated,
          { signal: controller.signal }
        ).then(
          value => ({ value }),
          error => ({ error })
        )
        const pid = await Promise.race([
          entered.promise,
          outcome.then(early => {
            if ('error' in early) throw early.error
            throw new Error('transaction ended before the physical fixture was ready')
          }),
        ])
        const backendState = async () => {
          const result = await observer.query<{ state: string; query: string }>(
            'SELECT state, query FROM pg_stat_activity WHERE pid = $1 AND datname = $2',
            [pid, database]
          )
          return result.rows[0]
        }
        await observe(async () => {
          const backend = await backendState()
          return mode === 'idle'
            ? backend?.state === 'idle in transaction'
            : backend?.state === 'active' && backend.query === 'SELECT pg_sleep(30)'
        })
        proxy.dropRepliesAndDisconnects()
        controller.abort(reason)
        expect(((await outcome) as { error: unknown }).error).toBe(reason)
        expect(isolated.totalCount).toBe(0)
        await observe(async () => proxy.suppressedDisconnects > 0)
        // Local completion cannot establish remote cleanup. The backend still
        // exists with its socket held by the fault proxy, independently observed.
        expect(await backendState()).toBeDefined()
        if (mode === 'active-then-aborted') {
          // The server statement timeout first enters TRANS_ABORT. A SET LOCAL
          // idle limit would have reverted to zero here and leave this backend
          // alive indefinitely; the pre-BEGIN session backstop must remain.
          await observe(
            async () => (await backendState())?.state === 'idle in transaction (aborted)',
            4_000
          )
          expect(proxy.discardedReplyBytes).toBeGreaterThan(0)
        }
        await observe(async () => !(await backendState()), 6_000)
        await lockAvailable(lock)
        expect(await rowCounts(invocationId)).toEqual({ attempts: 0, tickets: 0, reservations: 0 })
        proxy.allowReplies()

        // With max one, a fresh physical backend still serves and commits the
        // real writer after both the local checkout and remote backend ended.
        const later = randomUUID()
        await withTransaction(db => writeAuthorizeRows(db, later), isolated, {
          signal: new AbortController().signal,
        })
        expect(await rowCounts(later)).toEqual({ attempts: 1, tickets: 1, reservations: 1 })
        await pool.query('DELETE FROM budget_pending_reservations WHERE task_ref = $1', [later])
      } finally {
        controller.abort(reason)
        proxy.allowReplies()
        await endPoolAndWaitForClients(isolated)
        await proxy.close()
      }
    },
    20_000
  )
})