Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
11.6 KB
Raw Blame
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'
import { randomBytes } from 'node:crypto'
import { Pool } from 'pg'
import type { ImageInputCapability } from '@clerum/llm-providers'
import { initDb } from '../src/db.js'
import { type CatalogSyncResult, syncDiscoveredModels } from '../src/services/llmCatalogSync.js'
import { loadStub, trimSnapshot, withModalities } from './helpers/modelsDevFixtures.js'
import { endPoolAndWaitForClients } from './helpers/realPostgresTeardown.js'
import './realPostgres.requirement.ts'

// R1-M2: run `syncDiscoveredModels` against a REAL PostgreSQL. The in-memory
// fake used by services.llmCatalogSync.invariants.test.ts models the statement
// semantics; it cannot prove the parts only Postgres decides:
//   - the `UPDATE ... FROM llm_allowed_models AS prev ... RETURNING
//     (t.image_input IS DISTINCT FROM prev.image_input)` self-join, which must
//     compare the NEW target row with the PRE-UPDATE row, so the returned
//     enabledImageInputChanged count (and the ConfigMap republish) is exact;
//   - the `(checkedAt)::timestamptz <= $capturedAt::timestamptz` monotonic
//     guard, which must compare instants, not strings.
// Skipped without CONTROL_API_REAL_PG_ADMIN_URL, like every sibling suite.

const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL
const describeRealPostgres = adminUrl ? describe : describe.skip

function databaseUrl(baseUrl: string, database: string): string {
  const url = new URL(baseUrl)
  url.pathname = `/${database}`
  return url.toString()
}

function quoteIdent(value: string): string {
  return `"${value.replace(/"/g, '""')}"`
}

const CAPTURED_AT = '2026-08-19T16:16:10.000Z'
const TTL_MS = 30 * 24 * 60 * 60 * 1000
const LOW_FLOOR = { minPlausibleLiveTotal: 1 }

const curated: ImageInputCapability = {
  state: 'supported',
  evidence: {
    source: 'curated',
    reference: 'https://docs.z.ai/guides/vlm/glm-5.3-flash',
    checkedAt: '2026-09-16T00:00:00.000Z',
  },
}
const oldDiscovery: ImageInputCapability = {
  state: 'unknown',
  evidence: {
    source: 'discovery',
    reference: 'https://models.dev/api.json',
    checkedAt: '2026-07-01T00:00:00.000Z',
  },
}
// Captured AFTER this run's catalog. Spelled without milliseconds (CAPTURED_AT
// carries `.000Z`) so the real `::timestamptz` cast parses both spellings.
const newerDiscovery: ImageInputCapability = {
  state: 'supported',
  evidence: {
    source: 'discovery',
    reference: 'https://models.dev/api.json',
    checkedAt: '2026-09-01T00:00:00Z',
    validUntil: '2026-10-01T00:00:00Z',
  },
}

function discovered(state: 'supported' | 'unsupported') {
  return {
    state,
    evidence: {
      source: 'discovery',
      reference: 'https://models.dev/api.json',
      checkedAt: CAPTURED_AT,
      validUntil: new Date(Date.parse(CAPTURED_AT) + TTL_MS).toISOString(),
    },
  }
}

interface SeedRow {
  model: string
  source: 'discovery' | 'manual'
  enabled: boolean
  stale?: boolean
  display_name?: string | null
  context_window_tokens?: number | null
  image_input: ImageInputCapability | null
}

// Same seed as the #654 invariant in services.llmCatalogSync.invariants.test.ts.
const SEED: SeedRow[] = [
  // Enabled + curated: published, must stay byte-identical.
  { model: 'claude-opus-4-5', source: 'discovery', enabled: true, image_input: curated },
  // Disabled + curated: an operator decision discovery must not undo.
  { model: 'claude-sonnet-5', source: 'discovery', enabled: false, image_input: curated },
  // Disabled + older discovery provenance: refreshed.
  {
    model: 'claude-haiku-4-5-20251001',
    source: 'discovery',
    enabled: false,
    image_input: oldDiscovery,
  },
  // ENABLED + older discovery provenance: refreshed → counts as a change.
  { model: 'claude-opus-4-6', source: 'discovery', enabled: true, image_input: oldDiscovery },
  // ENABLED + NEWER discovery provenance: untouched (monotonic guard).
  { model: 'claude-sonnet-4-5', source: 'discovery', enabled: true, image_input: newerDiscovery },
  // MANUAL, enabled, no evidence: gets image_input and NOTHING else → counts
  // as a change. Seeded stale with null metadata so any leak of the discovery
  // branch's other writes is observable.
  {
    model: 'claude-fable-5-1',
    source: 'manual',
    enabled: true,
    stale: true,
    display_name: null,
    context_window_tokens: null,
    image_input: null,
  },
]

const CATALOG = withModalities(
  trimSnapshot({
    anthropic: [
      'claude-opus-4-5',
      'claude-sonnet-5',
      'claude-haiku-4-5-20251001',
      'claude-opus-4-6',
      'claude-sonnet-4-5',
      'claude-fable-5-1',
      // Absent from the DB → the first run INSERTs it as a disabled row.
      'claude-opus-4-7',
    ],
  }),
  {
    'claude-haiku-4-5-20251001': ['text', 'image'],
    'claude-opus-4-6': ['text'],
    'claude-sonnet-4-5': ['text', 'image'],
    'claude-fable-5-1': ['text', 'image'],
    'claude-opus-4-7': ['text', 'image'],
  }
)

type DbRow = Record<string, unknown> & {
  model: string
  enabled: boolean
  source: string
  image_input: ImageInputCapability | null
}

describeRealPostgres('syncDiscoveredModels on real PostgreSQL (#654 image_input)', () => {
  const database = `control_api_catalog_sync_${randomBytes(6).toString('hex')}`
  let adminPool: Pool
  let dbPool: Pool
  const materializer = { materialize: vi.fn(async () => {}) }
  let manualBefore: DbRow
  let firstRun: CatalogSyncResult

  async function readRows(): Promise<Map<string, DbRow>> {
    const { rows } = await dbPool.query<DbRow>(
      `SELECT * FROM llm_allowed_models WHERE provider = 'claude' ORDER BY model`
    )
    return new Map(rows.map(row => [row.model, row]))
  }

  async function countRuns(): Promise<number> {
    const { rows } = await dbPool.query<{ n: string }>(
      'SELECT count(*)::text AS n FROM llm_catalog_sync_runs'
    )
    return Number(rows[0].n)
  }

  function runSync(capturedAt: string, source: 'live' | 'vendored' = 'live') {
    return syncDiscoveredModels(
      {
        materializer,
        loadCatalog: loadStub(CATALOG, source, capturedAt),
        imageEvidenceTtlMs: TTL_MS,
        ...LOW_FLOOR,
      },
      dbPool
    )
  }

  beforeAll(async () => {
    adminPool = new Pool({ connectionString: adminUrl })
    await adminPool.query(`CREATE DATABASE ${quoteIdent(database)}`)
    dbPool = new Pool({ connectionString: databaseUrl(adminUrl!, database) })
    await initDb({ connect: () => dbPool.connect() })

    // initDb seeds the static allowlist as `source='manual'` rows. Start from
    // an empty table so the seed below is the whole universe the sync sees.
    await dbPool.query('DELETE FROM llm_allowed_models')
    for (const row of SEED) {
      await dbPool.query(
        `INSERT INTO llm_allowed_models
           (provider, model, enabled, source, stale, display_name,
            context_window_tokens, image_input)
         VALUES ('claude', $1, $2, $3, $4, $5, $6, $7::jsonb)`,
        [
          row.model,
          row.enabled,
          row.source,
          row.stale ?? false,
          row.display_name ?? null,
          row.context_window_tokens ?? null,
          row.image_input === null ? null : JSON.stringify(row.image_input),
        ]
      )
    }
    const seeded = await readRows()
    expect(seeded.size).toBe(SEED.length)
    manualBefore = seeded.get('claude-fable-5-1')!
  })

  afterAll(async () => {
    try {
      await endPoolAndWaitForClients(dbPool)
      if (adminPool) {
        await adminPool.query(
          `SELECT pg_terminate_backend(pid)
         FROM pg_stat_activity
        WHERE datname = $1
          AND pid <> pg_backend_pid()`,
          [database]
        )
        await adminPool.query(`DROP DATABASE IF EXISTS ${quoteIdent(database)}`)
      }
    } finally {
      await adminPool?.end()
    }
  })

  it('counts exactly the enabled rows whose image_input changed and republishes once', async () => {
    firstRun = await runSync(CAPTURED_AT)

    // claude-opus-4-6 (discovery) and claude-fable-5-1 (manual). The other
    // enabled rows are curated or newer, so the self-join must see them as
    // unchanged; the disabled rows that did change must not be counted.
    expect(firstRun.enabledImageInputChanged).toBe(2)
    expect(firstRun.materialized).toBe(true)
    expect(firstRun.added).toBe(1)
    expect(materializer.materialize).toHaveBeenCalledTimes(1)

    const rows = await readRows()
    expect(rows.get('claude-opus-4-6')!.image_input).toEqual(discovered('unsupported'))
    expect(rows.get('claude-haiku-4-5-20251001')!.image_input).toEqual(discovered('supported'))
    const inserted = rows.get('claude-opus-4-7')!
    expect(inserted.enabled).toBe(false)
    expect(inserted.source).toBe('discovery')
    expect(inserted.image_input).toEqual(discovered('supported'))
  })

  it('an immediate re-run reports no change and does not republish', async () => {
    const runsBefore = await countRuns()
    const rerun = await runSync(CAPTURED_AT)

    // Liveness witness: this run really happened and really visited the rows.
    // It persisted its own summary row, and refreshed the discovery rows
    // (`<=` rewrites evidence stamped with the same capture).
    expect(await countRuns()).toBe(runsBefore + 1)
    expect(rerun.ranAt).not.toBe(firstRun.ranAt)
    expect(rerun.updated).toBe(SEED.filter(r => r.source === 'discovery').length + 1)

    expect(rerun.enabledImageInputChanged).toBe(0)
    expect(rerun.materialized).toBe(false)
    expect(materializer.materialize).toHaveBeenCalledTimes(1)
    const rows = await readRows()
    expect(rows.get('claude-opus-4-6')!.image_input).toEqual(discovered('unsupported'))
    expect(rows.get('claude-fable-5-1')!.image_input).toEqual(discovered('supported'))
  })

  it('never regresses stored discovery evidence with a newer checkedAt', async () => {
    const rows = await readRows()
    expect(rows.get('claude-sonnet-4-5')!.image_input).toEqual(newerDiscovery)

    // An OLDER capture (a vendored fallback after the live run) must not
    // overwrite anything this live run stamped either.
    const olderRun = await runSync('2026-08-01T00:00:00Z', 'vendored')
    expect(olderRun.updated).toBeGreaterThan(0)
    expect(olderRun.enabledImageInputChanged).toBe(0)
    expect(materializer.materialize).toHaveBeenCalledTimes(1)
    const after = await readRows()
    expect(after.get('claude-sonnet-4-5')!.image_input).toEqual(newerDiscovery)
    expect(after.get('claude-opus-4-6')!.image_input).toEqual(discovered('unsupported'))
    expect(after.get('claude-fable-5-1')!.image_input).toEqual(discovered('supported'))
  })

  it('writes only image_input on a manual row', async () => {
    const manual = (await readRows()).get('claude-fable-5-1')!
    expect(manual.image_input).toEqual(discovered('supported'))
    expect(manualBefore.image_input).toBeNull()

    // Every other column — including stale, last_seen_at, discovered_at,
    // display_name and context_window_tokens, which the discovery branch
    // would write — is exactly what the seed stored.
    const { image_input: _afterImage, ...afterRest } = manual
    const { image_input: _beforeImage, ...beforeRest } = manualBefore
    expect(afterRest).toEqual(beforeRest)
    expect(afterRest.source).toBe('manual')
    expect(afterRest.stale).toBe(true)
    expect(afterRest.last_seen_at).toBeNull()
  })

  it('leaves curated evidence untouched, enabled or not', async () => {
    const rows = await readRows()
    expect(rows.get('claude-opus-4-5')!.image_input).toEqual(curated)
    expect(rows.get('claude-sonnet-5')!.image_input).toEqual(curated)
    // Witness: the sync did visit both rows (last_seen_at is only set by it).
    expect(rows.get('claude-opus-4-5')!.last_seen_at).toBeInstanceOf(Date)
    expect(rows.get('claude-sonnet-5')!.last_seen_at).toBeInstanceOf(Date)
  })
})