Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
21.7 KB
Raw Blame
import { beforeEach, describe, expect, it, vi } from 'vitest'
import { registry } from '../src/observability/metrics.js'
import { syncDiscoveredModels } from '../src/services/llmCatalogSync.js'
import type { RawModelsDevCatalog } from '../src/services/modelsDevClient.js'

type Row = Record<string, unknown>

/**
 * The sync REQUIRES a ConfigMap materializer (#654) — it can change what runtime
 * is served. Most tests here assert SQL shape on rows the run never publishes,
 * so this spy stays uncalled; the publication tests assert on it directly.
 */
const materializer = { materialize: vi.fn(async () => {}) }

beforeEach(() => {
  materializer.materialize.mockReset()
})

/**
 * A fake transactional client that routes queries by SQL shape. `existing` maps
 * provider → the rows the per-provider SELECT should return (id/model/source).
 * It records the INSERT/UPDATE/stale calls so tests can assert branch behavior.
 */
function makeConnector(
  existing: Record<string, Array<{ id: string; model: string; source: string }>>,
  // What the discovery UPDATE's RETURNING clause reports. The default says the
  // touched row is disabled, which is what every SQL-shape test below wants: a
  // disabled row can never force a ConfigMap write.
  updateReturning: Row = { enabled: false, image_input_changed: false }
) {
  const calls: Array<{ sql: string; params: unknown[] }> = []
  const query = vi.fn(async (sql: string, params: unknown[] = []) => {
    calls.push({ sql, params })
    if (/SELECT id, model, source FROM llm_allowed_models WHERE provider/.test(sql)) {
      const provider = String(params[0])
      const rows: Row[] = (existing[provider] ?? []).map(r => ({ ...r }))
      return { rows, rowCount: rows.length }
    }
    if (/INSERT INTO llm_allowed_models/.test(sql)) {
      return { rows: [], rowCount: 1 }
    }
    if (/UPDATE llm_allowed_models[\s\S]*SET last_seen_at/.test(sql)) {
      return { rows: [updateReturning], rowCount: 1 }
    }
    // The manual-row statement reports through the SAME RETURNING contract, so
    // it answers with the same row: a test that flips `updateReturning` to an
    // enabled row must be able to drive publication from either branch.
    if (/UPDATE llm_allowed_models[\s\S]*t\.source = 'manual'/.test(sql)) {
      return { rows: [updateReturning], rowCount: 1 }
    }
    if (/UPDATE llm_allowed_models[\s\S]*SET stale = true/.test(sql)) {
      // Report a stale transition for the vanished-row test (claude provider).
      const provider = String(params[0])
      return { rows: [], rowCount: provider === 'claude' ? 1 : 0 }
    }
    if (/INSERT INTO llm_catalog_sync_runs/.test(sql)) {
      return { rows: [{ ran_at: new Date('2026-07-13T00:00:00Z') }], rowCount: 1 }
    }
    return { rows: [], rowCount: 0 }
  })
  const release = vi.fn()
  const connector = { connect: vi.fn(async () => ({ query, release })) }
  return { connector, calls, query, release }
}

function catalogWith(
  models: Record<
    string,
    {
      id: string
      name?: string
      limit?: { context?: number }
      modalities?: { input?: unknown }
    }
  >
): RawModelsDevCatalog {
  return { anthropic: { name: 'Anthropic', models } }
}

function loadStub(
  catalog: RawModelsDevCatalog,
  source: 'live' | 'vendored' = 'vendored',
  // `fetchedAt` is the run's acquisition time; `capturedAt` is when the DATA was
  // captured (a vendored snapshot's baked date). Kept distinct on purpose.
  capturedAt = '2026-07-10T00:00:00.000Z'
) {
  return async () => ({
    source,
    fetchedAt: '2026-07-13T00:00:00.000Z',
    capturedAt,
    catalog,
  })
}

const CALL_FOR = (calls: Array<{ sql: string; params: unknown[] }>, re: RegExp) =>
  calls.filter(c => re.test(c.sql))

/**
 * Current value of `clerum_llm_allowlist_configmap_write_failures_total{phase}`.
 * Read from the real registry rather than a spy: the point of the assertion is
 * that the operator-visible counter moved.
 */
async function configMapWriteFailures(phase: string): Promise<number> {
  const metric = registry.getSingleMetric('clerum_llm_allowlist_configmap_write_failures_total')
  const collected = (await metric?.get()) as
    | { values: Array<{ labels: Record<string, unknown>; value: number }> }
    | undefined
  return collected?.values.find(v => v.labels.phase === phase)?.value ?? 0
}

/**
 * The exact provenance payload discovery persists for a model the catalog says
 * nothing about. A KNOWN state additionally carries `validUntil` — the shared
 * contract refuses a discovery claim without an expiry — and is asserted as a
 * parsed object below, since only key ORDER would differ here.
 */
function discoveryProvenance(capturedAt: string): string {
  return JSON.stringify({
    state: 'unknown',
    evidence: {
      source: 'discovery',
      reference: 'https://models.dev/api.json',
      checkedAt: capturedAt,
    },
  })
}

describe('syncDiscoveredModels — source-guarded reconciliation', () => {
  it('INSERTs a new discovered model as disabled discovery (enabled=false, ON CONFLICT DO NOTHING)', async () => {
    const { connector, calls } = makeConnector({}) // no existing rows anywhere
    const catalog = catalogWith({
      'claude-opus-4-5': { id: 'claude-opus-4-5', name: 'Opus', limit: { context: 200000 } },
    })
    const res = await syncDiscoveredModels(
      { materializer, loadCatalog: loadStub(catalog) },
      connector
    )

    const inserts = CALL_FOR(calls, /INSERT INTO llm_allowed_models/)
    expect(inserts).toHaveLength(1)
    expect(inserts[0].sql).toMatch(/enabled/)
    expect(inserts[0].sql).toMatch(/'discovery'/)
    expect(inserts[0].sql).toMatch(/false/)
    expect(inserts[0].sql).toMatch(/ON CONFLICT \(provider, model\) DO NOTHING/)
    // params: provider, model, ctx, display, image_input
    expect(inserts[0].params).toEqual([
      'claude',
      'claude-opus-4-5',
      200000,
      'Opus',
      discoveryProvenance('2026-07-10T00:00:00.000Z'),
    ])
    expect(res.added).toBe(1)
    expect(res.updated).toBe(0)
  })

  it('#654 stamps discovery provenance with the SOURCE capture time, never `fetchedAt`', async () => {
    const { connector, calls } = makeConnector({})
    const catalog = catalogWith({
      // Says nothing about modalities …
      'claude-opus-4-5': { id: 'claude-opus-4-5' },
      // … and one that does, so the KNOWN-state payload is covered too.
      'claude-sonnet-4-5': {
        id: 'claude-sonnet-4-5',
        modalities: { input: ['text', 'image'] },
      },
    })
    // The stub's fetchedAt (acquisition) deliberately differs from capturedAt.
    await syncDiscoveredModels(
      {
        materializer,
        loadCatalog: loadStub(catalog, 'vendored', '2026-08-19T16:16:10.000Z'),
        imageEvidenceTtlMs: 30 * 24 * 60 * 60 * 1000,
      },
      connector
    )

    const inserts = CALL_FOR(calls, /INSERT INTO llm_allowed_models/)
    const byModel = new Map(inserts.map(c => [String(c.params[1]), c]))
    const opus = byModel.get('claude-opus-4-5')!
    const persisted = JSON.parse(String(opus.params[4]))
    expect(persisted).toEqual({
      state: 'unknown',
      evidence: {
        source: 'discovery',
        reference: 'https://models.dev/api.json',
        checkedAt: '2026-08-19T16:16:10.000Z',
      },
    })
    // Reloading a stale snapshot must not manufacture freshness.
    expect(String(opus.params[4])).not.toContain('2026-07-13T00:00:00.000Z')
    // An entry the catalog said nothing about stays unknown — and, being
    // unknown, carries no expiry to state.
    expect(persisted.evidence.validUntil).toBeUndefined()

    // A KNOWN state must carry an expiry, and it is derived from the CAPTURE
    // time: a vendored snapshot may only claim support for as long as it is
    // fresh (capturedAt + 30d, NOT fetchedAt + 30d).
    const sonnet = JSON.parse(String(byModel.get('claude-sonnet-4-5')!.params[4]))
    expect(sonnet).toEqual({
      state: 'supported',
      evidence: {
        source: 'discovery',
        reference: 'https://models.dev/api.json',
        checkedAt: '2026-08-19T16:16:10.000Z',
        validUntil: '2026-09-18T16:16:10.000Z',
      },
    })
  })

  it('throws when the shared contract rejects the discovery evidence it built', async () => {
    const { connector, calls } = makeConnector({})
    const catalog = catalogWith({ 'claude-opus-4-5': { id: 'claude-opus-4-5' } })
    // `checkedAt` must be a real date, so the shared parser refuses the payload
    // this capture stamp produces. Storing NULL instead would hide a programming
    // error behind a row that merely looks unexamined.
    await expect(
      syncDiscoveredModels(
        { materializer, loadCatalog: loadStub(catalog, 'live', 'not-a-timestamp') },
        connector
      )
    ).rejects.toThrow(/discoveryImageInput: shared contract rejected/)

    // Witness the run actually started (so the throw is the contract's, not a
    // connector that never opened) and that nothing was written after it.
    expect(CALL_FOR(calls, /^BEGIN$/)).toHaveLength(1)
    expect(CALL_FOR(calls, /INSERT INTO llm_allowed_models/)).toHaveLength(0)
  })

  it('touches ONLY image_input on a colliding source=manual row, through the same guard', async () => {
    const { connector, calls } = makeConnector({
      claude: [{ id: 'm1', model: 'claude-opus-4-5', source: 'manual' }],
    })
    const catalog = catalogWith({
      'claude-opus-4-5': { id: 'claude-opus-4-5', name: 'Opus', modalities: { input: ['text'] } },
    })
    const res = await syncDiscoveredModels(
      {
        materializer,
        loadCatalog: loadStub(catalog, 'live', '2026-08-19T16:16:10.000Z'),
        imageEvidenceTtlMs: 30 * 24 * 60 * 60 * 1000,
      },
      connector
    )

    // The row is still invisible to everything an operator authored.
    expect(CALL_FOR(calls, /INSERT INTO llm_allowed_models/)).toHaveLength(0)
    expect(CALL_FOR(calls, /SET last_seen_at/)).toHaveLength(0)
    expect(res.added).toBe(0)
    expect(res.updated).toBe(0)

    // …except `image_input`, written by exactly one statement (#654). This is
    // the liveness witness for the four negative assertions above: without it
    // they would also hold for a sync that never reached this row at all.
    const manual = CALL_FOR(calls, /t\.source = 'manual'/)
    expect(manual).toHaveLength(1)
    const sql = manual[0]!.sql
    expect(sql).toMatch(/SET image_input = CASE/)
    // Same guard as the discovery branch: NULL, or discovery evidence not newer.
    expect(sql).toMatch(/WHEN t\.image_input IS NULL THEN \$2::jsonb/)
    expect(sql).toMatch(
      /\(t\.image_input->'evidence'->>'checkedAt'\)::timestamptz <= \$3::timestamptz/
    )
    expect(sql).toMatch(
      /RETURNING t\.enabled, \(t\.image_input IS DISTINCT FROM prev\.image_input\)/
    )
    // Nothing else is ASSIGNED — asserted on the assignment TARGETS, not on a
    // substring search. `source` and `enabled` legitimately appear elsewhere in
    // this statement (the WHERE, the RETURNING, and the guard's
    // `evidence->>'source'`), so a substring search could not tell "never
    // written" from "mentioned" and would fail on a correct statement.
    const setClause = sql.slice(sql.indexOf('SET '), sql.indexOf('FROM llm_allowed_models AS prev'))
    const assigned = [...setClause.matchAll(/(?:SET|,)\s+([a-z_]+)\s*=/g)].map(m => m[1])
    expect(assigned).toEqual(['image_input'])
    expect(manual[0]!.params).toEqual([
      'm1',
      JSON.stringify({
        state: 'unsupported',
        evidence: {
          source: 'discovery',
          reference: 'https://models.dev/api.json',
          checkedAt: '2026-08-19T16:16:10.000Z',
          validUntil: '2026-09-18T16:16:10.000Z',
        },
      }),
      '2026-08-19T16:16:10.000Z',
    ])
  })

  it('NULL-FILLs a present discovery row (COALESCE) and never overwrites enabled', async () => {
    const { connector, calls } = makeConnector({
      claude: [{ id: 'd1', model: 'claude-opus-4-5', source: 'discovery' }],
    })
    const catalog = catalogWith({
      'claude-opus-4-5': { id: 'claude-opus-4-5', name: 'Opus', limit: { context: 200000 } },
    })
    const res = await syncDiscoveredModels(
      { materializer, loadCatalog: loadStub(catalog) },
      connector
    )

    const updates = CALL_FOR(calls, /SET last_seen_at/)
    expect(updates).toHaveLength(1)
    const sql = updates[0].sql
    expect(sql).toMatch(/COALESCE\(t\.context_window_tokens/)
    expect(sql).toMatch(/COALESCE\(t\.display_name/)
    expect(sql).toMatch(/stale = false/)
    // The serialized columns are CASE-guarded to disabled rows only, so an
    // ENABLED discovery row (already in the CM) is never mutated → no CM drift.
    expect(sql).toMatch(/CASE\s+WHEN t\.enabled THEN t\.context_window_tokens/)
    expect(sql).toMatch(/CASE\s+WHEN t\.enabled THEN t\.display_name/)
    // `enabled` is never assigned in the SET clause.
    expect(/enabled\s*=/.test(sql)).toBe(false)
    expect(updates[0].params).toEqual([
      'd1',
      200000,
      'Opus',
      discoveryProvenance('2026-07-10T00:00:00.000Z'),
      '2026-07-10T00:00:00.000Z',
    ])
    expect(res.updated).toBe(1)
    expect(res.added).toBe(0)
  })

  it('refreshes image_input on enabled rows only when NULL or older discovery evidence', async () => {
    const { connector, calls } = makeConnector({
      claude: [{ id: 'd1', model: 'claude-opus-4-5', source: 'discovery' }],
    })
    const catalog = catalogWith({ 'claude-opus-4-5': { id: 'claude-opus-4-5' } })
    await syncDiscoveredModels({ materializer, loadCatalog: loadStub(catalog) }, connector)

    const sql = CALL_FOR(calls, /SET last_seen_at/)[0].sql
    // A row with no evidence gets the provenance …
    expect(sql).toMatch(/WHEN t\.image_input IS NULL THEN \$4::jsonb/)
    // … a discovery-sourced one is refreshed only by a capture at least as
    // recent as the one already recorded (compared as timestamptz, because
    // `…Z` and `….000Z` are the same instant but different strings) …
    expect(sql).toMatch(
      /t\.image_input->'evidence'->>'source' = 'discovery'[\s\S]*\(t\.image_input->'evidence'->>'checkedAt'\)::timestamptz <= \$5::timestamptz/
    )
    // … and anything else (operator-curated, or a newer capture) is untouched.
    expect(sql).toMatch(/ELSE t\.image_input/)
    // The run reports whether it actually changed an ENABLED row's evidence —
    // the only change that can make the served ConfigMap wrong.
    expect(sql).toMatch(
      /RETURNING t\.enabled, \(t\.image_input IS DISTINCT FROM prev\.image_input\)/
    )
    // image_input deliberately does NOT take the enabled-row freeze. Witness
    // that the freeze exists in this very statement for the other columns, so
    // its absence here is a choice and not a missing clause.
    expect(sql).toMatch(/WHEN t\.enabled THEN t\.context_window_tokens/)
    expect(/WHEN t\.enabled THEN t\.image_input/.test(sql)).toBe(false)
  })

  it('materializes the ConfigMap after COMMIT when an enabled row changed image_input', async () => {
    const { connector, calls, query } = makeConnector(
      { claude: [{ id: 'd1', model: 'claude-opus-4-5', source: 'discovery' }] },
      { enabled: true, image_input_changed: true }
    )
    const catalog = catalogWith({ 'claude-opus-4-5': { id: 'claude-opus-4-5' } })
    const res = await syncDiscoveredModels(
      { materializer, loadCatalog: loadStub(catalog) },
      connector
    )

    expect(materializer.materialize).toHaveBeenCalledTimes(1)
    // AFTER commit, never inside the transaction: a ConfigMap write held open
    // would keep the advisory lock for the length of a call to the cluster.
    const commitCall = query.mock.calls.findIndex(c => /^COMMIT$/.test(String(c[0])))
    expect(commitCall).toBeGreaterThanOrEqual(0)
    expect(materializer.materialize.mock.invocationCallOrder[0]).toBeGreaterThan(
      query.mock.invocationCallOrder[commitCall]
    )
    expect(res.enabledImageInputChanged).toBe(1)
    expect(res.materialized).toBe(true)
    expect(CALL_FOR(calls, /SET last_seen_at/)).toHaveLength(1)
  })

  it('does not materialize when only disabled rows changed', async () => {
    const { connector, calls } = makeConnector(
      { claude: [{ id: 'd1', model: 'claude-opus-4-5', source: 'discovery' }] },
      { enabled: false, image_input_changed: true }
    )
    const catalog = catalogWith({ 'claude-opus-4-5': { id: 'claude-opus-4-5' } })
    const res = await syncDiscoveredModels(
      { materializer, loadCatalog: loadStub(catalog) },
      connector
    )

    expect(materializer.materialize).not.toHaveBeenCalled()
    // Witness: the write the assertion above denies publishing DID happen — a
    // disabled row is not in the ConfigMap, so changing it publishes nothing.
    expect(CALL_FOR(calls, /SET last_seen_at/)).toHaveLength(1)
    expect(res.updated).toBe(1)
    expect(res.enabledImageInputChanged).toBe(0)
    expect(res.materialized).toBe(false)
  })

  it('reports and rethrows a ConfigMap write failure after the transaction committed', async () => {
    const { connector, calls } = makeConnector(
      { claude: [{ id: 'd1', model: 'claude-opus-4-5', source: 'discovery' }] },
      { enabled: true, image_input_changed: true }
    )
    const catalog = catalogWith({ 'claude-opus-4-5': { id: 'claude-opus-4-5' } })
    const before = await configMapWriteFailures('sync')
    materializer.materialize.mockRejectedValueOnce(new Error('apiserver unreachable'))

    await expect(
      syncDiscoveredModels({ materializer, loadCatalog: loadStub(catalog) }, connector)
    ).rejects.toThrow('apiserver unreachable')

    // The rows are durable: the failure happened after COMMIT, and nothing rolls
    // back (that COMMIT is also the witness that the run reached the publish).
    expect(CALL_FOR(calls, /^COMMIT$/)).toHaveLength(1)
    expect(CALL_FOR(calls, /^ROLLBACK$/)).toHaveLength(0)
    expect(await configMapWriteFailures('sync')).toBe(before + 1)
  })

  it('flags vanished discovery rows stale from a LIVE catalog (never delete, never disable)', async () => {
    const { connector, calls } = makeConnector({
      // A discovery row exists but the catalog no longer lists it.
      claude: [{ id: 'gone', model: 'claude-legacy', source: 'discovery' }],
    })
    const catalog = catalogWith({ 'claude-opus-4-5': { id: 'claude-opus-4-5' } })
    // This test isolates the vanished→stale reconciliation, not the §4.5
    // plausibility floor. The tiny 1-model catalog would otherwise trip the
    // global floor (default 100), so lower it here to exercise stale-marking.
    const res = await syncDiscoveredModels(
      { materializer, loadCatalog: loadStub(catalog, 'live'), minPlausibleLiveTotal: 1 },
      connector
    )

    const staleUpdates = CALL_FOR(calls, /SET stale = true/)
    // One stale UPDATE per provider is issued; assert none delete / disable.
    expect(staleUpdates.length).toBeGreaterThan(0)
    for (const u of staleUpdates) {
      expect(u.sql).not.toMatch(/DELETE/i)
      expect(/enabled\s*=\s*false/.test(u.sql)).toBe(false)
      expect(u.sql).toMatch(/source = 'discovery'/)
      expect(u.sql).toMatch(/model <> ALL/)
    }
    expect(res.staled).toBe(1)
  })

  it('does NOT stale anything on a VENDORED fallback (snapshot is not authoritative)', async () => {
    const { connector, calls } = makeConnector({
      // Same vanished discovery row, but the run fell back to the vendored
      // snapshot (live fetch failed) — a transient blip must not stale a live model.
      claude: [{ id: 'gone', model: 'claude-legacy', source: 'discovery' }],
    })
    const catalog = catalogWith({ 'claude-opus-4-5': { id: 'claude-opus-4-5' } })
    const res = await syncDiscoveredModels(
      { materializer, loadCatalog: loadStub(catalog, 'vendored') },
      connector
    )

    // No stale UPDATE is issued at all on a vendored run.
    expect(CALL_FOR(calls, /SET stale = true/)).toHaveLength(0)
    expect(res.staled).toBe(0)
    // Inserts / last_seen refresh still happen (harmless from vendored data).
    expect(res.source).toBe('vendored')
  })

  it('runs inside a transaction, takes the advisory lock, persists a run row, and returns the source', async () => {
    const { connector, calls, release } = makeConnector({})
    const catalog = catalogWith({ 'claude-opus-4-5': { id: 'claude-opus-4-5' } })
    const res = await syncDiscoveredModels(
      { materializer, loadCatalog: loadStub(catalog, 'live') },
      connector
    )

    expect(CALL_FOR(calls, /^BEGIN$/)).toHaveLength(1)
    expect(CALL_FOR(calls, /pg_advisory_xact_lock/)).toHaveLength(1)
    expect(CALL_FOR(calls, /^COMMIT$/)).toHaveLength(1)
    const runInsert = CALL_FOR(calls, /INSERT INTO llm_catalog_sync_runs/)
    expect(runInsert).toHaveLength(1)
    expect(runInsert[0].params[0]).toBe('live') // source persisted
    expect(runInsert[0].sql).toMatch(/RETURNING ran_at/)
    expect(res.source).toBe('live')
    // fetchedAt = catalog acquisition time (loader); ranAt = DB commit time (run row).
    expect(res.fetchedAt).toBe('2026-07-13T00:00:00.000Z')
    expect(res.ranAt).toBe('2026-07-13T00:00:00.000Z')
    expect(release).toHaveBeenCalledTimes(1)
  })

  it('rolls back and releases on a mid-transaction failure', async () => {
    const existing = { claude: [] as Array<{ id: string; model: string; source: string }> }
    const { connector, query, release } = makeConnector(existing)
    // Make the run-summary INSERT explode after the reconciliation writes.
    query.mockImplementation(async (sql: string) => {
      if (/INSERT INTO llm_catalog_sync_runs/.test(sql)) throw new Error('boom')
      if (/SELECT id, model, source/.test(sql)) return { rows: [], rowCount: 0 }
      return { rows: [], rowCount: 0 }
    })
    const catalog = catalogWith({ 'claude-opus-4-5': { id: 'claude-opus-4-5' } })
    await expect(
      syncDiscoveredModels({ materializer, loadCatalog: loadStub(catalog) }, connector)
    ).rejects.toThrow('boom')
    expect(query).toHaveBeenCalledWith('ROLLBACK')
    expect(release).toHaveBeenCalledTimes(1)
  })
})