Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
18.7 KB
Raw Blame
/**
 * Property-based coverage (T2) for the catalog-sync reconciliation — the decision
 * logic in `reconcileProvider` / `syncDiscoveredModels` (manual-invisible,
 * discovery-refresh, vanished→stale, per-provider + global sanity floors). The
 * example-based suites (services.llmCatalogSync.invariants / .guard) pin single
 * scenarios; these fuzz random LIVE catalogs against random pre-existing rows to
 * blind the invariants across the precedence combinations a human enumeration
 * misses. No behavior is exercised the example suites don't already cover — this
 * is coverage, not a fix.
 *
 * T1 (no hand-built fixtures): every catalog is cut from the REAL vendored
 * snapshot (via `trimSnapshot`, the same producer the example suites use), and
 * every pre-existing "present"/"vanished" row's `model` is the EXACT normalized
 * id that `mapCatalogToProviders` (the real mapping layer) emits for that catalog
 * — never an id we invent. If the producer can't emit a shape, no test asserts on
 * it.
 *
 * T4 (observable state): assertions are on the resulting row set (which rows are
 * `stale`, which are `enabled`) and the run's `staled`/`added` counts — never on
 * intermediate SQL or call counts.
 */
import { describe, expect, it, vi } from 'vitest'
import fc from 'fast-check'
import type { LlmProviderId } from '@clerum/llm-providers'
import { VENDORED_MODELS_DEV_SNAPSHOT } from '../src/data/modelsDevSnapshot.js'
import { syncDiscoveredModels } from '../src/services/llmCatalogSync.js'
import { PROVIDER_KEY_MAP, mapCatalogToProviders } from '../src/services/modelsDevClient.js'
import { type FakeDb, type SeedRow, makeFakeDb } from './helpers/llmCatalogSyncFakeDb.js'
import { loadStub, trimSnapshot, withModalities } from './helpers/modelsDevFixtures.js'

vi.mock('../src/observability/logger.js', () => ({
  rootLogger: { child: () => ({ info() {}, warn() {}, error() {}, debug() {} }) },
}))

const NUM_RUNS = 200

// models.dev provider KEY → our provider id (inverse of PROVIDER_KEY_MAP). The
// mapping is 1:1, so the inverse is well-defined.
const KEY_TO_ID = Object.fromEntries(
  Object.entries(PROVIDER_KEY_MAP).map(([id, key]) => [key, id as LlmProviderId])
) as Record<string, LlmProviderId>

// Snapshot provider keys we (a) map to a provider id and (b) that carry models —
// the only keys a live catalog cut from the snapshot could contain.
const MAPPED_KEYS = Object.keys(VENDORED_MODELS_DEV_SNAPSHOT).filter(
  k => KEY_TO_ID[k] && Object.keys(VENDORED_MODELS_DEV_SNAPSHOT[k]!.models).length > 0
)

/** One provider's slice of a generated scenario: which snapshot ids are live vs vanished. */
interface KeySelection {
  key: string
  liveIds: string[]
  /** snapshot ids seeded as discovery rows but ABSENT from the live catalog. */
  vanishedIds: string[]
}

/**
 * Arbitrary of a set of per-provider selections. `allowZeroLive` controls whether
 * a chosen provider may come back with 0 live models (needed for the per-provider
 * floor property; forbidden for the staling properties where every provider must
 * clear the floor so vanished→stale actually fires).
 */
function selectionsArb(opts: { allowZeroLive: boolean }): fc.Arbitrary<KeySelection[]> {
  return fc
    .uniqueArray(fc.constantFrom(...MAPPED_KEYS), { minLength: 1, maxLength: 4 })
    .chain(keys =>
      fc.tuple(
        ...keys.map(key => {
          const allIds = Object.keys(VENDORED_MODELS_DEV_SNAPSHOT[key]!.models)
          const liveArb = fc.subarray(allIds, {
            minLength: opts.allowZeroLive ? 0 : 1,
            maxLength: Math.min(allIds.length, 6),
          })
          return liveArb.chain(liveIds => {
            const remaining = allIds.filter(id => !liveIds.includes(id))
            return fc
              .subarray(remaining, { maxLength: Math.min(remaining.length, 4) })
              .map(vanishedIds => ({ key, liveIds, vanishedIds }) as KeySelection)
          })
        })
      )
    )
}

interface Scenario {
  selections: KeySelection[]
  /** deterministic per-(provider,model) flags — decouples flag gen from id counts. */
  enabledFor: (...args: unknown[]) => boolean
  staleFor: (...args: unknown[]) => boolean
  /** whether a still-listed (present) model was pre-seeded (else it is a NEW insert). */
  seedPresentFor: (...args: unknown[]) => boolean
}

function scenarioArb(opts: { allowZeroLive: boolean }): fc.Arbitrary<Scenario> {
  return fc.record({
    selections: selectionsArb(opts),
    enabledFor: fc.func(fc.boolean()),
    staleFor: fc.func(fc.boolean()),
    seedPresentFor: fc.func(fc.boolean()),
  })
}

/** Per-provider derived id sets, computed via the REAL mapping producer. */
interface ProviderPlan {
  providerId: LlmProviderId
  /** normalized ids the live catalog emits for this provider (still-listed). */
  presentIds: string[]
  /** normalized ids seeded as discovery rows but not in the live catalog. */
  vanishedIds: string[]
}

/**
 * Build the concrete catalog + seed rows a scenario stands for. Present/vanished
 * `model` values are the mapping layer's own output (T1) — never invented.
 */
function materialize(s: Scenario): {
  catalog: ReturnType<typeof trimSnapshot>
  seed: SeedRow[]
  plans: ProviderPlan[]
} {
  const liveSpec: Record<string, string[]> = {}
  for (const sel of s.selections) if (sel.liveIds.length) liveSpec[sel.key] = sel.liveIds
  const catalog = Object.keys(liveSpec).length ? trimSnapshot(liveSpec) : {}
  const mapped = mapCatalogToProviders(catalog)

  const seed: SeedRow[] = []
  const plans: ProviderPlan[] = []
  for (const sel of s.selections) {
    const providerId = KEY_TO_ID[sel.key]!
    const presentIds = mapped[providerId].map(m => m.model_id)
    const presentSet = new Set(presentIds)

    // Vanished ids: normalized through the SAME producer, then excluded from the
    // present set so present/vanished never collide.
    const vanCatalog = sel.vanishedIds.length ? trimSnapshot({ [sel.key]: sel.vanishedIds }) : {}
    const vanishedIds = mapCatalogToProviders(vanCatalog)
      [providerId].map(m => m.model_id)
      .filter(id => !presentSet.has(id))

    plans.push({ providerId, presentIds, vanishedIds })

    for (const id of presentIds) {
      // Some present models are pre-seeded (refresh path), some are absent (NEW
      // insert path) — both must converge to the same stable state.
      if (!s.seedPresentFor(providerId, id)) continue
      seed.push({
        provider: providerId,
        model: id,
        source: 'discovery',
        enabled: s.enabledFor(providerId, id),
        stale: s.staleFor(providerId, id),
      })
    }
    for (const id of vanishedIds) {
      seed.push({
        provider: providerId,
        model: id,
        source: 'discovery',
        enabled: s.enabledFor(providerId, id),
        stale: s.staleFor(providerId, id),
      })
    }
  }
  return { catalog, seed, plans }
}

const rowKey = (r: { provider: string; model: string }) => `${r.provider}|${r.model}`
const staleSet = (db: FakeDb) => new Set(db.rows.filter(r => r.stale).map(rowKey))
const enabledSet = (db: FakeDb) => new Set(db.rows.filter(r => r.enabled).map(rowKey))

// Low floors: keep the sanity guard OUT of the way so the stale mechanics run.
const LIVE_LOW_FLOOR = { minPlausibleLiveTotal: 1, providerMinLive: 0 } as const

describe('catalog sync — property-based reconciliation invariants (R1-M4)', () => {
  it('P1 idempotence: re-applying the same LIVE catalog stales nothing new and does not flip-flop', async () => {
    await fc.assert(
      fc.asyncProperty(scenarioArb({ allowZeroLive: false }), async scenario => {
        const { catalog, seed } = materialize(scenario)
        const db = makeFakeDb(seed)

        await syncDiscoveredModels(
          {
            materializer: db.materializer,
            loadCatalog: loadStub(catalog, 'live'),
            ...LIVE_LOW_FLOOR,
          },
          db.connector
        )
        const staleAfter1 = staleSet(db)
        const enabledAfter1 = enabledSet(db)

        const res2 = await syncDiscoveredModels(
          {
            materializer: db.materializer,
            loadCatalog: loadStub(catalog, 'live'),
            ...LIVE_LOW_FLOOR,
          },
          db.connector
        )

        // 2nd pass: nothing new staled, nothing new inserted, no stale flip-flop.
        expect(res2.staled).toBe(0)
        expect(res2.added).toBe(0)
        expect(staleSet(db)).toEqual(staleAfter1)
        expect(enabledSet(db)).toEqual(enabledAfter1)
      }),
      { numRuns: NUM_RUNS }
    )
  })

  it('P2 auto-heal: every still-listed discovery row ends stale=false, and stays healed on re-run', async () => {
    await fc.assert(
      fc.asyncProperty(scenarioArb({ allowZeroLive: false }), async scenario => {
        const { catalog, seed, plans } = materialize(scenario)
        const db = makeFakeDb(seed)

        await syncDiscoveredModels(
          {
            materializer: db.materializer,
            loadCatalog: loadStub(catalog, 'live'),
            ...LIVE_LOW_FLOOR,
          },
          db.connector
        )

        // A present (still-listed) model — whether seeded stale or freshly
        // inserted — must be non-stale after a live sync.
        for (const plan of plans) {
          for (const id of plan.presentIds) {
            expect(db.get(plan.providerId, id)?.stale).toBe(false)
          }
        }

        const res2 = await syncDiscoveredModels(
          {
            materializer: db.materializer,
            loadCatalog: loadStub(catalog, 'live'),
            ...LIVE_LOW_FLOOR,
          },
          db.connector
        )
        // Re-run must not re-stale a healed row.
        expect(res2.staled).toBe(0)
        for (const plan of plans) {
          for (const id of plan.presentIds) {
            expect(db.get(plan.providerId, id)?.stale).toBe(false)
          }
        }
      }),
      { numRuns: NUM_RUNS }
    )
  })

  it('P3 non-destructive: the set of enabled rows is invariant and no seeded row is dropped', async () => {
    await fc.assert(
      fc.asyncProperty(scenarioArb({ allowZeroLive: false }), async scenario => {
        const { catalog, seed } = materialize(scenario)
        const db = makeFakeDb(seed)

        const enabledBefore = new Set(seed.filter(r => r.enabled).map(rowKey))

        await syncDiscoveredModels(
          {
            materializer: db.materializer,
            loadCatalog: loadStub(catalog, 'live'),
            ...LIVE_LOW_FLOOR,
          },
          db.connector
        )

        // Reconciliation never enables or disables a row: the enabled set is
        // exactly what was seeded (new inserts are enabled=false).
        expect(enabledSet(db)).toEqual(enabledBefore)
        // No seeded row is ever deleted, and its enabled flag is untouched.
        for (const s of seed) {
          const row = db.get(s.provider, s.model)
          expect(row).toBeDefined()
          expect(row?.enabled).toBe(s.enabled ?? false)
        }
      }),
      { numRuns: NUM_RUNS }
    )
  })

  it('P4a global floor: a below-floor LIVE total marks nothing new stale (fail-safe)', async () => {
    await fc.assert(
      fc.asyncProperty(scenarioArb({ allowZeroLive: false }), async scenario => {
        const { catalog, seed } = materialize(scenario)
        const db = makeFakeDb(seed)

        // Rows that were stale BEFORE the run (the guard may still HEAL present
        // rows to non-stale, so the post set may only shrink — never grow).
        const staleBefore = new Set(seed.filter(r => r.stale).map(rowKey))

        const res = await syncDiscoveredModels(
          // Impossibly high global floor → every live total is "implausible".
          {
            materializer: db.materializer,
            loadCatalog: loadStub(catalog, 'live'),
            minPlausibleLiveTotal: 1_000_000,
          },
          db.connector
        )

        // No vanished→stale marking happened this run.
        expect(res.staled).toBe(0)
        // Fail-safe: no row transitioned false→true. Post-stale ⊆ pre-stale.
        for (const key of staleSet(db)) {
          expect(staleBefore.has(key)).toBe(true)
        }
      }),
      { numRuns: NUM_RUNS }
    )
  })

  it('P4b per-provider floor: a provider below its live floor keeps its rows unstaled', async () => {
    await fc.assert(
      fc.asyncProperty(scenarioArb({ allowZeroLive: true }), async scenario => {
        const { catalog, seed, plans } = materialize(scenario)

        // Providers below the floor (0 live here, providerMinLive=0 → floor=1)
        // that DO have pre-existing discovery rows are the protected set.
        const protectedProviders = plans.filter(
          p => p.presentIds.length === 0 && p.vanishedIds.length > 0
        )
        // Non-vacuous only when at least one such provider exists.
        fc.pre(protectedProviders.length > 0)

        const db = makeFakeDb(seed)
        const staleBeforeByKey = new Map(seed.map(r => [rowKey(r), r.stale ?? false]))

        await syncDiscoveredModels(
          {
            materializer: db.materializer,
            loadCatalog: loadStub(catalog, 'live'),
            minPlausibleLiveTotal: 1,
            providerMinLive: 0,
          },
          db.connector
        )

        // Every protected provider's rows: none transitions false→true stale.
        for (const plan of protectedProviders) {
          for (const id of plan.vanishedIds) {
            const before = staleBeforeByKey.get(`${plan.providerId}|${id}`) ?? false
            const after = db.get(plan.providerId, id)?.stale
            if (!before) expect(after).toBe(false)
          }
        }
      }),
      { numRuns: NUM_RUNS }
    )
  })

  it('P5 image_input evidence is monotonic in checkedAt and never overwrites curated', async () => {
    const CAPTURED_AT = '2026-08-19T16:16:10.000Z'
    const TTL_MS = 30 * 24 * 60 * 60 * 1000
    const KEY = 'anthropic'
    const IDS = Object.keys(VENDORED_MODELS_DEV_SNAPSHOT[KEY]!.models).slice(0, 6)
    const PROVIDER_ID = KEY_TO_ID[KEY]!

    /** The four evidence shapes a row can already carry when a sync arrives. */
    type EvidenceKind = 'none' | 'curated' | 'discovery-older' | 'discovery-newer'
    const evidenceFor = (kind: EvidenceKind, offsetMs: number) => {
      if (kind === 'none') return null
      if (kind === 'curated') {
        return {
          state: 'supported' as const,
          evidence: {
            source: 'curated' as const,
            reference: 'https://docs.z.ai/guides/vlm/glm-5.3-flash',
            checkedAt: new Date(Date.parse(CAPTURED_AT) - offsetMs).toISOString(),
          },
        }
      }
      const sign = kind === 'discovery-older' ? -1 : 1
      const checkedAt = new Date(Date.parse(CAPTURED_AT) + sign * offsetMs).toISOString()
      return {
        state: 'unsupported' as const,
        evidence: {
          source: 'discovery' as const,
          reference: 'https://models.dev/api.json',
          checkedAt,
          validUntil: new Date(Date.parse(checkedAt) + TTL_MS).toISOString(),
        },
      }
    }

    const rowArb = fc.record({
      source: fc.constantFrom('discovery', 'manual'),
      enabled: fc.boolean(),
      kind: fc.constantFrom<EvidenceKind>('none', 'curated', 'discovery-older', 'discovery-newer'),
      // Strictly positive so `discovery-older`/`-newer` are never AT the capture
      // instant, where `<=` makes the outcome a boundary case, not a class.
      offsetMs: fc.integer({ min: 1_000, max: 90 * 24 * 60 * 60 * 1000 }),
      supportsImages: fc.boolean(),
    })

    await fc.assert(
      fc.asyncProperty(
        fc.array(rowArb, { minLength: IDS.length, maxLength: IDS.length }),
        async specs => {
          const seed: SeedRow[] = IDS.map((model, i) => ({
            provider: PROVIDER_ID,
            model,
            source: specs[i]!.source,
            enabled: specs[i]!.enabled,
            // Seeded stale so the flag DISCRIMINATES: the discovery branch
            // clears it on every row it touches, so a manual row still stale
            // after the run proves the manual statement did not write it.
            stale: true,
            image_input: evidenceFor(specs[i]!.kind, specs[i]!.offsetMs),
          }))
          const db = makeFakeDb(seed)
          const catalog = withModalities(
            trimSnapshot({ [KEY]: IDS }),
            Object.fromEntries(
              IDS.map((id, i) => [id, specs[i]!.supportsImages ? ['text', 'image'] : ['text']])
            )
          )

          await syncDiscoveredModels(
            {
              materializer: db.materializer,
              loadCatalog: loadStub(catalog, 'live', CAPTURED_AT),
              imageEvidenceTtlMs: TTL_MS,
              ...LIVE_LOW_FLOOR,
            },
            db.connector
          )

          // Witness the run was not vacuous: every seeded row is still there.
          expect(db.rows.length).toBeGreaterThanOrEqual(IDS.length)

          for (const [i, model] of IDS.entries()) {
            const spec = specs[i]!
            const before = evidenceFor(spec.kind, spec.offsetMs)
            const after = db.get(PROVIDER_ID, model)!.image_input
            const derived = {
              state: spec.supportsImages ? 'supported' : 'unsupported',
              evidence: {
                source: 'discovery',
                reference: 'https://models.dev/api.json',
                checkedAt: CAPTURED_AT,
                validUntil: new Date(Date.parse(CAPTURED_AT) + TTL_MS).toISOString(),
              },
            }
            // The guard is the SAME on both branches (#654), so `source` drops
            // out of this rule entirely: what decides is the evidence already
            // on the row, never who owns the row or whether it is enabled.
            //
            // Curated evidence outranks discovery whatever its age.
            if (spec.kind === 'curated') expect(after).toEqual(before)
            // A capture newer than this run's is never regressed.
            else if (spec.kind === 'discovery-newer') expect(after).toEqual(before)
            // Everything else takes the catalog's verdict — manual or not,
            // enabled or not.
            else expect(after).toEqual(derived)

            // …but a manual row keeps every column an operator authored. The
            // seed leaves these null/false, so a leak from the discovery
            // branch (which writes all three) is observable.
            const row = db.get(PROVIDER_ID, model)!
            if (spec.source === 'manual') {
              expect(row.source).toBe('manual')
              expect(row.stale).toBe(true)
              expect(row.display_name).toBeNull()
              expect(row.context_window_tokens).toBeNull()
            } else {
              // The discriminator's other half: the discovery branch DOES clear
              // `stale`, so `stale === true` above is a choice, not an untouched
              // row in a run that wrote nothing.
              expect(row.stale).toBe(false)
            }
          }
        }
      ),
      { numRuns: NUM_RUNS }
    )
  })
})