Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
12.0 KB
Raw Blame
import { describe, expect, it, vi } from 'vitest'
import { PROVIDER_IDS } from '@clerum/llm-providers'
import { VENDORED_MODELS_DEV_SNAPSHOT_CAPTURED_AT } from '../src/data/modelsDevSnapshot.js'
import {
  MODELS_DEV_API_URL,
  PROVIDER_KEY_MAP,
  type RawModelsDevCatalog,
  imageInputStateFromModalities,
  loadModelsDevCatalog,
  mapCatalogToProviders,
} from '../src/services/modelsDevClient.js'

// A tiny catalog fixture shaped exactly like models.dev api.json (and the
// vendored snapshot): models.dev provider key → { name?, models: id → entry }.
const FIXTURE: RawModelsDevCatalog = {
  anthropic: {
    name: 'Anthropic',
    models: {
      'claude-opus-4-5': {
        id: 'claude-opus-4-5',
        name: 'Claude Opus 4.5',
        limit: { context: 200000 },
        modalities: { input: ['text', 'image'] },
      },
      'claude-haiku-4-5': {
        id: 'claude-haiku-4-5',
        name: 'Claude Haiku 4.5',
        modalities: { input: ['text'] },
      },
    },
  },
  google: {
    name: 'Google',
    models: {
      // Google-native `models/` prefix must be normalized away.
      'models/gemini-3.1-flash': {
        id: 'models/gemini-3.1-flash',
        name: 'Gemini 3.1 Flash',
        limit: { context: 1000000 },
      },
    },
  },
  openai: { name: 'OpenAI', models: {} },
}

describe('modelsDevClient — PROVIDER_KEY_MAP', () => {
  it('maps every static provider to a models.dev key and excludes oauth-broker ids', () => {
    const staticIds = PROVIDER_IDS.filter(
      id => id !== 'codex-subscription' && id !== 'grok-subscription'
    )
    for (const id of staticIds) {
      expect(typeof PROVIDER_KEY_MAP[id]).toBe('string')
      expect(PROVIDER_KEY_MAP[id].length).toBeGreaterThan(0)
    }
    expect(Object.keys(PROVIDER_KEY_MAP).sort()).toEqual([...staticIds].sort())
    expect('codex-subscription' in PROVIDER_KEY_MAP).toBe(false)
    expect('grok-subscription' in PROVIDER_KEY_MAP).toBe(false)
  })

  it('pins the non-obvious / ambiguous choices (zai coding-plan, bailian→alibaba)', () => {
    expect(PROVIDER_KEY_MAP.zai).toBe('zai-coding-plan')
    expect(PROVIDER_KEY_MAP.bailian).toBe('alibaba')
    expect(PROVIDER_KEY_MAP.claude).toBe('anthropic')
    expect(PROVIDER_KEY_MAP.gemini).toBe('google')
    expect(PROVIDER_KEY_MAP.vertex).toBe('google-vertex')
    expect(PROVIDER_KEY_MAP.bedrock).toBe('amazon-bedrock')
    expect(PROVIDER_KEY_MAP.together).toBe('togetherai')
    expect(PROVIDER_KEY_MAP.fireworks).toBe('fireworks-ai')
    expect(PROVIDER_KEY_MAP.moonshot).toBe('moonshotai')
    expect(PROVIDER_KEY_MAP.novita).toBe('novita-ai')
  })
})

describe('modelsDevClient — mapCatalogToProviders', () => {
  it('maps a provider key to our provider id with ctx + display_name, no vendor', () => {
    const byProvider = mapCatalogToProviders(FIXTURE)
    const claude = byProvider.claude
    expect(claude).toEqual([
      {
        model_id: 'claude-haiku-4-5',
        display_name: 'Claude Haiku 4.5',
        image_input_state: 'unsupported',
      },
      {
        model_id: 'claude-opus-4-5',
        display_name: 'Claude Opus 4.5',
        context_window_tokens: 200000,
        image_input_state: 'supported',
      },
    ])
    // vendor is never derived from models.dev (no reliable per-model field).
    for (const m of claude) expect('vendor' in m).toBe(false)
  })

  it('normalizes a `models/` prefixed id (gemini native style)', () => {
    const byProvider = mapCatalogToProviders(FIXTURE)
    expect(byProvider.gemini).toEqual([
      {
        model_id: 'gemini-3.1-flash',
        display_name: 'Gemini 3.1 Flash',
        context_window_tokens: 1000000,
        // No `modalities` in the fixture — the catalog said nothing, so neither do we.
        image_input_state: 'unknown',
      },
    ])
  })

  it('yields an empty list for a mapped provider absent from the catalog or with no models', () => {
    const byProvider = mapCatalogToProviders(FIXTURE)
    expect(byProvider.openai).toEqual([]) // present but empty models
    expect(byProvider.groq).toEqual([]) // absent from fixture
  })

  it('is defined for every provider id (never undefined)', () => {
    const byProvider = mapCatalogToProviders(FIXTURE)
    for (const id of PROVIDER_IDS) expect(Array.isArray(byProvider[id])).toBe(true)
  })

  it('SKIPs a model whose id exceeds the operator-API length cap (400) or has control chars', () => {
    const longId = 'x'.repeat(401)
    const catalog: RawModelsDevCatalog = {
      anthropic: {
        name: 'Anthropic',
        models: {
          [longId]: { id: longId, name: 'too long' },
          'bad\nid': { id: 'bad\nid', name: 'newline id' },
          'has\u0000null': { id: 'has\u0000null', name: 'null byte' },
          good: { id: 'good', name: 'ok' },
        },
      },
    }
    const claude = mapCatalogToProviders(catalog).claude
    // Only the well-formed id survives; the three malformed ones are dropped.
    expect(claude).toEqual([{ model_id: 'good', display_name: 'ok', image_input_state: 'unknown' }])
  })

  it('CLAMPs an over-long display_name to 400 chars (keeps the model)', () => {
    const longName = 'n'.repeat(401)
    const catalog: RawModelsDevCatalog = {
      anthropic: { name: 'Anthropic', models: { 'm-1': { id: 'm-1', name: longName } } },
    }
    const [model] = mapCatalogToProviders(catalog).claude
    expect(model.model_id).toBe('m-1')
    expect(model.display_name).toBe('n'.repeat(400))
    expect(model.display_name!.length).toBe(400)
  })

  it('keeps a 400-char id (boundary) — only >400 is rejected', () => {
    const id400 = 'a'.repeat(400)
    const catalog: RawModelsDevCatalog = {
      anthropic: { name: 'Anthropic', models: { [id400]: { id: id400 } } },
    }
    expect(mapCatalogToProviders(catalog).claude).toEqual([
      { model_id: id400, image_input_state: 'unknown' },
    ])
  })

  it('carries image_input_state per discovered model', () => {
    const catalog: RawModelsDevCatalog = {
      anthropic: {
        name: 'Anthropic',
        models: {
          'sees-images': { id: 'sees-images', modalities: { input: ['text', 'image'] } },
          'text-only': { id: 'text-only', modalities: { input: ['text'] } },
          'no-modalities': { id: 'no-modalities' },
        },
      },
    }
    const claude = mapCatalogToProviders(catalog).claude
    // Witness: all three entries survived the mapping, so the states below are
    // read off real rows and not off an empty list.
    expect(claude.map(m => m.model_id)).toEqual(['no-modalities', 'sees-images', 'text-only'])
    expect(claude.map(m => m.image_input_state)).toEqual(['unknown', 'supported', 'unsupported'])
  })
})

describe('modelsDevClient — imageInputStateFromModalities', () => {
  it('returns supported when modalities.input includes "image"', () => {
    expect(imageInputStateFromModalities({ modalities: { input: ['text', 'image'] } })).toBe(
      'supported'
    )
    expect(imageInputStateFromModalities({ modalities: { input: ['image'] } })).toBe('supported')
  })

  it('returns unsupported for a non-empty string array without "image"', () => {
    expect(imageInputStateFromModalities({ modalities: { input: ['text'] } })).toBe('unsupported')
    expect(imageInputStateFromModalities({ modalities: { input: ['text', 'audio'] } })).toBe(
      'unsupported'
    )
    // The token is matched exactly: models.dev writes it lowercase.
    expect(imageInputStateFromModalities({ modalities: { input: ['text', 'Image'] } })).toBe(
      'unsupported'
    )
  })

  it('returns unknown when modalities is absent', () => {
    expect(imageInputStateFromModalities({})).toBe('unknown')
    expect(imageInputStateFromModalities({ id: 'm-1', name: 'M1' })).toBe('unknown')
  })

  it('returns unknown when modalities.input is absent, not an array, or empty', () => {
    expect(imageInputStateFromModalities({ modalities: {} })).toBe('unknown')
    expect(imageInputStateFromModalities({ modalities: { input: 'image' } })).toBe('unknown')
    expect(imageInputStateFromModalities({ modalities: { input: null } })).toBe('unknown')
    expect(imageInputStateFromModalities({ modalities: { input: {} } })).toBe('unknown')
    expect(imageInputStateFromModalities({ modalities: { input: [] } })).toBe('unknown')
  })

  it('returns unknown when any element is not a string', () => {
    expect(imageInputStateFromModalities({ modalities: { input: ['text', 1] } })).toBe('unknown')
    // Load-bearing: the array DOES contain 'image', and the answer is still
    // unknown — a malformed list is not evidence of anything.
    expect(imageInputStateFromModalities({ modalities: { input: ['image', null] } })).toBe(
      'unknown'
    )
  })
})

describe('modelsDevClient — loadModelsDevCatalog', () => {
  function jsonResponse(body: unknown, init?: ResponseInit): Response {
    return new Response(JSON.stringify(body), {
      status: 200,
      headers: { 'content-type': 'application/json' },
      ...init,
    })
  }

  it('returns source=live on a successful, well-shaped fetch', async () => {
    const fetchImpl = vi.fn().mockResolvedValue(jsonResponse(FIXTURE))
    const res = await loadModelsDevCatalog({ fetchImpl: fetchImpl as unknown as typeof fetch })
    expect(res.source).toBe('live')
    expect(res.catalog.anthropic?.models['claude-opus-4-5']?.id).toBe('claude-opus-4-5')
    expect(fetchImpl).toHaveBeenCalledWith(
      MODELS_DEV_API_URL,
      expect.objectContaining({ redirect: 'error' })
    )
    expect(typeof res.fetchedAt).toBe('string')
    // Live data IS the observation: acquisition and capture coincide.
    expect(res.capturedAt).toBe(res.fetchedAt)
  })

  it('falls back to the vendored snapshot when the fetch throws (network error)', async () => {
    const fetchImpl = vi.fn().mockRejectedValue(new Error('ECONNREFUSED'))
    const res = await loadModelsDevCatalog({
      fetchImpl: fetchImpl as unknown as typeof fetch,
      // A vendored fallback loaded today is NOT a fresh observation: the capture
      // stamp stays the snapshot's baked date.
      now: () => new Date('2026-09-16T12:00:00.000Z'),
    })
    expect(res.source).toBe('vendored')
    expect(res.fetchedAt).toBe('2026-09-16T12:00:00.000Z')
    expect(res.capturedAt).toBe(VENDORED_MODELS_DEV_SNAPSHOT_CAPTURED_AT)
    expect(res.capturedAt).not.toBe(res.fetchedAt)
    // The vendored snapshot must carry our mapped providers so the sync has data.
    const byProvider = mapCatalogToProviders(res.catalog)
    expect(byProvider.claude.length).toBeGreaterThan(0)
    expect(byProvider.openai.length).toBeGreaterThan(0)
    // #654: it must also carry `modalities.input`. The offline path is the one
    // taken when there is NO network — exactly where an operator cannot go look
    // the answer up — so a snapshot regenerated without modalities would report
    // every model's image capability as `unknown` and make the feature inert
    // precisely where it is least recoverable. Both verdicts are asserted: a
    // snapshot that only ever answers `supported` would be just as wrong.
    const states = Object.values(byProvider)
      .flat()
      .map(m => m.image_input_state)
    expect(states.filter(s => s === 'supported').length).toBeGreaterThan(0)
    expect(states.filter(s => s === 'unsupported').length).toBeGreaterThan(0)
  })

  it('falls back to vendored on a non-2xx response', async () => {
    const fetchImpl = vi.fn().mockResolvedValue(new Response('nope', { status: 503 }))
    const res = await loadModelsDevCatalog({ fetchImpl: fetchImpl as unknown as typeof fetch })
    expect(res.source).toBe('vendored')
  })

  it('falls back to vendored on malformed JSON', async () => {
    const fetchImpl = vi.fn().mockResolvedValue(new Response('{not json', { status: 200 }))
    const res = await loadModelsDevCatalog({ fetchImpl: fetchImpl as unknown as typeof fetch })
    expect(res.source).toBe('vendored')
  })

  it('falls back to vendored on a well-formed but wrong-shape body (parse-guard)', async () => {
    const fetchImpl = vi.fn().mockResolvedValue(jsonResponse([1, 2, 3]))
    const res = await loadModelsDevCatalog({ fetchImpl: fetchImpl as unknown as typeof fetch })
    expect(res.source).toBe('vendored')
  })
})