Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
15.1 KB
Raw Blame
import { describe, expect, it } from 'vitest'
import { PROVIDER_IDS } from '@clerum/llm-providers'
import {
  BEDROCK_CREDENTIAL_KEYS,
  LLM_CREDENTIAL_GROUPS,
  LLM_DEFAULT_MODEL_BY_PROVIDER,
  LLM_SECRET_EDITOR_GROUPS,
  type LlmModelCatalogEntry,
  brokerBackedRecipeAuthoringError,
  budgetUnitAllowedForProviders,
  catalogGroupKey,
  getAllModelOptions,
  getLlmGroupCompleteness,
  getModelOptions,
  getProviderDisplayLabel,
  getProviderLabel,
  inferProviderFromModels,
  isOpenAiFamily,
  llmChainRequiresSecret,
  operatorProviderOptions,
  providerRequiresLlmSecret,
  resolveDefaultModel,
  validateLlmSecretData,
} from '../llm'

// The model catalog now comes from the operator allowlist (`/admin/llm-models`),
// so the helpers take the rows as an argument instead of reading a static map.
const CATALOG: LlmModelCatalogEntry[] = [
  { provider: 'openai', model: 'gpt-5.4', enabled: true },
  { provider: 'openai', model: 'gpt-5.4-mini', enabled: true },
  { provider: 'openai', model: 'gpt-5.1-codex', enabled: false },
  { provider: 'claude', model: 'claude-opus-4-7', enabled: true },
  { provider: 'claude', model: 'claude-sonnet-4-6', enabled: true },
  { provider: 'zai', model: 'glm-5.2', enabled: true },
  { provider: 'zai', model: 'glm-5.1', enabled: true },
  { provider: 'bailian', model: 'qwen3-coder-plus', enabled: true },
  { provider: 'bailian', model: 'glm-5.1', enabled: true },
]

describe('getModelOptions', () => {
  it('returns only enabled models for a provider by default', () => {
    expect(getModelOptions(CATALOG, 'openai')).toEqual(['gpt-5.4', 'gpt-5.4-mini'])
  })

  it('includes disabled models when asked', () => {
    expect(getModelOptions(CATALOG, 'openai', { includeDisabled: true })).toEqual([
      'gpt-5.4',
      'gpt-5.4-mini',
      'gpt-5.1-codex',
    ])
  })

  it('returns an empty list for a provider with no rows', () => {
    expect(getModelOptions(CATALOG, 'vertex')).toEqual([])
  })

  it('hides stale models from new picks unless includeStale is set', () => {
    const catalog: LlmModelCatalogEntry[] = [
      { provider: 'codex-subscription', model: 'gpt-5.1', enabled: true },
      { provider: 'codex-subscription', model: 'old-codex', enabled: true, stale: true },
      { provider: 'codex-subscription', model: 'disabled-codex', enabled: false },
    ]
    expect(getModelOptions(catalog, 'codex-subscription')).toEqual(['gpt-5.1'])
    expect(getModelOptions(catalog, 'codex-subscription', { includeStale: true })).toEqual([
      'gpt-5.1',
      'old-codex',
    ])
  })
})

describe('OpenAI family presentation', () => {
  // The agent-side credential picker offers a credential PATH, so an API key and
  // a subscription are peers there and both families are treated alike. One row
  // per family is the rule of the /llm-models catalog, which collapses its rows
  // with `collapseFamilyRows` and never calls this function.
  it('offers an API key and a subscription at the same level, in both families', () => {
    const options = operatorProviderOptions({ grokEnabled: true })
    expect(options.some(option => option.value === 'openai')).toBe(true)
    expect(options.some(option => option.value === 'codex-subscription')).toBe(true)
    expect(options.some(option => option.value === 'xai')).toBe(true)
    expect(options.some(option => option.value === 'grok-subscription')).toBe(true)
  })

  // The Grok filter is a deployment capability gate, not a statement about
  // families: Codex has no such signal in control-ui and is never gated.
  it('hides only Grok, and only until the deployment gate is proven on', () => {
    const gated = operatorProviderOptions()
    expect(gated.some(option => option.value === 'grok-subscription')).toBe(false)
    expect(gated.some(option => option.value === 'codex-subscription')).toBe(true)
    expect(gated.some(option => option.value === 'xai')).toBe(true)
  })

  it('labels the subscription runtime id as OpenAI for operators', () => {
    expect(getProviderLabel('codex-subscription')).toBe('OpenAI')
    expect(getProviderDisplayLabel('codex-subscription')).toBe('OpenAI')
    expect(getProviderLabel('openai')).toBe('OpenAI')
  })

  it('groups subscription catalog rows under OpenAI', () => {
    expect(catalogGroupKey('codex-subscription')).toBe('openai')
    expect(catalogGroupKey('openai')).toBe('openai')
    expect(catalogGroupKey('claude')).toBe('claude')
    expect(isOpenAiFamily('codex-subscription')).toBe(true)
  })

  it('groups Grok subscription catalog rows under xAI, not under OpenAI', () => {
    expect(catalogGroupKey('grok-subscription')).toBe('xai')
    expect(catalogGroupKey('xai')).toBe('xai')
    expect(isOpenAiFamily('grok-subscription')).toBe(false)
    expect(getProviderLabel('grok-subscription')).not.toBe('OpenAI')
  })

  it('keeps the broker label distinct from its family label', () => {
    // grok-subscription is offered in the operator picker beside xai, so the two
    // must stay tellable apart. Only the catalog GROUPING is unified.
    expect(getProviderLabel('grok-subscription')).toBe('xAI Grok Subscription')
    expect(getProviderDisplayLabel('grok-subscription')).toBe('xAI Grok Subscription')
    expect(getProviderLabel('xai')).toBe('xAI (Grok)')
  })

  it('passes an unknown provider through as its own group', () => {
    // The prices table carries free-form providers; they must not throw and must
    // not be folded into anything.
    expect(catalogGroupKey('not-a-provider')).toBe('not-a-provider')
  })
})

describe('getAllModelOptions', () => {
  it('flattens and de-duplicates enabled models across providers', () => {
    const all = getAllModelOptions(CATALOG)
    // glm-5.1 exists for both zai and bailian — appears once.
    expect(all.filter(m => m === 'glm-5.1')).toHaveLength(1)
    expect(all).toContain('gpt-5.4')
    expect(all).toContain('claude-opus-4-7')
    expect(all).not.toContain('gpt-5.1-codex') // disabled
  })
})

describe('resolveDefaultModel', () => {
  it('prefers the static provider default when it is enabled', () => {
    const enabled = getModelOptions(CATALOG, 'claude')
    // Seed the static default into the enabled list to exercise the happy path.
    const claudeDefault = LLM_DEFAULT_MODEL_BY_PROVIDER.claude
    expect(claudeDefault).toBeDefined()
    const withDefault = [...enabled, claudeDefault!]
    expect(resolveDefaultModel('claude', withDefault)).toBe(claudeDefault)
  })

  it('falls back to the first enabled model when the default is not allowed', () => {
    // 'gpt-5.4-mini' (the static openai default) is intentionally absent here.
    expect(resolveDefaultModel('openai', ['gpt-5.4', 'gpt-5.1-codex'])).toBe('gpt-5.4')
  })

  it('returns an empty string when the provider has no enabled models', () => {
    expect(resolveDefaultModel('openai', [])).toBe('')
  })
})

describe('inferProviderFromModels', () => {
  it('recovers the provider from the first model found in the catalog', () => {
    expect(inferProviderFromModels(['glm-5.2'], CATALOG)).toBe('zai')
    expect(inferProviderFromModels(['claude-sonnet-4-6'], CATALOG)).toBe('claude')
    expect(inferProviderFromModels(['gpt-5.4-mini'], CATALOG)).toBe('openai')
    expect(inferProviderFromModels(['qwen3-coder-plus'], CATALOG)).toBe('bailian')
  })

  it('uses the first recognized model when the list is mixed', () => {
    expect(inferProviderFromModels(['unknown-model', 'glm-5.2'], CATALOG)).toBe('zai')
  })

  it('defaults to the first provider option for an empty or unknown list', () => {
    expect(inferProviderFromModels([], CATALOG)).toBe('openai')
    expect(inferProviderFromModels(['some-future-model'], CATALOG)).toBe('openai')
  })
})

describe('LLM_CREDENTIAL_GROUPS (spec R4.5.1/R4.5.2)', () => {
  it('renders one group per canonical provider, derived from the package', () => {
    // Derived from the package so this never drifts as providers are added
    // (R6 expanded the set to 21). One group per canonical id, in order.
    expect(LLM_CREDENTIAL_GROUPS.map(g => g.provider)).toEqual([...PROVIDER_IDS])
  })

  it('models single-slot providers as one field', () => {
    const openai = LLM_CREDENTIAL_GROUPS.find(g => g.provider === 'openai')!
    expect(openai.slots.map(s => s.dataKey)).toEqual(['openai-api-key'])
    expect(openai.slots[0].multiline).toBe(false)
  })

  it('models Bedrock as the access-key pair', () => {
    const bedrock = LLM_CREDENTIAL_GROUPS.find(g => g.provider === 'bedrock')!
    expect(bedrock.slots.map(s => s.dataKey)).toEqual([
      'aws-access-key-id',
      'aws-secret-access-key',
    ])
    expect(bedrock.nonSecretEnv).toContain('AWS_REGION')
  })

  it('models Vertex as a multiline service-account JSON slot with env hints', () => {
    const vertex = LLM_CREDENTIAL_GROUPS.find(g => g.provider === 'vertex')!
    expect(vertex.slots.map(s => s.dataKey)).toEqual(['vertex-service-account-json'])
    expect(vertex.slots[0].multiline).toBe(true)
    expect(vertex.nonSecretEnv).toEqual(expect.arrayContaining(['VERTEX_PROJECT_ID']))
  })

  it('models Codex as a zero-slot broker with no static default model', () => {
    const codex = LLM_CREDENTIAL_GROUPS.find(g => g.provider === 'codex-subscription')!
    expect(codex.slots).toEqual([])
    expect(codex.nonSecretEnv).toEqual([])
    expect(LLM_DEFAULT_MODEL_BY_PROVIDER['codex-subscription']).toBeUndefined()
    expect(resolveDefaultModel('codex-subscription', ['gpt-5.1'])).toBe('')
  })

  it('models Grok as a zero-slot broker with no invented default model', () => {
    const grok = LLM_CREDENTIAL_GROUPS.find(g => g.provider === 'grok-subscription')!
    expect(grok.slots).toEqual([])
    expect(LLM_DEFAULT_MODEL_BY_PROVIDER['grok-subscription']).toBeUndefined()
  })

  it('keeps oauth-broker providers off the Kubernetes Secret editor list', () => {
    expect(LLM_SECRET_EDITOR_GROUPS.some(group => group.provider === 'codex-subscription')).toBe(
      false
    )
    expect(LLM_SECRET_EDITOR_GROUPS.some(group => group.provider === 'grok-subscription')).toBe(
      false
    )
    expect(LLM_SECRET_EDITOR_GROUPS.every(group => group.slots.length > 0)).toBe(true)
  })
})

describe('getLlmGroupCompleteness (spec R4.5.5)', () => {
  const bedrock = LLM_CREDENTIAL_GROUPS.find(g => g.provider === 'bedrock')!
  const openai = LLM_CREDENTIAL_GROUPS.find(g => g.provider === 'openai')!

  it('marks a single-slot provider usable once its key is present', () => {
    expect(getLlmGroupCompleteness(openai, () => false)).toEqual({
      present: 0,
      total: 1,
      usable: false,
    })
    expect(getLlmGroupCompleteness(openai, () => true).usable).toBe(true)
  })

  it('treats a zero-slot oauth-broker provider as usable without any keys', () => {
    const codex = LLM_CREDENTIAL_GROUPS.find(g => g.provider === 'codex-subscription')!
    expect(getLlmGroupCompleteness(codex, () => false)).toEqual({
      present: 0,
      total: 0,
      usable: true,
    })
  })

  it('is usable for Bedrock only when both required slots are present', () => {
    expect(getLlmGroupCompleteness(bedrock, k => k === 'aws-access-key-id')).toEqual({
      present: 1,
      total: 2,
      usable: false,
    })
    expect(getLlmGroupCompleteness(bedrock, () => true)).toEqual({
      present: 2,
      total: 2,
      usable: true,
    })
  })
})

describe('validateLlmSecretData (spec R4.5.3)', () => {
  it('accepts an empty payload and single-key providers', () => {
    expect(validateLlmSecretData({})).toEqual([])
    expect(validateLlmSecretData({ 'openai-api-key': 'sk-x' })).toEqual([])
  })

  it('rejects a half-written Bedrock pair', () => {
    const errors = validateLlmSecretData({ 'aws-access-key-id': 'AKIA' })
    expect(errors).toHaveLength(1)
    expect(errors[0]).toMatch(/aws-secret-access-key/)
  })

  it('accepts the full Bedrock pair', () => {
    expect(
      validateLlmSecretData({
        'aws-access-key-id': 'AKIA',
        'aws-secret-access-key': 'secret',
      })
    ).toEqual([])
    // Guard against a divergent hardcoded key list.
    expect(BEDROCK_CREDENTIAL_KEYS).toEqual(['aws-access-key-id', 'aws-secret-access-key'])
  })

  it('rejects malformed Vertex JSON', () => {
    expect(validateLlmSecretData({ 'vertex-service-account-json': 'nope' })[0]).toMatch(/JSON/)
  })

  it('rejects Vertex JSON missing required fields', () => {
    const errors = validateLlmSecretData({
      'vertex-service-account-json': JSON.stringify({ type: 'service_account' }),
    })
    expect(errors[0]).toMatch(/client_email/)
  })

  it('accepts a well-formed Vertex service-account JSON', () => {
    expect(
      validateLlmSecretData({
        'vertex-service-account-json': JSON.stringify({
          client_email: '[email protected]',
          private_key: '-----BEGIN PRIVATE KEY-----\nx\n-----END PRIVATE KEY-----\n',
        }),
      })
    ).toEqual([])
  })
})

describe('broker-backed authoring helpers', () => {
  it('requires a Secret only when any static-credentials provider is in the chain', () => {
    expect(providerRequiresLlmSecret('codex-subscription')).toBe(false)
    expect(providerRequiresLlmSecret('openai')).toBe(true)
    expect(llmChainRequiresSecret('codex-subscription')).toBe(false)
    expect(llmChainRequiresSecret('codex-subscription', [{ provider: 'openai' }])).toBe(true)
    expect(llmChainRequiresSecret('openai', [{ provider: 'codex-subscription' }])).toBe(true)
  })

  it('rejects cost-unit budgets when a broker provider is in scope', () => {
    expect(budgetUnitAllowedForProviders('tokens', ['codex-subscription'])).toBe(true)
    expect(budgetUnitAllowedForProviders('cost', ['openai'])).toBe(true)
    expect(budgetUnitAllowedForProviders('cost', ['codex-subscription'])).toBe(false)
    expect(budgetUnitAllowedForProviders('cost', ['openai', 'codex-subscription'])).toBe(false)
  })

  it('rejects Codex recipe authoring that omits the model, ships a secretRef, or uses cost', () => {
    expect(
      brokerBackedRecipeAuthoringError(
        { agent: { provider: 'codex-subscription', model: 'gpt-5.1' } },
        'team-plus'
      )
    ).toBeNull()
    expect(
      brokerBackedRecipeAuthoringError({
        agent: { provider: 'codex-subscription', model: 'gpt-5.1' },
      })
    ).toMatch(/ChatGPT grant/)
    expect(
      brokerBackedRecipeAuthoringError({
        agent: { provider: 'codex-subscription' },
      })
    ).toMatch(/explicit catalog model/)
    expect(
      brokerBackedRecipeAuthoringError({
        agent: {
          provider: 'codex-subscription',
          model: 'gpt-5.1',
          secretRef: { name: 'codex-oauth', key: 'refresh' },
        },
      })
    ).toMatch(/must not declare an LLM secretRef/)
    expect(
      brokerBackedRecipeAuthoringError({
        agent: { provider: 'codex-subscription', model: 'gpt-5.1' },
        budget: { unit: 'cost' },
      })
    ).toMatch(/unit tokens/)
  })

  it('requires a Grok grant when only a step agent uses grok-subscription', () => {
    expect(
      brokerBackedRecipeAuthoringError({
        agent: { provider: 'openai', model: 'gpt-5.1' },
        steps: [{ agent: { provider: 'grok-subscription', model: 'grok-4.6' } }],
      })
    ).toMatch(/Grok grant/)
    expect(
      brokerBackedRecipeAuthoringError(
        {
          agent: { provider: 'openai', model: 'gpt-5.1' },
          steps: [{ agent: { provider: 'grok-subscription', model: 'grok-4.6' } }],
        },
        'team-grok'
      )
    ).toBeNull()
  })
})