Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
4.1 KB
Raw Blame
import { describe, expect, it } from 'vitest'
import { readFileSync } from 'node:fs'
import { dirname, resolve } from 'node:path'
import { fileURLToPath } from 'node:url'
import { parse } from 'yaml'
import { PROVIDER_AUTH_MODE, PROVIDER_IDS } from '@clerum/llm-providers'

// These are CRD-enum (K8s admission) invariants, not control-api code paths:
// a provider missing from a CRD enum is rejected by the apiserver on write. We
// assert them here so the enum edits can't silently drift from the shared
// provider list. `bailian` is unique to the LLM provider enums, so collecting
// every enum array that contains it yields exactly the provider enums.
const crdsDir = resolve(dirname(fileURLToPath(import.meta.url)), '../../charts/clerum-crds/crds')

const NEW_SINGLE_KEY = [
  'openrouter',
  'gemini',
  'deepseek',
  'groq',
  'together',
  'fireworks',
  'mistral',
  'xai',
  'cerebras',
  'deepinfra',
  'perplexity',
  'moonshot',
  'nebius',
  'novita',
  'minimax',
] as const

function collectProviderEnums(crdFile: string): string[][] {
  const doc = parse(readFileSync(resolve(crdsDir, crdFile), 'utf8'))
  const found: string[][] = []
  const walk = (node: unknown): void => {
    if (Array.isArray(node)) {
      node.forEach(walk)
      return
    }
    if (node && typeof node === 'object') {
      for (const [key, value] of Object.entries(node as Record<string, unknown>)) {
        if (
          key === 'enum' &&
          Array.isArray(value) &&
          value.includes('bailian') &&
          value.includes('openai')
        ) {
          found.push(value as string[])
        }
        walk(value)
      }
    }
  }
  walk(doc)
  return found
}

describe('LLM provider CRD enums', () => {
  it('host.yaml carries all 16 new providers (incl. azure) in both provider enums', () => {
    const enums = collectProviderEnums('host.yaml')
    // spec.model.provider + spec.llmPolicy.fallbacks[].provider
    expect(enums).toHaveLength(2)
    for (const e of enums) {
      for (const id of [...NEW_SINGLE_KEY, 'azure', 'codex-subscription', 'grok-subscription']) {
        expect(e).toContain(id)
      }
      // additive: original providers preserved
      expect(e).toContain('bedrock')
      expect(e).toContain('vertex')
    }
  })

  it('workflowrecipe.yaml carries the 15 single-key providers but EXCLUDES azure and bedrock', () => {
    const enums = collectProviderEnums('workflowrecipe.yaml')
    // the two WRC provider enums (model.provider + a second one ~line 543)
    expect(enums).toHaveLength(2)
    for (const e of enums) {
      for (const id of [...NEW_SINGLE_KEY, 'codex-subscription', 'grok-subscription']) {
        expect(e).toContain(id)
      }
      // azure and bedrock must fail at admission (mono-credential WRC transport).
      expect(e).not.toContain('azure')
      expect(e).not.toContain('bedrock')
    }
  })

  it('Host and WorkflowRecipe CEL mention every oauth-broker provider', () => {
    const host = readFileSync(resolve(crdsDir, 'host.yaml'), 'utf8')
    const recipe = readFileSync(resolve(crdsDir, 'workflowrecipe.yaml'), 'utf8')
    const brokers = PROVIDER_IDS.filter(id => PROVIDER_AUTH_MODE[id] === 'oauth-broker')
    expect(brokers.length).toBeGreaterThan(0)
    for (const id of brokers) {
      expect(host).toMatch(new RegExp(`provider == '${id}'`))
      expect(recipe).toMatch(new RegExp(`provider (==|!=) '${id}'`))
    }
  })

  it('WorkflowRecipe CEL rejects mixed oauth-broker agent and step providers', () => {
    const recipe = readFileSync(resolve(crdsDir, 'workflowrecipe.yaml'), 'utf8')
    expect(recipe).toContain('a WorkflowRecipe may declare at most one oauth-broker provider')
    expect(recipe).toContain(
      "!(((has(self.agent) && has(self.agent.provider) && self.agent.provider == 'codex-subscription') || (has(self.steps) && self.steps.exists(s, has(s.agent) && has(s.agent.provider) && s.agent.provider == 'codex-subscription'))) && ((has(self.agent) && has(self.agent.provider) && self.agent.provider == 'grok-subscription') || (has(self.steps) && self.steps.exists(s, has(s.agent) && has(s.agent.provider) && s.agent.provider == 'grok-subscription'))))"
    )
  })
})