Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
5.5 KB
Raw Blame
/**
 * LLM Provider Benchmark — E2E test suite.
 *
 * Sends identical prompts at 3 difficulty levels to each configured LLM provider,
 * measuring response quality, latency, and tool-use accuracy.
 *
 * The test swaps providers live via Host CRD patches (no pod restart) and collects
 * results as JSON for post-hoc analysis with report.ts.
 *
 * Prerequisites:
 *   1. Minikube running with all services deployed
 *   2. All API keys in mcp-host-keys secret (openai, claude, zai, bailian)
 *   3. Mock MCP server deployed (for MCP tool tests)
 *   4. Run setup: bash tests/e2e/benchmark/setup/configure-benchmark.sh
 *   5. Port-forward: kubectl port-forward -n mcp-host svc/chatllm 8080:8080
 *
 * Run:
 *   cd tests/e2e && npx vitest run benchmark/benchmark.test.ts
 *
 * Filter providers via env:
 *   BENCHMARK_PROVIDERS=openai,claude npx vitest run benchmark/benchmark.test.ts
 */
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
import { healthCheck, waitForIdle } from '../helpers.js'
import {
  type BenchmarkResult,
  cleanWorkspace,
  getActiveProviders,
  printSummaryTable,
  runBenchmarkPrompt,
  saveResults,
  switchProvider,
} from './helpers.js'
import { ALL_PROMPTS, type Category } from './prompts.js'
import { scoreAll } from './scoring.js'

// ---------------------------------------------------------------------------
// Configuration — timeouts vary by category complexity
// ---------------------------------------------------------------------------

const CATEGORY_TIMEOUTS: Record<Category, number> = {
  reasoning: 120_000, // 2 min — pure text, no tools
  instruction_following: 120_000, // 2 min — pure text
  tool_single: 300_000, // 5 min — single tool call + rate limit retries
  tool_chain: 420_000, // 7 min — multi-step chains
  mcp_tool: 300_000, // 5 min — MCP tool calls
}

const TIMEOUT_PROVIDER_SWITCH = 30_000

// ---------------------------------------------------------------------------
// Global state
// ---------------------------------------------------------------------------

const allResults: BenchmarkResult[] = []
const providers = getActiveProviders()

// ---------------------------------------------------------------------------
// Pre-flight check
// ---------------------------------------------------------------------------

describe('LLM Provider Benchmark', () => {
  beforeAll(async () => {
    const healthy = await healthCheck()
    if (!healthy) {
      throw new Error(
        'mcp-host is not reachable at http://localhost:8080. ' +
          'Ensure port-forward is active: kubectl port-forward -n mcp-host svc/chatllm 8080:8080'
      )
    }
    await waitForIdle(10_000)
    console.log(
      `\n[Benchmark] Starting benchmark with ${providers.length} providers, ${ALL_PROMPTS.length} prompts each`
    )
    console.log(`[Benchmark] Providers: ${providers.map(p => p.label).join(', ')}`)
    console.log(`[Benchmark] Total test runs: ${providers.length * ALL_PROMPTS.length}\n`)
  }, 30_000)

  // -------------------------------------------------------------------------
  // Per-provider test blocks
  // -------------------------------------------------------------------------

  for (const provider of providers) {
    describe(`Provider: ${provider.label} (${provider.provider}/${provider.model})`, () => {
      beforeAll(async () => {
        await switchProvider(provider)
        cleanWorkspace()
        await waitForIdle(5_000).catch(() => {})
      }, TIMEOUT_PROVIDER_SWITCH)

      for (const category of [
        'reasoning',
        'tool_single',
        'tool_chain',
        'mcp_tool',
        'instruction_following',
      ] as const) {
        const categoryPrompts = ALL_PROMPTS.filter(p => p.category === category)
        const catTimeout = CATEGORY_TIMEOUTS[category]

        describe(`Category: ${category}`, () => {
          for (const prompt of categoryPrompts) {
            it(
              `[${prompt.id}] ${prompt.difficulty}: ${prompt.label}`,
              async () => {
                const result = await runBenchmarkPrompt({
                  prompt: prompt.prompt,
                  testId: prompt.id,
                  category: prompt.category,
                  difficulty: prompt.difficulty,
                  label: prompt.label,
                  provider,
                  timeoutMs: catTimeout - 15_000, // leave buffer for vitest
                })

                allResults.push(result)

                if (result.error) {
                  console.warn(`  [${prompt.id}] Error: ${result.error}`)
                }

                // Record the result regardless, but assert we got a response.
                // Failed assertions still record the result in allResults (push above).
                expect(
                  result.response,
                  `${provider.label} should return a response for ${prompt.id}`
                ).toBeTruthy()
              },
              catTimeout
            )
          }
        })
      }
    })
  }

  // -------------------------------------------------------------------------
  // Scoring & reporting — runs even if some tests failed
  // -------------------------------------------------------------------------

  afterAll(() => {
    if (allResults.length === 0) {
      console.log('[Benchmark] No results collected')
      return
    }

    // Score all results
    scoreAll(allResults)

    // Save to disk
    const filepath = saveResults(allResults)

    // Print console summary
    printSummaryTable(allResults)

    console.log(`\nGenerate full report with: npx tsx benchmark/report.ts ${filepath}`)
  })
})