Skip to content

Project NOMAD is an offline-first knowledge and education server. Wikipedia, thousands of books, courses, maps, and optional local AI, all running on hardware you own with no internet required.

TypeScript
7.2 KB
Raw Blame
import { BaseCommand, flags } from '@adonisjs/core/ace'
import type { CommandOptions } from '@adonisjs/core/types/ace'
import type { GenerationAggregate } from '../../app/services/eval_generation_service.js'

type Row = {
  model: string
  oracle: GenerationAggregate | null
  e2e: GenerationAggregate | null
  error?: string
}

/**
 * Build the model capability table.
 *
 * This is the artifact that answers a GitHub issue. When a user reports "the
 * AI gave me a bad answer", look up their model in this table:
 *
 *   - scoring at or near its row  -> the model is at its ceiling. The honest
 *                                    answer is "run a larger model", not a bug.
 *   - scoring well below its row  -> something is wrong with their config or
 *                                    our code, and it is worth investigating.
 *
 * Without it, every quality concern is unfalsifiable.
 *
 *   node ace eval:matrix --models=llama3.2:latest,llama3:8b --limit=30
 *   node ace eval:matrix --models=... --promote   # commit as the reference table
 */
export default class EvalMatrix extends BaseCommand {
  static commandName = 'eval:matrix'
  static description = 'Score several models side by side to build the capability reference table'

  @flags.string({ description: 'Comma-separated Ollama model names' })
  declare models: string

  @flags.string({ description: 'Repeats per question (default: 1 — the matrix is already long)' })
  declare repeats: string

  @flags.string({ description: 'Limit to the first N goldens' })
  declare limit: string

  @flags.string({ description: 'Only run goldens carrying this tag' })
  declare tag: string

  @flags.boolean({ description: 'Write the table to tests/eval/baselines/<fingerprint>/matrix.json' })
  declare promote: boolean

  @flags.boolean({ description: 'Leave application debug logging on (very noisy)' })
  declare debug: boolean

  static options: CommandOptions = {
    startApp: true,
  }

  async run() {
    const { EvalCorpusService } = await import('#services/eval_corpus_service')
    const { EvalGenerationService } = await import('#services/eval_generation_service')
    const { quietLogging } = await import('../../app/utils/eval/quiet.js')
    const { mkdir, writeFile } = await import('node:fs/promises')
    const { dirname, join, resolve } = await import('node:path')

    const corpusService = await this.app.container.make(EvalCorpusService)
    const generationService = await this.app.container.make(EvalGenerationService)
    const restoreLogging = quietLogging(this.debug)

    try {
      if (!this.models) {
        this.logger.error('--models is required, e.g. --models=llama3.2:latest,llama3:8b')
        this.exitCode = 1
        return
      }
      const models = this.models.split(',').map((m) => m.trim()).filter(Boolean)

      let goldens = await corpusService.loadGoldens()
      if (this.tag) goldens = goldens.filter((g) => g.tags.includes(this.tag))
      if (this.limit) goldens = goldens.slice(0, Number.parseInt(this.limit, 10))

      const fingerprint = await corpusService.fingerprint()
      const repeats = this.repeats ? Number.parseInt(this.repeats, 10) : 1
      this.logger.info(
        `Corpus ${fingerprint} · ${goldens.length} goldens · ${models.length} models · repeats=${repeats}`
      )
      this.logger.info('Running oracle and e2e for each model. This takes a while.')
      this.logger.info('')

      const rows: Row[] = []
      for (const model of models) {
        this.logger.info(`--- ${model} ---`)
        const row: Row = { model, oracle: null, e2e: null }
        try {
          // Only oracle and e2e: those two are what the triage rule needs, and
          // adding noretrieval would half again the runtime of an already long
          // command for a number that does not change the verdict.
          for (const mode of ['oracle', 'e2e'] as const) {
            const started = Date.now()
            const result = await generationService.run(goldens, { mode, model, repeats })
            row[mode] = result.overall
            this.logger.info(
              `  ${mode.padEnd(6)} correctness=${fmt(result.overall.correctness)} refusal=${fmt(result.overall.refusalCorrectness)} (${((Date.now() - started) / 1000).toFixed(0)}s)`
            )
            if (result.overall.errors > 0) {
              this.logger.warning(`  ${result.overall.errors} question(s) errored in ${mode}`)
            }
          }
        } catch (error) {
          row.error = error instanceof Error ? error.message : String(error)
          this.logger.error(`  failed: ${row.error}`)
        }
        rows.push(row)
      }

      this.logger.info('')
      this.printTable(rows)

      if (this.promote) {
        const path = resolve(join(process.cwd(), 'tests/eval/baselines', fingerprint, 'matrix.json'))
        await mkdir(dirname(path), { recursive: true })
        await writeFile(
          path,
          JSON.stringify(
            {
              corpusFingerprint: fingerprint,
              createdAt: new Date().toISOString(),
              goldens: goldens.length,
              repeats,
              tag: this.tag ?? null,
              rows,
            },
            null,
            2
          )
        )
        this.logger.success(`Capability table written: ${path}`)
        this.logger.info('Commit it — this is the reference a support triage compares against.')
      }
    } catch (error) {
      this.logger.error(error instanceof Error ? error.message : String(error))
      this.exitCode = 1
    } finally {
      restoreLogging()
    }
  }

  private printTable(rows: Row[]) {
    this.logger.info('=== Capability matrix ===')
    this.logger.info('')
    const width = Math.max(20, ...rows.map((r) => r.model.length + 2))
    this.logger.info(
      `${'model'.padEnd(width)}${'oracle'.padStart(8)}${'e2e'.padStart(8)}${'ret.cost'.padStart(10)}${'refusal'.padStart(9)}${'leakage'.padStart(9)}${'ground'.padStart(8)}`
    )
    for (const row of rows) {
      if (row.error) {
        this.logger.info(`${row.model.padEnd(width)}  ERROR: ${row.error}`)
        continue
      }
      const cost =
        row.oracle?.correctness !== null && row.oracle?.correctness !== undefined &&
        row.e2e?.correctness !== null && row.e2e?.correctness !== undefined
          ? row.oracle.correctness - row.e2e.correctness
          : null
      this.logger.info(
        row.model.padEnd(width) +
          fmt(row.oracle?.correctness ?? null).padStart(8) +
          fmt(row.e2e?.correctness ?? null).padStart(8) +
          (cost === null ? '   n/a' : `${cost >= 0 ? '+' : ''}${cost.toFixed(3)}`).padStart(10) +
          fmt(row.e2e?.refusalCorrectness ?? null).padStart(9) +
          fmt(row.e2e?.leakageRate ?? null).padStart(9) +
          fmt(row.e2e?.groundedness?.mean ?? null).padStart(8)
      )
    }
    this.logger.info('')
    this.logger.info('oracle   = correctness with perfect context (the model\'s ceiling)')
    this.logger.info('e2e      = correctness with real retrieval (what a user gets)')
    this.logger.info('ret.cost = oracle - e2e (what imperfect retrieval costs this model)')
    this.logger.info('leakage  = rate of narrating retrieval; lower is better')
  }
}

const fmt = (v: number | null) => (v === null ? ' n/a' : v.toFixed(3))