Skip to content

Project NOMAD is an offline-first knowledge and education server. Wikipedia, thousands of books, courses, maps, and optional local AI, all running on hardware you own with no internet required.

TypeScript
18.8 KB
Raw Blame
import { EvalCorpusService } from '#services/eval_corpus_service'
import { OllamaService, type NomadChatUsage } from '#services/ollama_service'
import { RagPipelineService } from '#services/rag_pipeline_service'
import { inject } from '@adonisjs/core'
import logger from '@adonisjs/core/services/logger'
import { KB_EVAL_COLLECTION } from '../../constants/kb_collections.js'
import { RAG_MIN_FINAL_SCORE } from '../../constants/ollama.js'
import { resolveSamplerProfile } from '../utils/sampler.js'
import type { OllamaChatMessage, ResponseStyle, SamplerProfile } from '../../types/ollama.js'
import type { PipelineOptions, RetrievedChunk } from '../../types/rag.js'
import { docIdFromSource } from '../utils/eval/corpus_source.js'
import type { Golden } from '../utils/eval/golden_set.js'
import {
  scoreAnswer,
  summarizeNumeric,
  summarizeRepeats,
  type GenerationScores,
  type NumericSummary,
  type RepeatStats,
} from '../utils/eval/generation_metrics.js'

/**
 * The three ways to run a question, and what each one isolates.
 *
 * This is the part of the harness that answers "code bug or weak model?".
 * Running the same question all three ways turns one ambiguous score into a
 * decomposition:
 *
 * - `oracle` gives the model perfect context by construction. A low score here
 *   is the model (or the prompt) failing to use good context — retrieval is
 *   provably not at fault.
 * - `e2e` is the real product. `oracle - e2e` is the cost of imperfect retrieval.
 * - `noretrieval` is the model's parametric baseline. `e2e - noretrieval` is
 *   what RAG is actually buying, and on the fictional questions it should be
 *   nearly everything, since no model can know them.
 */
export type GenerationMode = 'oracle' | 'e2e' | 'noretrieval'

/**
 * Sentinel model name for the extractive reference run.
 *
 * `--model=mock` needs no Ollama at all: it answers by echoing whatever context
 * the pipeline injected, and refuses when nothing was injected. That makes it
 * two useful things at once — a way to exercise scoring and reporting with zero
 * models installed (so the harness itself is CI-testable), and a genuine
 * *ceiling* line: the best a perfectly extractive model could score given this
 * retrieval. A real model below the mock line is the bottleneck; a mock line
 * that is itself low means retrieval is.
 */
export const MOCK_MODEL = 'mock'

/**
 * Fixed sampling for eval runs. Never used by production chat.
 *
 * Temperature 0 is the default because a repeatable number is worth more than a
 * realistic one for most of what this harness gates. But greedy decoding makes
 * top_p, top_k and min_p inert — the highest-probability token wins whatever
 * they say — so a run left at 0 cannot see a Response Style change at all. That
 * is what `--style` is for: it sets `samplerStyle`, which replaces this
 * temperature with the style's and sends the real truncation settings, so the
 * chat path can actually be measured. The seed stays fixed either way, which
 * makes a sampled run approximately rather than exactly reproducible; `--repeats`
 * and the `unstable` counter are how that gets accounted for.
 */
export const EVAL_TEMPERATURE = 0
export const EVAL_SEED = 42

/**
 * What a run sends to the decoder: a fixed temperature, a sampler profile, or
 * neither. Exactly one of the three, which is why it is a shape rather than two
 * loose optional arguments threaded through runCase.
 */
type EvalSampling = { temperature?: number; sampler?: SamplerProfile }

function resolveEvalSampling(style: ResponseStyle | undefined): EvalSampling {
  if (!style) return { temperature: EVAL_TEMPERATURE }
  const sampler = resolveSamplerProfile(style)
  // `off` resolves to no profile, and sends no temperature either — that is the
  // pre-setting behaviour this harness needs to be able to reproduce.
  return sampler ? { sampler } : {}
}

export type GenerationRunOptions = {
  mode: GenerationMode
  model: string
  repeats?: number
  topK?: number
  scoreThreshold?: number
  /** Post-rerank relevance floor. Defaults to the constant, never to the setting. */
  minFinalScore?: number
  /** Skip the history-aware rewrite even on multi-turn goldens. */
  skipQueryRewrite?: boolean
  /**
   * Sample the way real chat does, under this Response Style, instead of at
   * temperature 0.
   *
   * Three states, not two. Unset is the deterministic default and the right
   * choice for anything being gated on. A style is the chat path as a user
   * experiences it. `off` sends no sampler settings *and* no temperature, which
   * is the pre-setting behaviour and therefore the honest control to compare a
   * style against: `--style=auto` against `--style=off` measures the change,
   * where `--style=auto` against a temperature-0 run measures the change plus
   * the difference between greedy and sampled decoding.
   *
   * Model-declared samplers are deliberately not merged in even under `auto`:
   * the harness scores a style, and a number that moved because one machine's
   * model shipped a different modelfile is not a number worth committing.
   */
  samplerStyle?: ResponseStyle
  onProgress?: (id: string, index: number, total: number) => void
}

export type GenerationCaseResult = {
  id: string
  tags: string[]
  expectRefusal: boolean
  /** One entry per repeat. */
  answers: string[]
  scores: GenerationScores[]
  correctness: RepeatStats
  refusalCorrectness: RepeatStats
  leakageFree: RepeatStats
  groundedness: NumericSummary | null
  retrievedDocIds: string[]
  injectedChunks: number
  /**
   * What the budget planner did with the window on this case.
   *
   * Without this a low score is ambiguous: it could mean the model answered
   * badly, or it could mean the model never saw the material because the
   * relevant turns were evicted. These fields tell the two apart.
   */
  budget?: {
    contextWindow: number
    estimatedPromptTokens: number
    promptBudget: number
    turnsDropped: number
    chunksDropped: number
    historyElided: boolean
    queryTruncated: boolean
  }
  /**
   * Estimated vs. the backend's real prompt token count. The estimator is the
   * foundation every budget decision rests on, so its error is worth gating on.
   */
  promptTokenError?: number
  /** Non-null only when something went wrong talking to the model. */
  error?: string
}

export type GenerationAggregate = {
  cases: number
  /** Cases whose pass/fail flipped across repeats; excluded from gating. */
  unstable: number
  errors: number
  correctness: number | null
  refusalCorrectness: number | null
  leakageRate: number | null
  thinkTagLeakRate: number | null
  markdownRate: number | null
  groundedness: NumericSummary | null
  meanAnswerLength: number | null
  /** Share of cases where history had to be trimmed to fit the window. */
  historyElidedRate: number | null
  /** Share of cases where a retrieved chunk didn't fit the context budget. */
  chunksDroppedRate: number | null
  /** Mean absolute error of the token estimator against real prompt counts. */
  promptTokenError: NumericSummary | null
  meanPromptTokens: number | null
}

export type GenerationRunResult = {
  params: {
    mode: GenerationMode
    model: string
    repeats: number
    /** Null when the run sent no temperature at all, under `--style=off`. */
    temperature: number | null
    seed: number
    /** The Response Style sampled under; null for the deterministic default. */
    samplerStyle: ResponseStyle | null
    topK?: number
    scoreThreshold?: number
    minFinalScore: number
  }
  overall: GenerationAggregate
  byTag: Record<string, GenerationAggregate>
  cases: GenerationCaseResult[]
  elapsedMs: number
}

@inject()
export class EvalGenerationService {
  constructor(
    private ollamaService: OllamaService,
    private pipeline: RagPipelineService,
    private corpusService: EvalCorpusService
  ) {}

  async run(goldens: Golden[], options: GenerationRunOptions): Promise<GenerationRunResult> {
    const repeats = Math.max(1, options.repeats ?? 3)
    const started = Date.now()
    const isMock = options.model === MOCK_MODEL
    const sampling = resolveEvalSampling(options.samplerStyle)

    // Oracle mode needs the corpus text on hand to synthesize perfect context.
    const corpusText = options.mode === 'oracle' ? await this.loadCorpusText() : null

    if (!isMock) await this.prepareModel(options.model)

    const cases: GenerationCaseResult[] = []
    for (const [index, golden] of goldens.entries()) {
      options.onProgress?.(golden.id, index + 1, goldens.length)
      cases.push(await this.runCase(golden, options, repeats, corpusText, isMock, sampling))
    }

    return {
      params: {
        mode: options.mode,
        model: options.model,
        repeats,
        temperature: sampling.sampler?.temperature ?? sampling.temperature ?? null,
        seed: EVAL_SEED,
        samplerStyle: options.samplerStyle ?? null,
        topK: options.topK,
        scoreThreshold: options.scoreThreshold,
        minFinalScore: options.minFinalScore ?? RAG_MIN_FINAL_SCORE,
      },
      overall: aggregateGeneration(cases),
      byTag: aggregateGenerationByTag(cases),
      cases,
      elapsedMs: Date.now() - started,
    }
  }

  /**
   * Evict other resident models and burn one throwaway generation.
   *
   * Both borrowed from BenchmarkService, and for the same reasons it added
   * them: a cold first run is dramatically slower and behaves differently, and
   * leftover models in VRAM change how the one under test is scheduled. Quality
   * runs are less timing-sensitive than throughput runs, but a first-token
   * timeout or an OOM-driven CPU fallback absolutely does change the answer.
   */
  private async prepareModel(model: string): Promise<void> {
    try {
      await this.ollamaService.unloadAllChatModelsExcept(model)
    } catch (error) {
      logger.warn(`[Eval] Could not evict resident models: ${errorText(error)}`)
    }
    try {
      await this.ollamaService.chat({
        model,
        messages: [{ role: 'user', content: 'Reply with the single word: ready' }],
        temperature: EVAL_TEMPERATURE,
        seed: EVAL_SEED,
      })
    } catch (error) {
      // Best-effort: if warm-up fails the scored runs will surface the real
      // problem with a better message than we could produce here.
      logger.warn(`[Eval] Warm-up generation failed: ${errorText(error)}`)
    }
  }

  private async loadCorpusText(): Promise<Map<string, { text: string; path: string }>> {
    const docs = await this.corpusService.loadCorpus()
    return new Map(docs.map((d) => [d.docId, { text: d.text, path: d.path }]))
  }

  private async runCase(
    golden: Golden,
    options: GenerationRunOptions,
    repeats: number,
    corpusText: Map<string, { text: string; path: string }> | null,
    isMock: boolean,
    sampling: EvalSampling
  ): Promise<GenerationCaseResult> {
    const messages: OllamaChatMessage[] = [
      ...golden.turns.map((t) => ({ role: t.role, content: t.content })),
      { role: 'user' as const, content: golden.query },
    ]

    const pipelineOptions: PipelineOptions = {
      collection: KB_EVAL_COLLECTION,
      topK: options.topK,
      scoreThreshold: options.scoreThreshold,
      // Pinned to the constant, not left to fall through to resolveMinFinalScore()
      // — otherwise a `rag.minRelevance` slider set on one developer's machine
      // would silently move every generation score. Same reasoning as the
      // retrieval tier; see EvalRetrievalService.run.
      minFinalScore: options.minFinalScore ?? RAG_MIN_FINAL_SCORE,
      // The mock run must not touch Ollama at all — that is what makes it
      // usable with no models installed. The rewrite is a chat-model call, so
      // it is always skipped there (it would 404 and silently fall back, which
      // works but quietly makes the "no model needed" claim untrue).
      skipQueryRewrite: options.skipQueryRewrite || isMock,
      // A developer's personal NOMAD.md would silently skew every score.
      skipNomadMd: true,
      // Pinned off rather than read from `rag.relevanceCheck`, for the same
      // reason as minFinalScore. eval:retrieval --judge-model measures it.
      relevanceCheckModel: null,
    }

    if (options.mode === 'oracle') {
      pipelineOptions.oracleContext = golden.relevantDocIds.map((docId) => {
        const doc = corpusText?.get(docId)
        return {
          text: doc?.text ?? '',
          score: 1,
          metadata: { source: doc?.path },
        } satisfies RetrievedChunk
      })
    } else if (options.mode === 'noretrieval') {
      // An empty oracle context short-circuits retrieval without injecting
      // anything — the model answers from parametric memory alone.
      pipelineOptions.oracleContext = []
    }

    const answers: string[] = []
    const scores: GenerationScores[] = []
    let retrievedDocIds: string[] = []
    let injectedChunks = 0
    let budget: GenerationCaseResult['budget']
    let promptTokenError: number | undefined
    let error: string | undefined

    for (let attempt = 0; attempt < repeats; attempt++) {
      try {
        const trace = await this.pipeline.buildPrompt(messages, options.model, pipelineOptions)
        retrievedDocIds = uniqueDocIds(trace.retrieved)
        injectedChunks = trace.injected.length
        const context = trace.injected.map((c) => c.text).join('\n\n')
        if (trace.budget) {
          budget = {
            contextWindow: trace.budget.contextWindow,
            estimatedPromptTokens: trace.budget.estimatedPromptTokens,
            promptBudget: trace.budget.promptBudget,
            turnsDropped: trace.budget.turnsDropped,
            chunksDropped: trace.budget.chunksDropped,
            historyElided: trace.budget.historyElided,
            queryTruncated: trace.budget.queryTruncated,
          }
        }

        const generated = isMock
          ? { answer: mockAnswer(context), usage: undefined }
          : await this.generate(options.model, trace.messages, trace.numCtx, trace.numPredict, sampling)
        const answer = generated.answer

        // The backend just reported the prompt's real token count. Comparing it
        // to the estimate is the only way to know whether the budget above was
        // built on a number worth trusting.
        if (generated.usage?.promptTokens && trace.uncalibratedPromptTokens) {
          promptTokenError =
            Math.abs(generated.usage.promptTokens - trace.uncalibratedPromptTokens) /
            generated.usage.promptTokens
        }

        answers.push(answer)
        scores.push(
          scoreAnswer({
            answer,
            context,
            mustInclude: golden.mustInclude,
            mustNotInclude: golden.mustNotInclude,
            expectRefusal: golden.expectRefusal,
          })
        )
      } catch (err) {
        error = errorText(err)
        break
      }
    }

    return {
      id: golden.id,
      tags: golden.tags,
      expectRefusal: golden.expectRefusal,
      answers,
      scores,
      correctness: summarizeRepeats(scores.map((s) => s.correct)),
      refusalCorrectness: summarizeRepeats(scores.map((s) => s.refusalCorrect)),
      leakageFree: summarizeRepeats(scores.map((s) => s.leakage.length === 0)),
      groundedness: summarizeNumeric(scores.map((s) => s.numericGroundedness)),
      retrievedDocIds,
      injectedChunks,
      budget,
      promptTokenError,
      error,
    }
  }

  private async generate(
    model: string,
    messages: OllamaChatMessage[],
    numCtx?: number,
    numPredict?: number,
    sampling: EvalSampling = { temperature: EVAL_TEMPERATURE }
  ): Promise<{ answer: string; usage?: NomadChatUsage }> {
    const response = await this.ollamaService.chat({
      model,
      messages,
      numCtx,
      numPredict,
      // Spread rather than passed as fields: an explicit temperature outranks a
      // profile's inside OllamaService, so under a style it has to be absent
      // entirely or the run would carry the style's truncation settings at
      // temperature 0 and measure a decoder nobody uses.
      ...sampling,
      seed: EVAL_SEED,
    })
    return { answer: response.message.content.trim(), usage: response.usage }
  }
}

/**
 * The extractive reference answer: echo the context, or decline when there is
 * none. Deterministic, model-free, and an honest ceiling for the current
 * retrieval.
 */
export function mockAnswer(context: string): string {
  if (!context.trim()) return "I don't have information about that."
  return context
}

function uniqueDocIds(chunks: RetrievedChunk[]): string[] {
  const ids: string[] = []
  const seen = new Set<string>()
  for (const chunk of chunks) {
    const docId = docIdFromSource(chunk.metadata?.source)
    if (docId && !seen.has(docId)) {
      seen.add(docId)
      ids.push(docId)
    }
  }
  return ids
}

function errorText(error: unknown): string {
  return error instanceof Error ? error.message : String(error)
}

const rate = (values: boolean[]): number | null =>
  values.length === 0 ? null : values.filter(Boolean).length / values.length

export function aggregateGeneration(cases: GenerationCaseResult[]): GenerationAggregate {
  const scored = cases.filter((c) => c.scores.length > 0)
  const allScores = scored.flatMap((c) => c.scores)

  return {
    cases: cases.length,
    unstable: scored.filter((c) => c.correctness.unstable).length,
    errors: cases.filter((c) => c.error).length,
    // Mean pass-rate rather than all-or-nothing, so a case that passes 2 of 3
    // is reported as 0.67 instead of being silently rounded either way.
    correctness: scored.length === 0 ? null : mean(scored.map((c) => c.correctness.passRate)),
    refusalCorrectness:
      scored.length === 0 ? null : mean(scored.map((c) => c.refusalCorrectness.passRate)),
    leakageRate: rate(allScores.map((s) => s.leakage.length > 0)),
    thinkTagLeakRate: rate(allScores.map((s) => s.thinkTagLeak)),
    markdownRate: rate(allScores.map((s) => s.markdownFormatted)),
    groundedness: summarizeNumeric(allScores.map((s) => s.numericGroundedness)),
    meanAnswerLength: allScores.length === 0 ? null : mean(allScores.map((s) => s.length)),
    historyElidedRate: rate(cases.filter((c) => c.budget).map((c) => c.budget!.historyElided)),
    chunksDroppedRate: rate(cases.filter((c) => c.budget).map((c) => c.budget!.chunksDropped > 0)),
    promptTokenError: summarizeNumeric(
      cases.map((c) => c.promptTokenError).filter((e): e is number => e !== undefined)
    ),
    meanPromptTokens: (() => {
      const values = cases
        .map((c) => c.budget?.estimatedPromptTokens)
        .filter((v): v is number => v !== undefined)
      return values.length === 0 ? null : mean(values)
    })(),
  }
}

export function aggregateGenerationByTag(
  cases: GenerationCaseResult[]
): Record<string, GenerationAggregate> {
  const tags = new Set(cases.flatMap((c) => c.tags))
  const out: Record<string, GenerationAggregate> = {}
  for (const tag of [...tags].sort()) {
    out[tag] = aggregateGeneration(cases.filter((c) => c.tags.includes(tag)))
  }
  return out
}

const mean = (values: number[]) => values.reduce((a, b) => a + b, 0) / values.length