Skip to content

Project NOMAD is an offline-first knowledge and education server. Wikipedia, thousands of books, courses, maps, and optional local AI, all running on hardware you own with no internet required.

TypeScript
15.4 KB
Raw Blame
import { BaseCommand, flags } from '@adonisjs/core/ace'
import type { CommandOptions } from '@adonisjs/core/types/ace'
import type { RetrievalAggregate } from '../../app/utils/eval/retrieval_metrics.js'

/**
 * Score NOMAD's retrieval against the frozen golden set.
 *
 * No chat model is involved, so this is deterministic and hardware-independent:
 * a movement in these numbers is a code change, not a slow machine or an unlucky
 * sample. It is the fast inner loop for anything touching chunking, embedding,
 * thresholds, or reranking.
 *
 *   node ace eval:retrieval
 *   node ace eval:retrieval --ablate                 # is the reranker helping?
 *   node ace eval:retrieval --threshold=0.5          # sweep the Qdrant cutoff
 *   node ace eval:retrieval --min-final-score=0.6    # sweep the post-rerank floor
 *   node ace eval:retrieval --tag=multi-hop          # one slice only
 */
export default class EvalRetrieval extends BaseCommand {
  static commandName = 'eval:retrieval'
  static description = 'Score RAG retrieval against the golden set (deterministic, no chat model)'

  @flags.string({ description: 'Chunks to retrieve per query (default: the production value)' })
  declare topK: string

  @flags.string({ description: 'Minimum similarity score (default: the production value)' })
  declare threshold: string

  @flags.string({
    description:
      'Post-rerank relevance floor; chunks below it are dropped (default: the production value). ' +
      'Reads RAG_MIN_FINAL_SCORE, never the rag.minRelevance setting — see EvalRetrievalService.',
  })
  declare minFinalScore: string

  @flags.string({
    description:
      'Run the retrieval relevance check on this model after the floor (default: off, keeping this tier model-free)',
  })
  declare judgeModel: string

  @flags.boolean({ description: 'Also score the raw dense, reranked, and diversified orderings' })
  declare ablate: boolean

  @flags.string({
    description:
      'Comma-separated golden suites to run, by filename without .jsonl (default: core). ' +
      'Extra suites are opt-in so they cannot move the numbers a baseline was recorded against.',
  })
  declare suite: string

  @flags.string({ description: 'Only run goldens carrying this tag' })
  declare tag: string

  @flags.boolean({ description: 'Print each failing question and what it retrieved' })
  declare verbose: boolean

  @flags.boolean({ description: 'Leave application debug logging on (very noisy)' })
  declare debug: boolean

  @flags.boolean({ description: 'Write a JSON + Markdown report to tests/eval/reports/' })
  declare report: boolean

  @flags.string({
    description:
      "Write every golden's full pre-floor candidate pool (boosted + cosine scores) to this JSON path, " +
      'for simulating a relevance gate offline',
  })
  declare dump: string

  static options: CommandOptions = {
    startApp: true,
  }

  async run() {
    const { EvalCorpusService } = await import('#services/eval_corpus_service')
    const { EvalRetrievalService } = await import('#services/eval_retrieval_service')
    const { EvalReportService } = await import('#services/eval_report_service')
    const { quietLogging } = await import('../../app/utils/eval/quiet.js')

    const corpusService = await this.app.container.make(EvalCorpusService)
    const retrievalService = await this.app.container.make(EvalRetrievalService)
    const reportService = await this.app.container.make(EvalReportService)
    const restoreLogging = quietLogging(this.debug)

    try {
      const chunks = await retrievalService.assertCorpusReady()
      const fingerprint = await corpusService.fingerprint()
      let goldens = await corpusService.loadGoldens(this.suite ? this.suite.split(',').map((s) => s.trim()) : undefined)

      if (this.tag) {
        goldens = goldens.filter((g) => g.tags.includes(this.tag))
        if (goldens.length === 0) {
          this.logger.error(`No goldens carry the tag "${this.tag}"`)
          this.exitCode = 1
          return
        }
      }

      this.logger.info(`Corpus ${fingerprint} · ${chunks} chunks · ${goldens.length} goldens`)
      this.logger.info('')

      const started = Date.now()
      const result = await retrievalService.run(goldens, {
        topK: this.topK ? Number.parseInt(this.topK, 10) : undefined,
        scoreThreshold: this.threshold ? Number.parseFloat(this.threshold) : undefined,
        minFinalScore: this.minFinalScore ? Number.parseFloat(this.minFinalScore) : undefined,
        ablate: this.ablate,
        dump: Boolean(this.dump),
        judgeModel: this.judgeModel || undefined,
      })
      const elapsed = ((Date.now() - started) / 1000).toFixed(1)

      this.logger.info(
        `Params: topK=${result.params.topK} threshold=${result.params.scoreThreshold} ` +
          `minFinalScore=${result.params.minFinalScore}  (${elapsed}s)`
      )
      if (result.judge) {
        const j = result.judge
        this.logger.info(
          `Relevance check: ${j.model} · ${j.calls} calls · rejected ${j.rejected} chunk(s) · ` +
            `${j.failures} failed open · mean ${Math.round(j.meanMs)}ms max ${Math.round(j.maxMs)}ms`
        )
      }
      this.logger.info('')
      this.printAggregate('OVERALL', result.overall, result.params.kValues)

      if (result.unresolvedChunks > 0) {
        // Every retrieved chunk should belong to the eval corpus. Anything else
        // means the collection filter leaked and the numbers above describe a
        // corpus nobody chose.
        this.logger.error(
          `${result.unresolvedChunks} retrieved chunk(s) did not belong to the eval corpus — the collection filter leaked.`
        )
        this.exitCode = 1
      }

      this.printThresholdGuidance(result.overall)

      if (result.ablation) {
        this.logger.info('')
        this.logger.info('=== Stage ablation (does each heuristic earn its place?) ===')
        const k = result.params.kValues.includes(5) ? 5 : result.params.kValues[0]
        const row = (name: string, agg: RetrievalAggregate) =>
          this.logger.info(
            `  ${name.padEnd(14)} recall@${k}=${fmt(agg.recall[k])}  ndcg@${k}=${fmt(agg.ndcg[k])}  mrr=${fmt(agg.mrr)}  prec@${k}=${fmt(agg.precision[k])}`
          )
        row('dense only', result.ablation.dense)
        row('+ rerank', result.ablation.reranked)
        row('+ diversity', result.ablation.diversified)
        this.explainAblation(result.ablation, k)
      }

      this.logger.info('')
      this.logger.info('=== By tag ===')
      for (const [tag, agg] of Object.entries(result.byTag).sort()) {
        const k = result.params.kValues.includes(5) ? 5 : result.params.kValues[0]
        // A refusal-only tag has no recall to report; what it has is the leak rate,
        // which is the number #1341 is about.
        const leak =
          agg.nonEmptyRateOnRefusal === null ? '' : `  leaked=${pct(agg.nonEmptyRateOnRefusal)}`
        this.logger.info(
          `  ${tag.padEnd(20)} n=${String(agg.cases).padStart(3)}  recall@${k}=${fmt(agg.recall[k])}  ndcg@${k}=${fmt(agg.ndcg[k])}${leak}`
        )
      }

      const k = result.params.kValues.includes(5) ? 5 : result.params.kValues[0]
      const misses = result.cases.filter((c) => !c.expectRefusal && (c.recall[k] ?? 1) < 1)
      this.logger.info('')
      this.logger.info(`${misses.length} of ${result.overall.answerable} answerable questions missed at k=${k}`)

      if (this.verbose && misses.length > 0) {
        this.logger.info('')
        this.logger.info('=== Misses ===')
        for (const miss of misses) {
          this.logger.info(`  ${miss.id}`)
          this.logger.info(`    wanted:   ${miss.relevantDocIds.join(', ') || '(none)'}`)
          this.logger.info(`    retrieved: ${miss.retrievedDocIds.join(', ') || '(nothing)'}`)
        }
      } else if (misses.length > 0) {
        this.logger.info('Re-run with --verbose to see which questions and what they retrieved.')
      }

      // The other failure: a question the corpus cannot answer that retrieved
      // something anyway. Each of these becomes a citation the user sees.
      const leaks = result.cases.filter((c) => c.expectRefusal && !c.empty)
      if (leaks.length > 0) {
        this.logger.info('')
        this.logger.info(`${leaks.length} out-of-corpus question(s) retrieved something anyway`)
        if (this.verbose) {
          this.logger.info('')
          this.logger.info('=== Leaks ===')
          for (const leak of leaks) {
            this.logger.info(`  ${leak.id}`)
            const top = leak.top
              ? `  (top final=${leak.top.score.toFixed(3)} semantic=${leak.top.semanticScore?.toFixed(3) ?? 'n/a'})`
              : ''
            this.logger.info(`    retrieved: ${leak.retrievedDocIds.join(', ')}${top}`)
          }
        }
      }

      if (this.dump && result.pools) {
        const { writeFile } = await import('node:fs/promises')
        await writeFile(this.dump, JSON.stringify({ fingerprint, params: result.params, pools: result.pools }, null, 2))
        this.logger.info('')
        this.logger.success(`Candidate pools written: ${this.dump}`)
      }

      if (this.report) {
        const meta = await reportService.buildMeta('retrieval', fingerprint, {
          ...result.params,
          tag: this.tag ?? null,
        })
        const doc = reportService.fromRetrieval(meta, result)
        const slug = `retrieval-${meta.createdAt.replace(/[:.]/g, '-')}`
        const path = await reportService.write(doc, slug, renderRetrievalMarkdown(doc, result, misses))
        this.logger.info('')
        this.logger.success(`Report written: ${path}`)
      }
    } catch (error) {
      this.logger.error(error instanceof Error ? error.message : String(error))
      this.exitCode = 1
    } finally {
      restoreLogging()
    }
  }

  private printAggregate(label: string, agg: RetrievalAggregate, kValues: number[]) {
    this.logger.info(`=== ${label} (${agg.answerable} answerable of ${agg.cases}) ===`)
    const row = (name: string, values: Record<number, number | null>) =>
      this.logger.info(
        `  ${name.padEnd(10)} ${kValues.map((k) => `@${k}=${fmt(values[k])}`).join('  ')}`
      )
    row('recall', agg.recall)
    row('hit rate', agg.hitRate)
    row('precision', agg.precision)
    row('ndcg', agg.ndcg)
    this.logger.info(`  mrr        ${fmt(agg.mrr)}`)
  }

  /**
   * Turn the score distributions into an actual recommendation. The raw
   * percentiles are the evidence; this is the reading of them, which is what
   * the threshold constants have never had.
   *
   * Printed per axis, because the two cutoffs filter different numbers:
   * `--threshold` is applied by Qdrant to the raw cosine score, and
   * `--min-final-score` is applied after reranking. Reading one off the other's
   * percentiles is wrong by the reranker's boost factor.
   */
  private printThresholdGuidance(agg: RetrievalAggregate) {
    this.logger.info('')
    this.logger.info('=== Score separation (how to calibrate the cutoffs) ===')
    this.printAxis(
      'semantic, pre-rerank  — calibrates --threshold',
      agg.relevantScores,
      agg.irrelevantScores
    )
    this.printAxis(
      'final, post-rerank    — calibrates --min-final-score',
      agg.relevantFinalScores,
      agg.irrelevantFinalScores
    )
    if (agg.emptyRateOnAnswerable !== null && agg.emptyRateOnAnswerable > 0) {
      this.logger.warning(
        `  ${pct(agg.emptyRateOnAnswerable)} of answerable questions retrieved nothing — threshold may be too high.`
      )
    }
    if (agg.nonEmptyRateOnRefusal !== null && agg.nonEmptyRateOnRefusal > 0) {
      this.logger.warning(
        `  ${pct(agg.nonEmptyRateOnRefusal)} of out-of-corpus questions retrieved something anyway — that context is what produces confident wrong answers.`
      )
    }
  }

  /** One axis of the separation table, plus the reading of it. */
  private printAxis(
    label: string,
    rel: RetrievalAggregate['relevantScores'],
    irr: RetrievalAggregate['irrelevantScores']
  ) {
    if (!rel && !irr) return
    this.logger.info(`  ${label}`)
    const row = (name: string, d: NonNullable<typeof rel>) =>
      this.logger.info(
        `    ${name.padEnd(18)} n=${d.count} min=${d.min.toFixed(3)} p10=${d.p10.toFixed(3)} median=${d.median.toFixed(3)} p90=${d.p90.toFixed(3)}`
      )
    if (rel) row('relevant chunks', rel)
    if (irr) row('irrelevant chunks', irr)
    if (rel && irr) {
      if (rel.p10 > irr.p90) {
        this.logger.success(
          `    Clean separation: a cutoff between ${irr.p90.toFixed(3)} and ${rel.p10.toFixed(3)} splits them.`
        )
      } else {
        this.logger.warning(
          `    Overlapping: relevant p10 (${rel.p10.toFixed(3)}) sits below irrelevant p90 (${irr.p90.toFixed(3)}); ` +
            `a cutoff near ${rel.min.toFixed(3)} keeps every relevant chunk seen here.`
        )
      }
    }
  }

  private explainAblation(ablation: { dense: RetrievalAggregate; reranked: RetrievalAggregate; diversified: RetrievalAggregate }, k: number) {
    const verdict = (name: string, before: number | null, after: number | null) => {
      if (before === null || after === null) return
      const delta = after - before
      if (Math.abs(delta) < 1e-6) {
        this.logger.warning(`  ${name} changed nothing at k=${k} — it is complexity with no measured benefit.`)
      } else if (delta < 0) {
        this.logger.warning(`  ${name} made ndcg@${k} worse by ${Math.abs(delta).toFixed(4)}.`)
      } else {
        this.logger.success(`  ${name} improved ndcg@${k} by ${delta.toFixed(4)}.`)
      }
    }
    verdict('Reranking', ablation.dense.ndcg[k], ablation.reranked.ndcg[k])
    verdict('Source diversity', ablation.reranked.ndcg[k], ablation.diversified.ndcg[k])
  }
}

const fmt = (v: number | null) => (v === null ? '  n/a' : v.toFixed(3))
const pct = (v: number) => `${(v * 100).toFixed(0)}%`

/**
 * The human-readable half of a report. Leads with the misses, because when a
 * number moves the next question is always "which questions?" and a table of
 * aggregates cannot answer it.
 */
function renderRetrievalMarkdown(doc: any, result: any, misses: any[]): string {
  const lines: string[] = ['# Retrieval eval', '']
  lines.push(`- corpus: \`${doc.meta.corpusFingerprint}\``)
  lines.push(`- commit: \`${doc.meta.gitSha ?? 'unknown'}\`${doc.meta.gitDirty ? ' (dirty tree)' : ''}`)
  lines.push(`- when: ${doc.meta.createdAt}`)
  lines.push(
    `- params: topK=${result.params.topK} threshold=${result.params.scoreThreshold} minFinalScore=${result.params.minFinalScore}`
  )
  lines.push('')
  lines.push('## Metrics', '')
  lines.push('| metric | value |', '|---|---:|')
  for (const [name, value] of Object.entries(doc.metrics)) {
    lines.push(`| ${name} | ${value === null ? 'n/a' : (value as number).toFixed(4)} |`)
  }
  lines.push('')
  lines.push(`## Misses (${misses.length})`, '')
  if (misses.length === 0) {
    lines.push('None.')
  } else {
    for (const m of misses) {
      lines.push(`### \`${m.id}\``)
      lines.push(`- wanted: ${m.relevantDocIds.join(', ') || '_none_'}`)
      lines.push(`- retrieved: ${m.retrievedDocIds.join(', ') || '_nothing_'}`)
      lines.push('')
    }
  }
  lines.push('## By tag', '')
  lines.push('| tag | recall@5 | ndcg@5 |', '|---|---:|---:|')
  for (const [tag, metrics] of Object.entries(doc.byTag).sort()) {
    const m = metrics as Record<string, number | null>
    const cell = (v: number | null | undefined) => (v === null || v === undefined ? 'n/a' : v.toFixed(4))
    lines.push(`| ${tag} | ${cell(m['recall@5'])} | ${cell(m['ndcg@5'])} |`)
  }
  return lines.join('\n') + '\n'
}