Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
8.8 KB
Raw Blame
/**
 * qa-accuracy.ts — the `qa_accuracy` summary block for the judged-answer
 * lane, rebuilt from ALL rows (prior + new) on every run.
 *
 * INVARIANT: pure over row objects; no I/O, no LLM. The LAST row per
 * question_id wins (a resume file may carry an older error row above a
 * retried row for the same question).
 *
 * Denominators (plan D16, outside-voice round 2):
 *   - `accuracy_headline` = correct / total_questions — EVERY question in
 *     the run counts, including `_abs`; a judge_error, a budget-skipped row,
 *     a reader-error row and a never-judged row all score INCORRECT. This is
 *     the number that is published (and it is never more lenient than the
 *     official scorer, which sees exactly one label per hypothesis).
 *   - `accuracy_excluding_errors` = correct / judged — secondary; errors out
 *     of the denominator.
 *   - `accuracy_470` = headline-style over the non-`_abs` questions (the
 *     retrieval-metric denominator), for the like-for-like reader.
 *   - `complete` = no judge_error, no skipped_budget, no reader_error, no
 *     unjudged row: the publishability bit the run-end gate reads (a
 *     reader-error row scores incorrect without ever being judged, so a run
 *     that carries one is not publishable either).
 *   - `ci95_bootstrap` = percentile bootstrap over the headline 0/1 vector,
 *     labelled question-sampling only (D8/D17).
 */

import { bootstrapMeanCi, type BootstrapCi } from '../shared/bootstrap.ts';
import { hasJudgeAttempt } from './judge-lane.ts';
import { isAbstentionQuestion } from './metrics.ts';

export interface QaRowLike {
  question_id?: unknown;
  question_type?: unknown;
  kind?: unknown;
  hypothesis?: unknown;
  error?: unknown;
  abstention?: unknown;
  judge_correct?: unknown;
  judge_error?: unknown;
  judge_skipped?: unknown;
  judge_cost_usd?: unknown;
  [k: string]: unknown;
}

export interface QaTypeStats {
  total: number;
  judged: number;
  correct: number;
  judge_errors: number;
  skipped_budget: number;
  reader_errors: number;
  unjudged: number;
  accuracy_headline: number | null;
  accuracy_excluding_errors: number | null;
}

export type QaAccuracyBlock = {
  /** Every question row in the run (deduped by question_id; _abs included). */
  total_questions: number;
  judged: number;
  correct: number;
  judge_errors: number;
  skipped_budget: number;
  /** Rows whose reader call failed (empty hypothesis + error); never judged. */
  reader_errors: number;
  /** Rows with a hypothesis that carry no judge attempt at all. */
  unjudged: number;
  /** Alias of accuracy_headline (the ONE published accuracy). */
  accuracy: number | null;
  accuracy_headline: number | null;
  accuracy_excluding_errors: number | null;
  /** Headline-style over non-_abs questions. */
  accuracy_470: number | null;
  non_abstention_total: number;
  by_type: Record<string, QaTypeStats>;
  abstention: { total: number; judged: number; correct: number; judge_errors: number; accuracy_headline: number | null };
  ci95_bootstrap: BootstrapCi;
  /** judge_error class → count. */
  judge_error_classes: Record<string, number>;
  complete: boolean;
  judge_model: string;
  judge_prompt_version: string;
  /**
   * The ONE `judge_config_hash` stamped on the judged rows when they agree;
   * this run's hash when no row carries one (or the rows disagree — see
   * `mixed_judge_config`). Derived from the ROWS, never from the launch flags
   * alone, so a backfill launched without the original `--model` cannot
   * relabel a homogeneous file.
   */
  judge_config_hash: string;
  /** More than one distinct `judge_config_hash` across the judged rows. */
  mixed_judge_config: boolean;
  est_cost_usd: number | null;
  /** Summed over every row's `judge_cost_usd` (the receipt's cumulative judge spend across resumes). */
  actual_cost_usd: number;
  /** THIS run's judge spend from the ledger (null when the lane did not run). */
  run_cost_usd: number | null;
  methodology_note: string;
};

export interface QaAccuracyOpts {
  judgeModel: string;
  judgePromptVersion: string;
  /** This run's hash — published only when no row carries a hash or the rows disagree. */
  judgeConfigHash: string;
  estCostUsd: number | null;
  /** Summed over the rows' own `judge_cost_usd` when omitted. */
  actualCostUsd?: number;
  runCostUsd?: number | null;
  methodologyNote: string;
  seed?: number;
  resamples?: number;
}

function isQuestionRow(row: QaRowLike): boolean {
  return row.kind !== 'by_type_summary' && typeof row.question_id === 'string';
}

function newTypeStats(): QaTypeStats {
  return { total: 0, judged: 0, correct: 0, judge_errors: 0, skipped_budget: 0, reader_errors: 0, unjudged: 0, accuracy_headline: null, accuracy_excluding_errors: null };
}

function finalizeType(t: QaTypeStats): QaTypeStats {
  t.accuracy_headline = t.total === 0 ? null : t.correct / t.total;
  t.accuracy_excluding_errors = t.judged === 0 ? null : t.correct / t.judged;
  return t;
}

/** Last row per question_id wins; summary rows ignored. */
export function dedupeQuestionRows<T extends QaRowLike>(rows: ReadonlyArray<T>): T[] {
  const byId = new Map<string, T>();
  for (const row of rows) {
    if (!isQuestionRow(row)) continue;
    byId.set(row.question_id as string, row);
  }
  return [...byId.values()];
}

export function buildQaAccuracy(rows: ReadonlyArray<QaRowLike>, opts: QaAccuracyOpts): QaAccuracyBlock {
  const deduped = dedupeQuestionRows(rows);
  const byType: Record<string, QaTypeStats> = {};
  const agg = newTypeStats();
  const abs = { total: 0, judged: 0, correct: 0, judge_errors: 0 };
  const errorClasses: Record<string, number> = {};
  const headline: number[] = [];
  let nonAbsTotal = 0;
  let nonAbsCorrect = 0;
  let actual = 0;
  const hashes = new Set<string>();

  for (const row of deduped) {
    const type = typeof row.question_type === 'string' ? row.question_type : 'unknown';
    const t = byType[type] ?? (byType[type] = newTypeStats());
    const isAbs = isAbstentionQuestion(row.question_id as string) || row.abstention === true;
    const readerError = typeof row.error === 'string' && (typeof row.hypothesis !== 'string' || row.hypothesis === '');
    const correct = row.judge_correct === true;
    const judged = typeof row.judge_correct === 'boolean';
    const judgeError = typeof row.judge_error === 'string';
    const skipped = row.judge_skipped === 'budget';
    if (typeof row.judge_cost_usd === 'number' && Number.isFinite(row.judge_cost_usd)) actual += row.judge_cost_usd;
    if (typeof row.judge_config_hash === 'string') hashes.add(row.judge_config_hash);

    for (const s of [t, agg]) {
      s.total++;
      if (judged) { s.judged++; if (correct) s.correct++; }
      else if (judgeError) s.judge_errors++;
      else if (skipped) s.skipped_budget++;
      else if (readerError) s.reader_errors++;
      else s.unjudged++;
    }
    if (judgeError) errorClasses[row.judge_error as string] = (errorClasses[row.judge_error as string] ?? 0) + 1;
    headline.push(correct ? 1 : 0);
    if (isAbs) {
      abs.total++;
      if (judged) { abs.judged++; if (correct) abs.correct++; }
      else if (judgeError) abs.judge_errors++;
    } else {
      nonAbsTotal++;
      if (correct) nonAbsCorrect++;
    }
  }

  const sortedTypes: Record<string, QaTypeStats> = {};
  for (const k of Object.keys(byType).sort()) sortedTypes[k] = finalizeType(byType[k]);
  finalizeType(agg);
  const headlineAcc = agg.accuracy_headline;
  const [onlyHash] = hashes;
  return {
    total_questions: agg.total,
    judged: agg.judged,
    correct: agg.correct,
    judge_errors: agg.judge_errors,
    skipped_budget: agg.skipped_budget,
    reader_errors: agg.reader_errors,
    unjudged: agg.unjudged,
    accuracy: headlineAcc,
    accuracy_headline: headlineAcc,
    accuracy_excluding_errors: agg.accuracy_excluding_errors,
    accuracy_470: nonAbsTotal === 0 ? null : nonAbsCorrect / nonAbsTotal,
    non_abstention_total: nonAbsTotal,
    by_type: sortedTypes,
    abstention: { ...abs, accuracy_headline: abs.total === 0 ? null : abs.correct / abs.total },
    ci95_bootstrap: bootstrapMeanCi(headline, { seed: opts.seed, resamples: opts.resamples }),
    judge_error_classes: errorClasses,
    complete: agg.judge_errors === 0 && agg.skipped_budget === 0 && agg.reader_errors === 0 && agg.unjudged === 0,
    judge_model: opts.judgeModel,
    judge_prompt_version: opts.judgePromptVersion,
    judge_config_hash: hashes.size === 1 ? onlyHash : opts.judgeConfigHash,
    mixed_judge_config: hashes.size > 1,
    est_cost_usd: opts.estCostUsd,
    actual_cost_usd: opts.actualCostUsd ?? actual,
    run_cost_usd: opts.runCostUsd ?? null,
    methodology_note: opts.methodologyNote,
  };
}

/** True when any question row carries a judge attempt (`hasJudgeAttempt`: verdict, error, or budget skip). */
export function anyRowJudged(rows: ReadonlyArray<QaRowLike>): boolean {
  return rows.some(r => isQuestionRow(r) && hasJudgeAttempt(r));
}