Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
11.1 KB
Raw Blame
/**
 * judge-lane.ts — harness-side orchestration for `gbrain eval longmemeval
 * --judge`: the `--max-usd` parser, the per-row `judge_config_hash` (D33),
 * backfill row selection + the mixed-config gate, the spend preflight, and
 * the concurrent judge-only backfill. Keeps src/commands/eval-longmemeval.ts
 * under its size cap; stderr writing and exit codes stay in the harness.
 *
 * INVARIANT: pure except `runJudgeBackfill`, whose only side effects are the
 * injected judge client and in-place row updates. No engine, no file I/O.
 *
 * INVARIANT (D33): a row's `judge_config_hash` covers the judge pins (model,
 * prompt version, max_tokens, temperature) AND the reader pins the row was
 * produced under (reader model, reader prompt sha, k, reader max_tokens). A
 * backfill hashes each prior row from the row's OWN recorded reader pins when
 * present, so a file answered by another reader is never relabelled as this
 * run's; rows already judged under a different hash are refused unless
 * `--allow-mixed-run-config`.
 */

import { runWithLimit } from '../../core/worker-pool.ts';
import { estimateTokens } from '../../core/search/token-budget.ts';
import { BudgetLedger, isJudgeModelPriced } from '../shared/judge-runner.ts';
import {
  JUDGE_MAX_TOKENS,
  JUDGE_PROMPT_VERSION,
  JUDGE_TEMPERATURE,
  errorJudgeFields,
  estimateJudgeRunUsd,
  judgeConfigHash,
  judgePromptKind,
  judgeRow,
  stripJudgeFields,
  type JudgeLaneContext,
  type JudgePromptInput,
} from './judge.ts';
import type { LongMemEvalQuestion } from './adapter.ts';
import { parseCapFlag } from '../../core/budget/cap-flag.ts';

/** `--max-usd N|off`: `off` disables the cap (and lets an unpriced judge run). */
export function parseMaxUsd(flag: string, v: string): number | null {
  // D19 shared parser; 0 keeps its meaning here, a $0 judge cap.
  return parseCapFlag(flag, v, { zero: 'cap' }).usd;
}

/** This run's reader pins (the hash's reader half for live rows). */
export interface ReaderPins {
  model: string;
  prompt_sha: string;
  max_tokens: number;
  k: number;
}

export type RowLike = Record<string, unknown>;

/**
 * Per-row hasher: a prior row's recorded `reader_model` / `reader_prompt_sha`
 * / `reader_max_tokens` win over this run's pins; k is the run's `--top-k`
 * (already gated by `retrieval_config_hash`).
 */
export function makeJudgeConfigHasher(judgeModel: string, run: ReaderPins): (row: RowLike) => string {
  return (row) => judgeConfigHash({
    judge_model: judgeModel,
    prompt_version: JUDGE_PROMPT_VERSION,
    max_tokens: JUDGE_MAX_TOKENS,
    temperature: JUDGE_TEMPERATURE,
    reader_model: typeof row.reader_model === 'string' ? row.reader_model : run.model,
    reader_prompt_sha: typeof row.reader_prompt_sha === 'string' ? row.reader_prompt_sha : run.prompt_sha,
    context: {
      k: run.k,
      max_tokens: typeof row.reader_max_tokens === 'number' ? row.reader_max_tokens : run.max_tokens,
    },
  });
}

/** True when the row carries a judge attempt (verdict, error, or budget skip). */
export function hasJudgeAttempt(row: RowLike): boolean {
  return typeof row.judge_correct === 'boolean' || typeof row.judge_error === 'string' || row.judge_skipped === 'budget';
}

/** A judged verdict that needs no re-judge. */
export function hasSettledVerdict(row: RowLike): boolean {
  return typeof row.judge_correct === 'boolean' && typeof row.judge_error !== 'string' && row.judge_skipped !== 'budget';
}

export interface BackfillSelection {
  /** Prior rows with a reader hypothesis and no settled verdict — judged from their stored hypothesis. */
  candidates: RowLike[];
  /** Prior rows whose verdict stands. */
  settled: number;
  /** Rows already stamped with a DIFFERENT judge_config_hash than this run would stamp (refuse unless allowed). */
  mismatched: number;
  foreign: string[];
  /** Candidates produced by --retrieval-only (no reader hypothesis) — refused. */
  retrievalOnly: number;
  /** Candidates whose question_id is not in the dataset (cannot be judged: no gold). */
  missingFromDataset: number;
}

export function selectBackfillRows(
  priorRows: ReadonlyArray<RowLike>,
  ctx: { questionByQid: ReadonlyMap<string, LongMemEvalQuestion>; hashFor: (row: RowLike) => string },
): BackfillSelection {
  const sel: BackfillSelection = { candidates: [], settled: 0, mismatched: 0, foreign: [], retrievalOnly: 0, missingFromDataset: 0 };
  const foreign = new Set<string>();
  for (const row of priorRows) {
    if (row.kind === 'by_type_summary' || typeof row.question_id !== 'string') continue;
    if (typeof row.hypothesis !== 'string' || row.hypothesis === '') continue; // reader error rows are re-run, not judged
    if (typeof row.error === 'string') continue;
    if (typeof row.judge_config_hash === 'string' && row.judge_config_hash !== ctx.hashFor(row)) {
      sel.mismatched++;
      foreign.add(row.judge_config_hash);
    }
    if (hasSettledVerdict(row)) { sel.settled++; continue; }
    if (row.retrieval_only === true) { sel.retrievalOnly++; continue; }
    if (!ctx.questionByQid.has(row.question_id)) { sel.missingFromDataset++; continue; }
    sel.candidates.push(row);
  }
  sel.foreign = [...foreign].sort();
  return sel;
}

export interface JudgePreflightInput {
  judgeModel: string;
  maxUsd: number | null;
  yes: boolean;
  /** `isAvailable('chat', judgeModel)` — or true when a client is injected. */
  available: boolean;
  /** Questions the reader will answer live (hypothesis unknown → assume `readerMaxTokens`). */
  live: ReadonlyArray<{ question: string; answer: string }>;
  readerMaxTokens: number;
  /** Prior rows to judge from their stored hypothesis. */
  backfill: ReadonlyArray<RowLike>;
}

export type JudgePreflightResult =
  | { ok: true; estUsd: number | null; ledger: BudgetLedger; lines: string[] }
  | { ok: false; exitCode: 1 | 2; message: string };

/**
 * Availability → pricing → estimate → cap. Exit 1 when the judge model has
 * no usable provider; exit 2 when a budget is set against an unpriced model
 * or the estimate exceeds the cap without `--yes` (eval-cross-modal pattern).
 */
export function judgePreflight(input: JudgePreflightInput): JudgePreflightResult {
  if (!input.available) {
    return {
      ok: false, exitCode: 1,
      message: `--judge: no usable chat provider for judge model ${input.judgeModel}. Set OPENAI_API_KEY (or pass --judge-model <provider:model> for a configured provider).`,
    };
  }
  const priced = isJudgeModelPriced(input.judgeModel);
  if (!priced && input.maxUsd !== null) {
    return {
      ok: false, exitCode: 2,
      message: `--judge: --max-usd requires a priced judge model; "${input.judgeModel}" has no CANONICAL_PRICING entry (src/core/model-pricing.ts). Pick a priced model or pass --max-usd off to accept unbounded spend.`,
    };
  }
  const items = [
    ...input.live.map(q => ({ question: q.question, answer: String(q.answer ?? ''), hypothesisTokens: input.readerMaxTokens })),
    ...input.backfill.map(r => ({
      question: typeof r.question === 'string' ? r.question : '',
      answer: r.answer === undefined || r.answer === null ? '' : String(r.answer),
      hypothesisTokens: estimateTokens(typeof r.hypothesis === 'string' ? r.hypothesis : ''),
    })),
  ];
  const estUsd = priced ? estimateJudgeRunUsd(input.judgeModel, items) : null;
  const lines: string[] = [];
  lines.push(
    `[longmemeval] judge: ${input.judgeModel}, ${input.live.length} live + ${input.backfill.length} backfill question(s), ` +
    `estimated judge spend ${estUsd === null ? 'unpriced' : `~$${estUsd.toFixed(4)}`}` +
    `${input.maxUsd === null ? ' (no cap: --max-usd off)' : ` (cap $${input.maxUsd.toFixed(2)})`}`,
  );
  if (input.maxUsd !== null && estUsd !== null && estUsd > input.maxUsd && !input.yes) {
    return {
      ok: false, exitCode: 2,
      message: `--judge: estimated judge spend $${estUsd.toFixed(4)} exceeds --max-usd $${input.maxUsd.toFixed(2)}; pass --yes to proceed under the cap (the run soft-stops at the cap), raise --max-usd, or lower --limit.`,
    };
  }
  return { ok: true, estUsd, ledger: new BudgetLedger(input.maxUsd, estUsd), lines };
}

export interface BackfillResult {
  judged: number;
  errors: number;
  skipped: number;
}

/** The judge input for a prior row: the dataset's question/answer win; the row's own fields are the fallback. */
function backfillInput(row: RowLike, q: LongMemEvalQuestion | undefined): JudgePromptInput & RowLike {
  return {
    ...row,
    question_id: row.question_id as string,
    question_type: q?.question_type ?? (typeof row.question_type === 'string' ? row.question_type : 'unknown'),
    question: q?.question ?? (typeof row.question === 'string' ? row.question : ''),
    answer: String(q?.answer ?? row.answer ?? ''), // integer golds (32 in LongMemEval-S) are graded as their decimal string, as Python's f-string does
    hypothesis: row.hypothesis as string,
  };
}

/**
 * Judge prior rows from their stored hypothesis, `concurrency` at a time,
 * updating each row IN PLACE (prior judge_* keys stripped first so a
 * re-judge leaves no stale field behind). The dataset's answer is the
 * reference; the row's own `answer` is the fallback.
 *
 * INVARIANT: no candidate is left untouched. `judgeRow` never throws for a
 * transport failure, but a throw from anywhere else (a hasher bug, an
 * aborted signal, a malformed row) is stamped `judge_error: 'provider_error'`
 * with the redacted message — the SAME field set `judgeRow` stamps
 * (`errorJudgeFields`), minus `judge_config_hash` when the hasher itself
 * threw — so qa_accuracy counts it and the run-end gate refuses to publish
 * instead of the row silently keeping its old state.
 */
export async function runJudgeBackfill(
  candidates: ReadonlyArray<RowLike>,
  ctx: JudgeLaneContext,
  opts: { concurrency: number; questionByQid: ReadonlyMap<string, LongMemEvalQuestion>; onRow?: (row: RowLike) => void },
): Promise<BackfillResult> {
  const result: BackfillResult = { judged: 0, errors: 0, skipped: 0 };
  const settled = await runWithLimit({
    items: candidates,
    limit: Math.max(1, opts.concurrency),
    fn: async (row) => {
      const fields = await judgeRow(backfillInput(row, opts.questionByQid.get(row.question_id as string)), ctx);
      stripJudgeFields(row);
      Object.assign(row, fields);
      opts.onRow?.(row); // a throwing sink lands in the catch-all below; tally only once the row was accepted
      if (typeof fields.judge_correct === 'boolean') result.judged++;
      else if (fields.judge_error) result.errors++;
      else result.skipped++;
    },
  });
  for (const s of settled) {
    if (s.ok) continue;
    const row = candidates[s.idx];
    const err = s.error as { message?: unknown } | undefined;
    const input = backfillInput(row, opts.questionByQid.get(row.question_id as string));
    let configHash: string | null = null;
    try { configHash = ctx.configHashFor(input); } catch { /* hasher failed: leave unstamped (re-judged on the next resume) */ }
    stripJudgeFields(row);
    Object.assign(row, errorJudgeFields(
      ctx, judgePromptKind(input.question_id, input.question_type), configHash,
      { judge_error: 'provider_error', detail: String(err?.message ?? s.error) },
    ));
    result.errors++;
    opts.onRow?.(row);
  }
  return result;
}