Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
9.1 KB
Raw Blame
#!/usr/bin/env bun
/**
 * Take-grading eval runner (W6, wave 4b).
 *
 * CHECK mode ($0, no key):
 *   bun evals/take-grading/harness.ts --check
 * Verifies fixtures.jsonl matches cases.ts and runs the instrument checks
 * (set counts, placeholder names, the dates each set promises, the production
 * evidence block). Exit 1 on any violation.
 *
 * SCORE mode ($0):
 *   bun evals/take-grading/harness.ts --score a.jsonl b.jsonl ... [--json summary.json] [--md summary.md]
 *
 * LIVE mode (spends tokens):
 *   bun evals/take-grading/harness.ts --model <provider:model> --split dev|counted --out <rows.jsonl>
 *     --max-usd <n> [--concurrency 4] [--only id1,id2] [--preregistration <commit>]
 * Each fixture goes through the production path of grade_takes: the evidence
 * block from `formatEvidenceBlock` and the verdict from `defaultJudge` (the
 * production prompt, one gateway `chat` call with the production options
 * and the production parser), with the model named the way the phase names
 * it. The harness only wraps the chat call to read the provider's usage. A
 * parse failure stands (production retries it on the next cycle); a provider
 * error is retried up to twice after a pause. The counted split refuses to
 * run without `--preregistration <commit>`, the commit that froze
 * PREREGISTRATION.md. The run stops once `--max-usd` is spent.
 *
 * Exit codes: 0 done, 1 check violation, 2 usage or no provider key, 3 budget stop.
 */
import { appendFileSync, readFileSync, writeFileSync } from 'node:fs';
import { createHash } from 'node:crypto';
import { join } from 'node:path';
import { FIXTURES_PATH, fixturesJsonl, type Fixture } from './generate-fixtures.ts';
import type { ResultRow } from './score.ts';

const here = import.meta.dir;
const args = process.argv.slice(2);
const flag = (name: string) => { const i = args.indexOf(name); return i === -1 ? null : args[i + 1] ?? null; };

export function loadFixtures(): Fixture[] {
  return readFileSync(FIXTURES_PATH, 'utf8').trim().split('\n').map(line => JSON.parse(line));
}

if (args.includes('--check')) {
  const { checkFixtures } = await import('./check.ts');
  const violations = readFileSync(FIXTURES_PATH, 'utf8') === fixturesJsonl() ? [] : ['fixtures.jsonl does not match cases.ts (regenerate with bun evals/take-grading/generate-fixtures.ts)'];
  const fixtures = loadFixtures();
  violations.push(...checkFixtures(fixtures));
  const bySet: Record<string, number> = {};
  for (const f of fixtures) bySet[`${f.split}/${f.set}`] = (bySet[`${f.split}/${f.set}`] ?? 0) + 1;
  for (const v of violations) console.error(`VIOLATION ${v}`);
  console.log(`check: ${fixtures.length} fixture(s) (${Object.entries(bySet).map(([s, n]) => `${s} ${n}`).join(', ')}), ${violations.length} violation(s)`);
  process.exit(violations.length ? 1 : 0);
}

if (args.includes('--score')) {
  const skip = new Set([flag('--json'), flag('--md')]);
  const files = args.slice(args.indexOf('--score') + 1).filter(a => !a.startsWith('--') && !skip.has(a));
  const { summarize } = await import('./score.ts');
  const { buildReport, renderMarkdown } = await import('./report.ts');
  const rows: ResultRow[] = files.flatMap(file => readFileSync(file, 'utf8').trim().split('\n').filter(Boolean).map(line => JSON.parse(line)));
  const report = buildReport(summarize(rows), files.map(f => f.slice(f.lastIndexOf('/') + 1)));
  const md = renderMarkdown(report);
  if (flag('--json')) writeFileSync(flag('--json')!, JSON.stringify(report, null, 2) + '\n');
  if (flag('--md')) writeFileSync(flag('--md')!, md);
  console.log(md);
  process.exit(0);
}

const model = flag('--model');
const split = flag('--split');
const outPath = flag('--out');
const maxUsd = Number(flag('--max-usd'));
const concurrency = Number(flag('--concurrency') ?? 4);
const preregistration = flag('--preregistration');
if (!model || !outPath || (split !== 'dev' && split !== 'counted') || !(maxUsd > 0) || !(concurrency >= 1)) {
  console.error('usage: bun evals/take-grading/harness.ts --check | --score <rows.jsonl ...> [--json f] [--md f] | --model <provider:model> --split dev|counted --out <rows.jsonl> --max-usd <n> [--concurrency 4] [--only ids] [--preregistration <commit>]');
  process.exit(2);
}
if (split === 'counted' && !preregistration) {
  console.error('take-grading: the counted split runs only after PREREGISTRATION.md is frozen; pass --preregistration <commit>.');
  process.exit(2);
}

const { configureEvalGateway } = await import('../../src/eval/shared/gateway-bootstrap.ts');
const gateway = await import('../../src/core/ai/gateway.ts');
const { chatCallUsd } = await import('../../src/core/budget/daily-ledger.ts');
const { defaultJudge, GRADE_TAKE_PROMPT, GRADE_TAKES_PROMPT_VERSION, GRADE_TAKES_AUTO_RESOLVE_THRESHOLD } = await import('../../src/core/cycle/grade-takes.ts');
const { evidenceBlock, productionTake } = await import('./check.ts');
const { grade } = await import('./score.ts');

configureEvalGateway({ chatModel: model });
if (!gateway.isAvailable('chat', model)) {
  console.error(`take-grading: no chat provider configured for ${model}; set its key. Refusing to run keyless.`);
  process.exit(2);
}

const only = flag('--only')?.split(',');
const selected = loadFixtures().filter(f => f.split === split && (!only || only.includes(f.id)));
const sha = (s: string | Buffer) => createHash('sha256').update(s).digest('hex');
const meta = {
  model, split, started_at: new Date().toISOString(),
  gbrain_commit: Bun.spawnSync(['git', 'rev-parse', 'HEAD'], { cwd: here }).stdout.toString().trim(),
  preregistration, bun: Bun.version, prompt_version: GRADE_TAKES_PROMPT_VERSION, prompt_sha256: sha(GRADE_TAKE_PROMPT),
  threshold: GRADE_TAKES_AUTO_RESOLVE_THRESHOLD, fixtures_sha256: sha(readFileSync(FIXTURES_PATH)),
  harness_sha256: sha(['harness.ts', 'check.ts', 'score.ts', 'cases.ts'].map(f => readFileSync(join(here, f), 'utf8')).join('\u0000')),
  max_usd: maxUsd, concurrency, fixtures: selected.length,
};
writeFileSync(`${outPath}.meta.json`, JSON.stringify(meta, null, 2) + '\n');
writeFileSync(outPath, '');
console.error(`take-grading: ${selected.length} fixture(s) of the ${split} split on ${model}, cap $${maxUsd}`);

let spent = 0;
let stopped = false;

async function judgeOne(f: Fixture): Promise<void> {
  const take = productionTake(f);
  const evidence = evidenceBlock(f);
  let attempts = 0;
  let usage: { input_tokens: number; output_tokens: number } | null = null;
  let verdict: Awaited<ReturnType<typeof defaultJudge>> | null = null;
  let failure: ResultRow['failure'] = null;
  let answerText: string | null = null;
  const t0 = performance.now();
  for (;;) {
    attempts++;
    const chatFn = (async (opts: Parameters<typeof gateway.chat>[0]) => {
      const result = await gateway.chat(opts);
      answerText = result.text;
      usage = { input_tokens: (usage?.input_tokens ?? 0) + result.usage.input_tokens, output_tokens: (usage?.output_tokens ?? 0) + result.usage.output_tokens };
      return result;
    }) as typeof gateway.chat;
    try {
      verdict = await defaultJudge({ take, evidence, modelHint: model! }, chatFn);
      failure = null;
      break;
    } catch (err) {
      const message = (err as Error).message ?? String(err);
      if (message.includes('judge output parse failed')) { failure = 'parse'; break; }
      failure = 'unavailable';
      if (attempts >= 3) break;
      console.error(`${f.id}: provider error (${message.slice(0, 120)}), attempt ${attempts}; retrying after a pause`);
      await new Promise(r => setTimeout(r, attempts * 20_000));
    }
  }
  const latency = Math.round(performance.now() - t0);
  const u = usage as { input_tokens: number; output_tokens: number } | null;
  const usd = u ? chatCallUsd(model!, { inputTokens: u.input_tokens, outputTokens: u.output_tokens }).usd : 0;
  spent += usd;
  const g = grade(f, verdict?.verdict ?? null, verdict?.confidence ?? null, GRADE_TAKES_AUTO_RESOLVE_THRESHOLD);
  const row: ResultRow = {
    model: model!, split: f.split, id: f.id, set: f.set, cls: f.cls, rule: f.rule, gold: f.gold, world_truth: f.world_truth,
    verdict: verdict?.verdict ?? null, confidence: verdict?.confidence ?? null, failure, applied: g.applied, wrong: g.wrong,
    input_tokens: u?.input_tokens ?? null, output_tokens: u?.output_tokens ?? null, usd, latency_ms: latency,
    reasoning: verdict?.reasoning ?? null, attempts, answer_text: failure === 'parse' ? answerText : null,
  };
  appendFileSync(outPath!, JSON.stringify(row) + '\n');
  console.error(`${f.id}: ${row.verdict ?? `no answer (${failure})`}@${row.confidence?.toFixed(2) ?? '-'} ${g.applied ? (g.wrong ? 'APPLIED-WRONG' : 'applied') : 'held'} $${usd.toFixed(4)} ${latency}ms`);
}

const queue = [...selected];
await Promise.all(Array.from({ length: Math.min(concurrency, queue.length) }, async () => {
  for (let f = queue.shift(); f; f = queue.shift()) {
    if (spent >= maxUsd) { stopped = true; return; }
    await judgeOne(f);
  }
}));

writeFileSync(`${outPath}.meta.json`, JSON.stringify({ ...meta, finished_at: new Date().toISOString(), spent_usd: spent, budget_stop: stopped }, null, 2) + '\n');
console.error(`spend: $${spent.toFixed(4)} on ${model} (${split})${stopped ? '; stopped at the budget cap' : ''}`);
process.exit(stopped ? 3 : 0);