Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
25.3 KB
Raw Blame
import { closeSync, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, unlinkSync, writeFileSync } from 'node:fs';
import { join, resolve } from 'node:path';
import { chat, type ChatOpts, type ChatResult } from '../src/core/ai/gateway.ts';
import { withAIInvocationGuard, type AIInvocationUsage } from '../src/core/ai/invocation-guard.ts';
import { BudgetTracker } from '../src/core/budget/budget-tracker.ts';
import { canonicalLookup } from '../src/core/model-pricing.ts';
import { normalizeSessions, haystackToPages, opaqueSessionId, type LongMemEvalQuestion } from '../src/eval/longmemeval/adapter.ts';
import { buildEvidencePacket, renderFullSessions, type EvidenceSession } from '../src/eval/longmemeval/evidence-packet.ts';
import { buildReaderUserText, READER_SYSTEM_TEXT, READER_MAX_TOKENS, READER_MAX_SESSION_CHARS } from '../src/eval/longmemeval/reader.ts';
import { sanitizeChatContent } from '../src/eval/longmemeval/sanitize.ts';
import { buildJudgePrompt, classifyJudgeResponse, DEFAULT_JUDGE_MODEL, JUDGE_MAX_TOKENS, JUDGE_TEMPERATURE } from '../src/eval/longmemeval/judge.ts';
import { runJudge } from '../src/eval/shared/judge-runner.ts';
import { configureEvalGateway } from '../src/eval/shared/gateway-bootstrap.ts';
import { bootstrapMeanCi } from '../src/eval/shared/bootstrap.ts';
import { redactSecrets, sha256Hex, stableStringify } from '../src/eval/longmemeval/run-config.ts';

export const READER = 'anthropic:claude-sonnet-4-6';
export const VARIANTS = { v1: { anchorRounds: 2, adjacentRounds: 1 }, v2: { anchorRounds: 4, adjacentRounds: 2 } } as const;
const DATA_SHA = 'd6f21ea9d60a0d56f34a05b609c79c88a451d2ae03597821ea3d5a9678c3a442';
const RECEIPT_SHA = '65ffeaa8299586e5a1dd6a9e10f269bab33462ab5470a95e329fbf189a703140';
const CAP = 20;
const HOLDOUT_RESERVE = 12;
const CODE_PATHS = ['scripts/eval-answer-packet.ts', 'src/eval/longmemeval/evidence-packet.ts', 'src/eval/longmemeval/reader.ts', 'src/eval/longmemeval/sanitize.ts', 'src/eval/longmemeval/adapter.ts', 'src/eval/longmemeval/judge.ts', 'src/eval/shared/judge-runner.ts', 'src/eval/shared/bootstrap.ts', 'src/eval/longmemeval/run-config.ts', 'src/core/search/title-match.ts', 'src/core/search/token-budget.ts', 'src/core/think/sanitize.ts', 'src/core/ai/gateway.ts', 'src/core/ai/invocation-guard.ts', 'src/core/budget/budget-tracker.ts', 'src/core/model-pricing.ts', 'bun.lock'];
type Question = LongMemEvalQuestion & { question_date?: string };
type Receipt = { question_id: string; retrieved_session_ids: string[] };
type Variant = keyof typeof VARIANTS;
type JsonRecord = Record<string, any>;

export function saveJson(path: string, value: unknown): void {
  const fd = openSync(path, 'wx', 0o600);
  try { writeFileSync(fd, JSON.stringify(value, null, 2) + '\n'); fsyncSync(fd); } finally { closeSync(fd); }
}

export function appendRecord(path: string, value: unknown): void {
  const fd = openSync(path, 'a', 0o600);
  try { writeFileSync(fd, JSON.stringify(value) + '\n'); fsyncSync(fd); } finally { closeSync(fd); }
}

export function readRecords(path: string): JsonRecord[] {
  if (!existsSync(path)) return [];
  const text = readFileSync(path, 'utf8');
  if (text && !text.endsWith('\n')) throw new Error('Interrupted journal tail; inspect it before resuming');
  return text.split('\n').filter(Boolean).map(line => JSON.parse(line));
}

export function freezeEvidence(q: Question, row: Receipt): EvidenceSession[] {
  if (q.question_id !== row.question_id || !Array.isArray(row.retrieved_session_ids)
    || new Set(row.retrieved_session_ids).size !== row.retrieved_session_ids.length) throw new Error('Invalid frozen receipt identity');
  const sessions = normalizeSessions(q);
  if (sessions.length !== q.haystack_sessions.length) throw new Error('Missing or ambiguous dataset sessions');
  const pages = haystackToPages(q);
  return row.retrieved_session_ids.map(id => {
    const i = sessions.findIndex(s => s.session_id === id);
    if (i < 0) throw new Error(`Missing saved session: ${id}`);
    if (sessions.filter(s => s.session_id === id).length !== 1 || pages.filter(p => p.slug === pages[i].slug).length !== 1) throw new Error('Missing or ambiguous saved session');
    // Reader-visible id is opaque (gold ids start with `answer_`); scoring uses the receipt's raw ids.
    return { source_id: 'longmemeval-public', session_id: opaqueSessionId(q.question_id, id), date: q.haystack_dates?.[i], body: pages[i].content, turns: sessions[i].turns.map(t => ({ role: t.role, content: t.content })) };
  });
}

export function selectCohorts(questions: readonly Question[], splits: { dev40: string[]; decision430: string[] }) {
  const sorted = [...questions].sort((a, b) => sha256Hex(`answer-packet-seed42:${a.question_id}`).localeCompare(sha256Hex(`answer-packet-seed42:${b.question_id}`)));
  const parent = new Map(questions.map(q => [q.question_id, q.question_id]));
  const root = (id: string): string => { const p = parent.get(id)!; return p === id ? id : root(p); };
  const seen = new Map<string, string>();
  for (const q of questions) {
    const keys = [`id:${q.question_id.replace(/_abs$/, '')}`, `question:${q.question.trim().toLowerCase()}`,
      `history:${sha256Hex(stableStringify((q.haystack_session_ids ?? normalizeSessions(q).map(s => s.session_id)).slice().sort()))}`];
    for (const key of keys) {
      const previous = seen.get(key);
      if (previous) parent.set(root(q.question_id), root(previous));
      seen.set(key, q.question_id);
    }
  }
  const groups = Object.fromEntries(questions.map(q => [q.question_id, root(q.question_id)]));
  const used = new Set<string>();
  const categories = [...new Set(questions.filter(q => !q.question_id.endsWith('_abs')).map(q => q.question_type))].sort();
  const pick = (pool: Set<string> | null, n: number, category?: string): string[] => {
    const ids: string[] = [];
    for (const q of sorted) {
      if (pool ? !pool.has(q.question_id) || q.question_type !== category : !q.question_id.endsWith('_abs')) continue;
      if (used.has(groups[q.question_id])) continue;
      ids.push(q.question_id); used.add(groups[q.question_id]);
      if (ids.length === n) break;
    }
    if (ids.length !== n) throw new Error(`Insufficient leakage-safe cohort: ${category ?? 'abstention'}`);
    return ids;
  };
  const dev = categories.flatMap(c => pick(new Set(splits.dev40), 2, c));
  dev.push(...pick(null, 6));
  const holdout = categories.flatMap(c => pick(new Set(splits.decision430), 8, c));
  holdout.push(...pick(null, 12));
  return { dev, holdout, groups, categories };
}

export function readerRequest(q: Pick<Question, 'question' | 'question_date'>, rendered: string): ChatOpts {
  return { model: READER, system: READER_SYSTEM_TEXT, messages: [{ role: 'user', content: buildReaderUserText({ question: q.question, questionDate: q.question_date, rendered }) }], maxTokens: READER_MAX_TOKENS };
}

export function completedReaderText(response: Pick<ChatResult, 'text' | 'stopReason'>, id: string): string {
  if (response.stopReason !== 'end' || !response.text.trim()) throw new Error(`Incomplete reader: ${id} (${response.stopReason})`);
  return response.text.trim();
}

export function inputUpperBound(opts: ChatOpts): number {
  if (!opts.model || opts.tools?.length || opts.messages.some(m => typeof m.content !== 'string') || opts.providerOptions) throw new Error('Only pinned text-only requests are supported');
  return Buffer.byteLength((opts.system ?? '') + opts.messages.map(m => m.content).join(''), 'utf8') + 1024;
}

export function usageCost(model: string, usage: AIInvocationUsage): number {
  const price = canonicalLookup(model);
  if (!price || [usage.inputTokens, usage.outputTokens, usage.cacheReadTokens ?? 0, usage.cacheWriteTokens ?? 0].some(n => !Number.isFinite(n) || n < 0)) throw new Error('Unknown pricing or invalid usage');
  return (usage.inputTokens * price.input + usage.outputTokens * price.output
    + (usage.cacheReadTokens ?? 0) * (price.cache_read ?? price.input)
    + (usage.cacheWriteTokens ?? 0) * (price.cache_write ?? price.input * 2)) / 1e6;
}

export async function recordedCall(
  path: string, id: string, opts: ChatOpts, maxUsd: number, client: (opts: ChatOpts) => Promise<ChatResult> = chat,
): Promise<ChatResult> {
  if (!Number.isFinite(maxUsd) || maxUsd <= 0 || maxUsd > CAP) throw new Error('Invalid total spending cap');
  const records = readRecords(path);
  const fingerprint = sha256Hex(stableStringify(opts));
  const same = records.filter(r => r.id === id);
  if (same.some(r => r.fingerprint !== fingerprint)) throw new Error('Changed request on resume');
  const admitted = records.filter(r => r.event === 'admit');
  if (admitted.some(r => !records.some(s => s.id === r.id && s.attempt === r.attempt && s.event === 'settle' && s.usage))) throw new Error('Unknown paid usage; refuse automatic replay');
  if (same.some(r => r.event === 'error')) throw new Error('Failed paid call; inspect before any retry');
  const prior = same.find(r => r.event === 'response');
  if (prior) {
    if (prior.accepted !== true) throw new Error('Unaccepted paid response; inspect before reuse');
    return prior.response;
  }
  if (same.length) throw new Error('Interrupted or failed paid call; inspect before any retry');
  const tracker = new BudgetTracker({ maxCostUsd: maxUsd, label: 'answer-packet', auditPath: `${path}.budget` });
  for (const r of records.filter(r => r.event === 'settle')) {
    const price = canonicalLookup(r.model)!;
    tracker.record({ modelId: r.model, inputTokens: r.cost_usd * 1e6 / price.input, outputTokens: 0 });
  }
  let attempts = 0;
  const start = performance.now();
  let unknown = false;
  let settlementError: unknown;
  try {
    const result = await withAIInvocationGuard(async call => {
      if (call.model !== opts.model || call.kind !== 'chat' || call.maxOutputTokens !== opts.maxTokens || call.cacheWriteTtl === '1h') throw new Error('Unpinned provider invocation');
      const input = inputUpperBound(opts);
      const price = canonicalLookup(call.model);
      if (!price) throw new Error('Unknown model pricing');
      const reserveInput = input * Math.max(1, (price.cache_write ?? price.input * 2) / price.input);
      tracker.reserve({ modelId: call.model, kind: 'chat', estimatedInputTokens: reserveInput, maxOutputTokens: opts.maxTokens! });
      const attempt = ++attempts;
      appendRecord(path, { event: 'admit', id, attempt, fingerprint, model: call.model, request: opts, input_upper_bound: input,
        reserved_usd: (reserveInput * price.input + opts.maxTokens! * price.output) / 1e6, timestamp: new Date().toISOString() });
      return { settle: async usage => {
        try {
          const cost = usage ? usageCost(call.model, usage) : null;
          appendRecord(path, { event: 'settle', id, attempt, fingerprint, model: call.model, usage, cost_usd: cost, timestamp: new Date().toISOString() });
          if (!usage) { unknown = true; return; }
          tracker.record({ modelId: call.model, inputTokens: cost! * 1e6 / price.input, outputTokens: 0 });
          if (usage.inputTokens + (usage.cacheReadTokens ?? 0) + (usage.cacheWriteTokens ?? 0) > input || usage.outputTokens > opts.maxTokens!) throw new Error('Provider exceeded the admitted token bound');
        } catch (error) { settlementError = error; }
      } };
    }, () => client({ ...opts, abortSignal: AbortSignal.timeout(180_000) }));
    const accepted = !settlementError && attempts === 1 && !unknown;
    appendRecord(path, { event: 'response', id, fingerprint, response: result, accepted, elapsed_ms: performance.now() - start });
    if (!accepted) throw settlementError ?? new Error('Expected one accounted provider attempt');
    return result;
  } catch (error) {
    appendRecord(path, { event: 'error', id, fingerprint, error: redactSecrets(String(error)), elapsed_ms: performance.now() - start });
    throw error;
  }
}

export function pairedSummary(rows: JsonRecord[]) {
  const complete = rows.filter(r => typeof r.baseline_correct === 'boolean' && typeof r.candidate_correct === 'boolean');
  const wins = complete.filter(r => !r.baseline_correct && r.candidate_correct).map(r => r.question_id);
  const losses = complete.filter(r => r.baseline_correct && !r.candidate_correct).map(r => r.question_id);
  return { n: rows.length, complete: complete.length, incomplete: rows.length - complete.length,
    baseline_correct: complete.filter(r => r.baseline_correct).length, candidate_correct: complete.filter(r => r.candidate_correct).length,
    both_right: complete.filter(r => r.baseline_correct && r.candidate_correct).length,
    both_wrong: complete.filter(r => !r.baseline_correct && !r.candidate_correct).length,
    wins, losses, net: wins.length - losses.length,
    delta_ci95: bootstrapMeanCi(complete.map(r => Number(r.candidate_correct) - Number(r.baseline_correct))),
  };
}

export function reportSummaries(manifest: JsonRecord, pairs: JsonRecord[], calls: JsonRecord[]) {
  const runs = new Map<string, { phase: string; variant: string }>();
  for (const r of [...calls.filter(r => r.event === 'phase_start'), ...pairs]) {
    if (['dev', 'holdout'].includes(r.phase) && r.variant in VARIANTS) runs.set(`${r.phase}-${r.variant}`, { phase: r.phase, variant: r.variant });
  }
  return Object.fromEntries([...runs].map(([key, { phase, variant }]) => {
    const observed = pairs.filter(r => r.phase === phase && r.variant === variant);
    const rows = (manifest.cohorts[phase] as string[]).map(id => {
      const row = observed.find(r => r.question_id === id) ?? { question_id: id, ...manifest.case_metadata?.[id] };
      if (!row.category || typeof row.abstention !== 'boolean' || typeof row.retrieval_complete !== 'boolean') throw new Error(`Missing expected case metadata: ${id}`);
      return row;
    });
    return [key, { ...pairedSummary(rows),
      by_category: Object.fromEntries(manifest.cohorts.categories.map((c: string) => [c, pairedSummary(rows.filter(r => r.category === c))])),
      by_abstention: Object.fromEntries([false, true].map(v => [String(v), pairedSummary(rows.filter(r => r.abstention === v))])),
      by_retrieval_complete: Object.fromEntries([false, true].map(v => [String(v), pairedSummary(rows.filter(r => r.retrieval_complete === v))])) }];
  }));
}

async function countReader(opts: ChatOpts): Promise<number> {
  const response = await fetch('https://api.anthropic.com/v1/messages/count_tokens', {
    method: 'POST', headers: { 'content-type': 'application/json', 'anthropic-version': '2023-06-01', 'x-api-key': process.env.ANTHROPIC_API_KEY! },
    body: JSON.stringify({ model: READER.split(':')[1], system: opts.system, messages: opts.messages }), signal: AbortSignal.timeout(60_000),
  });
  if (!response.ok) throw new Error(`Free token-count preflight failed: HTTP ${response.status}`);
  const result = await response.json() as { input_tokens: number };
  if (!Number.isSafeInteger(result.input_tokens) || result.input_tokens < 0) throw new Error('Invalid provider token count');
  return result.input_tokens;
}

function codePins() {
  return Object.fromEntries(CODE_PATHS.map(p => [p, sha256Hex(readFileSync(new URL(`../${p}`, import.meta.url), 'utf8'))]));
}

async function main(args: string[]) {
  const [command, outputArg, datasetPath, receiptPath, variantArg] = args;
  if (!['prepare', 'dev', 'freeze', 'holdout', 'report'].includes(command) || !outputArg) {
    throw new Error('Usage: bun scripts/eval-answer-packet.ts <prepare|dev|freeze|holdout|report> OUT DATASET RECEIPT [v1|v2]. prepare/report make no generation calls; dev/holdout are paid ($20 shared maximum).');
  }
  const out = resolve(outputArg);
  mkdirSync(out, { recursive: true, mode: 0o700 });
  const lock = join(out, 'writer.lock');
  const fd = openSync(lock, 'wx', 0o600);
  writeFileSync(fd, String(process.pid)); closeSync(fd);
  try {
    const manifestPath = join(out, 'manifest.json');
    const journal = join(out, 'calls.ndjson');
    if (command === 'report') {
      const manifest = JSON.parse(readFileSync(manifestPath, 'utf8'));
      const cases = readRecords(join(out, 'pairs.ndjson'));
      const calls = readRecords(journal);
      const summaries = reportSummaries(manifest, cases, calls);
      const settled = calls.filter(r => r.event === 'settle' && r.usage);
      const costByLane: Record<string, number> = {};
      for (const r of settled) { const [phase, , arm, lane] = r.id.split('/'); const key = `${phase}/${arm}/${lane}`; costByLane[key] = (costByLane[key] ?? 0) + r.cost_usd; }
      const unknown = calls.filter(r => r.event === 'admit' && !settled.some(s => s.id === r.id && s.attempt === r.attempt));
      const report = { summaries, measured_cost_usd: settled.reduce((n, r) => n + r.cost_usd, 0), cost_by_lane: costByLane,
        conservative_unsettled_usd: unknown.reduce((n, r) => n + r.reserved_usd, 0), errors: calls.filter(r => r.event === 'error'),
        pricing_note: 'Usage-priced estimate at pinned canonical rates; not an invoice. Unpriced automatic cache reads use full input rate.', manifest_sha: sha256Hex(stableStringify(manifest)) };
      console.log(JSON.stringify(report, null, 2));
      return;
    }
    if (!datasetPath || !receiptPath) throw new Error('Dataset and receipt paths are required');
    const datasetText = readFileSync(datasetPath, 'utf8');
    const receiptText = readFileSync(receiptPath, 'utf8');
    if (sha256Hex(datasetText) !== DATA_SHA || sha256Hex(receiptText) !== RECEIPT_SHA) throw new Error('Pinned input hash mismatch');
    const dataset: Question[] = JSON.parse(datasetText);
    const receipts: Receipt[] = receiptText.trim().split('\n').map(l => JSON.parse(l)).filter(r => typeof r.question_id === 'string');
    const splits = JSON.parse(readFileSync(new URL('../evals/longmemeval/splits-seed42.json', import.meta.url), 'utf8'));
    const cohorts = selectCohorts(dataset, splits);
    const caseMetadata = Object.fromEntries([...cohorts.dev, ...cohorts.holdout].map(id => {
      const q = dataset.find(q => q.question_id === id)!;
      const row = receipts.find(r => r.question_id === id);
      if (!row) throw new Error('Missing saved question');
      return [id, { category: q.question_type, abstention: id.endsWith('_abs'),
        retrieval_complete: q.answer_session_ids.length > 0 && q.answer_session_ids.every(id => row.retrieved_session_ids.includes(id)) }];
    }));
    const pins = { dataset_sha: DATA_SHA, receipt_sha: RECEIPT_SHA, split_sha: sha256Hex(stableStringify(splits)), code: codePins(), cohorts,
      case_metadata: caseMetadata,
      reader: READER, judge: DEFAULT_JUDGE_MODEL, variants: VARIANTS, cap_usd: CAP, holdout_reserve_usd: HOLDOUT_RESERVE,
      pricing: { [READER]: canonicalLookup(READER), [DEFAULT_JUDGE_MODEL]: canonicalLookup(DEFAULT_JUDGE_MODEL) },
      accounting: 'UTF-8 byte upper bound + 1024 input overhead; max output; worst cache-write reserve; one guarded attempt, SDK retries disabled',
      token_ceiling: 'baseline provider token-count estimate + 1024 safety overhead + 512 output; packet provider count must not exceed baseline count' };
    if (command === 'prepare') {
      saveJson(manifestPath, pins);
      for (const id of [...cohorts.dev, ...cohorts.holdout]) {
        const q = dataset.find(q => q.question_id === id)!;
        const rows = receipts.filter(r => r.question_id === id);
        if (rows.length !== 1) throw new Error('Missing or duplicate question receipt');
        const sources = freezeEvidence(q, rows[0]);
        renderFullSessions(sources);
        saveJson(join(out, `source-${id}.json`), { question_id: id, question: q.question, question_date: q.question_date, sources,
          source_sha: sha256Hex(stableStringify(sources)), baseline_sanitization: sources.map(s => sanitizeChatContent(s.body, READER_MAX_SESSION_CHARS).matched) });
      }
      console.log(JSON.stringify({ prepared: true, dev: cohorts.dev.length, holdout: cohorts.holdout.length, generation_cost_usd: 0 }));
      return;
    }
    if (stableStringify(JSON.parse(readFileSync(manifestPath, 'utf8'))) !== stableStringify(pins)) throw new Error('Manifest/code/config drift; refuse paid resume');
    const variant: Variant = (variantArg ?? 'v1') as Variant;
    if (!(variant in VARIANTS)) throw new Error('Expected v1 or v2');
    const freezePath = join(out, 'freeze.json');
    const pairsPath = join(out, 'pairs.ndjson');
    if (command === 'freeze') {
      const dev = readRecords(pairsPath).filter(r => r.phase === 'dev' && r.variant === variant);
      if (dev.length !== cohorts.dev.length || pairedSummary(dev).incomplete) throw new Error('Complete development before freezing');
      saveJson(freezePath, { variant, manifest_sha: sha256Hex(stableStringify(pins)), dev_result_sha: sha256Hex(stableStringify(dev)), frozen_at: new Date().toISOString() });
      console.log(JSON.stringify({ frozen: variant, development: pairedSummary(dev) }));
      return;
    }
    const phase = command as 'dev' | 'holdout';
    if (phase === 'dev' && existsSync(freezePath)) throw new Error('Development is closed after freeze');
    if (phase === 'holdout') {
      const frozen = JSON.parse(readFileSync(freezePath, 'utf8'));
      if (frozen.variant !== variant || frozen.manifest_sha !== sha256Hex(stableStringify(pins))) throw new Error('Holdout differs from frozen algorithm');
    }
    if (!process.env.ANTHROPIC_API_KEY || !process.env.OPENAI_API_KEY) throw new Error('Both reader and judge credentials are required');
    configureEvalGateway();
    appendRecord(journal, { event: 'phase_start', phase, variant, timestamp: new Date().toISOString() });
    for (const [index, id] of cohorts[phase].entries()) {
      if (readRecords(pairsPath).some(r => r.phase === phase && r.variant === variant && r.question_id === id)) continue;
      const q = dataset.find(q => q.question_id === id)!;
      const frozen = JSON.parse(readFileSync(join(out, `source-${id}.json`), 'utf8'));
      const sources: EvidenceSession[] = frozen.sources;
      if (sha256Hex(stableStringify(sources)) !== frozen.source_sha || frozen.question !== q.question || frozen.question_date !== q.question_date
        || stableStringify(sources) !== stableStringify(freezeEvidence(q, receipts.find(r => r.question_id === id)!))) throw new Error('Frozen source drift');
      const baseline = readerRequest(q, renderFullSessions(sources));
      const packet = buildEvidencePacket(q.question, sources, VARIANTS[variant]);
      let candidate = readerRequest(q, packet.rendered);
      const promptPath = join(out, `${phase}-${variant}-${id}-prompts.json`);
      let countBaseline: number, countCandidate: number;
      let providerCountFallback = false;
      if (existsSync(promptPath)) {
        const p = JSON.parse(readFileSync(promptPath, 'utf8'));
        countBaseline = p.count_baseline; countCandidate = p.count_candidate; providerCountFallback = p.provider_count_fallback;
        if (providerCountFallback) candidate = baseline;
        if (stableStringify(p.baseline) !== stableStringify(baseline) || stableStringify(p.candidate) !== stableStringify(candidate)) throw new Error('Saved prompt drift');
      } else {
        countBaseline = await countReader(baseline);
        countCandidate = stableStringify(baseline) === stableStringify(candidate) ? countBaseline : await countReader(candidate);
        if (countCandidate > countBaseline) { candidate = baseline; countCandidate = countBaseline; providerCountFallback = true; }
        saveJson(promptPath, { baseline, candidate, packet, provider_count_fallback: providerCountFallback, count_baseline: countBaseline, count_candidate: countCandidate,
          common_input_output_ceiling: countBaseline + 1024 + READER_MAX_TOKENS });
      }
      const outcomes: JsonRecord = {};
      const order = index % 2 ? ['candidate', 'baseline'] : ['baseline', 'candidate'];
      for (const arm of order) {
        const armKey = arm === 'baseline' ? 'baseline' : variant;
        const prefix = `${phase}/${id}/${armKey}`;
        const maxUsd = phase === 'dev' ? CAP - HOLDOUT_RESERVE : CAP;
        const response = await recordedCall(journal, `${prefix}/reader`, arm === 'baseline' ? baseline : candidate, maxUsd);
        const answer = completedReaderText(response, prefix);
        if (response.usage.input_tokens + READER_MAX_TOKENS > countBaseline + 1024 + READER_MAX_TOKENS) throw new Error('Reader exceeded common input/output ceiling');
        const judge = await runJudge({ model: DEFAULT_JUDGE_MODEL, prompt: buildJudgePrompt({ ...q, answer: String(q.answer), hypothesis: answer }).prompt,
          maxTokens: JUDGE_MAX_TOKENS, temperature: JUDGE_TEMPERATURE, parse: classifyJudgeResponse, retries: 0,
          client: opts => recordedCall(journal, `${prefix}/judge`, opts, maxUsd) });
        if (judge.kind !== 'verdict') { appendRecord(journal, { event: 'judge_error', id: `${prefix}/judge`, judge }); throw new Error(`Incomplete judge: ${prefix} ${judge.judge_error}`); }
        outcomes[`${arm}_correct`] = judge.verdict === 'correct';
      }
      const row = { phase, variant, question_id: id, category: q.question_type, abstention: id.endsWith('_abs'),
        retrieval_complete: q.answer_session_ids.length > 0 && q.answer_session_ids.every(gold => receipts.find(r => r.question_id === id)!.retrieved_session_ids.includes(gold)),
        ...outcomes, arm_order: order, budget_fallback: packet.budget_fallback, provider_count_fallback: providerCountFallback,
        baseline_input_count: countBaseline, candidate_input_count: countCandidate };
      appendRecord(pairsPath, row);
      console.log(JSON.stringify({ progress: index + 1, total: cohorts[phase].length, ...row }));
    }
  } finally { unlinkSync(lock); }
}

if (import.meta.main) main(process.argv.slice(2)).catch(error => { console.error(redactSecrets(String(error))); process.exitCode = 1; });