Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* v0.32.3 — `gbrain eval compare` — render the per-mode comparison.
*
* Reads `<repo>/.gbrain-evals/eval-results.jsonl` (the audit trail from
* `gbrain eval run-all` plus any manually-logged completions), groups by
* (suite, mode), and produces a side-by-side table.
*
* Statistics are computed only from per-query rows. A run record that
* points at its per-question output (`params.output`, written by
* `gbrain eval longmemeval --record --output`) contributes rows; pairs of
* runs are joined on question_id and compared with a paired cluster
* bootstrap (95% CI), a two-sided sign-flip randomization p-value, and Holm
* correction across every comparison in the report
* (src/core/eval/paired-bootstrap.ts). When no per-query rows are
* available the report says it is aggregate-only and computes nothing.
*
* Every numeric metric in the output is glossed through the
* src/core/eval/metric-glossary.ts module per [CDX-25]: ONE
* _meta.metric_glossary block per response, NOT sibling _gloss fields.
*/
import { readFileSync, existsSync, realpathSync } from 'fs';
import { isAbsolute, join, relative, sep } from 'path';
import { buildMetricGlossaryMeta } from '../core/eval/metric-glossary.ts';
import { holmAdjusted, pairedClusterStatistics } from '../core/eval/paired-bootstrap.ts';
import { SEARCH_MODES, type SearchMode } from '../core/search/mode.ts';
export interface CompareOpts {
help: boolean;
runIds: string[] | 'all';
modes: SearchMode[] | 'all';
suite?: string;
json: boolean;
md: boolean;
inputPath?: string;
baseline?: string;
candidate?: string;
draws: number;
seed: number;
}
function parseCompareArgs(args: string[]): CompareOpts {
const opts: CompareOpts = {
help: false,
runIds: 'all',
modes: 'all',
json: false,
md: true,
draws: 10_000,
seed: 42,
};
for (let i = 0; i < args.length; i++) {
const a = args[i];
if (a === '--help' || a === '-h') { opts.help = true; continue; }
if (a === '--runs') {
const list = (args[++i] ?? '').split(',').map(s => s.trim()).filter(Boolean);
opts.runIds = list.length > 0 ? list : 'all';
continue;
}
if (a === '--modes') {
const list = (args[++i] ?? '').split(',').map(s => s.trim()).filter(Boolean);
const validated: SearchMode[] = [];
for (const m of list) {
if (m === 'conservative' || m === 'balanced' || m === 'tokenmax') {
validated.push(m);
} else {
throw new Error(`--modes: ${m} is not valid`);
}
}
opts.modes = validated.length > 0 ? validated : 'all';
continue;
}
if (a === '--suite') { opts.suite = args[++i]; continue; }
if (a === '--json') { opts.json = true; opts.md = false; continue; }
if (a === '--md') { opts.md = true; opts.json = false; continue; }
if (a === '--input') { opts.inputPath = args[++i]; continue; }
if (a === '--baseline') { opts.baseline = args[++i]; continue; }
if (a === '--candidate') { opts.candidate = args[++i]; continue; }
if (a === '--draws' || a === '--seed') {
const v = Number(args[++i]);
if (!Number.isInteger(v) || v < (a === '--draws' ? 1000 : 0)) throw new Error(`${a}: expected an integer${a === '--draws' ? ' >= 1000' : ''}`);
if (a === '--draws') opts.draws = v; else opts.seed = v;
continue;
}
}
if ((opts.baseline === undefined) !== (opts.candidate === undefined)) throw new Error('--baseline and --candidate go together');
return opts;
}
function printHelp(): void {
process.stderr.write(
`gbrain eval compare [flags]\n\n` +
`Render a per-mode comparison from <repo>/.gbrain-evals/eval-results.jsonl\n\n` +
`Flags:\n` +
` --runs id1,id2,id3 Pick specific run_ids (default: all).\n` +
` --modes M1,M2,M3 Filter to these modes (default: all).\n` +
` --suite S Filter to one suite (longmemeval / replay / brainbench).\n` +
` --md Markdown output (default; CHANGELOG-paste-ready).\n` +
` --json JSON output (CI / programmatic consumption).\n` +
` --input PATH Override eval-results.jsonl location.\n` +
` --baseline RUN_ID Compare exactly this run ...\n` +
` --candidate RUN_ID ... against this one (default: every mode pair).\n` +
` --draws N Bootstrap resamples (default 10000).\n` +
` --seed N Resampling seed (default 42).\n\n` +
`Paired statistics need per-query rows: a run record whose params.output\n` +
`names its per-question JSONL. Without them the report is aggregate-only.\n` +
` -h, --help Show this help.\n`,
);
}
interface ParsedRecord {
run_id: string;
ran_at: string;
suite: string;
mode: SearchMode;
commit: string;
seed: number;
status: string;
duration_ms?: number;
error?: string;
metrics?: Record<string, number>;
params?: Record<string, unknown>;
}
type QueryRow = Record<string, unknown>;
/** Per-query metrics read from per-question rows, named as in the glossary. */
const PER_QUERY_METRICS: Array<{ name: string; value: (row: QueryRow) => number | undefined }> = [
{ name: 'recall_all@k', value: r => typeof r.recall_all_hit === 'boolean' ? Number(r.recall_all_hit) : undefined },
{ name: 'recall_any@k', value: r => typeof r.recall_any_hit === 'boolean' ? Number(r.recall_any_hit) : undefined },
{ name: 'qa_accuracy', value: r => typeof r.judge_correct === 'boolean' && typeof r.judge_error !== 'string' ? Number(r.judge_correct) : undefined },
];
export interface PairedComparison {
suite: string;
baseline_run: string;
candidate_run: string;
baseline_mode: string;
candidate_mode: string;
metric: string;
n: number;
clusters: number;
baseline_mean: number;
candidate_mean: number;
delta: number;
ci95: [number, number];
p_value: number;
holm_p_value: number;
wins: number;
losses: number;
ties: number;
significant: boolean;
}
/**
* `candidate` resolved against `root`, following symlinks, or null when it
* lands outside the root (through `..`, an absolute path, or a symlink).
*/
export function pathWithinRoot(root: string, candidate: string): string | null {
const base = realpathSync(root);
const target = isAbsolute(candidate) ? candidate : `${base}${sep}${candidate}`;
const real = existsSync(target) ? realpathSync(target) : target;
// relative() normalizes `..` segments of a path that does not exist yet.
const rel = relative(base, real);
return rel === '' || rel === '..' || rel.startsWith(`..${sep}`) || isAbsolute(rel) ? null : real;
}
/** Per-question rows of a run (last row per question_id), or why there are none. */
function readPerQueryRows(record: ParsedRecord, repoRoot: string): { rows: Map<string, QueryRow> } | { reason: string } {
const output = record.params?.output;
if (typeof output !== 'string' || !output) return { reason: 'record has no params.output per-query file' };
const path = pathWithinRoot(repoRoot, output);
if (!path) return { reason: `refused per-query file outside the repository root (${repoRoot}): ${output}` };
if (!existsSync(path)) return { reason: `per-query file not found: ${output}` };
const rows = new Map<string, QueryRow>();
for (const line of readFileSync(path, 'utf-8').split('\n')) {
if (!line.trim()) continue;
try {
const row = JSON.parse(line) as QueryRow;
if (row && typeof row.question_id === 'string' && row.kind !== 'by_type_summary') rows.set(row.question_id, row);
} catch {
// Same tolerance as the run ledger: a torn line never tanks the report.
}
}
return rows.size ? { rows } : { reason: `per-query file has no question rows: ${output}` };
}
/**
* Paired comparisons for each (baseline, candidate) run pair: rows joined on
* question_id, one comparison per metric both rows carry, questions as
* clusters, Holm across the whole family.
*/
export function pairedComparisons(
pairs: Array<[ParsedRecord, ParsedRecord]>,
repoRoot: string,
opts: { draws: number; seed: number },
): { comparisons: PairedComparison[]; unavailable: Array<{ run_id: string; reason: string }> } {
const unavailable = new Map<string, string>();
const cache = new Map<string, Map<string, QueryRow> | null>();
const rowsOf = (r: ParsedRecord) => {
if (!cache.has(r.run_id)) {
const loaded = readPerQueryRows(r, repoRoot);
if ('reason' in loaded) unavailable.set(r.run_id, loaded.reason);
cache.set(r.run_id, 'rows' in loaded ? loaded.rows : null);
}
return cache.get(r.run_id) ?? null;
};
const raw: Array<Omit<PairedComparison, 'holm_p_value' | 'significant'>> = [];
for (const [b, c] of pairs) {
const bRows = rowsOf(b), cRows = rowsOf(c);
if (!bRows || !cRows) continue;
for (const metric of PER_QUERY_METRICS) {
const observations = [...bRows].flatMap(([qid, row]) => {
const other = cRows.get(qid);
const bv = metric.value(row), cv = other ? metric.value(other) : undefined;
return bv === undefined || cv === undefined ? [] : [{ cluster: qid, baseline: bv, candidate: cv }];
});
if (!observations.length) continue;
const st = pairedClusterStatistics(observations, opts);
raw.push({
suite: c.suite, baseline_run: b.run_id, candidate_run: c.run_id, baseline_mode: b.mode, candidate_mode: c.mode,
metric: metric.name, n: st.n, clusters: st.clusters, baseline_mean: st.baseline_mean, candidate_mean: st.candidate_mean,
delta: st.delta, ci95: [st.lower95, st.upper95], p_value: st.p_value, wins: st.wins, losses: st.losses, ties: st.ties,
});
}
}
const holm = holmAdjusted(raw.map(r => r.p_value));
const comparisons = raw.map((r, i) => ({
...r,
holm_p_value: holm[i],
significant: holm[i] <= 0.05 && (r.ci95[0] > 0 || r.ci95[1] < 0),
}));
return { comparisons, unavailable: [...unavailable].map(([run_id, reason]) => ({ run_id, reason })) };
}
function methodologyText(comparisons: PairedComparison[], opts: { draws: number; seed: number }): string {
if (!comparisons.length) {
return 'Aggregate-only: no per-query rows were available for the selected runs, so no confidence intervals or significance tests were computed. Aggregate metrics are shown as recorded.';
}
return `Paired cluster bootstrap over per-query rows joined on question_id (${opts.draws} resamples, seed ${opts.seed}; clusters = questions); `
+ 'two-sided sign-flip randomization p-values (exact at 16 or fewer clusters); '
+ `Holm correction across the ${comparisons.length} comparison(s) in this report. A difference is significant only when the Holm p-value is at most 0.05 and the 95% CI excludes 0. `
+ 'See docs/eval/SEARCH_MODE_METHODOLOGY.md.';
}
function fmt(x: number): string {
return Number.isFinite(x) ? x.toFixed(4) : String(x);
}
function renderPaired(comparisons: PairedComparison[], unavailable: Array<{ run_id: string; reason: string }>, methodology: string): string {
const lines = ['## Paired comparisons', ''];
if (comparisons.length) {
lines.push('| Suite | Baseline → Candidate | Metric | Pairs (clusters) | Baseline | Candidate | Δ | 95% CI | p | Holm p | Wins / losses / ties | Verdict |');
lines.push('|---|---|---|---|---|---|---|---|---|---|---|---|');
for (const c of comparisons) {
lines.push(`| ${c.suite} | \`${c.baseline_run}\` (${c.baseline_mode}) → \`${c.candidate_run}\` (${c.candidate_mode}) | ${c.metric} | ${c.n} (${c.clusters}) | ${fmt(c.baseline_mean)} | ${fmt(c.candidate_mean)} | ${fmt(c.delta)} | [${fmt(c.ci95[0])}, ${fmt(c.ci95[1])}] | ${fmt(c.p_value)} | ${fmt(c.holm_p_value)} | ${c.wins} / ${c.losses} / ${c.ties} | ${c.significant ? 'significant' : 'not significant'} |`);
}
lines.push('');
}
for (const u of unavailable) lines.push(`- \`${u.run_id}\`: ${u.reason}`);
if (unavailable.length) lines.push('');
lines.push(`_${methodology}_`, '');
return lines.join('\n');
}
function readEvalResults(repoRoot: string, override?: string): ParsedRecord[] {
const path = override
? (override.endsWith('.jsonl') ? override : join(override, 'eval-results.jsonl'))
: join(repoRoot, '.gbrain-evals', 'eval-results.jsonl');
if (!existsSync(path)) return [];
const content = readFileSync(path, 'utf-8');
const records: ParsedRecord[] = [];
for (const line of content.split('\n')) {
if (!line.trim()) continue;
try {
const r = JSON.parse(line);
if (r && typeof r === 'object' && r.run_id && r.mode && r.suite) {
records.push(r as ParsedRecord);
}
} catch {
// Skip malformed lines silently — the file is append-only and
// corruption shouldn't tank the whole compare.
}
}
return records;
}
function getRepoRoot(): string {
try {
const { execSync } = require('../core/spawn.ts') as typeof import('../core/spawn.ts');
return execSync('git rev-parse --show-toplevel', { encoding: 'utf-8' }).trim();
} catch {
return process.cwd();
}
}
/**
* Group records into a (mode → record) map per suite. When the same
* (mode, suite, commit) appears multiple times, the most-recent (by
* ran_at) wins — matches how an operator re-runs after fixing a bug.
*/
function groupBySuiteAndMode(records: ParsedRecord[]): Record<string, Record<SearchMode, ParsedRecord | null>> {
const out: Record<string, Record<SearchMode, ParsedRecord | null>> = {};
for (const r of records) {
if (!out[r.suite]) {
out[r.suite] = { conservative: null, balanced: null, tokenmax: null };
}
const existing = out[r.suite][r.mode];
if (!existing || new Date(r.ran_at).getTime() > new Date(existing.ran_at).getTime()) {
out[r.suite][r.mode] = r;
}
}
return out;
}
function renderMarkdown(grouped: Record<string, Record<SearchMode, ParsedRecord | null>>, glossary: Record<string, string>): string {
const lines: string[] = [];
lines.push('# Search Mode Comparison');
lines.push('');
lines.push('_Auto-generated from `<repo>/.gbrain-evals/eval-results.jsonl`. Industry terms preserved verbatim so users searching the literature find what we report._');
lines.push('');
for (const [suite, modes] of Object.entries(grouped)) {
lines.push(`## ${suite}`);
lines.push('');
const present = SEARCH_MODES.filter(m => modes[m] !== null);
if (present.length === 0) {
lines.push('_No completed runs for this suite._');
lines.push('');
continue;
}
lines.push(`| Mode | Status | Run ID | Ran at |`);
lines.push(`|------|--------|--------|--------|`);
for (const m of SEARCH_MODES) {
const r = modes[m];
if (!r) {
lines.push(`| ${m} | N/A — no run | — | — |`);
continue;
}
lines.push(`| ${m} | ${r.status} | \`${r.run_id}\` | ${r.ran_at} |`);
}
lines.push('');
// Per-metric breakdown (only when metrics are populated; v0.32.3
// run-all logs skipped stubs without metrics yet).
const hasMetrics = present.some(m => modes[m]?.metrics && Object.keys(modes[m]!.metrics!).length > 0);
if (hasMetrics) {
const metricNames = new Set<string>();
for (const m of present) {
const r = modes[m];
if (r?.metrics) Object.keys(r.metrics).forEach(k => metricNames.add(k));
}
for (const metric of metricNames) {
lines.push(`### ${metric}`);
lines.push('');
for (const m of present) {
const v = modes[m]?.metrics?.[metric];
if (v === undefined) {
lines.push(` ${m}: _no value_`);
} else {
lines.push(` ${m}: **${v.toFixed(4)}**`);
}
}
const gloss = glossary[metric];
if (gloss) {
lines.push('');
lines.push(`Plain English: ${gloss}`);
}
lines.push('');
}
} else {
lines.push('_No metric data yet — orchestrator stubs only. Metric population lands in v0.32.4._');
lines.push('');
}
}
return lines.join('\n');
}
export async function runEvalCompare(args: string[]): Promise<void> {
const opts = parseCompareArgs(args);
if (opts.help) {
printHelp();
return;
}
const repoRoot = getRepoRoot();
const records = readEvalResults(repoRoot, opts.inputPath);
let filtered = records;
if (opts.suite) filtered = filtered.filter(r => r.suite === opts.suite);
if (opts.runIds !== 'all') filtered = filtered.filter(r => (opts.runIds as string[]).includes(r.run_id));
if (opts.modes !== 'all') filtered = filtered.filter(r => (opts.modes as SearchMode[]).includes(r.mode));
const grouped = groupBySuiteAndMode(filtered);
const allMetrics = new Set<string>();
for (const modes of Object.values(grouped)) {
for (const m of SEARCH_MODES) {
const r = modes[m];
if (r?.metrics) Object.keys(r.metrics).forEach(k => allMetrics.add(k));
}
}
let runPairs: Array<[ParsedRecord, ParsedRecord]> = [];
if (opts.baseline && opts.candidate) {
const find = (id: string) => records.find(r => r.run_id === id);
const b = find(opts.baseline), c = find(opts.candidate);
if (!b || !c) throw new Error(`run not found in eval-results.jsonl: ${!b ? opts.baseline : opts.candidate}`);
runPairs = [[b, c]];
} else {
for (const modes of Object.values(grouped)) {
const present = SEARCH_MODES.map(m => modes[m]).filter((r): r is ParsedRecord => r !== null);
for (let i = 0; i < present.length; i++) for (let j = i + 1; j < present.length; j++) runPairs.push([present[i], present[j]]);
}
}
const { comparisons, unavailable } = pairedComparisons(runPairs, repoRoot, opts);
comparisons.forEach(c => allMetrics.add(c.metric));
if (comparisons.length) ['p_value', 'confidence_interval'].forEach(k => allMetrics.add(k));
const glossary = buildMetricGlossaryMeta(Array.from(allMetrics));
const methodology = methodologyText(comparisons, opts);
if (opts.json) {
process.stdout.write(JSON.stringify({
schema_version: 2,
records: filtered,
grouped,
paired: comparisons,
paired_unavailable: unavailable,
_meta: {
metric_glossary: glossary,
methodology,
},
}, null, 2));
return;
}
if (records.length === 0) {
process.stdout.write(`_No eval-results.jsonl found at <repo>/.gbrain-evals/eval-results.jsonl._\n`);
process.stdout.write(`_Run, once per mode (conservative, balanced, tokenmax): gbrain eval longmemeval <dataset.jsonl> --mode <mode> --record_\n`);
return;
}
process.stdout.write(renderMarkdown(grouped, glossary));
process.stdout.write(renderPaired(comparisons, unavailable, methodology));
}