Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* BrainBench scoreboard: markdown render, canonical committed baseline, and
* the compare gate with main-baseline governance (decisions 4, 8, 10).
*
* Gate semantics (deterministic corpus ⇒ any flip is a real behavior change):
* same fixtures_hash → count-aware gate: any cell whose gold_failed rose,
* any adverse gated-metric move, or ANY
* source_isolation_violations > 0 is a breach.
* different hash → corpus-bless mode: the committed baseline in the
* working tree must EXACTLY match the current run
* (the file can't lie); adverse moves vs main's
* baseline additionally require a `justification`
* string in the committed baseline.
* --allow-regression → local/one-off escape hatch; reason recorded in the
* outcome notes (and the run output).
*
* The committed baseline is diff-stable by construction: metrics rounded to 4
* decimals, keys sorted, receipts excluded (decision 10).
*/
import {
round4,
type BrainBenchBaseline,
type BrainBenchResult,
type CompareOutcome,
type SuiteMetrics,
} from './types.ts';
/** Gated metrics + their good direction. Anything absent is diagnostic-only. */
export const GATED_METRICS: Readonly<Record<string, 'lower' | 'higher'>> = {
know_to_ask_failure_rate: 'lower',
false_fire_rate: 'lower',
source_isolation_violations: 'lower',
push_precision: 'higher',
push_recall: 'higher',
write_back_fidelity: 'higher',
provenance_accuracy: 'higher',
continuity_rate: 'higher',
extraction_recall: 'higher',
extraction_precision: 'higher',
// #5575 memory-trust suites.
trust_label_accuracy: 'higher',
laundering_violations: 'lower',
self_promotion_violations: 'lower',
current_fact_accuracy: 'higher',
stale_surfaced_as_current: 'lower',
history_preserved: 'higher',
lower_tier_supersede_violations: 'lower',
poison_persist_rate: 'lower',
flagged_and_labeled_rate: 'higher',
unconfirmed_preference_activation_rate: 'lower',
agent_relayed_activation_rate: 'lower',
poison_activation_rate: 'lower',
benign_retention: 'higher',
false_quarantine_rate: 'lower',
default_persist_unlabeled_rate: 'lower',
default_activation_unlabeled_rate: 'lower',
default_benign_retention: 'higher',
residual_after_purge: 'lower',
receipt_completeness: 'higher',
resurrection_after_resync: 'lower',
};
/**
* Invariants that gate at ZERO in every run, whatever any baseline says (like
* source isolation): a single laundered tier, self-promotion, stale fact
* served as current, lower-tier supersession, persisted or activated poison,
* purge residual or resurrection is a breach. Absolute floors for the rates
* live in test/brainbench-floors.test.ts.
*/
export const ZERO_GATED_METRICS: readonly string[] = [
'source_isolation_violations',
'laundering_violations',
'self_promotion_violations',
'stale_surfaced_as_current',
'lower_tier_supersede_violations',
'poison_persist_rate',
'unconfirmed_preference_activation_rate',
'agent_relayed_activation_rate',
'poison_activation_rate',
'default_persist_unlabeled_rate',
'default_activation_unlabeled_rate',
'residual_after_purge',
'resurrection_after_resync',
];
const BASELINE_SCHEMA_VERSION = 1;
/** The canonical `${harness}/${suite}` cell key — baseline + run-all records share it. */
export function cellKey(c: Pick<SuiteMetrics, 'harness' | 'suite'>): string {
return `${c.harness}/${c.suite}`;
}
function sortedRecord<T>(entries: Array<[string, T]>): Record<string, T> {
const out: Record<string, T> = {};
for (const [k, v] of entries.sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))) out[k] = v;
return out;
}
/** Build the canonical, diff-stable committed-baseline shape from a run. */
export function toCanonicalBaseline(
result: Pick<BrainBenchResult, 'cells'> & {
receipt: Pick<
BrainBenchResult['receipt'],
'fixtures_hash' | 'include_holdout' | 'llm'
> & { harnesses?: string[]; suites?: string[] };
},
justification?: string,
config?: { harnesses: string[]; suites: string[] },
): BrainBenchBaseline {
const cells: Array<[string, Record<string, number>]> = [];
const counts: Array<[string, { gold_total: number; gold_failed: number }]> = [];
for (const c of result.cells) {
cells.push([
cellKey(c),
sortedRecord(Object.entries(c.metrics).map(([k, v]) => [k, round4(v)] as [string, number])),
]);
counts.push([cellKey(c), { gold_total: c.gold_total, gold_failed: c.gold_failed }]);
}
const baseline: BrainBenchBaseline = {
schema_version: BASELINE_SCHEMA_VERSION,
fixtures_hash: result.receipt.fixtures_hash,
config: {
include_holdout: result.receipt.include_holdout,
llm: result.receipt.llm,
harnesses: [...(config?.harnesses ?? [])].sort(),
suites: [...(config?.suites ?? [])].sort(),
},
cells: sortedRecord(cells),
counts: sortedRecord(counts),
};
if (justification) baseline.justification = justification;
return baseline;
}
/** Deterministic serialization for the committed file + bless-mode equality. */
export function serializeBaseline(b: BrainBenchBaseline): string {
// Field order is fixed by construction; metrics/cells already sorted.
const ordered: Record<string, unknown> = {
schema_version: b.schema_version,
fixtures_hash: b.fixtures_hash,
config: b.config,
};
if (b.justification) ordered.justification = b.justification;
ordered.cells = b.cells;
ordered.counts = b.counts;
return JSON.stringify(ordered, null, 2) + '\n';
}
/** Equality on the receipts-backed content (justification excluded — it's the bless note, not data). */
function baselineDataEquals(a: BrainBenchBaseline, b: BrainBenchBaseline): boolean {
return (
serializeBaseline({ ...a, justification: undefined }) ===
serializeBaseline({ ...b, justification: undefined })
);
}
export function parseBaseline(raw: string, file: string): BrainBenchBaseline {
let parsed: unknown;
try {
parsed = JSON.parse(raw);
} catch (err) {
throw new Error(`brainbench baseline ${file}: invalid JSON (${(err as Error).message})`);
}
const b = parsed as Partial<BrainBenchBaseline>;
if (b.schema_version !== BASELINE_SCHEMA_VERSION) {
throw new Error(`brainbench baseline ${file}: schema_version must be ${BASELINE_SCHEMA_VERSION}`);
}
if (typeof b.fixtures_hash !== 'string' || !b.cells || !b.counts || !b.config) {
throw new Error(`brainbench baseline ${file}: missing fixtures_hash/config/cells/counts`);
}
return b as BrainBenchBaseline;
}
/**
* `current` is comparable to `main` when holdout, llm and harnesses match and
* the suites match exactly. In corpus-bless mode (the fixtures changed) a run
* may ADD suites: every suite main gates is still present and compared cell by
* cell, and the new suites' cells enter through the committed baseline the
* run must reproduce, its justification and the absolute floors. Dropping a
* suite stays incomparable.
*/
function configsMatch(current: BrainBenchBaseline['config'], main: BrainBenchBaseline['config'], allowAddedSuites: boolean): boolean {
const suitesOk = allowAddedSuites
? main.suites.every(s => current.suites.includes(s))
: JSON.stringify(current.suites) === JSON.stringify(main.suites);
return (
current.include_holdout === main.include_holdout &&
current.llm === main.llm &&
JSON.stringify(current.harnesses) === JSON.stringify(main.harnesses) &&
suitesOk
);
}
export interface CompareOpts {
/** Local escape hatch — reason recorded, breaches downgraded to notes. */
allowRegression?: string;
/**
* Corpus-bless verification target: the committed baseline from the WORKING
* TREE (decision 4). Required for bless mode to pass; must exactly match the
* current run.
*/
committedBaseline?: BrainBenchBaseline | null;
}
export function compareBaselines(
current: BrainBenchBaseline,
main: BrainBenchBaseline,
opts: CompareOpts = {},
): CompareOutcome {
const breaches: CompareOutcome['breaches'] = [];
const notes: string[] = [];
// Source isolation (and the memory-trust invariants) gate at zero regardless of what any baseline says.
for (const [cell, metrics] of Object.entries(current.cells)) {
for (const metric of ZERO_GATED_METRICS) {
const v = metrics[metric];
if (v !== undefined && v > 0) {
breaches.push({
cell,
metric,
baseline: 0,
current: v,
detail: metric === 'source_isolation_violations'
? 'cross-source injection — gates at zero (data-leak invariant)'
: 'memory-trust invariant — gates at zero',
});
}
}
}
const sameHash = current.fixtures_hash === main.fixtures_hash;
const mode: CompareOutcome['mode'] = sameHash ? 'same-hash' : 'corpus-bless';
// Run-config binding (red-team finding: fixtures_hash covers files only —
// a holdout-inclusive or --llm baseline is incomparable under the same hash).
if (!configsMatch(current.config, main.config, !sameHash)) {
return {
verdict: 'inconclusive',
mode,
breaches,
notes: [
`run config mismatch: current=${JSON.stringify(current.config)} vs baseline=${JSON.stringify(main.config)} — compare like with like`,
],
};
}
// Adverse metric/count movement vs main's baseline (both modes; in bless
// mode it's what the justification must answer for).
for (const [cell, mainCounts] of Object.entries(main.counts)) {
const cur = current.counts[cell];
if (!cur) {
breaches.push({
cell,
metric: 'gold_failed',
baseline: mainCounts.gold_failed,
current: NaN,
detail: 'cell missing from current run (suite/harness coverage disappeared)',
});
continue;
}
if (sameHash && cur.gold_failed > mainCounts.gold_failed) {
breaches.push({
cell,
metric: 'gold_failed',
baseline: mainCounts.gold_failed,
current: cur.gold_failed,
detail: `${cur.gold_failed - mainCounts.gold_failed} newly-failed gold item(s)`,
});
}
// Same corpus MUST yield identical gold_total (adversarial finding: a
// code-only change that breaks the scorer/loader could silently hollow
// scoring coverage to zero failures). Free invariant — enforce it.
if (sameHash && cur.gold_total !== mainCounts.gold_total) {
breaches.push({
cell,
metric: 'gold_total',
baseline: mainCounts.gold_total,
current: cur.gold_total,
detail: 'scoring coverage changed WITHOUT a fixture change — scorer/loader behavior shifted',
});
}
// Corpus hollowing (red-team finding): in bless mode, deleting failing
// fixtures (or flipping them holdout) shrinks gold_total and IMPROVES
// every rate — require a justification for shrunken coverage exactly like
// an adverse metric move.
if (!sameHash && cur.gold_total < mainCounts.gold_total) {
breaches.push({
cell,
metric: 'gold_total',
baseline: mainCounts.gold_total,
current: cur.gold_total,
detail: `gold coverage shrank by ${mainCounts.gold_total - cur.gold_total} item(s) — corpus hollowing requires justification`,
});
}
}
for (const [cell, mainMetrics] of Object.entries(main.cells)) {
const curMetrics = current.cells[cell];
if (!curMetrics) continue; // covered by the counts check above
for (const [metric, direction] of Object.entries(GATED_METRICS)) {
const was = mainMetrics[metric];
const now = curMetrics[metric];
if (was === undefined || now === undefined) continue;
if (ZERO_GATED_METRICS.includes(metric)) continue; // gated at zero above
const adverse = direction === 'lower' ? now > was : now < was;
if (adverse) {
breaches.push({
cell,
metric,
baseline: was,
current: now,
detail: `${direction === 'lower' ? 'rose' : 'fell'} ${round4(Math.abs(now - was))}`,
});
}
}
}
if (mode === 'corpus-bless') {
notes.push(
`fixtures_hash changed (${main.fixtures_hash.slice(0, 12)} → ${current.fixtures_hash.slice(0, 12)}) — corpus-bless mode`,
);
const committed = opts.committedBaseline;
if (!committed) {
return {
verdict: 'inconclusive',
mode,
breaches,
notes: [
...notes,
'no committed baseline to verify — run `gbrain eval brainbench --update-baseline` and commit the result',
],
};
}
// The committed file must match the actual run (it can't lie).
const runSer = serializeBaseline({ ...current, justification: committed.justification });
const committedSer = serializeBaseline(committed);
if (runSer !== committedSer) {
return {
verdict: 'inconclusive',
mode,
breaches,
notes: [
...notes,
'committed baseline does not match this run — re-run `--update-baseline` and commit',
],
};
}
if (breaches.length > 0 && !committed.justification && !opts.allowRegression) {
return {
verdict: 'regression',
mode,
breaches,
notes: [
...notes,
'metrics regressed vs main — add a `justification` to the committed baseline (the reviewable reason)',
],
};
}
if (breaches.length > 0) {
notes.push(
`regression blessed: ${committed.justification ?? opts.allowRegression ?? ''}`.trim(),
);
}
return { verdict: 'pass', mode, breaches, notes };
}
// SAME-HASH committed-baseline drift (red-team finding: two-PR gate
// poisoning). Any edit to the committed baseline without a fixture change
// must be receipts-backed by THIS run — otherwise a PR can doctor the file
// main's future gates compare against. NO hash carve-out (adversarial
// finding: a doctored committed file with a FOREIGN fixtures_hash slipped
// through a hash-matched check) — callers running foreign corpora opt out
// by pointing --committed-baseline elsewhere.
const committed = opts.committedBaseline;
if (sameHash && committed && !baselineDataEquals(committed, main)) {
if (!baselineDataEquals(committed, current)) {
return {
verdict: 'inconclusive',
mode,
breaches,
notes: [
...notes,
'committed baseline differs from main WITHOUT a fixture change and does not match this run — a baseline edit must be receipts-backed (`--update-baseline`) or reverted',
],
};
}
notes.push('committed baseline updated (matches this run) — the diff vs main is the visible delta');
// Receipts-backed update: regressions still need blessing below.
if (breaches.length > 0 && committed.justification && !opts.allowRegression) {
notes.push(`regression blessed: ${committed.justification}`);
return { verdict: 'pass', mode, breaches, notes };
}
}
if (breaches.length > 0) {
if (opts.allowRegression) {
notes.push(`regression allowed: ${opts.allowRegression}`);
return { verdict: 'pass', mode, breaches, notes };
}
return { verdict: 'regression', mode, breaches, notes };
}
return { verdict: 'pass', mode, breaches, notes };
}
// ---------------------------------------------------------------------------
// Markdown scoreboard
// ---------------------------------------------------------------------------
const METRIC_ORDER = [
'know_to_ask_failure_rate',
'false_fire_rate',
'push_precision',
'push_recall',
'write_back_fidelity',
'provenance_accuracy',
'continuity_rate',
'extraction_recall',
'extraction_precision',
'trust_label_accuracy',
'laundering_violations',
'self_promotion_violations',
'current_fact_accuracy',
'stale_surfaced_as_current',
'history_preserved',
'lower_tier_supersede_violations',
'poison_persist_rate',
'flagged_and_labeled_rate',
'unconfirmed_preference_activation_rate',
'agent_relayed_activation_rate',
'poison_activation_rate',
'benign_retention',
'false_quarantine_rate',
'default_persist_unlabeled_rate',
'default_activation_unlabeled_rate',
'default_benign_retention',
'residual_after_purge',
'receipt_completeness',
'resurrection_after_resync',
'source_isolation_violations',
'avg_injected_tokens',
];
export function renderScoreboardMarkdown(
result: BrainBenchResult,
compare?: CompareOutcome | null,
): string {
const lines: string[] = [];
lines.push('# BrainBench scoreboard');
lines.push('');
lines.push(
`fixtures \`${result.receipt.fixtures_hash.slice(0, 12)}\` · ${result.receipt.include_holdout ? 'holdout INCLUDED (published-run mode)' : 'holdout excluded (gate mode)'} · ${result.receipt.llm ? 'LLM extractor' : 'deterministic (hermetic)'}`,
);
lines.push('');
lines.push('| harness | seam | suite | failed/gold | ' + METRIC_ORDER.map((m) => `\`${m}\``).join(' | ') + ' |');
lines.push('|---|---|---|---|' + METRIC_ORDER.map(() => '---').join('|') + '|');
for (const c of result.cells) {
const vals = METRIC_ORDER.map((m) => (c.metrics[m] !== undefined ? String(c.metrics[m]) : '—'));
lines.push(
`| ${c.harness} | ${c.seam} | ${c.suite} | ${c.gold_failed}/${c.gold_total} | ${vals.join(' | ')} |`,
);
}
lines.push('');
lines.push(
'_seam: `production` rows exercise a shipped integration seam; `contract` rows grade the same gbrain primitives through a harness-shaped injection contract (see docs/eval/BRAINBENCH.md)._',
);
if (result.seed_failures.length > 0) {
lines.push('');
lines.push(`**SEED FAILURES (${result.seed_failures.length})** — run is invalid (exit 2):`);
for (const f of result.seed_failures) lines.push(`- ${f.fixture_id}: ${f.error}`);
}
if (compare) {
lines.push('');
lines.push(`## Gate: ${compare.verdict.toUpperCase()} (${compare.mode})`);
for (const n of compare.notes) lines.push(`- ${n}`);
if (compare.breaches.length > 0) {
lines.push('');
lines.push('| cell | metric | main | current | detail |');
lines.push('|---|---|---|---|---|');
for (const b of compare.breaches) {
lines.push(
`| ${b.cell} | ${b.metric} | ${Number.isNaN(b.baseline) ? '—' : b.baseline} | ${Number.isNaN(b.current) ? '—' : b.current} | ${b.detail} |`,
);
}
}
}
lines.push('');
return lines.join('\n');
}