Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* BrainBench fixture loader + strict validator + corpus hash.
*
* Strictness is the seal: a fixture turn carrying a `gold` key (or any unknown
* key) is a hard validation error, because the fixture file is the
* adapter-visible surface — gold lives in the sealed gold dir only
* (decision 22, gbrain-evals sealed-gold discipline).
*
* fixtures_hash covers BOTH fixture and gold files (sorted relative path +
* content), so a gold-only edit invalidates baseline comparisons exactly like
* a fixture edit (decision 4's same-hash vs corpus-bless modes key off it).
*/
import { createHash } from 'node:crypto';
import { readdir, readFile } from 'node:fs/promises';
import { join } from 'node:path';
import { isTrustTier } from '../../core/trust/tier.ts';
import {
ALL_SUITES,
FIXTURE_SCHEMA_VERSION,
TRUST_ACTOR_OPS,
TRUST_ACTORS,
TRUST_CHECKS,
TRUST_OPS,
isTrustSuite,
type TrustCheck,
type TrustSuite,
type BrainBenchFixture,
type BrainBenchSuite,
type FixtureGold,
type FixtureTurn,
type LoadedCorpus,
type LoadedFixture,
} from './types.ts';
export class FixtureValidationError extends Error {
constructor(
public readonly file: string,
message: string,
) {
super(`${file}: ${message}`);
this.name = 'FixtureValidationError';
}
}
const FIXTURE_KEYS = new Set([
'schema_version', 'fixture_id', 'suites', 'category', 'holdout', 'sources',
'active_source', 'seed_pages', 'seed_facts', 'turns', 'continuity', 'trust_steps',
]);
const TURN_KEYS = new Set(['turn_id', 'role', 'text', 'ts']);
const SEED_PAGE_KEYS = new Set(['slug', 'content', 'source_id']);
const SEED_FACT_KEYS = new Set(['fact', 'entity_slug', 'source', 'source_session', 'source_id']);
const GOLD_KEYS = new Set(['fixture_id', 'turns', 'continuity', 'trust']);
const TRUST_STEP_KEYS = new Set(['step_id', 'actor', 'op', 'slug', 'content', 'fact', 'entity', 'kind', 'content_origin', 'target', 'target_claim']);
const TRUST_ITEM_KEYS = new Set(['item_id', 'check', 'step', 'claim', 'tier', 'arm', 'canary', 'preference', 'probe']);
/** Which gold checks each memory-trust suite scores. */
export const TRUST_SUITE_CHECKS: Readonly<Record<TrustSuite, readonly TrustCheck[]>> = {
trust: ['tier', 'max_tier', 'refused'],
'state-resolution': ['current', 'stale', 'history', 'guarded'],
poisoning: ['poison', 'benign'],
deletion: ['purged'],
};
const TURN_GOLD_KEYS = new Set(['should_retrieve', 'gold_slugs', 'acceptable_slugs', 'gold_facts']);
const GOLD_FACT_KEYS = new Set(['gist', 'fact', 'entity_slug', 'match_keywords', 'kind']);
function assertOnlyKeys(
file: string,
obj: Record<string, unknown>,
allowed: Set<string>,
where: string,
): void {
for (const k of Object.keys(obj)) {
if (!allowed.has(k)) {
const hint =
k === 'gold'
? ' — gold is SEALED: it belongs in the gold dir (<fixture_id>.gold.json), never inline'
: '';
throw new FixtureValidationError(file, `unknown key "${k}" in ${where}${hint}`);
}
}
}
function isStringArray(v: unknown): v is string[] {
return Array.isArray(v) && v.every((x) => typeof x === 'string');
}
export function validateFixture(file: string, raw: unknown): BrainBenchFixture {
if (typeof raw !== 'object' || raw === null || Array.isArray(raw)) {
throw new FixtureValidationError(file, 'fixture must be a JSON object');
}
const f = raw as Record<string, unknown>;
assertOnlyKeys(file, f, FIXTURE_KEYS, 'fixture');
if (f.schema_version !== FIXTURE_SCHEMA_VERSION) {
throw new FixtureValidationError(
file,
`schema_version must be ${FIXTURE_SCHEMA_VERSION} (got ${JSON.stringify(f.schema_version)})`,
);
}
if (typeof f.fixture_id !== 'string' || !/^[a-z0-9][a-z0-9-]*$/.test(f.fixture_id)) {
throw new FixtureValidationError(file, 'fixture_id must be a kebab-case string');
}
if (!isStringArray(f.suites) || f.suites.length === 0) {
throw new FixtureValidationError(file, 'suites must be a non-empty string array');
}
for (const s of f.suites) {
if (!(ALL_SUITES as readonly string[]).includes(s)) {
throw new FixtureValidationError(file, `unknown suite "${s}" (valid: ${ALL_SUITES.join(', ')})`);
}
}
if (f.holdout !== undefined && typeof f.holdout !== 'boolean') {
throw new FixtureValidationError(file, 'holdout must be boolean');
}
if (f.sources !== undefined && !isStringArray(f.sources)) {
throw new FixtureValidationError(file, 'sources must be a string array');
}
if (f.active_source !== undefined && typeof f.active_source !== 'string') {
throw new FixtureValidationError(file, 'active_source must be a string');
}
if (f.seed_pages !== undefined) {
if (!Array.isArray(f.seed_pages)) throw new FixtureValidationError(file, 'seed_pages must be an array');
for (const p of f.seed_pages as Array<Record<string, unknown>>) {
assertOnlyKeys(file, p, SEED_PAGE_KEYS, 'seed_pages[]');
if (typeof p.slug !== 'string' || !p.slug) throw new FixtureValidationError(file, 'seed_pages[].slug required');
if (typeof p.content !== 'string' || !p.content) {
throw new FixtureValidationError(file, `seed_pages[${p.slug}].content required`);
}
}
}
if (f.seed_facts !== undefined) {
if (!Array.isArray(f.seed_facts)) throw new FixtureValidationError(file, 'seed_facts must be an array');
for (const sf of f.seed_facts as Array<Record<string, unknown>>) {
assertOnlyKeys(file, sf, SEED_FACT_KEYS, 'seed_facts[]');
if (typeof sf.fact !== 'string' || !sf.fact) throw new FixtureValidationError(file, 'seed_facts[].fact required');
}
}
if (!Array.isArray(f.turns) || f.turns.length === 0) {
throw new FixtureValidationError(file, 'turns must be a non-empty array');
}
const seenTurnIds = new Set<number>();
for (const t of f.turns as Array<Record<string, unknown>>) {
assertOnlyKeys(file, t, TURN_KEYS, 'turns[]');
if (typeof t.turn_id !== 'number' || !Number.isInteger(t.turn_id)) {
throw new FixtureValidationError(file, 'turns[].turn_id must be an integer');
}
if (seenTurnIds.has(t.turn_id)) {
throw new FixtureValidationError(file, `duplicate turn_id ${t.turn_id}`);
}
seenTurnIds.add(t.turn_id);
if (t.role !== 'user' && t.role !== 'assistant') {
throw new FixtureValidationError(file, `turns[${t.turn_id}].role must be user|assistant`);
}
if (typeof t.text !== 'string' || !t.text) {
throw new FixtureValidationError(file, `turns[${t.turn_id}].text required`);
}
if (t.ts !== undefined && (typeof t.ts !== 'string' || Number.isNaN(Date.parse(t.ts)))) {
throw new FixtureValidationError(file, `turns[${t.turn_id}].ts must be ISO 8601`);
}
}
if (f.continuity !== undefined) {
const c = f.continuity as Record<string, unknown>;
assertOnlyKeys(file, c, new Set(['pair_id', 'pair_role']), 'continuity');
if (typeof c.pair_id !== 'string' || !c.pair_id) {
throw new FixtureValidationError(file, 'continuity.pair_id required');
}
if (c.pair_role !== 'writer' && c.pair_role !== 'reader') {
throw new FixtureValidationError(file, 'continuity.pair_role must be writer|reader');
}
}
// write-back fixtures (and continuity WRITERS, which run the write-back
// pipeline to persist their decisions) need timestamps on every turn —
// segmentation is time-based.
const continuityRole = (f.continuity as Record<string, unknown> | undefined)?.pair_role;
if ((f.suites as string[]).includes('write-back') || continuityRole === 'writer') {
for (const t of f.turns as Array<Record<string, unknown>>) {
if (t.ts === undefined) {
throw new FixtureValidationError(
file,
`write-back fixture requires ts on every turn (missing on turn_id ${t.turn_id})`,
);
}
}
}
validateTrustSteps(file, f);
return raw as BrainBenchFixture;
}
function validateTrustSteps(file: string, f: Record<string, unknown>): void {
const suites = f.suites as string[];
const trustSuites = suites.filter(isTrustSuite);
if (trustSuites.length === 0) {
if (f.trust_steps !== undefined) throw new FixtureValidationError(file, 'trust_steps belong to memory-trust suite fixtures only');
return;
}
if (suites.length !== 1) {
throw new FixtureValidationError(file, `a memory-trust fixture declares exactly one suite (got ${suites.join(', ')})`);
}
if (f.seed_pages !== undefined || f.seed_facts !== undefined || f.continuity !== undefined) {
throw new FixtureValidationError(file, 'memory-trust fixtures write through trust_steps, never seed_pages/seed_facts/continuity');
}
if (!Array.isArray(f.trust_steps) || f.trust_steps.length === 0) {
throw new FixtureValidationError(file, 'memory-trust fixtures require a non-empty trust_steps array');
}
const seen = new Set<string>();
for (const st of f.trust_steps as Array<Record<string, unknown>>) {
assertOnlyKeys(file, st, TRUST_STEP_KEYS, 'trust_steps[]');
if (typeof st.step_id !== 'string' || !st.step_id) throw new FixtureValidationError(file, 'trust_steps[].step_id required');
if (seen.has(st.step_id)) throw new FixtureValidationError(file, `duplicate step_id ${st.step_id}`);
if (!(TRUST_ACTORS as readonly unknown[]).includes(st.actor)) {
throw new FixtureValidationError(file, `trust_steps[${st.step_id}].actor must be one of ${TRUST_ACTORS.join(', ')}`);
}
if (!(TRUST_OPS as readonly unknown[]).includes(st.op) || !TRUST_ACTOR_OPS[st.actor as keyof typeof TRUST_ACTOR_OPS].includes(st.op as never)) {
throw new FixtureValidationError(file, `trust_steps[${st.step_id}]: actor ${String(st.actor)} cannot perform op ${String(st.op)}`);
}
for (const k of ['slug', 'content', 'fact', 'entity', 'kind', 'content_origin', 'target', 'target_claim'] as const) {
if (st[k] !== undefined && typeof st[k] !== 'string') throw new FixtureValidationError(file, `trust_steps[${st.step_id}].${k} must be a string`);
}
if (st.target !== undefined && !seen.has(st.target as string)) {
throw new FixtureValidationError(file, `trust_steps[${st.step_id}].target "${String(st.target)}" names no earlier step`);
}
const needs: Partial<Record<string, string[]>> = {
write_file: ['slug', 'content'], put_page: ['slug', 'content'], capture: ['slug', 'content'], remember: ['fact'],
forget: ['target'], confirm: ['target'], purge: ['target'], raise_tier: ['target'],
};
for (const k of needs[st.op as string] ?? []) {
if (typeof st[k] !== 'string' || !st[k]) throw new FixtureValidationError(file, `trust_steps[${st.step_id}] (${String(st.op)}) requires ${k}`);
}
seen.add(st.step_id);
}
}
function validateTrustGold(file: string, g: Record<string, unknown>, fixture: BrainBenchFixture): void {
const suite = fixture.suites.find(isTrustSuite);
if (!suite) {
if (g.trust !== undefined) throw new FixtureValidationError(file, 'gold.trust present but the fixture runs no memory-trust suite');
return;
}
const t = g.trust as Record<string, unknown> | undefined;
if (!t || !Array.isArray(t.items) || t.items.length === 0) {
throw new FixtureValidationError(file, 'memory-trust fixtures require gold.trust.items');
}
assertOnlyKeys(file, t, new Set(['items']), 'gold.trust');
const steps = new Set((fixture.trust_steps ?? []).map(s => s.step_id));
const ids = new Set<string>();
for (const it of t.items as Array<Record<string, unknown>>) {
assertOnlyKeys(file, it, TRUST_ITEM_KEYS, 'gold.trust.items[]');
if (typeof it.item_id !== 'string' || !it.item_id || ids.has(it.item_id)) {
throw new FixtureValidationError(file, 'gold.trust.items[].item_id must be a unique non-empty string');
}
ids.add(it.item_id);
if (!(TRUST_CHECKS as readonly unknown[]).includes(it.check) || !TRUST_SUITE_CHECKS[suite].includes(it.check as TrustCheck)) {
throw new FixtureValidationError(file, `gold.trust.items[${it.item_id}].check must be one of ${TRUST_SUITE_CHECKS[suite].join(', ')} for suite ${suite}`);
}
if (typeof it.step !== 'string' || !steps.has(it.step)) {
throw new FixtureValidationError(file, `gold.trust.items[${it.item_id}].step names no trust step`);
}
if ((it.check === 'tier' || it.check === 'max_tier') && !isTrustTier(it.tier)) {
throw new FixtureValidationError(file, `gold.trust.items[${it.item_id}].tier must name a trust tier`);
}
if (it.check === 'poison' || it.check === 'benign') {
if (it.arm !== 'external' && it.arm !== 'agent_relayed') throw new FixtureValidationError(file, `gold.trust.items[${it.item_id}].arm must be external|agent_relayed`);
if (typeof it.canary !== 'string' || it.canary.length < 8) throw new FixtureValidationError(file, `gold.trust.items[${it.item_id}].canary must be a unique token (8+ chars)`);
}
if (it.preference !== undefined && typeof it.preference !== 'boolean') throw new FixtureValidationError(file, 'preference must be boolean');
if (it.check === 'purged' && (typeof it.probe !== 'string' || !it.probe)) {
throw new FixtureValidationError(file, `gold.trust.items[${it.item_id}].probe required`);
}
}
}
export function validateGold(file: string, raw: unknown, fixture: BrainBenchFixture): FixtureGold {
if (typeof raw !== 'object' || raw === null || Array.isArray(raw)) {
throw new FixtureValidationError(file, 'gold must be a JSON object');
}
const g = raw as Record<string, unknown>;
assertOnlyKeys(file, g, GOLD_KEYS, 'gold');
if (g.fixture_id !== fixture.fixture_id) {
throw new FixtureValidationError(
file,
`gold.fixture_id "${String(g.fixture_id)}" does not match fixture "${fixture.fixture_id}"`,
);
}
if (typeof g.turns !== 'object' || g.turns === null) {
throw new FixtureValidationError(file, 'gold.turns must be an object keyed by turn_id');
}
const turnIds = new Set(fixture.turns.map((t: FixtureTurn) => String(t.turn_id)));
const turnRoleById = new Map(fixture.turns.map((t: FixtureTurn) => [String(t.turn_id), t.role]));
for (const [key, val] of Object.entries(g.turns as Record<string, unknown>)) {
if (!turnIds.has(key)) {
throw new FixtureValidationError(file, `gold.turns["${key}"] has no matching fixture turn`);
}
const tg = val as Record<string, unknown>;
assertOnlyKeys(file, tg, TURN_GOLD_KEYS, `gold.turns["${key}"]`);
if (typeof tg.should_retrieve !== 'boolean') {
throw new FixtureValidationError(file, `gold.turns["${key}"].should_retrieve must be boolean`);
}
// Retrieval gold on ASSISTANT turns would never be scored — the harness
// replays user turns only (adversarial finding: silent coverage loss).
// Assistant-turn gold may carry gold_facts (write-back reads every turn)
// but never a retrieval expectation.
if (
turnRoleById.get(key) === 'assistant' &&
(tg.should_retrieve === true || tg.gold_slugs !== undefined || tg.acceptable_slugs !== undefined)
) {
throw new FixtureValidationError(
file,
`gold.turns["${key}"] carries retrieval gold on an ASSISTANT turn — only user turns are replayed for retrieval scoring`,
);
}
{
// On a retrieval-suite fixture, should_retrieve=true REQUIRES non-empty
// gold_slugs — a slug-less retrieve-turn is a guaranteed miss no harness
// can pass (review finding: it silently poisons the failure rate).
const retrievalSuites = fixture.suites.some(
(s: BrainBenchSuite) => s === 'know-to-ask' || s === 'push',
);
if (
retrievalSuites &&
tg.should_retrieve &&
(!isStringArray(tg.gold_slugs) || tg.gold_slugs.length === 0)
) {
throw new FixtureValidationError(
file,
`gold.turns["${key}"].gold_slugs must be non-empty when should_retrieve=true on a retrieval-suite fixture (a slug-less retrieve turn is an unpassable gold item)`,
);
}
}
if (tg.gold_slugs !== undefined && !isStringArray(tg.gold_slugs)) {
throw new FixtureValidationError(file, `gold.turns["${key}"].gold_slugs must be a string array`);
}
if (tg.acceptable_slugs !== undefined && !isStringArray(tg.acceptable_slugs)) {
throw new FixtureValidationError(file, `gold.turns["${key}"].acceptable_slugs must be a string array`);
}
if (tg.gold_facts !== undefined) {
if (!Array.isArray(tg.gold_facts)) {
throw new FixtureValidationError(file, `gold.turns["${key}"].gold_facts must be an array`);
}
for (const gf of tg.gold_facts as Array<Record<string, unknown>>) {
assertOnlyKeys(file, gf, GOLD_FACT_KEYS, `gold.turns["${key}"].gold_facts[]`);
if (typeof gf.gist !== 'string' || !gf.gist) {
throw new FixtureValidationError(file, 'gold_facts[].gist required');
}
if (typeof gf.fact !== 'string' || !gf.fact) {
throw new FixtureValidationError(file, 'gold_facts[].fact required');
}
if (gf.entity_slug !== null && typeof gf.entity_slug !== 'string') {
throw new FixtureValidationError(file, 'gold_facts[].entity_slug must be string or null');
}
if (!isStringArray(gf.match_keywords) || gf.match_keywords.length === 0) {
throw new FixtureValidationError(file, 'gold_facts[].match_keywords must be non-empty');
}
}
}
}
if (fixture.continuity) {
const gc = g.continuity as Record<string, unknown> | undefined;
if (fixture.continuity.pair_role === 'reader') {
if (!gc) throw new FixtureValidationError(file, 'reader continuity fixture requires gold.continuity');
}
if (gc) {
assertOnlyKeys(file, gc, new Set(['pair_id', 'decisions']), 'gold.continuity');
if (gc.pair_id !== fixture.continuity.pair_id) {
throw new FixtureValidationError(file, 'gold.continuity.pair_id mismatch');
}
if (!Array.isArray(gc.decisions) || gc.decisions.length === 0) {
throw new FixtureValidationError(file, 'gold.continuity.decisions must be non-empty');
}
for (const d of gc.decisions as Array<Record<string, unknown>>) {
assertOnlyKeys(file, d, new Set(['decision_id', 'expected_slugs', 'match_keywords']), 'decisions[]');
if (typeof d.decision_id !== 'string') throw new FixtureValidationError(file, 'decisions[].decision_id required');
if (!isStringArray(d.expected_slugs)) throw new FixtureValidationError(file, 'decisions[].expected_slugs required');
if (!isStringArray(d.match_keywords)) throw new FixtureValidationError(file, 'decisions[].match_keywords required');
}
}
} else if (g.continuity !== undefined) {
throw new FixtureValidationError(file, 'gold.continuity present but fixture has no continuity block');
}
validateTrustGold(file, g, fixture);
return raw as FixtureGold;
}
/**
* Load + validate the corpus. Every *.fixture.json must have a matching
* <fixture_id>.gold.json in goldDir; orphan gold files are an error too
* (a renamed fixture must rename its gold).
*/
export async function loadCorpus(fixtureDir: string, goldDir: string): Promise<LoadedCorpus> {
let fixtureFiles: string[];
try {
fixtureFiles = (await readdir(fixtureDir)).filter((f) => f.endsWith('.fixture.json')).sort();
} catch (err) {
throw new Error(`brainbench: cannot read fixtures dir ${fixtureDir}: ${(err as Error).message}`);
}
let goldFiles: string[];
try {
goldFiles = (await readdir(goldDir)).filter((f) => f.endsWith('.gold.json')).sort();
} catch (err) {
throw new Error(`brainbench: cannot read gold dir ${goldDir}: ${(err as Error).message}`);
}
const hash = createHash('sha256');
const fixtures: LoadedFixture[] = [];
const goldByFixtureId = new Map<string, { raw: unknown; file: string }>();
// Parallel I/O; hashing stays in sorted order over the resolved array
// (determinism needs ordered hash.update, not serial reads).
const goldContents = await Promise.all(goldFiles.map((gf) => readFile(join(goldDir, gf), 'utf-8')));
const fixtureContents = await Promise.all(
fixtureFiles.map((ff) => readFile(join(fixtureDir, ff), 'utf-8')),
);
for (let i = 0; i < goldFiles.length; i++) {
const gf = goldFiles[i];
const content = goldContents[i];
hash.update(`gold/${gf}\n`);
hash.update(content);
let parsed: unknown;
try {
parsed = JSON.parse(content);
} catch (err) {
throw new FixtureValidationError(gf, `invalid JSON: ${(err as Error).message}`);
}
const id = (parsed as Record<string, unknown>)?.fixture_id;
if (typeof id !== 'string') throw new FixtureValidationError(gf, 'gold.fixture_id required');
if (goldByFixtureId.has(id)) throw new FixtureValidationError(gf, `duplicate gold for fixture_id ${id}`);
goldByFixtureId.set(id, { raw: parsed, file: gf });
}
const seenIds = new Set<string>();
for (let i = 0; i < fixtureFiles.length; i++) {
const ff = fixtureFiles[i];
const path = join(fixtureDir, ff);
const content = fixtureContents[i];
hash.update(`fixtures/${ff}\n`);
hash.update(content);
let parsed: unknown;
try {
parsed = JSON.parse(content);
} catch (err) {
throw new FixtureValidationError(ff, `invalid JSON: ${(err as Error).message}`);
}
const fixture = validateFixture(ff, parsed);
if (seenIds.has(fixture.fixture_id)) {
throw new FixtureValidationError(ff, `duplicate fixture_id ${fixture.fixture_id}`);
}
seenIds.add(fixture.fixture_id);
const goldEntry = goldByFixtureId.get(fixture.fixture_id);
if (!goldEntry) {
throw new FixtureValidationError(ff, `no gold file for fixture_id ${fixture.fixture_id} in ${goldDir}`);
}
const gold = validateGold(goldEntry.file, goldEntry.raw, fixture);
fixtures.push({ fixture, gold, path });
}
for (const [id, entry] of goldByFixtureId) {
if (!seenIds.has(id)) {
throw new FixtureValidationError(entry.file, `orphan gold file: no fixture with fixture_id ${id}`);
}
}
// Continuity pairing integrity: every pair_id has exactly one writer + one reader.
const pairs = new Map<string, { writer?: string; reader?: string }>();
for (const { fixture } of fixtures) {
if (!fixture.continuity) continue;
const p = pairs.get(fixture.continuity.pair_id) ?? {};
const role = fixture.continuity.pair_role;
if (p[role]) {
throw new FixtureValidationError(
fixture.fixture_id,
`continuity pair ${fixture.continuity.pair_id} has two ${role}s`,
);
}
p[role] = fixture.fixture_id;
pairs.set(fixture.continuity.pair_id, p);
}
for (const [pairId, p] of pairs) {
if (!p.writer || !p.reader) {
throw new FixtureValidationError(
pairId,
`continuity pair ${pairId} incomplete (writer=${p.writer ?? '∅'}, reader=${p.reader ?? '∅'})`,
);
}
}
return {
fixtures,
fixtures_hash: hash.digest('hex'),
fixture_dir: fixtureDir,
gold_dir: goldDir,
};
}