Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* BrainBench CLI e2e — subprocess runs against a SMALL tmp corpus so the
* literal exit codes (the CI product, decision 9) are asserted end-to-end:
* 0 pass · 1 regression · 2 error/inconclusive. Also pins: the --out artifact
* is complete valid JSON with the _meta.metric_glossary block, --update-baseline
* is byte-deterministic across runs, anti-vacuous-pass, and the run-all
* once-per-sweep record semantics.
*
* Spawn budget: every independent CLI run executes ONCE in beforeAll through a
* width-2 pool (each child boots its own PGLite; wider would multiply across
* shards); tests assert on the cached results. The runs were order-independent
* by construction already (each depends only on the beforeAll artifacts).
*
* The former "run-all wiring — full corpus, in-process" describe was deleted:
* CI's dedicated brainbench job (scripts/ci-brainbench-gate.sh) runs the same
* committed corpus fresh on every PR and its --compare against the committed
* baseline is the authoritative completion + fixtures-hash drift gate; the
* once-per-sweep test below still runs the full corpus once for the ledger
* contract.
*/
import { beforeAll, describe, expect, test } from 'bun:test';
import { cpSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join, resolve } from 'node:path';
import { PAGE_MAX_BYTES } from '../src/core/page-size-policy.ts';
const REPO = process.cwd();
const CLI = join(REPO, 'src', 'cli.ts');
let root: string;
let fixtures: string;
let gold: string;
let cliSnapshot = process.env.GBRAIN_TEST_DEFAULT_SNAPSHOT;
type RunResult = { exitCode: number; stdout: string; stderr: string };
function brainBenchEnv(): NodeJS.ProcessEnv {
const snapshot = process.env.GBRAIN_NO_SNAPSHOT === '1' ? undefined : cliSnapshot;
return {
...process.env,
GBRAIN_QUIET: '1',
// Bare CLI defaults are voyage-4/1024; its parent bun test uses legacy/1536.
// An absent default fixture falls back to cold init without a wrong-shape tar.
GBRAIN_PGLITE_SNAPSHOT: snapshot ? resolve(REPO, snapshot) : '',
};
}
function withDefaultCommittedBaseline(args: string[]): string[] {
// Foreign-corpus runs opt OUT of the repo's committed baseline: the
// poisoning defense requires any committed-vs-main divergence to match the
// current run, and the repo's main.json never matches a tmp-corpus run.
return args.includes('--committed-baseline')
? args
: [...args, '--committed-baseline', join(root, 'no-committed-baseline.json')];
}
function run(args: string[], cwd = REPO): RunResult {
const proc = Bun.spawnSync(['bun', CLI, 'eval', 'brainbench', ...withDefaultCommittedBaseline(args)], {
cwd,
env: brainBenchEnv(),
stdout: 'pipe',
stderr: 'pipe',
});
return {
exitCode: proc.exitCode ?? -1,
stdout: proc.stdout.toString(),
stderr: proc.stderr.toString(),
};
}
async function runAsync(args: string[], cwd = REPO): Promise<RunResult> {
const proc = Bun.spawn(['bun', CLI, 'eval', 'brainbench', ...withDefaultCommittedBaseline(args)], {
cwd,
env: brainBenchEnv(),
stdout: 'pipe',
stderr: 'pipe',
});
// A wedged child (the synchronous PGLite-WASM block class) would otherwise
// hang beforeAll to its 300s cap and orphan a ~1.5GB process behind the CI
// job timeout. SIGTERM at 120s (healthy runs finish in ~5-15s), SIGKILL +2s.
const term = setTimeout(() => { try { proc.kill(); } catch { /* exited */ } }, 120_000);
const hardKill = setTimeout(() => { try { proc.kill(9); } catch { /* exited */ } }, 122_000);
try {
const [stdout, stderr, exitCode] = await Promise.all([
new Response(proc.stdout).text(),
new Response(proc.stderr).text(),
proc.exited,
]);
return { exitCode: exitCode ?? -1, stdout, stderr };
} finally {
clearTimeout(term);
clearTimeout(hardKill);
}
}
/**
* The independent CLI runs, executed once through a width-2 pool.
*
* Local pool rather than test/helpers/cli-spawn.ts#runCliBatch: that helper
* takes ONE shared opts for the whole batch, and this batch needs a per-job
* cwd (the outside-cwd case) plus this file's own argv wrapper (the
* --committed-baseline default). If runCliBatch grows per-argv opts, fold
* this into it.
*/
const batch: Record<string, RunResult> = {};
async function runBatch(jobs: Array<[string, string[], string?]>, width = 2): Promise<void> {
let next = 0;
async function worker(): Promise<void> {
while (next < jobs.length) {
const idx = next++;
const [key, args, cwd] = jobs[idx];
batch[key] = await runAsync(args, cwd);
}
}
await Promise.all(Array.from({ length: Math.min(width, jobs.length) }, () => worker()));
}
beforeAll(async () => {
// Direct `bun test` invocations need the same profile as the slow-lane runner.
if (!cliSnapshot && process.env.GBRAIN_NO_SNAPSHOT !== '1') {
const build = Bun.spawnSync([
process.execPath, join(REPO, 'scripts/build-pglite-snapshot.ts'), '--profile', 'default',
], { cwd: REPO, env: { ...process.env }, stdout: 'pipe', stderr: 'pipe' });
if (build.exitCode === 0) cliSnapshot = join(REPO, 'test/fixtures/pglite-snapshot-default.tar');
else console.warn(`[brainbench] Default snapshot unavailable; using cold initialization. ${build.stderr.toString().trim()}`);
}
root = mkdtempSync(join(tmpdir(), 'bb-e2e-'));
fixtures = join(root, 'fixtures');
gold = join(root, 'gold');
mkdirSync(fixtures, { recursive: true });
mkdirSync(gold, { recursive: true });
for (const id of ['kta-001-deal-recall', 'kta-002-quiet-smalltalk']) {
cpSync(join(REPO, 'evals/brainbench/fixtures', `${id}.fixture.json`), join(fixtures, `${id}.fixture.json`));
cpSync(join(REPO, 'evals/brainbench/gold', `${id}.gold.json`), join(gold, `${id}.gold.json`));
}
// Shared artifacts every consumer test depends on, built ONCE here so each
// test is self-sufficient under -t filters / sharding (review finding:
// order-dependent file production across describe blocks).
const r = run(['--fixtures', fixtures, '--gold', gold, '--update-baseline', join(root, 'base1.json')]);
if (r.exitCode !== 0) throw new Error(`beforeAll baseline build failed: ${r.stderr}`);
const doctored = JSON.parse(readFileSync(join(root, 'base1.json'), 'utf-8'));
const cellKey = Object.keys(doctored.counts).find((k) => doctored.counts[k].gold_total > 0)!;
doctored.counts[cellKey].gold_failed = -1; // pretends fewer failures than any run can match
writeFileSync(join(root, 'doctored.json'), JSON.stringify(doctored, null, 2));
const foreign = JSON.parse(readFileSync(join(root, 'base1.json'), 'utf-8'));
foreign.fixtures_hash = 'f'.repeat(64);
writeFileSync(join(root, 'foreign.json'), JSON.stringify(foreign, null, 2));
// Error-path corpora (independent tmp trees, built up front for the batch).
const empty = join(root, 'empty-fixtures');
const emptyGold = join(root, 'empty-gold');
mkdirSync(empty, { recursive: true });
mkdirSync(emptyGold, { recursive: true });
const badRoot = mkdtempSync(join(tmpdir(), 'bb-bad-'));
mkdirSync(join(badRoot, 'fixtures'));
mkdirSync(join(badRoot, 'gold'));
writeFileSync(join(badRoot, 'fixtures', 'bad.fixture.json'), '{ not json');
const dupRoot = mkdtempSync(join(tmpdir(), 'bb-dup-'));
mkdirSync(join(dupRoot, 'fixtures'));
mkdirSync(join(dupRoot, 'gold'));
// A page whose content exceeds importFromContent's size cap → status 'skipped' → SeedError.
const huge = 'x'.repeat(PAGE_MAX_BYTES + 1);
writeFileSync(
join(dupRoot, 'fixtures', 'seedfail-001.fixture.json'),
JSON.stringify({
schema_version: 1,
fixture_id: 'seedfail-001',
suites: ['know-to-ask'],
seed_pages: [{ slug: 'people/too-big', content: `---\ntitle: Too Big\n---\n${huge}` }],
turns: [{ turn_id: 1, role: 'user', text: 'Hello Too Big' }],
}),
);
writeFileSync(
join(dupRoot, 'gold', 'seedfail-001.gold.json'),
JSON.stringify({ fixture_id: 'seedfail-001', turns: { '1': { should_retrieve: false } } }),
);
const outsideCwd = mkdtempSync(join(tmpdir(), 'bb-outside-cwd-'));
await runBatch([
['outside-cwd', ['--harness', 'openclaw', '--suite', 'know-to-ask', '--json'], outsideCwd],
['clean-out', ['--fixtures', fixtures, '--gold', gold, '--harness', 'openclaw', '--out', join(root, 'r1.json')]],
['base2', ['--fixtures', fixtures, '--gold', gold, '--update-baseline', join(root, 'base2.json')]],
['compare-own', ['--fixtures', fixtures, '--gold', gold, '--compare', join(root, 'base1.json')]],
['compare-doctored', ['--fixtures', fixtures, '--gold', gold, '--compare', join(root, 'doctored.json')]],
['allow-regression', ['--fixtures', fixtures, '--gold', gold, '--compare', join(root, 'doctored.json'), '--allow-regression', 'e2e test bless']],
['foreign-hash', ['--fixtures', fixtures, '--gold', gold, '--compare', join(root, 'foreign.json'), '--committed-baseline', join(root, 'nonexistent.json')]],
['json-complete', ['--fixtures', fixtures, '--gold', gold, '--json', '--compare', join(root, 'base1.json')]],
['empty-fixtures', ['--fixtures', empty, '--gold', emptyGold]],
['suite-zero', ['--fixtures', fixtures, '--gold', gold, '--suite', 'continuity']],
['malformed', ['--fixtures', join(badRoot, 'fixtures'), '--gold', join(badRoot, 'gold')]],
['usage-error', ['--frobnicate']],
['seed-failure', ['--fixtures', join(dupRoot, 'fixtures'), '--gold', join(dupRoot, 'gold')]],
]);
// The baseline build + 13 pooled runs each spawn a full brainbench child;
// bun's default 5s hook timeout would SIGTERM them on a loaded shard.
}, 300_000);
describe('exit contract over a multi-brain run (PGLite exitCode-hijack guard)', () => {
test('bundled defaults resolve outside the package working directory', () => {
const r = batch['outside-cwd'];
expect(r.exitCode).toBe(0);
const doc = JSON.parse(r.stdout);
expect(doc.cells.length).toBeGreaterThan(0);
expect(doc.seed_failures).toEqual([]);
expect(r.stderr).not.toContain('not a git repository');
if (process.env.GBRAIN_TEST_DEFAULT_SNAPSHOT && process.env.GBRAIN_NO_SNAPSHOT !== '1') {
expect(r.stderr).not.toContain('embedding shape mismatch');
expect(r.stderr).not.toContain('migration(s) applied');
}
}, 60_000);
test('clean run: exit 0, --out is complete valid JSON with the glossary block', () => {
const r = batch['clean-out'];
expect(r.exitCode).toBe(0);
const doc = JSON.parse(readFileSync(join(root, 'r1.json'), 'utf-8'));
expect(doc.receipt.result_schema_version).toBe(1);
expect(doc.cells.length).toBeGreaterThan(0);
expect(doc._meta.metric_glossary.know_to_ask_failure_rate).toContain('thesis failure mode');
expect(doc.seed_failures).toEqual([]);
expect(r.stdout).toContain('# BrainBench scoreboard');
}, 30_000);
test('--update-baseline is byte-deterministic across two runs (decision 10)', () => {
expect(batch['base2'].exitCode).toBe(0);
// base1.json was produced by an entirely separate run in beforeAll.
expect(readFileSync(join(root, 'base2.json'), 'utf-8')).toBe(readFileSync(join(root, 'base1.json'), 'utf-8'));
}, 60_000);
test('--compare against own baseline: exit 0 PASS', () => {
const r = batch['compare-own'];
expect(r.exitCode).toBe(0);
expect(r.stdout).toContain('## Gate: PASS (same-hash)');
}, 30_000);
test('doctored main baseline (pretends fewer failures): exit 1 REGRESSION with named breach', () => {
const r = batch['compare-doctored'];
expect(r.exitCode).toBe(1);
expect(r.stdout).toContain('## Gate: REGRESSION');
expect(r.stdout).toContain('newly-failed');
}, 30_000);
test('--allow-regression flips the same comparison to exit 0 and records the reason', () => {
const r = batch['allow-regression'];
expect(r.exitCode).toBe(0);
expect(r.stdout).toContain('regression allowed: e2e test bless');
}, 30_000);
test('fixtures_hash mismatch without a committed baseline: exit 2 INCONCLUSIVE', () => {
const r = batch['foreign-hash'];
expect(r.exitCode).toBe(2);
expect(r.stdout).toContain('corpus-bless');
}, 30_000);
});
describe('anti-vacuous-pass + error paths (always exit 2, never 0)', () => {
test('empty fixtures dir: exit 2', () => {
const r = batch['empty-fixtures'];
expect(r.exitCode).toBe(2);
expect(r.stderr).toContain('vacuous');
}, 30_000);
test('suite filter matching zero fixtures: exit 2', () => {
const r = batch['suite-zero'];
expect(r.exitCode).toBe(2);
expect(r.stderr).toContain('vacuous');
}, 30_000);
test('malformed fixture JSON: exit 2 with the validation error named', () => {
const r = batch['malformed'];
expect(r.exitCode).toBe(2);
expect(r.stderr).toContain('invalid JSON');
}, 30_000);
test('usage error (unknown flag): exit 2 with usage', () => {
const r = batch['usage-error'];
expect(r.exitCode).toBe(2);
expect(r.stderr).toContain('Usage: gbrain eval brainbench');
}, 30_000);
test('seed failure (duplicate slug across seed pages in one source): exit 2, fixture named', () => {
const r = batch['seed-failure'];
expect(r.exitCode).toBe(2);
expect(r.stderr).toContain('SEED FAILURES');
expect(r.stderr).toContain('seedfail-001');
}, 30_000);
});
describe('holdout discipline (decision 22)', () => {
test('gate mode excludes holdout fixtures; --include-holdout scores them', async () => {
const { loadCorpus } = await import('../src/eval/brainbench/fixtures.ts');
const { runBrainBench } = await import('../src/eval/brainbench/harness.ts');
const corpus = await loadCorpus('evals/brainbench/fixtures', 'evals/brainbench/gold');
const holdoutIds = new Set(
corpus.fixtures.filter((f) => f.fixture.holdout).map((f) => f.fixture.fixture_id),
);
expect(holdoutIds.size).toBeGreaterThan(0);
const pick = corpus.fixtures
.filter((f) => f.fixture.category === 'kta-pos')
.filter((f, i, arr) => f.fixture.holdout || arr.findIndex((x) => !x.fixture.holdout) === i)
.slice(0, 4);
const sub = { ...corpus, fixtures: pick };
const gateRun = await runBrainBench(sub, {
harnesses: ['openclaw'], suites: ['know-to-ask'], includeHoldout: false, llm: false,
});
for (const r of gateRun.turn_rows) expect(holdoutIds.has(r.fixture_id)).toBe(false);
const pubRun = await runBrainBench(sub, {
harnesses: ['openclaw'], suites: ['know-to-ask'], includeHoldout: true, llm: false,
});
expect(pubRun.turn_rows.length).toBeGreaterThan(gateRun.turn_rows.length);
}, 60_000);
});
describe('file-vs-file --compare (pure diff, no run, no DB)', () => {
test('identical files: exit 0 with a JSON outcome on stdout', () => {
const b = join(root, 'base1.json');
const r = run(['--compare', b, b]);
expect(r.exitCode).toBe(0);
const outcome = JSON.parse(r.stdout);
expect(outcome.verdict).toBe('pass');
}, 30_000);
test('doctored current vs base: exit 1 with breaches listed', () => {
const r = run(['--compare', join(root, 'doctored.json'), join(root, 'base1.json')]);
// base1 (current) has MORE failures than doctored (main pretends fewer)…
// order: --compare BASE CURRENT → current=base1, main=doctored.
expect(r.exitCode).toBe(1);
const outcome = JSON.parse(r.stdout);
expect(outcome.verdict).toBe('regression');
expect(outcome.breaches.length).toBeGreaterThan(0);
}, 30_000);
});
describe('--json stdout completeness', () => {
test('stdout parses as a full result doc with compare embedded', () => {
// All harnesses: base1.json carries cells for all three seams, and a
// narrower run would (correctly) trip the disappeared-coverage breach.
const r = batch['json-complete'];
expect(r.exitCode).toBe(0);
const doc = JSON.parse(r.stdout);
expect(doc.receipt.result_schema_version).toBe(1);
expect(doc.cells.length).toBeGreaterThan(0);
expect(doc.compare.verdict).toBe('pass');
expect(doc._meta.metric_glossary).toBeDefined();
}, 30_000);
});
describe('--llm availability gate', () => {
test('no config + no keys: exit 2 with an actionable message, before any run', () => {
const bareHome = mkdtempSync(join(tmpdir(), 'bb-home-'));
// Minimal explicit env (review finding): spreading process.env and
// deleting a hardcoded key list leaks other provider keys (GOOGLE_*,
// per-recipe ${ID}_API_KEY) — on a dev machine that could flip the gate
// open and make REAL API calls from a test.
const env: Record<string, string> = { PATH: process.env.PATH ?? '', HOME: bareHome };
const proc = Bun.spawnSync(
['bun', 'src/cli.ts', 'eval', 'brainbench', '--fixtures', fixtures, '--gold', gold, '--llm'],
{ cwd: REPO, env, stdout: 'pipe', stderr: 'pipe' },
);
expect(proc.exitCode).toBe(2);
expect(proc.stderr.toString()).toContain('requires a configured chat model');
}, 30_000);
});
describe('render-brainbench-delta.ts (the CI step-summary block)', () => {
test('renders verdict header + per-cell headline from the --out artifact', () => {
const proc = Bun.spawnSync(['bun', 'scripts/render-brainbench-delta.ts', join(root, 'r1.json')], {
cwd: REPO, stdout: 'pipe', stderr: 'pipe',
});
expect(proc.exitCode).toBe(0);
const md = proc.stdout.toString();
expect(md).toContain('## BrainBench:');
expect(md).toContain('| openclaw | production |');
expect(md).toContain('know_to_ask_failure_rate=');
}, 30_000);
test('missing path argument: exit 2 with usage', () => {
const proc = Bun.spawnSync(['bun', 'scripts/render-brainbench-delta.ts'], {
cwd: REPO, stdout: 'pipe', stderr: 'pipe',
});
expect(proc.exitCode).toBe(2);
}, 30_000);
});
describe('privacy guard violation branches (negative path)', () => {
test('a fixture with a real dollar amount + out-of-range year fails the scan (gold dir scanned too)', () => {
const dirty = mkdtempSync(join(tmpdir(), 'bb-privacy-'));
mkdirSync(join(dirty, 'fixtures'), { recursive: true });
mkdirSync(join(dirty, 'gold'), { recursive: true });
writeFileSync(
join(dirty, 'fixtures', 'leak.fixture.json'),
JSON.stringify({ turns: [{ text: 'They raised $50M for the series B' }] }),
);
// The year violation lives in GOLD — pins that the year scan covers the
// gold dir as well (review finding: it previously scanned fixtures only).
writeFileSync(
join(dirty, 'gold', 'leak.gold.json'),
JSON.stringify({ fixture_id: 'leak', turns: { '1': { gold_facts: [{ fact: 'raised back in 2019' }] } } }),
);
const proc = Bun.spawnSync(['bash', 'scripts/check-synthetic-corpus-privacy.sh'], {
cwd: REPO,
env: { ...process.env, BRAINBENCH_PRIVACY_DIR: dirty },
stdout: 'pipe', stderr: 'pipe',
});
expect(proc.exitCode).toBe(1);
const out = proc.stdout.toString();
expect(out).toContain('explicit dollar amount');
expect(out).toContain('out-of-range year');
}, 30_000);
});
describe('run-all once-per-sweep semantics (decision 16)', () => {
test('--suites brainbench with TWO modes writes exactly ONE n/a record with cells', () => {
const outDir = mkdtempSync(join(tmpdir(), 'bb-runall-'));
const proc = Bun.spawnSync(
['bun', 'src/cli.ts', 'eval', 'run-all', '--suites', 'brainbench', '--modes', 'conservative,balanced', '--output', outDir],
{ cwd: REPO, env: brainBenchEnv(), stdout: 'pipe', stderr: 'pipe' },
);
expect(proc.exitCode).toBe(0);
const lines = readFileSync(join(outDir, 'eval-results.jsonl'), 'utf-8').trim().split('\n');
expect(lines.length).toBe(1); // NOT multiplied by the two modes
const record = JSON.parse(lines[0]);
expect(record.schema_version).toBe(3);
expect(record.suite).toBe('brainbench');
expect(record.mode).toBe('n/a');
expect(record.status).toBe('completed');
// 3 harnesses x 8 suites (the four memory-trust suites, #5575, joined the four retrieval/write suites).
expect(Object.keys(record.params.cells).length).toBe(24);
}, 240_000);
});