Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* v0.32.3 — eval-run-all orchestrator unit tests.
* Pins arg parsing + cost-guard semantics + persist hook + audit trail.
*/
import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'bun:test';
import { mkdtempSync, rmSync, readFileSync, existsSync } from 'fs';
import { tmpdir } from 'os';
import { join } from 'path';
import { runCli } from './helpers/cli-spawn.ts';
import {
parseRunAllArgs,
estimateRunCost,
evaluateCostGuard,
persistRunRecord,
type EvalRunRecord,
} from '../src/commands/eval-run-all.ts';
let tmp: string;
beforeAll(() => {
tmp = mkdtempSync(join(tmpdir(), 'gbrain-eval-runall-'));
});
afterAll(() => {
rmSync(tmp, { recursive: true, force: true });
});
describe('parseRunAllArgs', () => {
test('defaults: all modes, the wired suites (brainbench), seed=42', () => {
const opts = parseRunAllArgs([]);
expect(opts.modes).toEqual(['conservative', 'balanced', 'tokenmax']);
expect(opts.suites).toEqual(['brainbench']);
expect(opts.suitesExplicit).toBe(false);
expect(opts.seed).toBe(42);
expect(opts.parallel).toBe(1);
expect(opts.budgetUsdRetrieval).toBe(5);
expect(opts.budgetUsdAnswer).toBe(20);
expect(opts.yes).toBe(false);
});
test('--modes filters to a subset', () => {
const opts = parseRunAllArgs(['--modes', 'conservative,tokenmax']);
expect(opts.modes).toEqual(['conservative', 'tokenmax']);
});
test('--modes rejects invalid mode', () => {
expect(() => parseRunAllArgs(['--modes', 'frontier'])).toThrow(/not a valid mode/);
});
test('--suites filters; rejects unknown', () => {
const opts = parseRunAllArgs(['--suites', 'longmemeval']);
expect(opts.suites).toEqual(['longmemeval']);
expect(opts.suitesExplicit).toBe(true);
expect(() => parseRunAllArgs(['--suites', 'foo'])).toThrow(/not a recognized suite/);
});
test('--budget-usd-retrieval + --budget-usd-answer override defaults', () => {
const opts = parseRunAllArgs(['--budget-usd-retrieval', '15', '--budget-usd-answer', '50']);
expect(opts.budgetUsdRetrieval).toBe(15);
expect(opts.budgetUsdAnswer).toBe(50);
});
test('--parallel clamps to mode count', () => {
const opts = parseRunAllArgs(['--parallel', '10']);
expect(opts.parallel).toBe(3); // clamped to SEARCH_MODES.length
});
test('--parallel rejects 0 / negative', () => {
expect(() => parseRunAllArgs(['--parallel', '0'])).toThrow(/must be >= 1/);
expect(() => parseRunAllArgs(['--parallel', '-1'])).toThrow(/must be >= 1/);
});
test('--yes flag toggles', () => {
expect(parseRunAllArgs(['--yes']).yes).toBe(true);
expect(parseRunAllArgs(['-y']).yes).toBe(true);
});
// #5933 (absorbed in agent contract v1 D4): NaN/out-of-range numeric flags
// are invalid_params usage errors, so the cost guard can't be bypassed.
test('--limit requires a positive safe integer', () => {
for (const value of ['NaN', 'Infinity', '1.5', '0', '-1', '9007199254740992']) {
expect(() => parseRunAllArgs(['--limit', value]), `--limit ${value}`).toThrow(/--limit/);
}
expect(() => parseRunAllArgs(['--limit', 'NaN'])).toThrow(expect.objectContaining({ code: 'invalid_params' }));
});
test('--seed requires a safe integer and allows zero', () => {
expect(parseRunAllArgs(['--seed', '0']).seed).toBe(0);
for (const value of ['NaN', 'Infinity', '1.5', '9007199254740992']) {
expect(() => parseRunAllArgs(['--seed', value]), `--seed ${value}`).toThrow(/--seed/);
}
});
test('budget caps require finite non-negative amounts', () => {
expect(parseRunAllArgs(['--budget-usd-answer', '0']).budgetUsdAnswer).toBe(0);
expect(parseRunAllArgs(['--budget-usd-retrieval', '0.25']).budgetUsdRetrieval).toBe(0.25);
for (const flag of ['--budget-usd-retrieval', '--budget-usd-answer']) {
for (const value of ['NaN', 'Infinity', '-Infinity', '-1']) {
expect(() => parseRunAllArgs([flag, value]), `${flag} ${value}`).toThrow(new RegExp(flag));
}
}
});
});
describe('estimateRunCost', () => {
test('returns retrieval/answer/total breakdown', () => {
const est = estimateRunCost({ suites: ['longmemeval'], modes: ['balanced'], limit: 100 });
expect(est.total_usd).toBe(est.retrieval_usd + est.answer_usd);
expect(est.answer_usd).toBeGreaterThan(0); // longmemeval has answer-gen
expect(est.retrieval_usd).toBeGreaterThan(0);
});
test('expansion cost follows the mode bundle: no mode expands by default since GBRA-94', () => {
const bal = estimateRunCost({ suites: ['replay'], modes: ['balanced'], limit: 100 });
const tok = estimateRunCost({ suites: ['replay'], modes: ['tokenmax'], limit: 100 });
expect(tok.retrieval_usd).toBe(bal.retrieval_usd);
});
test('replay suite has zero answer cost', () => {
const est = estimateRunCost({ suites: ['replay'], modes: ['balanced'], limit: 100 });
expect(est.answer_usd).toBe(0);
});
test('multi-suite multi-mode estimate sums correctly', () => {
const est = estimateRunCost({ suites: ['longmemeval', 'replay'], modes: ['conservative', 'balanced'], limit: 100 });
expect(Object.keys(est.per_suite).length).toBe(4); // 2 × 2
});
});
describe('evaluateCostGuard', () => {
test('within-budget proceeds without --yes', () => {
const g = evaluateCostGuard(
{ retrieval_usd: 1, answer_usd: 10, total_usd: 11 },
{ budgetUsdRetrieval: 5, budgetUsdAnswer: 20, yes: false, isTty: true },
);
expect(g.proceed).toBe(true);
});
test('over-cap TTY without --yes refuses', () => {
const g = evaluateCostGuard(
{ retrieval_usd: 30, answer_usd: 30, total_usd: 60 },
{ budgetUsdRetrieval: 5, budgetUsdAnswer: 20, yes: false, isTty: true },
);
expect(g.proceed).toBe(false);
expect(g.reason).toContain('--yes');
});
test('over-cap TTY with --yes proceeds', () => {
const g = evaluateCostGuard(
{ retrieval_usd: 30, answer_usd: 30, total_usd: 60 },
{ budgetUsdRetrieval: 5, budgetUsdAnswer: 20, yes: true, isTty: true },
);
expect(g.proceed).toBe(true);
});
test('over-cap non-TTY without --yes refuses (exit 2 path)', () => {
const g = evaluateCostGuard(
{ retrieval_usd: 30, answer_usd: 30, total_usd: 60 },
{ budgetUsdRetrieval: 5, budgetUsdAnswer: 20, yes: false, isTty: false },
);
expect(g.proceed).toBe(false);
expect(g.reason).toContain('Non-TTY requires --yes');
});
test('only-retrieval-over OR only-answer-over both trigger guard', () => {
const a = evaluateCostGuard(
{ retrieval_usd: 100, answer_usd: 1, total_usd: 101 },
{ budgetUsdRetrieval: 5, budgetUsdAnswer: 20, yes: false, isTty: false },
);
expect(a.proceed).toBe(false);
const b = evaluateCostGuard(
{ retrieval_usd: 1, answer_usd: 100, total_usd: 101 },
{ budgetUsdRetrieval: 5, budgetUsdAnswer: 20, yes: false, isTty: false },
);
expect(b.proceed).toBe(false);
});
});
describe('persistRunRecord audit trail', () => {
beforeEach(() => {
rmSync(join(tmp, '.gbrain-evals'), { recursive: true, force: true });
rmSync(join(tmp, 'eval-results.jsonl'), { force: true });
});
test('appends to eval-results.jsonl, creates dir if missing', () => {
const record: EvalRunRecord = {
schema_version: 3,
run_id: 'abc123-longmemeval-conservative-42',
ran_at: '2026-05-12T12:00:00Z',
suite: 'longmemeval',
mode: 'conservative',
commit: 'abc123',
seed: 42,
params: {},
status: 'completed',
duration_ms: 12_345,
};
persistRunRecord(tmp, record, tmp);
const path = join(tmp, 'eval-results.jsonl');
expect(existsSync(path)).toBe(true);
const content = readFileSync(path, 'utf-8');
expect(content).toContain('abc123-longmemeval-conservative-42');
const parsed = JSON.parse(content.trim());
expect(parsed.mode).toBe('conservative');
expect(parsed.schema_version).toBe(3);
});
test('appends multiple records (NDJSON)', () => {
const base = {
schema_version: 3 as const,
ran_at: '2026-05-12T12:00:00Z',
suite: 'longmemeval' as const,
commit: 'abc',
seed: 42,
params: {},
status: 'completed' as const,
duration_ms: 1,
};
persistRunRecord(tmp, { ...base, run_id: 'a', mode: 'conservative' }, tmp);
persistRunRecord(tmp, { ...base, run_id: 'b', mode: 'balanced' }, tmp);
persistRunRecord(tmp, { ...base, run_id: 'c', mode: 'tokenmax' }, tmp);
const content = readFileSync(join(tmp, 'eval-results.jsonl'), 'utf-8');
const lines = content.trim().split('\n');
expect(lines.length).toBe(3);
});
test('secrets in error text AND params string leaves are redacted on the write path (DB URL password, provider keys)', () => {
const record: EvalRunRecord = {
schema_version: 3,
run_id: 'abc123-longmemeval-balanced-zz',
ran_at: '2026-09-06T12:00:00Z',
suite: 'longmemeval',
mode: 'balanced',
commit: 'abc123',
seed: 0,
params: {
topK: 5,
output: 'postgres://brain_user:[email protected]:5432/brain?sslmode=require',
nested: { note: 'anthropic sk-ant-api03-ABCDEFGHIJKLMNOP failed', n: 3 },
list: ['Authorization: Bearer abcdefghijklmnop.qrstuv', 'plain text'],
},
status: 'failed',
duration_ms: 1,
error: 'exit 1 | q1: connect failed: postgres://brain_user:[email protected]:5432/brain; openai key sk-proj-abcdefghijklmnopqrstuvwxyz0123',
};
// No caller-side redaction here — the write path alone must scrub.
persistRunRecord(tmp, record, tmp);
const line = readFileSync(join(tmp, 'eval-results.jsonl'), 'utf-8').trim();
expect(line).not.toContain('s3cr3t-pw');
expect(line).not.toContain('brain_user:');
expect(line).not.toContain('sk-proj-abcdefghijklmnopqrstuvwxyz0123');
expect(line).not.toContain('ABCDEFGHIJKLMNOP');
expect(line).not.toContain('abcdefghijklmnop.qrstuv');
// The line is still valid JSON with the diagnostic shape + non-string leaves intact.
const parsed = JSON.parse(line);
expect(parsed.error).toContain('exit 1 | q1: connect failed: postgres://<redacted>@db.example.internal:5432/brain');
expect(parsed.error).toContain('sk-<redacted>');
expect(parsed.params.output).toBe('postgres://<redacted>@db.example.internal:5432/brain?sslmode=require');
expect(parsed.params.nested.note).toBe('anthropic sk-ant-<redacted> failed');
expect(parsed.params.nested.n).toBe(3);
expect(parsed.params.topK).toBe(5);
expect(parsed.params.list).toEqual(['Authorization: Bearer <redacted>', 'plain text']);
expect(parsed.status).toBe('failed');
});
test("v3: brainbench records carry mode 'n/a' (decision 16 — no params.mode_independent hack)", () => {
const record: EvalRunRecord = {
schema_version: 3,
run_id: 'abc123-brainbench-na-42',
ran_at: '2026-06-12T12:00:00Z',
suite: 'brainbench',
mode: 'n/a',
commit: 'abc123',
seed: 42,
params: { fixtures_hash: 'deadbeef', cells: {} },
status: 'completed',
duration_ms: 1,
};
persistRunRecord(tmp, record, tmp);
const parsed = JSON.parse(readFileSync(join(tmp, 'eval-results.jsonl'), 'utf-8').trim());
expect(parsed.mode).toBe('n/a');
expect(parsed.suite).toBe('brainbench');
expect(parsed.params.mode_independent).toBeUndefined();
});
});
describe('eval run-all with a suite it does not run (CLI)', () => {
test('explicit --suites longmemeval,replay exits 1 with eval_suite_unwired, the per-suite commands as fix, and no record', async () => {
const out = join(tmp, 'unwired');
const r = await runCli(['eval', 'run-all', '--suites', 'longmemeval,replay', '--modes', 'balanced', '--yes', '--json', '--output', out]);
expect(r.exitCode).toBe(1);
const env = JSON.parse(r.stdout);
expect(env.code).toBe('eval_suite_unwired');
expect(env.fix.argv.slice(0, 7)).toEqual(['gbrain', 'eval', 'longmemeval', '<dataset.jsonl>', '--mode', '<mode>', '--record']);
expect(env.fix.next).toBe('ask_user');
expect(env.fix.then.argv.slice(0, 5)).toEqual(['gbrain', 'eval', 'replay', '--mode', '<mode>']);
expect(r.stderr).toContain('Error [eval_suite_unwired]');
expect(existsSync(join(out, 'eval-results.jsonl'))).toBe(false);
});
test('gbrain errors eval_suite_unwired explains the code', async () => {
const r = await runCli(['errors', 'eval_suite_unwired']);
expect(r.exitCode).toBe(0);
expect(r.stdout).toContain('eval_suite_unwired (caller, exit 1)');
expect(r.stdout).toContain('does not run in-process');
});
});