Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* v0.32.3 — `gbrain eval run-all` orchestrator.
*
* Sweeps every requested mode × suite combination and writes per-run
* results to `<repo>/.gbrain-evals/eval-results.jsonl` [CDX-23]. Personal
* brain (~/.gbrain) is never touched; the repo's git history is the
* audit trail.
*
* Sequential default per D9: --parallel N is opt-in. Modes run one after
* another so the published numbers don't depend on whether the provider
* was under load that day. --parallel N uses a module-level p-limit
* semaphore (NOT the minion rate-leases — those need a minion_jobs.id FK
* that a CLI eval doesn't have, per [CDX-10]).
*
* Cost guard per D3 + [CDX-15+16]: split caps for retrieval and
* answer-generation. Default $5 retrieval / $20 answer. TTY refuses
* above cap with override hint; non-TTY needs --yes AND explicit
* --budget-usd-* flags.
*
* Wired suites (brainbench) run in-process; longmemeval and replay run only
* through their own commands. A bare run sweeps the wired suites and names
* the per-suite commands; an explicit `--suites` naming an unwired suite
* refuses with `eval_suite_unwired` (exit 1). Any wired suite whose record
* says `failed` makes the run exit 1, with the record already persisted.
*
* Per-suite implementations are documented in src/commands/eval-*.ts.
* This file is the dispatcher + bookkeeper.
*/
import { writeFileSync, mkdirSync, appendFileSync, existsSync } from 'fs';
import { dirname, join } from 'path';
import type { BrainEngine } from '../core/engine.ts';
import { MODE_BUNDLES, SEARCH_MODES, type SearchMode } from '../core/search/mode.ts';
import { redactSecrets } from '../eval/longmemeval/run-config.ts';
import { OperationError, opError } from '../core/ops/contract.ts';
import type { Action } from '../core/agent-output.ts';
import { usageError } from '../cli/cli-error.ts';
import { intFlagValue, numberFlagValue } from '../cli/flag-values.ts';
export interface RunAllOpts {
help: boolean;
modes: SearchMode[];
suites: string[];
/** True when `--suites` named the suites; a bare run sweeps the wired ones. */
suitesExplicit: boolean;
limit?: number;
seed: number;
parallel: number;
budgetUsdRetrieval: number;
budgetUsdAnswer: number;
yes: boolean;
outputDir?: string;
jsonOutput: boolean;
}
const VALID_SUITES = ['longmemeval', 'replay', 'brainbench'] as const;
type ValidSuite = (typeof VALID_SUITES)[number];
const WIRED_SUITES = ['brainbench'] as const;
type UnwiredSuite = Exclude<ValidSuite, (typeof WIRED_SUITES)[number]>;
/** The per-suite command for each suite run-all does not execute in-process. */
const UNWIRED_SUITE_ARGV: Record<UnwiredSuite, string[]> = {
longmemeval: ['gbrain', 'eval', 'longmemeval', '<dataset.jsonl>', '--mode', '<mode>', '--record'],
replay: ['gbrain', 'eval', 'replay', '--mode', '<mode>'],
};
const DEFAULT_BUDGET_USD_RETRIEVAL = 5;
const DEFAULT_BUDGET_USD_ANSWER = 20;
const DEFAULT_SEED = 42;
const DEFAULT_PARALLEL = 1;
export function parseRunAllArgs(args: string[]): RunAllOpts {
const opts: RunAllOpts = {
help: false,
modes: [...SEARCH_MODES],
suites: [...WIRED_SUITES],
suitesExplicit: false,
seed: DEFAULT_SEED,
parallel: DEFAULT_PARALLEL,
budgetUsdRetrieval: DEFAULT_BUDGET_USD_RETRIEVAL,
budgetUsdAnswer: DEFAULT_BUDGET_USD_ANSWER,
yes: false,
jsonOutput: false,
};
for (let i = 0; i < args.length; i++) {
const a = args[i];
if (a === '--help' || a === '-h') { opts.help = true; continue; }
if (a === '--modes') {
const list = (args[++i] ?? '').split(',').map(s => s.trim()).filter(Boolean);
const validated: SearchMode[] = [];
for (const m of list) {
if (m === 'conservative' || m === 'balanced' || m === 'tokenmax') {
validated.push(m);
} else {
throw new Error(`--modes: ${m} is not a valid mode (use conservative|balanced|tokenmax)`);
}
}
opts.modes = validated;
continue;
}
if (a === '--suite' || a === '--suites') {
const list = (args[++i] ?? '').split(',').map(s => s.trim()).filter(Boolean);
for (const s of list) {
if (!(VALID_SUITES as readonly string[]).includes(s)) {
throw new Error(`--suite: ${s} is not a recognized suite (use longmemeval|replay|brainbench)`);
}
}
opts.suites = list;
opts.suitesExplicit = true;
continue;
}
// #5933 (D4): NaN/out-of-range values would bypass the cost guard; reject them (usage error, exit 2).
if (a === '--limit') { opts.limit = intFlagValue(args[++i], '--limit', { min: 1, example: 50 }); continue; }
if (a === '--seed') { opts.seed = intFlagValue(args[++i], '--seed', { example: 42 }); continue; }
if (a === '--parallel') {
const n = Number(args[++i]);
if (!Number.isFinite(n) || n < 1) throw new Error('--parallel must be >= 1');
opts.parallel = Math.min(n, SEARCH_MODES.length);
continue;
}
if (a === '--budget-usd-retrieval') { opts.budgetUsdRetrieval = numberFlagValue(args[++i], a, { min: 0, example: 5 }); continue; }
if (a === '--budget-usd-answer') { opts.budgetUsdAnswer = numberFlagValue(args[++i], a, { min: 0, example: 5 }); continue; }
if (a === '--yes' || a === '-y') { opts.yes = true; continue; }
if (a === '--output' || a === '--output-dir') { opts.outputDir = args[++i]; continue; }
if (a === '--json') { opts.jsonOutput = true; continue; }
}
if (opts.modes.length === 0) throw new Error('--modes resolved to an empty list');
if (opts.suites.length === 0) throw new Error('--suites resolved to an empty list');
return opts;
}
function printHelp(): void {
process.stderr.write(
`gbrain eval run-all [flags]\n\n` +
`Runs the eval suites wired into the orchestrator (brainbench, once per sweep)\n` +
`and writes per-run results to <repo>/.gbrain-evals/eval-results.jsonl. Exits 1\n` +
`when a suite fails (its record is kept). Personal brain is never touched.\n\n` +
`Flags:\n` +
` --modes M1,M2,M3 Modes to evaluate (default: conservative,balanced,tokenmax).\n` +
` --suites S1,S2 Suites to run (default: brainbench).\n` +
` run-all runs brainbench in-process. longmemeval and\n` +
` replay run through their own commands, once per mode:\n` +
` ${UNWIRED_SUITE_ARGV.longmemeval.join(' ')}\n` +
` ${UNWIRED_SUITE_ARGV.replay.join(' ')}\n` +
` Naming them here exits 1 (eval_suite_unwired).\n` +
` --limit N Limit each suite to N questions (default: full split).\n` +
` --seed N Random seed (default: 42).\n` +
` --parallel N Run N modes in parallel (default: 1; max ${SEARCH_MODES.length}).\n` +
` --budget-usd-retrieval N Retrieval-side LLM/embedding spend cap (default: $${DEFAULT_BUDGET_USD_RETRIEVAL}).\n` +
` --budget-usd-answer N Answer-gen LLM spend cap (default: $${DEFAULT_BUDGET_USD_ANSWER}).\n` +
` --yes Required (alongside --budget-usd-*) in non-TTY for over-cap runs.\n` +
` --output DIR Override .gbrain-evals/ location.\n` +
` --json Emit run summary as JSON.\n` +
` -h, --help Show this help.\n\n` +
`Cost guard refuses non-TTY runs over the budget cap without --yes AND an\n` +
`explicit --budget-usd-* flag (defense against agent loops + cron jobs).\n`,
);
}
function unwiredSuiteFix(suites: UnwiredSuite[], modes: SearchMode[]): Action {
const [suite, ...rest] = suites;
return {
argv: UNWIRED_SUITE_ARGV[suite],
consent: ['paid'],
actor: 'agent',
why: `run-all does not run ${suite} in-process; its own command runs it, once per mode.`,
inputs: [
...(suite === 'longmemeval' ? [{ name: 'dataset.jsonl', how: 'The LongMemEval dataset file (https://huggingface.co/datasets/xiaowu0162/longmemeval).' }] : []),
{ name: 'mode', how: `Each requested mode in turn: ${modes.join(', ')}.` },
],
docs: 'docs/eval/SEARCH_MODE_METHODOLOGY.md#4-run-procedure',
requires_exclusive: false,
...(rest.length ? { then: unwiredSuiteFix(rest, modes) } : {}),
};
}
/** The refusal for an explicit `--suites` that names a suite run-all does not run. */
export function unwiredSuiteError(suites: string[], modes: SearchMode[]): OperationError | null {
const unwired = suites.filter((s): s is UnwiredSuite => s in UNWIRED_SUITE_ARGV);
if (unwired.length === 0) return null;
const commands = unwired.map(s => UNWIRED_SUITE_ARGV[s].join(' ')).join('; ');
return opError(
'eval_suite_unwired',
`gbrain eval run-all does not run ${unwired.join(' or ')}; nothing ran.`,
`Run ${unwired.length > 1 ? 'each suite' : 'it'} with its own command, once per mode: ${commands}. Or drop ${unwired.join(', ')} from --suites.`,
{ fix: unwiredSuiteFix(unwired, modes) },
);
}
export interface EvalRunRecord {
/**
* v3 (BrainBench wave, decision 16): `mode` widened to `SearchMode | 'n/a'`
* for search-mode-independent suites — brainbench runs once per sweep and
* records 'n/a' instead of fabricating a mode. v2 records (mode always a
* SearchMode) parse fine under v3 readers. NOTE: eval-compare's markdown
* renderer iterates SEARCH_MODES only, so 'n/a' rows surface in its --json
* `records` output, not the mode table (review finding; markdown surfacing
* is a follow-up).
*/
schema_version: 3;
run_id: string;
ran_at: string;
suite: ValidSuite;
mode: SearchMode | 'n/a';
commit: string;
seed: number;
limit?: number;
params: Record<string, unknown>;
status: 'completed' | 'failed' | 'skipped' | 'over_budget';
duration_ms: number;
error?: string;
}
function getRepoRoot(): string {
try {
const { execSync } = require('../core/spawn.ts') as typeof import('../core/spawn.ts');
return execSync('git rev-parse --show-toplevel', { encoding: 'utf-8' }).trim();
} catch {
return process.cwd();
}
}
function getCommitSha(): string {
try {
const { execSync } = require('../core/spawn.ts') as typeof import('../core/spawn.ts');
return execSync('git rev-parse --short HEAD', { encoding: 'utf-8' }).trim();
} catch {
return 'unknown';
}
}
function evalResultsPath(repoRoot: string, outputDirOverride?: string): string {
if (outputDirOverride) {
return join(outputDirOverride, 'eval-results.jsonl');
}
return join(repoRoot, '.gbrain-evals', 'eval-results.jsonl');
}
/** Apply `redactSecrets` to every string leaf (keys untouched); JSON stays valid. */
function redactDeep<T>(value: T): T {
if (typeof value === 'string') return redactSecrets(value) as unknown as T;
if (Array.isArray(value)) return value.map(redactDeep) as unknown as T;
if (value && typeof value === 'object') {
const out: Record<string, unknown> = {};
for (const [k, v] of Object.entries(value as Record<string, unknown>)) out[k] = redactDeep(v);
return out as T;
}
return value;
}
/**
* Secret-scrub a ledger record before it is written: `error` text and every
* string leaf of `params` pass through `redactSecrets` (provider keys, bearer
* tokens, DB connection strings). Lives HERE, on the one write path every
* suite shares, so a caller that forgets to redact cannot leak a connection
* string into the committed ledger. String leaves are redacted individually
* (not the serialized line) so the written JSON is always valid.
*/
export function redactRunRecord(record: EvalRunRecord): EvalRunRecord {
return {
...record,
params: redactDeep(record.params ?? {}),
...(typeof record.error === 'string' ? { error: redactSecrets(record.error) } : {}),
};
}
export function persistRunRecord(repoRoot: string, record: EvalRunRecord, outputDirOverride?: string): void {
const path = evalResultsPath(repoRoot, outputDirOverride);
mkdirSync(dirname(path), { recursive: true });
appendFileSync(path, JSON.stringify(redactRunRecord(record)) + '\n', 'utf-8');
}
/**
* Estimate per-run cost. Rough heuristic: per-mode estimates × #questions.
* The methodology doc documents the assumption. Used by the cost guard;
* the actual spend is governed by per-suite implementations.
*
* Returns total estimated USD across retrieval + answer-gen sides.
*/
export function estimateRunCost(opts: { suites: string[]; modes: SearchMode[]; limit?: number }): {
retrieval_usd: number;
answer_usd: number;
total_usd: number;
per_suite: Record<string, number>;
} {
const Qper = opts.limit ?? 500;
const perSuite: Record<string, number> = {};
let retrieval = 0;
let answer = 0;
for (const suite of opts.suites) {
for (const mode of opts.modes) {
// Retrieval side: 1 embed + (optional Haiku) per query.
const expansionCalls = MODE_BUNDLES[mode].expansion ? Qper : 0;
const retCost = Qper * 0.00001 + expansionCalls * 0.001;
// Answer side: 1 Sonnet/Opus call per query for longmemeval; 0 for replay/brainbench.
const answerCost = suite === 'longmemeval' ? Qper * 0.015 : 0;
retrieval += retCost;
answer += answerCost;
perSuite[`${suite}:${mode}`] = retCost + answerCost;
}
}
return {
retrieval_usd: retrieval,
answer_usd: answer,
total_usd: retrieval + answer,
per_suite: perSuite,
};
}
interface CostGuardResult {
proceed: boolean;
reason: string;
}
export function evaluateCostGuard(
estimate: { retrieval_usd: number; answer_usd: number; total_usd: number },
opts: { budgetUsdRetrieval: number; budgetUsdAnswer: number; yes: boolean; isTty: boolean },
): CostGuardResult {
const retOver = estimate.retrieval_usd > opts.budgetUsdRetrieval;
const ansOver = estimate.answer_usd > opts.budgetUsdAnswer;
if (!retOver && !ansOver) {
return { proceed: true, reason: 'within budget' };
}
if (!opts.isTty && !opts.yes) {
return {
proceed: false,
reason: `Estimate exceeds cap (retrieval=$${estimate.retrieval_usd.toFixed(2)} vs cap $${opts.budgetUsdRetrieval}, answer=$${estimate.answer_usd.toFixed(2)} vs cap $${opts.budgetUsdAnswer}). Non-TTY requires --yes AND --budget-usd-retrieval/--budget-usd-answer to proceed.`,
};
}
if (opts.isTty && !opts.yes) {
return {
proceed: false,
reason: `Estimate ($${estimate.total_usd.toFixed(2)}) exceeds caps. Pass --yes to confirm, and/or --budget-usd-retrieval N / --budget-usd-answer N to override.`,
};
}
return { proceed: true, reason: 'over cap but --yes acknowledged' };
}
/** Runs the sweep; returns the exit code (0 all wired suites completed, 1 one failed). */
export async function runEvalRunAll(_engine: BrainEngine | null, args: string[]): Promise<number> {
let opts: RunAllOpts;
try {
opts = parseRunAllArgs(args);
} catch (e) {
throw e instanceof OperationError ? e : usageError((e as Error).message, 'Run `gbrain eval run-all --help` for the accepted flags and examples.');
}
if (opts.help) {
printHelp();
return 0;
}
const unwired = unwiredSuiteError(opts.suites, opts.modes);
if (unwired) throw unwired;
const repoRoot = getRepoRoot();
const commit = getCommitSha();
const estimate = estimateRunCost({ suites: opts.suites, modes: opts.modes, limit: opts.limit });
const isTty = Boolean(process.stderr.isTTY);
if (opts.jsonOutput) {
process.stdout.write(JSON.stringify({
phase: 'estimate',
schema_version: 2,
modes: opts.modes,
suites: opts.suites,
limit: opts.limit,
seed: opts.seed,
commit,
estimate,
}) + '\n');
} else {
process.stderr.write(`[eval run-all] commit=${commit} modes=${opts.modes.join(',')} suites=${opts.suites.join(',')}\n`);
process.stderr.write(`[eval run-all] cost estimate: retrieval=$${estimate.retrieval_usd.toFixed(2)} answer=$${estimate.answer_usd.toFixed(2)} total=$${estimate.total_usd.toFixed(2)}\n`);
process.stderr.write(`[eval run-all] budget caps: retrieval=$${opts.budgetUsdRetrieval} answer=$${opts.budgetUsdAnswer}\n`);
}
if (!opts.suitesExplicit) {
process.stderr.write(
`[eval run-all] running the wired suites (${WIRED_SUITES.join(', ')}). longmemeval and replay run through their own commands, once per mode:\n` +
Object.values(UNWIRED_SUITE_ARGV).map(argv => `[eval run-all] ${argv.join(' ')}\n`).join(''),
);
}
const guard = evaluateCostGuard(estimate, {
budgetUsdRetrieval: opts.budgetUsdRetrieval,
budgetUsdAnswer: opts.budgetUsdAnswer,
yes: opts.yes,
isTty,
});
if (!guard.proceed) {
process.stderr.write(`[eval run-all] REFUSED: ${guard.reason}\n`);
process.exit(2);
}
if (!opts.jsonOutput) {
process.stderr.write(`[eval run-all] ${guard.reason}, proceeding.\n`);
}
const failed: EvalRunRecord[] = [];
for (const suite of opts.suites) {
// BrainBench is search-mode-independent (decision 16): run ONCE per
// sweep, in-process, recorded under mode 'n/a' — never multiplied by
// modes. Unwired suites never reach this loop (unwiredSuiteError).
if (suite === 'brainbench') {
const startedAt = Date.now();
const runId = `${commit}-brainbench-na-${opts.seed}`;
const { runBrainBenchCore } = await import('./eval-brainbench.ts');
const core = await runBrainBenchCore();
const record: EvalRunRecord = {
schema_version: 3,
run_id: runId,
ran_at: new Date().toISOString(),
suite: 'brainbench',
mode: 'n/a',
commit,
seed: opts.seed,
limit: opts.limit,
params: {
budget_usd_retrieval: opts.budgetUsdRetrieval,
budget_usd_answer: opts.budgetUsdAnswer,
parallel: opts.parallel,
fixtures_hash: core.fixtures_hash,
cells: core.cells,
},
status: core.status,
duration_ms: Date.now() - startedAt,
};
if (core.error) record.error = core.error;
persistRunRecord(repoRoot, record, opts.outputDir);
if (record.status === 'failed') failed.push(record);
if (!opts.jsonOutput) {
process.stderr.write(`[eval run-all] ${runId}: ${record.status}${record.error ? ` (${redactSecrets(record.error)})` : ''}\n`);
}
}
}
if (opts.jsonOutput) {
process.stdout.write(JSON.stringify({
phase: 'complete',
commit,
modes: opts.modes,
suites: opts.suites,
failed_runs: failed.map(r => r.run_id),
output_path: evalResultsPath(repoRoot, opts.outputDir),
}) + '\n');
} else if (failed.length > 0) {
process.stderr.write(`[eval run-all] FAILED: ${failed.map(r => r.run_id).join(', ')}. Records kept in ${evalResultsPath(repoRoot, opts.outputDir)}\n`);
} else {
process.stderr.write(`[eval run-all] complete. Audit trail: ${evalResultsPath(repoRoot, opts.outputDir)}\n`);
}
return failed.length > 0 ? 1 : 0;
}