Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* B1 benchmark hooks (GBRA-274): the batch rollout hook contract, explicit task
* lists, per-task scores, rollout tool declarations and train-only criteria.
* Pure: no engine work, no models.
*/
import { describe, expect, test } from 'bun:test';
import type { BrainEngine } from '../../src/core/engine.ts';
import { runValidationGate, scoreSkillPerTask } from '../../src/core/skillopt/validate-gate.ts';
import { isSkilloptMustAbort, SKILLOPT_HOOK_INFRA_ERROR, SKILLOPT_RUNTIME_EXCEEDED } from '../../src/core/skillopt/must-abort.ts';
import { loadRunTasks } from '../../src/core/skillopt/benchmark.ts';
import { describeJudges } from '../../src/core/skillopt/reflect.ts';
import { READ_ONLY_BRAIN_TOOLS, rolloutToolsForSkill } from '../../src/core/skillopt/rollout.ts';
import type { BenchmarkTask, RolloutBatchHook, RolloutBatchRequest, ScoredRollout } from '../../src/core/skillopt/types.ts';
import * as skillopt from '../../src/core/skillopt/index.ts';
import { getCurrentBudgetTracker, withBudgetTracker } from '../../src/core/ai/gateway.ts';
import { BudgetTracker } from '../../src/core/budget/budget-tracker.ts';
const engine = {} as BrainEngine;
const task = (id: string, arg = 'x'): BenchmarkTask => ({ task_id: id, task: `do ${id}`, judge: { kind: 'rule', checks: [{ op: 'contains', arg }] } } as BenchmarkTask);
const tasks = (prefix: string, n: number) => Array.from({ length: n }, (_, i) => task(`${prefix}-${i + 1}`));
const row = (score: number): ScoredRollout => ({ score, trajectory: { task_id: '', task: '', final_text: '', tool_calls: [], turns: 1, stop_reason: 'end', duration_ms: 0,
usage: { input_tokens: 0, output_tokens: 0, cache_read_tokens: 0, cache_creation_tokens: 0 } } } as ScoredRollout);
test('B1.1: the harness surface is exported from the skillopt index', () => {
for (const name of ['runSkillOpt', 'runRollout', 'scoreTrajectory', 'scoreSkillPerTask', 'scoreSkillOnTasks', 'rolloutToolsForSkill', 'describeJudges']) {
expect(typeof (skillopt as Record<string, unknown>)[name]).toBe('function');
}
expect(skillopt.READ_ONLY_BRAIN_TOOLS).toBe(READ_ONLY_BRAIN_TOOLS);
});
describe('rollout hook (B1.3)', () => {
test('one call for the whole batch, per-task medians and the request labels', async () => {
const calls: RolloutBatchRequest[] = [];
const hook: RolloutBatchHook = async (req) => { calls.push(req); return req.tasks.map((_, i) => [row(i % 2), row(1), row(0)]); };
const res = await scoreSkillPerTask({ engine, skillText: 'SKILL', tasks: tasks('s', 4), targetModel: 'm', rolloutHook: hook });
expect(calls).toHaveLength(1);
expect(calls[0]).toMatchObject({ phase: 'test', skillText: 'SKILL', runsPerTask: 3, epoch: 0, step: 0 });
expect(calls[0]!.skillSha256).toMatch(/^[0-9a-f]{64}$/);
expect(res.perTask.map((p) => p.median)).toEqual([0, 1, 0, 1]);
expect(res.score).toBe(0.5);
});
test.each([
['too few rows', (req: RolloutBatchRequest) => req.tasks.slice(1).map(() => [row(1), row(1), row(1)])],
['too few runs', (req: RolloutBatchRequest) => req.tasks.map(() => [row(1)])],
['not an array', () => ({}) as unknown as ScoredRollout[][]],
['a string score', (req: RolloutBatchRequest) => req.tasks.map(() => [row(1), row(1), { ...row(1), score: '0.9' } as unknown as ScoredRollout])],
['a NaN score', (req: RolloutBatchRequest) => req.tasks.map(() => [row(1), row(1), row(Number.NaN)])],
['a score above 1', (req: RolloutBatchRequest) => req.tasks.map(() => [row(7), row(7), row(7)])],
])('a malformed batch (%s) is an infra error that aborts, never a score', async (_name, reply) => {
const hook: RolloutBatchHook = async (req) => reply(req);
const err = await runValidationGate({ engine, candidateSkillText: 'S', selSet: tasks('s', 5), bestScore: 0, targetModel: 'm', rolloutHook: hook })
.then(() => null, (e) => e);
expect(String(err?.message)).toContain(SKILLOPT_HOOK_INFRA_ERROR);
expect(isSkilloptMustAbort(err)).toBe(true);
});
test('the hook runs outside the run budget, so in-process rollouts are never billed to the optimizer cap', async () => {
const seen: Array<BudgetTracker | null> = [];
const hook: RolloutBatchHook = async (req) => {
seen.push(getCurrentBudgetTracker());
await Promise.resolve();
seen.push(getCurrentBudgetTracker());
return req.tasks.map(() => [row(1), row(1), row(1)]);
};
const tracker = new BudgetTracker({ maxCostUsd: 1, label: 'skillopt:test' });
await withBudgetTracker(tracker, () => scoreSkillPerTask({ engine, skillText: 'S', tasks: tasks('s', 2), targetModel: 'm', rolloutHook: hook }));
expect(seen).toEqual([null, null]);
});
test('a batch still running at the runtime deadline aborts the run, and the hook is signalled', async () => {
let signal: AbortSignal | undefined;
const hook: RolloutBatchHook = (req, s) => { signal = s; return new Promise(() => {}); };
const err = await runValidationGate({ engine, candidateSkillText: 'S', selSet: tasks('s', 5), bestScore: 0, targetModel: 'm', rolloutHook: hook, deadlineMs: Date.now() + 50 })
.then(() => null, (e) => e);
expect(String(err?.message)).toContain(SKILLOPT_RUNTIME_EXCEEDED);
expect(isSkilloptMustAbort(err)).toBe(true);
expect(signal?.aborted).toBe(true);
});
test('a hook that throws synchronously fails the gate and leaves no deadline timer behind', async () => {
const unhandled: unknown[] = [];
const onUnhandled = (e: unknown) => { unhandled.push(e); };
process.on('unhandledRejection', onUnhandled);
try {
const hook: RolloutBatchHook = () => { throw new Error('sync hook failure'); };
const err = await runValidationGate({ engine, candidateSkillText: 'S', selSet: tasks('s', 5), bestScore: 0, targetModel: 'm', rolloutHook: hook, deadlineMs: Date.now() + 30 })
.then(() => null, (e) => e);
expect(String(err?.message)).toContain('sync hook failure');
await new Promise((r) => setTimeout(r, 80));
expect(unhandled).toEqual([]);
} finally {
process.off('unhandledRejection', onUnhandled);
}
});
test('a runtime cap beyond the timer range (or Infinity) does not abort the batch', async () => {
const hook: RolloutBatchHook = async (req) => { await new Promise((r) => setTimeout(r, 20)); return req.tasks.map(() => [row(1), row(1), row(1)]); };
for (const deadlineMs of [Date.now() + 30 * 24 * 3600 * 1000, Number.POSITIVE_INFINITY]) {
const res = await runValidationGate({ engine, candidateSkillText: 'S', selSet: tasks('s', 5), bestScore: 0, targetModel: 'm', rolloutHook: hook, deadlineMs });
expect(res.selScore).toBe(1);
}
});
test('a batch answered after the runtime deadline is not a score', async () => {
const hook: RolloutBatchHook = async (req) => { await new Promise((r) => setTimeout(r, 30)); return req.tasks.map(() => [row(1), row(1), row(1)]); };
const deadlineMs = Date.now() + 10;
const err = await runValidationGate({ engine, candidateSkillText: 'S', selSet: tasks('s', 5), bestScore: 0, targetModel: 'm', rolloutHook: hook, deadlineMs })
.then(() => null, (e) => e);
expect(String(err?.message)).toContain(SKILLOPT_RUNTIME_EXCEEDED);
});
test('a train_forward row without a trajectory is an infra error', async () => {
const hook: RolloutBatchHook = async (req) => req.tasks.map(() => [{ score: 1 } as ScoredRollout]);
const err = await scoreSkillPerTask({ engine, skillText: 'S', tasks: tasks('t', 2), targetModel: 'm', runsPerTask: 1, phase: 'train_forward', rolloutHook: hook })
.then(() => null, (e) => e);
expect(String(err?.message)).toContain(SKILLOPT_HOOK_INFRA_ERROR);
});
});
describe('explicit task lists (B1.2)', () => {
test('lists are used as given, with a stable sha', () => {
const t = { train: tasks('t', 2), sel: tasks('s', 5) };
const a = loadRunTasks({ tasks: t });
expect(a.split.train.map((x) => x.task_id)).toEqual(['t-1', 't-2']);
expect(a.split.sel).toHaveLength(5);
expect(a.split.test).toHaveLength(0);
expect(a.bench.benchmark_sha8).toBe(loadRunTasks({ tasks: t }).bench.benchmark_sha8);
expect(a.bench.benchmark_sha8).not.toBe(loadRunTasks({ tasks: { ...t, train: tasks('t', 3) } }).bench.benchmark_sha8);
});
test('a task id in two lists, an empty train list or a small sel list is refused', () => {
const code = (fn: () => unknown) => { try { fn(); return null; } catch (e) { return (e as { envelope?: { code: string } }).envelope?.code; } };
expect(code(() => loadRunTasks({ tasks: { train: [task('s-1')], sel: tasks('s', 5) } }))).toBe('benchmark_duplicate_task_id');
expect(code(() => loadRunTasks({ tasks: { train: [], sel: tasks('s', 5) } }))).toBe('benchmark_empty');
expect(code(() => loadRunTasks({ tasks: { train: tasks('t', 2), sel: tasks('s', 4) } }))).toBe('d_sel_too_small');
});
});
describe('criteria (defect 3 / B1.4)', () => {
test('a check every task shares is verbatim; a per-task answer is described, never quoted', () => {
const ts = ['a', 'b', 'c'].map((id) => ({ judge: { kind: 'rule' as const, checks: [{ op: 'contains', arg: `GOLD-${id}` }, { op: 'section_present', arg: 'Sources' }] } }));
const text = describeJudges(ts as never);
expect(text).toContain('"Sources"');
expect(text).not.toContain('GOLD-');
expect(text.split('\n')).toHaveLength(2);
});
test('a single task never counts as shared', () => {
expect(describeJudges([{ judge: { kind: 'rule', checks: [{ op: 'contains', arg: 'GOLD-1' }] } }] as never)).not.toContain('GOLD-1');
});
test('per-task LLM rubrics are not quoted', () => {
const text = describeJudges([{ judge: { kind: 'llm', rubric: 'RUBRIC-A' } }, { judge: { kind: 'llm', rubric: 'RUBRIC-B' } }] as never);
expect(text).not.toContain('RUBRIC-');
});
});
describe('rollout tools (B1.5)', () => {
const skill = (tools: string) => `---\nname: s\ntools: [${tools}]\n---\n# S\n`;
test('the base set never offers a write', () => {
expect(READ_ONLY_BRAIN_TOOLS.has('put_page')).toBe(false);
expect(READ_ONLY_BRAIN_TOOLS.has('add_timeline_entry')).toBe(false);
});
test('a skill adds reviewed read ops by declaring them; anything else it names is ignored', () => {
const got = rolloutToolsForSkill(skill('recall, get_timeline, put_page, add_timeline_entry, advisor, context_pack'));
expect(got.has('recall')).toBe(true);
expect(got.has('get_timeline')).toBe(true);
for (const n of ['put_page', 'add_timeline_entry', 'advisor', 'context_pack']) expect(got.has(n)).toBe(false);
expect(rolloutToolsForSkill(skill('')).size).toBe(READ_ONLY_BRAIN_TOOLS.size);
});
});