Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* H1 (test-gap plan) — keyless CI validation for the takes-bootstrap eval
* (evals/takes-bootstrap/, TODOS TODO-E). Two halves:
*
* 1. CORPUS integrity: >=100 cases, schema-valid, unique ids, category
* floors (incl. empty/attribution/adversarial precision classes),
* regenerator-fresh, placeholder-only names (privacy rule).
* 2. SCORER math: exact precision/recall arithmetic on synthetic
* predictions, malformed-case = FAILURE (never a skip), forbid
* violations fail graduation, and the graduation boundary bites.
*
* The LIVE classifier run stays opt-in (spends tokens):
* bun evals/takes-bootstrap/harness.mjs # needs a chat key
* bun evals/takes-bootstrap/harness.mjs --replay results.jsonl # $0
* Autopilot tier for takes-bootstrap remains manual_only until a live run
* GRADUATES (TODO-E) — this file guards the instrument, not the score.
*/
import { describe, test, expect } from 'bun:test';
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { execFileSync, spawnSync } from 'node:child_process';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { ALLOWED_PAGE_TYPES } from '../src/core/extract-takes-from-pages.ts';
import { scoreCorpus, GRADUATION, type CorpusCase } from '../evals/takes-bootstrap/scorer.ts';
const repoRoot = join(import.meta.dir, '..');
const corpusPath = join(repoRoot, 'evals/takes-bootstrap/corpus.jsonl');
const corpus: CorpusCase[] = readFileSync(corpusPath, 'utf8').trim().split('\n').map(l => JSON.parse(l));
describe('corpus integrity', () => {
test('>=100 schema-valid cases with unique ids', () => {
expect(corpus.length).toBeGreaterThanOrEqual(100);
const ids = new Set<string>();
for (const c of corpus) {
expect(typeof c.id).toBe('string');
expect(ids.has(c.id)).toBe(false);
ids.add(c.id);
expect(c.page.body.length).toBeGreaterThan(10);
expect(Array.isArray(c.expected)).toBe(true);
for (const e of c.expected) {
expect(['fact', 'take', 'bet', 'hunch']).toContain(e.kind);
expect(() => new RegExp(e.claim_re)).not.toThrow();
expect(e.weight_min).toBeLessThanOrEqual(e.weight_max);
}
for (const f of c.forbid) expect(() => new RegExp(f)).not.toThrow();
}
});
test('every case passes the production eligibility filter (allowed type, body over 200 chars)', () => {
// extractTakesFromPages selects only these pages; an ineligible case is
// never classified, so a live run could never score it.
for (const c of corpus) {
expect({ id: c.id, allowed: (ALLOWED_PAGE_TYPES as readonly string[]).includes(c.page.type) }).toEqual({ id: c.id, allowed: true });
expect({ id: c.id, long: c.page.body.length > 200 }).toEqual({ id: c.id, long: true });
}
expect(new Set(corpus.map(c => c.archetype)).size).toBe(41);
});
test('category floors: precision classes are represented', () => {
const byCat = new Map<string, number>();
for (const c of corpus) byCat.set(c.category, (byCat.get(c.category) ?? 0) + 1);
for (const cat of ['fact', 'take', 'bet', 'hunch', 'empty', 'attribution', 'adversarial', 'mixed']) {
expect(byCat.get(cat) ?? 0).toBeGreaterThanOrEqual(6);
}
// Precision cases (expect NOTHING) must be a real share of the corpus.
const emptyExpected = corpus.filter(c => c.expected.length === 0).length;
expect(emptyExpected).toBeGreaterThanOrEqual(20);
});
test('committed corpus is regenerator-fresh (deterministic builder)', () => {
// Read the committed bytes FIRST, regenerate in place, then demand FULL
// byte equality — a drifted generator (or hand-edited corpus) fails here,
// not just a changed line count or first id. The finally block restores
// the committed bytes on failure so a red run never leaves the tree dirty.
const committed = readFileSync(corpusPath, 'utf8');
try {
const out = execFileSync('node', ['evals/takes-bootstrap/generate-corpus.mjs'], { cwd: repoRoot, encoding: 'utf8' });
expect(out).toContain('wrote');
const regenerated = readFileSync(corpusPath, 'utf8');
expect(regenerated).toBe(committed);
} finally {
if (readFileSync(corpusPath, 'utf8') !== committed) {
writeFileSync(corpusPath, committed);
}
}
});
test('privacy: placeholder identities only', () => {
const text = readFileSync(corpusPath, 'utf8');
// Every person/company mention must come from the placeholder pools.
for (const required of ['Alice Example', 'Widget Co', 'fund-c']) {
expect(text).toContain(required);
}
// Negative side: the corpus lives under evals/, OUTSIDE the scan surface
// of the repo privacy guards (check-privacy.sh scans src/test/docs/skills/
// scripts only), so this test re-applies their banned tokens to the
// corpus. The tokens are read from the guard scripts — their one
// allowlisted home — because repeating the literals here would itself
// trip check-privacy / check-test-real-names on this file.
const lower = text.toLowerCase();
const realNamesSh = readFileSync(join(repoRoot, 'scripts/check-test-real-names.sh'), 'utf8');
// Line-state parse (a lazy paren regex truncates at ')' inside comments);
// entries only, not quoted words in comments.
const banned: string[] = [];
let inArray = false;
for (const line of realNamesSh.split('\n')) {
if (/^BANNED_(NAMES|EMAILS)=\($/.test(line.trim())) { inArray = true; continue; }
if (inArray && line.trim() === ')') { inArray = false; continue; }
const m = inArray ? line.match(/^\s*'([^']+)'/) : null;
if (m) banned.push(m[1]);
}
const privacySh = readFileSync(join(repoRoot, 'scripts/check-privacy.sh'), 'utf8');
const forkName = privacySh.match(/^BANNED_NAME='([^']+)'/m)?.[1];
expect(forkName).toBeTruthy();
banned.push(forkName!);
expect(banned.length).toBeGreaterThanOrEqual(5);
for (const b of banned) {
expect(lower.includes(b.toLowerCase())).toBe(false);
}
});
});
describe('scorer math', () => {
const mini: CorpusCase[] = [
{
id: 'a', archetype: 'a', category: 'take', page: { slug: 't/a', type: 'note', title: 'A', body: 'x' },
expected: [{ claim_re: 'strongest team', kind: 'take', weight_min: 0.4, weight_max: 1 }],
forbid: [], notes: '',
},
{
id: 'b', archetype: 'b', category: 'empty', page: { slug: 't/b', type: 'note', title: 'B', body: 'x' },
expected: [], forbid: ['fraudulent'], notes: '',
},
];
test('perfect run graduates with exact arithmetic', () => {
const r = scoreCorpus(mini, [
{ id: 'a', claims: [{ claim: 'the strongest team here', kind: 'take', weight: 0.8 }] },
{ id: 'b', claims: [] },
]);
expect(r.by_kind.find(k => k.kind === 'take')).toMatchObject({ expected: 1, matched: 1, predicted: 1, precise: 1, precision: 1, recall: 1 });
expect(r.overall).toEqual({ precision: 1, recall: 1, f1: 1 });
expect(r.malformed).toEqual([]);
expect(r.graduated).toBe(true);
});
test('malformed case is a FAILURE, never a skip', () => {
const r = scoreCorpus(mini, [
{ id: 'a', claims: null },
{ id: 'b', claims: [] },
]);
expect(r.malformed).toEqual(['a']);
expect(r.graduated).toBe(false);
// The malformed case's expectations still count in the denominator.
expect(r.by_kind.find(k => k.kind === 'take')!.expected).toBe(1);
expect(r.by_kind.find(k => k.kind === 'take')!.recall).toBe(0);
});
test('a MISSING prediction row is malformed too (no silent denominator shrink)', () => {
const r = scoreCorpus(mini, [{ id: 'b', claims: [] }]);
expect(r.malformed).toEqual(['a']);
expect(r.graduated).toBe(false);
});
test('forbid violation (injection/attribution leak) blocks graduation', () => {
const r = scoreCorpus(mini, [
{ id: 'a', claims: [{ claim: 'the strongest team here', kind: 'take', weight: 0.8 }] },
{ id: 'b', claims: [{ claim: 'acme is fraudulent', kind: 'fact', weight: 1 }] },
]);
expect(r.forbid_violations.length).toBe(1);
expect(r.graduated).toBe(false);
});
test('kind mismatch and out-of-range weight are recall misses; hallucination is a precision miss', () => {
const r = scoreCorpus(mini, [
// right words, wrong kind → neither recall nor precision credit
{ id: 'a', claims: [{ claim: 'strongest team', kind: 'fact', weight: 0.8 }, { claim: 'made-up thing', kind: 'take', weight: 0.5 }] },
{ id: 'b', claims: [] },
]);
const take = r.by_kind.find(k => k.kind === 'take')!;
expect(take.matched).toBe(0);
expect(take.precise).toBe(0);
expect(take.predicted).toBe(1);
const fact = r.by_kind.find(k => k.kind === 'fact')!;
expect(fact.predicted).toBe(1);
expect(fact.precise).toBe(0);
expect(r.graduated).toBe(false);
});
test('graduation boundary bites at the documented bars', () => {
// 5 expected takes; 4 matched (recall .8 >= .7 ok), 5 predicted 4 precise
// (precision .8 >= .8 ok) → graduates. Drop one precise → .6 precision → fails.
const cases: CorpusCase[] = Array.from({ length: 5 }, (_, i) => ({
id: `c${i}`, archetype: `c${i % 2}`, category: 'take', page: { slug: `t/c${i}`, type: 'note', title: 'C', body: 'x' },
expected: [{ claim_re: `claim-${i}`, kind: 'take', weight_min: 0, weight_max: 1 }],
forbid: [], notes: '',
}));
const good = cases.map((c, i) => ({
id: c.id,
claims: i === 4
? [{ claim: 'unrelated', kind: 'take', weight: 0.5 }]
: [{ claim: `claim-${i} text`, kind: 'take', weight: 0.5 }],
}));
const passing = scoreCorpus(cases, good);
expect(passing.by_kind.find(k => k.kind === 'take')!.precision).toBe(0.8);
expect(passing.by_kind.find(k => k.kind === 'take')!.recall).toBe(0.8);
expect(passing.graduated).toBe(true);
expect(GRADUATION.minPrecision).toBe(0.8);
});
test('per-variant and per-archetype rows explain the verdict', () => {
const cases: CorpusCase[] = [0, 1, 2].map(i => ({
id: `t-v${i}`, archetype: 't', category: 'take', page: { slug: `t/v${i}`, type: 'writing', title: 'T', body: 'x' },
expected: [{ claim_re: 'strongest team', kind: 'take', weight_min: 0.4, weight_max: 1 }],
forbid: ['fraudulent'], notes: '',
}));
const r = scoreCorpus(cases, [
{ id: 't-v0', claims: [{ claim: 'the strongest team', kind: 'take', weight: 0.8 }] },
{ id: 't-v1', claims: [{ claim: 'the strongest team', kind: 'take', weight: 0.8 }, { claim: 'acme is fraudulent', kind: 'fact', weight: 1 }] },
{ id: 't-v2', claims: null },
]);
expect(r.by_variant).toEqual([
{ id: 't-v0', archetype: 't', category: 'take', malformed: false, pass: true, expected: 1, matched: 1, predicted: 1, precise: 1, forbid_violations: 0 },
{ id: 't-v1', archetype: 't', category: 'take', malformed: false, pass: false, expected: 1, matched: 1, predicted: 2, precise: 1, forbid_violations: 1 },
{ id: 't-v2', archetype: 't', category: 'take', malformed: true, pass: false, expected: 1, matched: 0, predicted: 0, precise: 0, forbid_violations: 0 },
]);
expect(r.by_archetype).toEqual([
{ archetype: 't', category: 'take', variants: 3, variants_passed: 1, malformed: 1, expected: 3, matched: 2, predicted: 3, precise: 2, forbid_violations: 1 },
]);
});
test('replay --max N scores only the first N cases', () => {
const dir = mkdtempSync(join(tmpdir(), 'takes-eval-replay-'));
try {
const head = corpus.slice(0, 3);
const preds = head.map(c => ({
id: c.id,
claims: c.expected.map(e => ({ claim: new RegExp(e.claim_re, 'i').exec(c.page.body)![0], kind: e.kind, weight: (e.weight_min + e.weight_max) / 2 })),
}));
const file = join(dir, 'preds.jsonl');
writeFileSync(file, preds.map(p => JSON.stringify(p)).join('\n') + '\n');
const res = spawnSync(process.execPath, ['evals/takes-bootstrap/harness.mjs', '--replay', file, '--max', '3'], { cwd: repoRoot, encoding: 'utf8' });
expect(res.status).toBe(0);
const report = JSON.parse(res.stdout.slice(0, res.stdout.lastIndexOf('}') + 1));
expect(report.cases).toBe(3);
expect(report.malformed).toEqual([]);
expect(report.by_variant.map((v: { id: string }) => v.id)).toEqual(head.map(c => c.id));
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
test('the committed corpus scores 100% against its own oracle predictions (labels are satisfiable)', () => {
// Build oracle predictions straight from the labels: every expected
// regex must be satisfiable by SOME claim string (we synthesize one by
// using the page body, which each claim_re was authored against).
const oracle = corpus.map(c => ({
id: c.id,
claims: c.expected.map(e => {
const m = new RegExp(e.claim_re, 'i').exec(c.page.body) ?? new RegExp(e.claim_re, 'i').exec(c.page.title);
if (!m) throw new Error(`${c.id}: claim_re '${e.claim_re}' matches neither body nor title — unsatisfiable label`);
return { claim: m[0], kind: e.kind, weight: (e.weight_min + e.weight_max) / 2 };
}),
}));
const r = scoreCorpus(corpus, oracle);
expect(r.malformed).toEqual([]);
expect(r.forbid_violations).toEqual([]);
expect(r.overall.recall).toBe(1);
expect(r.overall.precision).toBe(1);
expect(r.graduated).toBe(true);
});
});