Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
#!/usr/bin/env bun
/**
* GBRA-377 latency bench for the skill resolver library (the plan's
* preregistered bars: cold load plus rank p95 at most 10 ms at 1,000
* skills and at most 40 ms at 7,500 skills).
*
* For each size it writes a synthetic SKILL.md directory catalog (made-up
* words, no real names), builds it, then times resolveSkills() turns. Every
* timed turn is cold: it opens the index file again, as each CLI or hook
* invocation does, and runs the turn-path stat freshness check over every
* skill file (the clock is advanced past the check interval each turn).
*
* Each size runs one discarded warm-up round, then --rounds short rounds of
* --turns turns, and is judged on the median round's p95. On a shared 2-vCPU
* runner the host stalls in bursts that last a second or more (a neighbour's
* job, not the resolver: the same tree measured one round at 50 ms beside
* rounds at 5 ms), and the first round after a build pays for cold caches.
* Three long rounds let one such burst land in two of them and fail the bar;
* nine short ones need a burst over most of the run, while a regression that
* slows every turn, or one turn in ten, still fails every round.
*
* Usage: bun scripts/bench-skill-resolver.ts [--sizes 1000,7500] [--turns 100] [--rounds 9] [--json]
* Exit 1 when a size misses its bar (--no-gate to only report).
*/
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { buildSkillIndex, resolveSkills } from '../src/core/skills/resolver/index.ts';
const BARS_MS: Record<number, number> = { 1000: 10, 7500: 40 };
const args = process.argv.slice(2);
const opt = (f: string, d: string) => { const i = args.indexOf(f); return i >= 0 && args[i + 1] ? args[i + 1]! : d; };
const sizes = opt('--sizes', '1000,7500').split(',').map(Number);
const turns = Number(opt('--turns', '100'));
const rounds = Number(opt('--rounds', '9'));
const json = args.includes('--json');
const gate = !args.includes('--no-gate');
function word(i: number, salt: number): string {
const C = 'bdfgklmnprtvz';
const V = 'aeiou';
let x = i * 7919 + salt * 104729 + 17;
let w = '';
for (let s = 0; s < 3; s++) { w += C[x % C.length]! + V[Math.floor(x / C.length) % V.length]!; x = Math.floor(x / (C.length * V.length)) + i + s * 31 + salt; }
return w + C[i % C.length]!;
}
const VOCAB = Array.from({ length: 3000 }, (_, i) => word(i, 7));
const pick = (seed: number, n: number) => Array.from({ length: n }, (_, k) => VOCAB[(seed * 131 + k * 977) % VOCAB.length]!);
const p95 = (xs: number[]) => { const s = [...xs].sort((a, b) => a - b); return s[Math.min(s.length - 1, Math.ceil(0.95 * s.length) - 1)]!; };
const p50 = (xs: number[]) => { const s = [...xs].sort((a, b) => a - b); return s[Math.floor(s.length / 2)]!; };
const cells: { skills: number; turns: number; rounds_p95_ms: number[]; build_ms: number; p50_ms: number; p95_ms: number; p95_all_ms: number; bar_ms: number | null; pass: boolean }[] = [];
const round2 = (x: number) => Math.round(x * 100) / 100;
for (const n of sizes) {
const root = mkdtempSync(join(tmpdir(), 'skres-bench-'));
try {
const skillsDir = join(root, 'skills');
for (let i = 0; i < n; i++) {
const d = join(skillsDir, `s${String(i).padStart(5, '0')}-${word(i, 3)}`);
mkdirSync(d, { recursive: true });
const trig = i % 3 === 0 ? `triggers:\n - "${pick(i, 3).join(' ')}"\n` : '';
writeFileSync(join(d, 'SKILL.md'), `---\nname: ${word(i, 3)}-${word(i, 4)}\ndescription: "${pick(i + 1, 18).join(' ')}"\n${trig}---\n${pick(i + 2, 220).join(' ')}\n`);
}
const home = join(root, 'home');
const t0 = performance.now();
await buildSkillIndex({ catalog: 'bench', dirs: [skillsDir] }, { home });
const buildMs = performance.now() - t0;
let clock = Date.now();
const all: number[] = [];
const roundsP95: number[] = [];
for (let round = -1; round < rounds; round++) {
const times: number[] = [];
for (let t = 0; t < turns + 10; t++) {
clock += 61_000;
const message = `please help with ${pick(t * 17 + round * 7919 + 5, 6).join(' ')} today`;
const s = performance.now();
const r = await resolveSkills({ message, session_id: `bench-${round}-${t}` }, { home, now: () => clock });
const e = performance.now() - s;
if (r.output.error) throw new Error(r.output.error.code);
if (t >= 10) times.push(e);
}
if (round < 0) continue;
all.push(...times);
roundsP95.push(p95(times));
}
const bar = BARS_MS[n] ?? null;
const p = p50(roundsP95);
cells.push({ skills: n, turns, rounds_p95_ms: roundsP95.map(round2), build_ms: Math.round(buildMs), p50_ms: round2(p50(all)), p95_ms: round2(p), p95_all_ms: round2(p95(all)), bar_ms: bar, pass: bar === null || p <= bar });
} finally {
rmSync(root, { recursive: true, force: true });
}
}
if (json) console.log(JSON.stringify({ bench: 'skill-resolver', cpus: navigator.hardwareConcurrency, cells }));
else for (const c of cells) console.log(`${c.skills} skills: build ${c.build_ms} ms, turn p50 ${c.p50_ms} ms, p95 ${c.p95_ms} ms (median of rounds ${c.rounds_p95_ms.join(', ')}; all turns ${c.p95_all_ms} ms)${c.bar_ms !== null ? ` (bar ${c.bar_ms} ms: ${c.pass ? 'pass' : 'FAIL'})` : ''}`);
if (gate && cells.some(c => !c.pass)) process.exit(1);