Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
5.3 KB
Raw Blame
#!/usr/bin/env bun
/**
 * GBRA-377 latency bench for the skill resolver library (the plan's
 * preregistered bars: cold load plus rank p95 at most 10 ms at 1,000
 * skills and at most 40 ms at 7,500 skills).
 *
 * For each size it writes a synthetic SKILL.md directory catalog (made-up
 * words, no real names), builds it, then times resolveSkills() turns. Every
 * timed turn is cold: it opens the index file again, as each CLI or hook
 * invocation does, and runs the turn-path stat freshness check over every
 * skill file (the clock is advanced past the check interval each turn).
 *
 * Each size runs one discarded warm-up round, then --rounds short rounds of
 * --turns turns, and is judged on the median round's p95. On a shared 2-vCPU
 * runner the host stalls in bursts that last a second or more (a neighbour's
 * job, not the resolver: the same tree measured one round at 50 ms beside
 * rounds at 5 ms), and the first round after a build pays for cold caches.
 * Three long rounds let one such burst land in two of them and fail the bar;
 * nine short ones need a burst over most of the run, while a regression that
 * slows every turn, or one turn in ten, still fails every round.
 *
 * Usage: bun scripts/bench-skill-resolver.ts [--sizes 1000,7500] [--turns 100] [--rounds 9] [--json]
 * Exit 1 when a size misses its bar (--no-gate to only report).
 */
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { buildSkillIndex, resolveSkills } from '../src/core/skills/resolver/index.ts';

const BARS_MS: Record<number, number> = { 1000: 10, 7500: 40 };
const args = process.argv.slice(2);
const opt = (f: string, d: string) => { const i = args.indexOf(f); return i >= 0 && args[i + 1] ? args[i + 1]! : d; };
const sizes = opt('--sizes', '1000,7500').split(',').map(Number);
const turns = Number(opt('--turns', '100'));
const rounds = Number(opt('--rounds', '9'));
const json = args.includes('--json');
const gate = !args.includes('--no-gate');

function word(i: number, salt: number): string {
  const C = 'bdfgklmnprtvz';
  const V = 'aeiou';
  let x = i * 7919 + salt * 104729 + 17;
  let w = '';
  for (let s = 0; s < 3; s++) { w += C[x % C.length]! + V[Math.floor(x / C.length) % V.length]!; x = Math.floor(x / (C.length * V.length)) + i + s * 31 + salt; }
  return w + C[i % C.length]!;
}
const VOCAB = Array.from({ length: 3000 }, (_, i) => word(i, 7));
const pick = (seed: number, n: number) => Array.from({ length: n }, (_, k) => VOCAB[(seed * 131 + k * 977) % VOCAB.length]!);

const p95 = (xs: number[]) => { const s = [...xs].sort((a, b) => a - b); return s[Math.min(s.length - 1, Math.ceil(0.95 * s.length) - 1)]!; };
const p50 = (xs: number[]) => { const s = [...xs].sort((a, b) => a - b); return s[Math.floor(s.length / 2)]!; };

const cells: { skills: number; turns: number; rounds_p95_ms: number[]; build_ms: number; p50_ms: number; p95_ms: number; p95_all_ms: number; bar_ms: number | null; pass: boolean }[] = [];
const round2 = (x: number) => Math.round(x * 100) / 100;
for (const n of sizes) {
  const root = mkdtempSync(join(tmpdir(), 'skres-bench-'));
  try {
    const skillsDir = join(root, 'skills');
    for (let i = 0; i < n; i++) {
      const d = join(skillsDir, `s${String(i).padStart(5, '0')}-${word(i, 3)}`);
      mkdirSync(d, { recursive: true });
      const trig = i % 3 === 0 ? `triggers:\n  - "${pick(i, 3).join(' ')}"\n` : '';
      writeFileSync(join(d, 'SKILL.md'), `---\nname: ${word(i, 3)}-${word(i, 4)}\ndescription: "${pick(i + 1, 18).join(' ')}"\n${trig}---\n${pick(i + 2, 220).join(' ')}\n`);
    }
    const home = join(root, 'home');
    const t0 = performance.now();
    await buildSkillIndex({ catalog: 'bench', dirs: [skillsDir] }, { home });
    const buildMs = performance.now() - t0;
    let clock = Date.now();
    const all: number[] = [];
    const roundsP95: number[] = [];
    for (let round = -1; round < rounds; round++) {
      const times: number[] = [];
      for (let t = 0; t < turns + 10; t++) {
        clock += 61_000;
        const message = `please help with ${pick(t * 17 + round * 7919 + 5, 6).join(' ')} today`;
        const s = performance.now();
        const r = await resolveSkills({ message, session_id: `bench-${round}-${t}` }, { home, now: () => clock });
        const e = performance.now() - s;
        if (r.output.error) throw new Error(r.output.error.code);
        if (t >= 10) times.push(e);
      }
      if (round < 0) continue;
      all.push(...times);
      roundsP95.push(p95(times));
    }
    const bar = BARS_MS[n] ?? null;
    const p = p50(roundsP95);
    cells.push({ skills: n, turns, rounds_p95_ms: roundsP95.map(round2), build_ms: Math.round(buildMs), p50_ms: round2(p50(all)), p95_ms: round2(p), p95_all_ms: round2(p95(all)), bar_ms: bar, pass: bar === null || p <= bar });
  } finally {
    rmSync(root, { recursive: true, force: true });
  }
}
if (json) console.log(JSON.stringify({ bench: 'skill-resolver', cpus: navigator.hardwareConcurrency, cells }));
else for (const c of cells) console.log(`${c.skills} skills: build ${c.build_ms} ms, turn p50 ${c.p50_ms} ms, p95 ${c.p95_ms} ms (median of rounds ${c.rounds_p95_ms.join(', ')}; all turns ${c.p95_all_ms} ms)${c.bar_ms !== null ? ` (bar ${c.bar_ms} ms: ${c.pass ? 'pass' : 'FAIL'})` : ''}`);
if (gate && cells.some(c => !c.pass)) process.exit(1);