Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
13.2 KB
Raw Blame
#!/usr/bin/env bun
/**
 * Hot-path bench (GBRA-66). Opt-in, never run in CI.
 *
 *   bun scripts/bench/efficiency/bench-hot.ts --engine pglite|postgres --label <name>
 *       [--n 20] [--warmup 3] [--only search,query_expand,...] [--sync-dir <dir>] [--out results/hot.jsonl]
 *
 * Runs against the brain bench-import.ts built at $BENCH_WORK/engines/<engine>-<label>.
 *   cold  one fresh `gbrain <cmd>` process per sample (what a CLI user or a
 *         shell-out agent pays: bun start, config, engine connect, op, exit)
 *   warm  one long-lived `gbrain serve` stdio MCP process, N tool calls after
 *         --warmup calls (what an MCP-connected agent pays per call)
 * Also: MCP stdio startup + first tool call, HTTP serve startup + first tool
 * call (fresh server per sample, bearer token), doctor, and a one-page sync.
 *
 * Query strings and slugs are sampled from the brain itself (three words of a
 * random page's body) and never printed or written; rows carry counts only.
 */
import { appendFileSync, existsSync, rmSync, writeFileSync } from 'node:fs';
import { join } from 'node:path';
import { CLI, REPO, WORK, appendRows, ensureDir, flag, has, lastJson, log, machine, readJsonl, rng, round, runCli, summarize, usageCost, type Row, type UsageRecord } from './lib.ts';

const engine = flag('engine', 'pglite') as 'pglite' | 'postgres';
const label = flag('label')!;
const N = Number(flag('n', '20'));
const WARMUP = Number(flag('warmup', '3'));
const only = flag('only')?.split(',');
const out = flag('out', join(WORK, 'results', 'hot.jsonl'))!;
const home = join(WORK, 'engines', `${engine}-${label}`);
if (!existsSync(home)) throw new Error(`no brain at ${home}; run bench-import.ts first`);
const env: Record<string, string> = { GBRAIN_HOME: home, GBRAIN_ALLOW_DEFAULT_WRITE: '1' };
const runDir = ensureDir(join(home, 'bench-run'));
const usageFile = join(runDir, 'hot.usage.jsonl');
rmSync(usageFile, { force: true });
const M = machine();
const want = (k: string) => !only || only.includes(k);

/** Minimal newline-delimited JSON-RPC client for `gbrain serve` over stdio. */
class StdioMcp {
  proc: ReturnType<typeof Bun.spawn>;
  private buf = '';
  private waiters = new Map<number, (v: any) => void>();
  private nextId = 1;
  constructor(extraEnv: Record<string, string> = {}) {
    this.proc = Bun.spawn([process.execPath, '--preload', join(import.meta.dir, 'preload-instrument.ts'), CLI, 'serve'], {
      cwd: REPO, env: { ...process.env, ...env, ...extraEnv }, stdin: 'pipe', stdout: 'pipe', stderr: 'ignore',
    });
    void this.pump();
  }
  private async pump() {
    const reader = (this.proc.stdout as ReadableStream<Uint8Array>).getReader();
    const dec = new TextDecoder();
    for (;;) {
      const { value, done } = await reader.read();
      if (done) break;
      this.buf += dec.decode(value, { stream: true });
      let i;
      while ((i = this.buf.indexOf('\n')) >= 0) {
        const line = this.buf.slice(0, i).trim();
        this.buf = this.buf.slice(i + 1);
        if (!line.startsWith('{')) continue;
        try {
          const msg = JSON.parse(line);
          if (typeof msg.id === 'number' && this.waiters.has(msg.id)) {
            this.waiters.get(msg.id)!(msg);
            this.waiters.delete(msg.id);
          }
        } catch { /* partial or non-JSON line */ }
      }
    }
  }
  request(method: string, params: unknown, timeoutMs = 120_000): Promise<any> {
    const id = this.nextId++;
    const p = new Promise<any>((resolve, reject) => {
      const t = setTimeout(() => { this.waiters.delete(id); reject(new Error(`timeout ${method}`)); }, timeoutMs);
      this.waiters.set(id, (v) => { clearTimeout(t); resolve(v); });
    });
    (this.proc.stdin as any).write(JSON.stringify({ jsonrpc: '2.0', id, method, params }) + '\n');
    (this.proc.stdin as any).flush?.();
    return p;
  }
  notify(method: string, params: unknown = {}) {
    (this.proc.stdin as any).write(JSON.stringify({ jsonrpc: '2.0', method, params }) + '\n');
  }
  async init() {
    await this.request('initialize', { protocolVersion: '2025-06-18', capabilities: {}, clientInfo: { name: 'gbra66-bench', version: '1' } });
    this.notify('notifications/initialized');
  }
  async call(name: string, args: Record<string, unknown>) {
    const r = await this.request('tools/call', { name, arguments: args });
    if (r.error) throw new Error(`rpc error ${r.error.code}`);
    if (r.result?.isError) throw new Error('tool error');
    return r.result;
  }
  async close() {
    try { (this.proc.stdin as any).end(); } catch { /* already closed */ }
    const t = setTimeout(() => this.proc.kill(), 3000);
    await this.proc.exited;
    clearTimeout(t);
  }
}

function toolJson(result: any): any {
  const text = result?.content?.find((c: any) => c.type === 'text')?.text;
  try { return JSON.parse(text); } catch { return null; }
}

async function sample<T>(n: number, fn: (i: number) => Promise<T>): Promise<{ ms: number[]; errors: number }> {
  const ms: number[] = [];
  let errors = 0;
  for (let i = 0; i < n; i++) {
    const t0 = performance.now();
    try { await fn(i); ms.push(performance.now() - t0); } catch { errors++; }
  }
  return { ms, errors };
}

async function main() {
  const stats = lastJson((await runCli(['stats', '--json'], env)).stdout) ?? {};
  const pages = Number(stats.page_count ?? 0);
  const chunks = Number(stats.chunk_count ?? 0);
  const rows: Row[] = [];
  const add = (command: string, mode: Row['mode'], s: { ms: number[]; errors: number }, extra?: Record<string, unknown>) => {
    if (s.ms.length === 0) { log(`${command} [${mode}] all ${s.errors} samples failed`); }
    const row: Row = { suite: 'hot', machine: M, engine, brain: label, pages, chunks, command, mode, ...summarize(s.ms.length ? s.ms : [NaN]), errors: s.errors, extra };
    row.n = s.ms.length;
    rows.push(row);
    log(`${command} [${mode}] n=${row.n} p50=${row.p50_ms} p95=${row.p95_ms} err=${s.errors}`);
  };

  // Sample slugs and queries from the brain through a warm server; never printed.
  const mcp = new StdioMcp({ BENCH_USAGE_LOG: usageFile });
  await mcp.init();
  const listed = toolJson(await mcp.call('list_pages', { limit: 500, sort: 'slug', source_id: '__all__' })) ?? [];
  const all: { slug: string; source_id: string }[] = (Array.isArray(listed) ? listed : listed.pages ?? []).map((p: any) => ({ slug: p.slug, source_id: p.source_id }));
  const r = rng(1234);
  const picks = Array.from({ length: Math.max(N, 20) }, () => all[Math.floor(r() * all.length)]!).filter(Boolean);
  const queries: string[] = [];
  for (const p of picks) {
    if (queries.length >= picks.length) break;
    const page = toolJson(await mcp.call('get_page', { slug: p.slug, source_id: p.source_id }));
    const words = String(page?.compiled_truth ?? '').replace(/[^\p{L}\p{N}\s]/gu, ' ').split(/\s+/).filter((w) => w.length > 3);
    if (words.length < 3) continue;
    const at = Math.floor(r() * (words.length - 3));
    queries.push(words.slice(at, at + 3).join(' '));
  }
  log(`sampled ${picks.length} pages, ${queries.length} queries (not printed)`);
  const q = (i: number) => queries[i % queries.length]!;
  const pk = (i: number) => picks[i % picks.length]!;

  const paths: { key: string; tool: string; args: (i: number) => Record<string, unknown>; cli: (i: number) => string[] }[] = [
    { key: 'search', tool: 'search', args: (i) => ({ query: q(i), limit: 10 }), cli: (i) => ['search', q(i)] },
    { key: 'query_expand', tool: 'query', args: (i) => ({ query: q(i), expand: true }), cli: (i) => ['query', q(i)] },
    { key: 'query_noexpand', tool: 'query', args: (i) => ({ query: q(i), expand: false }), cli: (i) => ['query', q(i), '--no-expand'] },
    { key: 'get_page', tool: 'get_page', args: (i) => ({ slug: pk(i).slug, source_id: pk(i).source_id }), cli: (i) => ['call', '--source', pk(i).source_id, 'get_page', JSON.stringify({ slug: pk(i).slug })] },
    { key: 'list_pages', tool: 'list_pages', args: () => ({ limit: 50 }), cli: () => ['list', '--limit', '50'] },
    { key: 'recall', tool: 'recall', args: (i) => ({ query: q(i), budget_tokens: 2000 }), cli: (i) => ['call', 'recall', JSON.stringify({ query: q(i), budget_tokens: 2000 })] },
  ];

  // Warm: the already-running stdio server.
  for (const p of paths) {
    if (!want(p.key)) continue;
    await sample(WARMUP, (i) => mcp.call(p.tool, p.args(i + 1000)));
    add(`mcp ${p.tool}${p.key.startsWith('query_') ? (p.key === 'query_expand' ? ' expand=true' : ' expand=false') : ''}`, 'warm', await sample(N, (i) => mcp.call(p.tool, p.args(i))));
  }
  await mcp.close();

  // Cold: one fresh CLI process per sample.
  for (const p of paths) {
    if (!want(p.key)) continue;
    add(`gbrain ${p.cli(0)[0]}${p.key === 'query_noexpand' ? ' --no-expand' : ''}${p.cli(0)[0] === 'call' ? ' ' + p.tool : ''}`, 'cold', await sample(N, async (i) => {
      const res = await runCli(p.cli(i), { ...env, BENCH_USAGE_LOG: usageFile }, { preload: true, timeoutMs: 120_000 });
      if (res.code !== 0) throw new Error(`exit ${res.code}`);
    }));
  }

  if (want('doctor')) {
    for (const args of [['doctor', '--json'], ['doctor', '--fast', '--json']]) {
      add(`gbrain ${args.join(' ')}`, 'cold', await sample(Math.min(N, Number(flag('doctor-n', String(N)))), async () => {
        const res = await runCli(args, env, { timeoutMs: 300_000 });
        if (res.code !== 0 && res.code !== 1) throw new Error(`exit ${res.code}`);
      }));
    }
  }

  if (want('mcp_startup')) {
    add('mcp stdio startup + get_page', 'cold', await sample(N, async (i) => {
      const c = new StdioMcp();
      try { await c.init(); await c.call('get_page', { slug: pk(i).slug, source_id: pk(i).source_id }); } finally { await c.close(); }
    }));
  }

  if (want('http_startup')) {
    const tokOut = await runCli(['auth', 'create', `bench-${Date.now()}`, '--scopes', 'read'], env);
    const token = tokOut.stdout.match(/gbrain_[A-Za-z0-9_]+/)?.[0];
    const port = 38100 + Math.floor(r() * 800);
    add('http serve startup + get_page', 'cold', await sample(token ? N : 0, async (i) => {
      const proc = Bun.spawn([process.execPath, CLI, 'serve', '--http', '--port', String(port), '--bind', '127.0.0.1'], { cwd: REPO, env: { ...process.env, ...env }, stdout: 'ignore', stderr: 'ignore' });
      try {
        const deadline = performance.now() + 60_000;
        for (;;) {
          try { if ((await fetch(`http://127.0.0.1:${port}/health`)).ok) break; } catch { /* not listening yet */ }
          if (performance.now() > deadline) throw new Error('http startup timeout');
          await Bun.sleep(20);
        }
        const res = await fetch(`http://127.0.0.1:${port}/mcp`, {
          method: 'POST',
          headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json', Accept: 'application/json, text/event-stream' },
          body: JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'tools/call', params: { name: 'get_page', arguments: { slug: pk(i).slug, source_id: pk(i).source_id } } }),
        });
        const body = await res.text();
        if (!res.ok || body.includes('"isError":true')) throw new Error(`http ${res.status}`);
      } finally {
        proc.kill();
        await proc.exited;
      }
    }));
  }

  const syncDir = flag('sync-dir');
  if (want('sync') && syncDir) {
    const git = async (...a: string[]) => {
      const p = Bun.spawn(['git', '-C', syncDir, ...a], { stdout: 'ignore', stderr: 'pipe' });
      if ((await p.exited) !== 0) throw new Error(`git ${a[0]} failed`);
    };
    if (!existsSync(join(syncDir, '.git'))) {
      await git('init', '-q');
      await git('add', '-A');
      await git('commit', '-q', '-m', 'bench baseline');
    }
    const syncArgs = ['sync', '--source', 'syncbench', '--no-pull', '--no-embed', '--json'];
    await runCli(['sources', 'add', 'syncbench', '--path', syncDir, '--no-federated'], env);
    const first = await runCli(syncArgs, env, { timeoutMs: 900_000 });
    add('gbrain sync --source (first sync of a git source, --no-embed)', 'once', { ms: first.code === 0 ? [first.ms] : [], errors: first.code === 0 ? 0 : 1 }, { source_pages: (await new Response(Bun.spawn(['git', '-C', syncDir, 'ls-files', '*.md'], { stdout: 'pipe' }).stdout).text()).split('\n').filter(Boolean).length });
    const files = (await new Response(Bun.spawn(['git', '-C', syncDir, 'ls-files', '*.md'], { stdout: 'pipe' }).stdout).text()).split('\n').filter(Boolean);
    add('gbrain sync (1-page change, --no-embed)', 'cold', await sample(N, async (i) => {
      const f = join(syncDir, files[Math.floor(r() * files.length)]!);
      appendFileSync(f, `\nBench edit ${i} ${Date.now()}.\n`);
      await git('commit', '-q', '-am', `bench edit ${i}`);
      const res = await runCli(syncArgs, env, { timeoutMs: 300_000 });
      if (res.code !== 0) throw new Error(`exit ${res.code}`);
    }));
    add('gbrain sync (no change)', 'cold', await sample(Math.min(N, 10), async () => {
      const res = await runCli(syncArgs, env, { timeoutMs: 300_000 });
      if (res.code !== 0) throw new Error(`exit ${res.code}`);
    }));
  }

  const usage = usageCost(readJsonl<UsageRecord>(usageFile));
  for (const row of rows) row.extra = { ...(row.extra ?? {}) };
  appendRows(out, rows);
  writeFileSync(join(WORK, 'results', `hot-${engine}-${label}-usage.json`), JSON.stringify(usage, null, 1));
  log(`done: ${rows.length} rows, provider spend so far for this brain $${usage.usd}`);
}

await main();