Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
19.7 KB
Raw Blame
/**
 * Helpers for scripts/bench/sync-freshness.ts (GBRA-133 lane A): the synthetic
 * corpus, commit generators, probes (in-process engine and MCP stdio), child
 * processes with per-line arrival timestamps, and the statistics. Everything
 * binds loopback and runs keyless.
 */
import { appendFileSync, cpSync, existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { dirname, join } from 'node:path';
import { execFileSync } from 'node:child_process';
import { cpus, totalmem } from 'node:os';

export const now = () => performance.timeOrigin + performance.now();
export const round1 = (n: number) => Math.round(n * 10) / 10;
export const sleep = (ms: number) => new Promise<void>(r => setTimeout(r, ms));

/** Linear-interpolated percentile. */
export function pct(xs: number[], p: number): number | null {
  if (!xs.length) return null;
  const s = [...xs].sort((a, b) => a - b);
  const rank = (p / 100) * (s.length - 1);
  const lo = Math.floor(rank), hi = Math.ceil(rank);
  return round1(s[lo]! + (s[hi]! - s[lo]!) * (rank - lo));
}
export interface Stats { n: number; p50: number | null; p95: number | null; max: number | null; min: number | null; mean: number | null; cv: number | null }
export function stats(xs: number[]): Stats {
  const n = xs.length;
  if (!n) return { n, p50: null, p95: null, max: null, min: null, mean: null, cv: null };
  const mean = xs.reduce((a, b) => a + b, 0) / n;
  const sd = n > 1 ? Math.sqrt(xs.reduce((a, b) => a + (b - mean) ** 2, 0) / (n - 1)) : 0;
  return { n, p50: pct(xs, 50), p95: pct(xs, 95), max: round1(Math.max(...xs)), min: round1(Math.min(...xs)), mean: round1(mean), cv: mean ? Math.round((sd / mean) * 1000) / 1000 : null };
}

export function git(root: string, ...args: string[]): string {
  return execFileSync('git', ['-C', root, ...args], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], maxBuffer: 256 * 1024 * 1024 }).trim();
}
export function commit(root: string, message: string, paths: string[] | 'all'): string {
  if (paths === 'all') git(root, 'add', '-A');
  else for (let i = 0; i < paths.length; i += 500) git(root, 'add', '-A', '--', ...paths.slice(i, i + 500));
  git(root, '-c', 'user.name=Bench Example', '-c', '[email protected]', 'commit', '-qm', message);
  return git(root, 'rev-parse', 'HEAD');
}

export function machineInfo(): Record<string, unknown> {
  let kernel = '';
  try { kernel = execFileSync('uname', ['-r'], { encoding: 'utf8' }).trim(); } catch { /* best effort */ }
  return { cores: cpus().length, cpu_model: cpus()[0]?.model ?? 'unknown', ram_gb: Math.round(totalmem() / 2 ** 30), kernel, bun: Bun.version };
}

// Synthetic corpus: deterministic pages, 1000 per directory.
const WORDS = ['ledger', 'harbor', 'quartz', 'meadow', 'signal', 'lantern', 'orbit', 'canvas', 'summit', 'thicket', 'ember', 'falcon',
  'garnet', 'hollow', 'island', 'juniper', 'kestrel', 'linden', 'marble', 'nectar', 'oracle', 'pepper', 'quiver', 'russet'];
export const pagePath = (i: number) => 'notes/d' + String(Math.floor(i / 1000)).padStart(3, '0') + '/n-' + String(i).padStart(6, '0') + '.md';
export const addedPath = (n: number) => 'notes/added/a-' + String(n).padStart(6, '0') + '.md';
export const slugOf = (path: string) => path.replace(/\.md$/, '');
export function pageContent(i: number, status = 'baseline'): string {
  const w = (k: number) => WORDS[(i * 7 + k * 13) % WORDS.length];
  const body = Array.from({ length: 6 }, (_, p) => 'Paragraph ' + p + ' of note ' + i + ' covers the ' + w(p) + ' ' + w(p + 1) + ' plan, the ' + w(p + 2) + ' follow-up and a decision about the ' + w(p + 3) + ' review.').join('\n\n');
  return '---\ntitle: Note ' + i + '\ntype: note\ntags: [bench, ' + w(0) + ']\n---\n# Note ' + i + '\n\nStatus: ' + status + '\n\n' + body + '\n\nSee [[' + slugOf(pagePath((i + 1) % 1000)) + ']].\n';
}
export function writeCorpus(root: string, pages: number): void {
  for (let i = 0; i < pages; i++) {
    const p = join(root, pagePath(i));
    if (i % 1000 === 0) mkdirSync(dirname(p), { recursive: true });
    writeFileSync(p, pageContent(i));
  }
  writeFileSync(join(root, 'README.md'), '# Bench brain (synthetic)\n');
}
/** Lowercase letters only, so every tokenizer keeps it as one word. */
export function token(): string {
  const a = 'abcdefghijklmnopqrstuvwxyz';
  let t = 'zq';
  for (let i = 0; i < 10; i++) t += a[Math.floor(Math.random() * 26)];
  return t;
}

// Corpus state between runs: which pages are deleted, how many were added.
export interface CorpusState { pages: number; deleted: number[]; added: number[]; nextAdd: number; cursor: number }
export function readState(dir: string): CorpusState { return JSON.parse(readFileSync(join(dir, 'corpus-state.json'), 'utf8')); }
export function writeState(dir: string, s: CorpusState): void { writeFileSync(join(dir, 'corpus-state.json'), JSON.stringify(s)); }

export type CommitKind = 'single' | 'batch' | 'backlog';
export interface Change { kind: CommitKind; token: string; sha: string; t_commit: number; t_commit_start: number; edited: string[]; added: string[]; deleted: string[]; source: string }

/** Writes and commits one change. single: 1 edit; batch: 50 edits + 5 adds + 5 deletes; backlog: 1000 edits. */
export function makeChange(root: string, dir: string, kind: CommitKind, source = 'bench'): Change {
  const st = readState(dir);
  const dead = new Set(st.deleted);
  const tok = token();
  const nEdit = kind === 'single' ? 1 : kind === 'batch' ? 50 : 1000;
  const nAdd = kind === 'batch' ? 5 : 0;
  const nDel = kind === 'batch' ? 5 : 0;
  const seen = new Set<number>();
  const pick = (): number => {
    for (;;) { const i = st.cursor; st.cursor = (st.cursor + 7919) % st.pages; if (!dead.has(i) && !seen.has(i)) { seen.add(i); return i; } }
  };
  const editIdx = Array.from({ length: nEdit }, pick);
  const delIdx = Array.from({ length: nDel }, pick);
  const edited = editIdx.map(pagePath);
  const added: string[] = [];
  const deleted = delIdx.map(pagePath);
  const t_commit_start = now();
  for (const i of editIdx) writeFileSync(join(root, pagePath(i)), pageContent(i, 'edited ' + tok));
  for (let k = 0; k < nAdd; k++) {
    const n = st.nextAdd++;
    const p = addedPath(n);
    mkdirSync(dirname(join(root, p)), { recursive: true });
    writeFileSync(join(root, p), '---\ntitle: Added ' + n + '\ntype: note\n---\n# Added ' + n + '\n\nStatus: added ' + tok + '\n\nA new bench note.\n');
    added.push(p); st.added.push(n);
  }
  for (const p of deleted) rmSync(join(root, p));
  for (const i of delIdx) st.deleted.push(i);
  const sha = commit(root, 'bench ' + kind + ' ' + tok, [...edited, ...added, ...deleted]);
  const t_commit = now();
  writeState(dir, st);
  return { kind, token: tok, sha, t_commit, t_commit_start, edited, added, deleted, source };
}

// Child processes with per-line arrival timestamps.
export interface Line { t: number; stream: 'out' | 'err'; text: string }
export interface Child { proc: ReturnType<typeof Bun.spawn>; lines: Line[]; exited: Promise<number>; onLine: (fn: (l: Line) => void) => void; exitAt: number | null; startedAt: number; stdoutRaw: () => string; write: (s: string) => void; kill: (sig?: NodeJS.Signals) => void }

export function spawnChild(cmd: string[], opts: { cwd: string; env: Record<string, string | undefined>; logFile?: string; stdin?: boolean }): Child {
  const startedAt = now();
  const proc = Bun.spawn(cmd, { cwd: opts.cwd, env: opts.env as Record<string, string>, stdin: opts.stdin ? 'pipe' : 'ignore', stdout: 'pipe', stderr: 'pipe' });
  const lines: Line[] = [];
  const listeners: Array<(l: Line) => void> = [];
  let raw = '';
  const emit = (line: Line) => {
    lines.push(line);
    if (opts.logFile) appendFileSync(opts.logFile, new Date(line.t).toISOString() + ' ' + line.stream + ' ' + line.text + '\n');
    for (const fn of listeners) fn(line);
  };
  const pump = async (stream: ReadableStream<Uint8Array>, name: 'out' | 'err') => {
    const dec = new TextDecoder();
    let buf = '';
    for await (const chunk of stream) {
      const t = now();
      const text = dec.decode(chunk, { stream: true });
      if (name === 'out') raw += text;
      buf += text;
      let nl: number;
      while ((nl = buf.indexOf('\n')) >= 0) { emit({ t, stream: name, text: buf.slice(0, nl) }); buf = buf.slice(nl + 1); }
    }
    if (buf) emit({ t: now(), stream: name, text: buf });
  };
  const child: Child = {
    proc, lines, exitAt: null, startedAt,
    exited: Promise.all([pump(proc.stdout as ReadableStream<Uint8Array>, 'out'), pump(proc.stderr as ReadableStream<Uint8Array>, 'err'), proc.exited])
      .then(([, , code]) => { child.exitAt = now(); return code as number; }),
    onLine: fn => { listeners.push(fn); },
    stdoutRaw: () => raw,
    write: s => { const w = proc.stdin as unknown as { write: (s: string) => void; flush?: () => void }; w.write(s); w.flush?.(); },
    kill: (sig = 'SIGTERM') => { try { proc.kill(sig); } catch { /* gone */ } },
  };
  return child;
}
export function waitLine(child: Child, re: RegExp, timeoutMs: number, fromT = 0): Promise<Line | null> {
  const hit = child.lines.find(l => l.t >= fromT && re.test(l.text));
  if (hit) return Promise.resolve(hit);
  return new Promise(resolve => {
    let done = false;
    const finish = (v: Line | null) => { if (!done) { done = true; clearTimeout(timer); resolve(v); } };
    const timer = setTimeout(() => finish(null), timeoutMs);
    child.onLine(l => { if (l.t >= fromT && re.test(l.text)) finish(l); });
    void child.exited.then(() => setTimeout(() => finish(null), 50));
  });
}

// MCP stdio client over a gbrain serve child.
export class McpClient {
  private id = 0;
  private pending = new Map<number, (v: { result?: any; error?: any }) => void>();
  constructor(private child: Child) {
    child.onLine(l => {
      if (l.stream !== 'out' || !l.text.startsWith('{')) return;
      try {
        const msg = JSON.parse(l.text) as { id?: number; result?: unknown; error?: unknown };
        const cb = typeof msg.id === 'number' ? this.pending.get(msg.id) : undefined;
        if (cb) { this.pending.delete(msg.id!); cb(msg); }
      } catch { /* not a JSON-RPC line */ }
    });
  }
  request(method: string, params: unknown, timeoutMs = 30_000): Promise<{ result?: any; error?: any }> {
    const id = ++this.id;
    return new Promise((resolve, reject) => {
      const timer = setTimeout(() => { this.pending.delete(id); reject(new Error('mcp ' + method + ' timed out')); }, timeoutMs);
      this.pending.set(id, v => { clearTimeout(timer); resolve(v); });
      this.child.write(JSON.stringify({ jsonrpc: '2.0', id, method, params }) + '\n');
    });
  }
  async init(): Promise<void> {
    const r = await this.request('initialize', { protocolVersion: '2025-06-18', capabilities: {}, clientInfo: { name: 'sync-freshness-bench', version: '1' } }, 120_000);
    if (r.error) throw new Error('mcp initialize failed: ' + JSON.stringify(r.error));
    this.child.write(JSON.stringify({ jsonrpc: '2.0', method: 'notifications/initialized' }) + '\n');
  }
  async call(name: string, args: Record<string, unknown>): Promise<string> {
    const r = await this.request('tools/call', { name, arguments: args });
    if (r.error) return 'ERROR ' + JSON.stringify(r.error);
    return (r.result?.content ?? []).map((c: { text?: string }) => c.text ?? '').join('\n');
  }
}

// Probes: has the change become visible?
export interface ProbeResult { getOk: boolean; searchHits: number; searchOk: boolean; partial: boolean; embedded?: boolean; vectorHit?: boolean; embeddedChunks?: number }

/** The vector the bench embedding stub returns for a text (managed-sync-catchup-lib.ts startEmbeddingStub), so a vector search can be aimed at a chunk. */
export function stubVector(text: string, dims: number): number[] {
  const v = new Array<number>(dims).fill(0);
  for (let i = 0; i < text.length; i++) v[(text.charCodeAt(i) * 31 + i) % dims] += 1;
  const norm = Math.sqrt(v.reduce((a, b) => a + b * b, 0)) || 1;
  return v.map(x => x / norm);
}
export interface Prober { check(c: Change): Promise<ProbeResult>; close(): Promise<void>; kind: string }

export const expectedHits = (c: Change) => c.edited.length + c.added.length;
/** get_page targets: the last edited path in path order, the last added page, and the first deleted page (must be gone). */
export function getTargets(c: Change): { present: string[]; gone: string[] } {
  return { present: [[...c.edited].sort().at(-1)!, ...(c.added.length ? [c.added.at(-1)!] : [])], gone: c.deleted.slice(0, 1) };
}

/**
 * `embeddedText` returns the input the embedding stub embedded for a token (the chunk text plus the context header gbrain
 * adds), so the vector check queries searchVector with exactly the vector the provider returned for that chunk.
 */
export async function engineProber(gbrainRoot: string, databaseUrl: string, embed = false, embeddedText?: (token: string) => string | undefined): Promise<Prober> {
  const { PostgresEngine } = await import(join(gbrainRoot, 'src/core/postgres-engine.ts'));
  const engine = new PostgresEngine();
  await engine.connect({ database_url: databaseUrl, poolSize: 2 });
  return {
    kind: 'engine',
    async check(c) {
      const { present, gone } = getTargets(c);
      const want = expectedHits(c);
      let getOk = true;
      const src = c.source;
      for (const path of present) {
        const p = await engine.getPage(slugOf(path), { sourceId: src }).catch(() => null);
        if (!p || !String(p.compiled_truth ?? '').includes(c.token)) { getOk = false; break; }
      }
      if (getOk) for (const path of gone) if (await engine.getPage(slugOf(path), { sourceId: src }).catch(() => null)) { getOk = false; break; }
      const hits = await engine.searchKeyword(c.token, { limit: 100, sourceId: src }).catch(() => []);
      const slugs = new Set((hits as Array<{ slug: string }>).map(h => h.slug));
      let embedded: boolean | undefined;
      let vectorHit: boolean | undefined;
      let embeddedChunks: number | undefined;
      if (embed) {
        const [{ total, done }] = await engine.executeRaw("SELECT count(*)::int AS total, count(*) FILTER (WHERE c.embedding IS NOT NULL)::int AS done FROM content_chunks c JOIN pages p ON p.id=c.page_id WHERE p.source_id=$2 AND p.deleted_at IS NULL AND c.chunk_text LIKE $1", ['%' + c.token + '%', src]);
        embedded = total >= Math.min(want, 1) && done === total && total > 0 && (want <= 100 || total >= want);
        embeddedChunks = done;
        const text = embeddedText?.(c.token);
        if (embedded && want <= 100 && text) {
          const target = slugOf([...c.edited].sort().at(-1)!);
          const vhits = await engine.searchVector(new Float32Array(stubVector(text, 1536)), { limit: 5, sourceId: src }).catch(() => []);
          vectorHit = (vhits as Array<{ slug: string }>).some(h => h.slug === target);
        }
      }
      if (want <= 100) return { getOk, searchHits: slugs.size, searchOk: slugs.size >= want, partial: slugs.size > 0, embedded, vectorHit, embeddedChunks };
      const [{ n }] = await engine.executeRaw("SELECT count(*)::int AS n FROM content_chunks c JOIN pages p ON p.id=c.page_id WHERE p.source_id=$2 AND p.deleted_at IS NULL AND c.chunk_text LIKE $1", ['%' + c.token + '%', src]);
      return { getOk, searchHits: n, searchOk: slugs.size >= 100 && n >= want, partial: slugs.size > 0, embedded, embeddedChunks };
    },
    async close() { await engine.disconnect(); },
  };
}

/**
 * Probes through a serve's MCP stdio. With `embedSeen` (the embedding stub returned a vector for an input carrying the
 * token) a single-file change also counts as embedded once get_page body:none reports every chunk embedded.
 */
export function mcpProber(client: McpClient, embedSeen?: (token: string) => boolean): Prober {
  return {
    kind: 'mcp',
    async check(c) {
      const { present, gone } = getTargets(c);
      const want = expectedHits(c);
      let getOk = true;
      for (const path of present) {
        const text = await client.call('get_page', { slug: slugOf(path), source_id: c.source }).catch(e => String(e));
        if (!text.includes(c.token)) { getOk = false; break; }
      }
      if (getOk) for (const path of gone) {
        const text = await client.call('get_page', { slug: slugOf(path), source_id: c.source }).catch(e => String(e));
        if (text.includes('Status: baseline')) { getOk = false; break; }
      }
      const text = await client.call('search', { query: c.token, limit: 100, source_id: c.source, match: 'keyword' }).catch(e => String(e));
      const slugs = new Set([...text.matchAll(/notes\/(?:d\d{3}\/n-\d{6}|added\/a-\d{6})/g)].map(m => m[0]));
      let embedded: boolean | undefined;
      if (embedSeen && want <= 100) {
        embedded = false;
        if (getOk && embedSeen(c.token)) {
          const none = await client.call('get_page', { slug: slugOf(present[0]!), source_id: c.source, body: 'none' }).catch(e => String(e));
          const m = /"chunks"\s*:\s*\{\s*"total"\s*:\s*(\d+)\s*,\s*"embedded"\s*:\s*(\d+)/.exec(none);
          embedded = !!m && Number(m[1]) > 0 && m[1] === m[2];
        }
      }
      return { getOk, searchHits: slugs.size, searchOk: slugs.size >= Math.min(want, 100), partial: slugs.size > 0, embedded };
    },
    async close() { /* the serve owns it */ },
  };
}

export interface Visibility { t_get: number | null; t_search: number | null; t_first_hit: number | null; t_embedded?: number | null; t_vector?: number | null; probes: number; probe_ms_mean: number }
/** Polls until both get_page and search see the change, or the deadline. */
export async function pollVisible(prober: Prober, c: Change, deadline: number, everyMs: number): Promise<Visibility> {
  let t_get: number | null = null, t_search: number | null = null, t_first: number | null = null, t_emb: number | null = null, t_vec: number | null = null;
  let probes = 0, probeMs = 0, wantEmb = false, wantVec = false;
  const pending = () => t_get === null || t_search === null || (wantEmb && t_emb === null) || (wantVec && t_vec === null);
  while (now() < deadline && pending()) {
    const t0 = now();
    const r = await prober.check(c);
    const t1 = now();
    probes++; probeMs += t1 - t0;
    if (r.getOk && t_get === null) t_get = t1;
    if (r.searchOk && t_search === null) t_search = t1;
    if (r.partial && t_first === null) t_first = t1;
    if (r.embedded !== undefined) { wantEmb = true; if (r.embedded && t_emb === null) t_emb = t1; }
    if (r.vectorHit !== undefined) { wantVec = true; if (r.vectorHit && t_vec === null) t_vec = t1; }
    if (pending()) await sleep(Math.max(0, everyMs - (now() - t0)));
  }
  return { t_get, t_search, t_first_hit: t_first, t_embedded: wantEmb ? t_emb : undefined, t_vector: wantVec ? t_vec : undefined, probes, probe_ms_mean: probes ? round1(probeMs / probes) : 0 };
}

/** CPU seconds (user + system, reaped children included) of a process and every live descendant, from /proc (Linux). */
export function treeCpuSeconds(pid: number): number {
  const tck = Number(execFileSync('getconf', ['CLK_TCK'], { encoding: 'utf8' }).trim()) || 100;
  const procs = new Map<number, { ppid: number; ticks: number }>();
  for (const name of readdirSync('/proc')) {
    if (!/^\d+$/.test(name)) continue;
    try {
      const stat = readFileSync('/proc/' + name + '/stat', 'utf8');
      const f = stat.slice(stat.lastIndexOf(')') + 2).split(' ');
      procs.set(Number(name), { ppid: Number(f[1]), ticks: Number(f[11]) + Number(f[12]) + Number(f[13]) + Number(f[14]) });
    } catch { /* exited while listing */ }
  }
  let ticks = 0;
  const queue = [pid];
  for (let q = queue.shift(); q !== undefined; q = queue.shift()) {
    ticks += procs.get(q)?.ticks ?? 0;
    for (const [child, p] of procs) if (p.ppid === q) queue.push(child);
  }
  return ticks / tck;
}

export function copyDir(from: string, to: string): void { rmSync(to, { recursive: true, force: true }); cpSync(from, to, { recursive: true }); }
export { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync };