Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
4.8 KB
Raw Blame
#!/usr/bin/env bun
/**
 * Vector search latency and recall per source scope, against an existing
 * brain. Opt-in, never run in CI, not shipped in the CLI; reads only.
 *
 *   bun scripts/bench/efficiency/vector-scope-share.ts <postgres-url | pglite-dir> <scope>... \
 *     [--queries 25] [--limit 20] [--remote] [--window] [--json]
 *
 * A scope is a source id, several joined with `+`, or `all` (unscoped). Query
 * vectors are stored chunk embeddings plus fixed noise (the same chunks every
 * run, by md5 of the chunk id), so two builds compare on identical queries.
 * Each scope runs one warm-up search, then every query through the engine's
 * `searchVector`, and reports p50/p95 latency (nearest rank), the mean result
 * count, recall of result pages against the exact statement (`exactSql`, no
 * index; `--window` bounds it to the candidate window instead of every
 * candidate), the share of pages the scope holds per planner statistics, and
 * underfilled exits. `--remote` adds the private-page rule (`excludePrivate`).
 */
import { PostgresEngine } from '../../../src/core/postgres-engine.ts';
import { PGLiteEngine } from '../../../src/core/pglite-engine.ts';
import { buildVectorSearchStatement, PAGE_SOURCE_STATS_SQL, sourceScope, type PageSourceStats } from '../../../src/core/search/vector-statement.ts';
import type { SearchOpts } from '../../../src/core/types.ts';

const args = process.argv.slice(2);
const flag = (name: string) => args.includes(`--${name}`);
const option = (name: string, fallback: number) => {
  const at = args.indexOf(`--${name}`);
  return at >= 0 ? Number(args[at + 1]) : fallback;
};
const positional = args.filter((arg, i) => !arg.startsWith('--') && !['--queries', '--limit'].includes(args[i - 1] ?? ''));
const [target, ...scopes] = positional;
if (!target || scopes.length === 0) {
  console.error('usage: bun scripts/bench/efficiency/vector-scope-share.ts <postgres-url | pglite-dir> <scope>... [--queries N] [--limit N] [--remote] [--window] [--json]');
  process.exit(2);
}
const queries = option('queries', 25);
const limit = option('limit', 20);
const pglite = !/^postgres(ql)?:\/\//.test(target);
const engine = pglite ? new PGLiteEngine() : new PostgresEngine();
await engine.connect(pglite ? { engine: 'pglite', database_path: target } : { engine: 'postgres', database_url: target });

const vectors: Float32Array[] = [];
let seed = 1;
const noise = () => (seed = (seed * 1103515245 + 12345) % 2147483648) / 2147483648 - 0.5;
for (const { v } of await engine.executeRaw<{ v: string }>(`SELECT embedding::text AS v FROM content_chunks WHERE embedding IS NOT NULL ORDER BY md5(id::text) LIMIT ${queries}`)) {
  vectors.push(Float32Array.from(JSON.parse(v) as number[], x => x + noise() * 0.8));
}
const [stats] = await engine.executeRaw<PageSourceStats>(PAGE_SOURCE_STATS_SQL);
const percentile = (values: number[], p: number) => {
  const sorted = [...values].sort((a, b) => a - b);
  return +sorted[Math.min(sorted.length - 1, Math.ceil(p * sorted.length) - 1)]!.toFixed(1);
};

const rows = [];
for (const scope of scopes) {
  const opts: SearchOpts = { limit, ...(flag('remote') ? { excludePrivate: true } : {}), ...(scope === 'all' ? {} : { sourceIds: scope.split('+') }) };
  let underfilled = 0;
  await engine.searchVector(vectors[0]!, opts);
  const ms: number[] = [];
  const results: number[][] = [];
  for (const vector of vectors) {
    const start = performance.now();
    const hits = await engine.searchVector(vector, { ...opts, onVectorPoolMeta: () => { underfilled++; } });
    ms.push(performance.now() - start);
    results.push(hits.map(hit => hit.page_id));
  }
  let hit = 0, total = 0;
  for (const [k, vector] of vectors.entries()) {
    const stmt = buildVectorSearchStatement({ dialect: pglite ? 'pglite' : 'postgres', embedding: vector, limit, offset: 0, opts });
    const bound = [...stmt.params];
    bound[stmt.innerLimitIdx] = flag('window') ? stmt.innerLimit : null;
    const truth = new Set((await engine.executeRaw<{ page_id: number | null }>(stmt.exactSql, bound)).filter(row => row.page_id != null).map(row => Number(row.page_id)));
    total += truth.size;
    hit += results[k]!.filter(id => truth.has(id)).length;
  }
  const routed = await (engine as unknown as { vectorScope: (o: SearchOpts) => Promise<{ chunks?: number } | undefined> }).vectorScope(opts);
  rows.push({
    scope, share: scope === 'all' ? 1 : +(sourceScope(stats, opts)?.share ?? NaN).toFixed(4), chunks: routed?.chunks === undefined ? null : Math.round(routed.chunks), n: ms.length,
    p50_ms: percentile(ms, 0.5), p95_ms: percentile(ms, 0.95),
    results: +(results.reduce((sum, r) => sum + r.length, 0) / results.length).toFixed(1),
    recall: +(hit / Math.max(1, total)).toFixed(3), underfilled,
  });
}
await engine.disconnect();
if (flag('json')) console.log(JSON.stringify(rows));
else console.table(rows);