Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
GBRA-218 A1: bench uses a distinct-word corpus, interleaved callers and a by-construction pre-filter oracle
garrytan37c256db7202
diff --git a/scripts/bench-member-access.ts b/scripts/bench-member-access.tsindex 942c92be97..81e9ff0a2b 100644--- a/scripts/bench-member-access.ts+++ b/scripts/bench-member-access.ts@@ -4,8 +4,9 @@* Seeds a synthetic one-source brain on in-process PGLite (repo-less), with a* share of pages naming readers, then runs remote search calls through MCP* dispatch as serve --http does, for an unbound token and a member token.- * Reports p50/p95 per cell and hit@10 against an oracle (the local caller's- * ranking filtered to the pages that caller may read). PGLite is in-process, so+ * Reports p50/p95 per cell and hit@10 against a pre-filter oracle: the pages+ * the member may read that contain every query word, known by construction+ * (the synthetic corpus gives each page distinct words; capped at 10). PGLite is in-process, so* runs on two revisions are directly comparable; on a revision without member* tables only the 0% cell runs.*@@ -27,7 +28,10 @@ const QUERIES = arg('--queries', 60);const JSON_MODE = process.argv.includes('--json');const DIMS = 1536;const SRC = 'bench-src';-const WORDS = 'harbor ledger quarry beacon orchard summit lantern canyon meadow glacier river forest delta prairie island coral'.split(' ');+const SYLLABLES = 'ba ce di fo gu ha je ki lo mu na pe qi ro su ta ve wi xo yu za'.split(' ');+const VOCAB = Array.from({ length: 4000 }, (_, i) => SYLLABLES[i % 21] + SYLLABLES[Math.floor(i / 21) % 21] + SYLLABLES[Math.floor(i / 441) % 21] + 'n');+/** Deterministic word j of page i (a fixed LCG), so both brains and both revisions see the same corpus. */+const wordOf = (i: number, j: number) => VOCAB[((i + 1) * 2654435761 + (j + 1) * 40503) % 4294967296 % VOCAB.length];function vectorFor(text: string): number[] {const v = new Array<number>(DIMS).fill(0);@@ -41,24 +45,35 @@ function vectorFor(text: string): number[] {}const pct = (xs: number[], p: number) => { const s = [...xs].sort((a, b) => a - b); return s[Math.min(s.length - 1, Math.floor(p * s.length))]; };-async function runCell(share: number) {+async function newEngine(): Promise<PGLiteEngine> {const engine = new PGLiteEngine();await engine.connect({ engine: 'pglite' } as never);await engine.initSchema();- const hasMembers = (await engine.executeRaw<{ t: string | null }>("SELECT to_regclass('access_members')::text AS t"))[0]?.t != null;- if (share > 0 && !hasMembers) { await engine.disconnect(); return null; }await engine.executeRaw('INSERT INTO sources (id, name) VALUES ($1, $1) ON CONFLICT (id) DO NOTHING', [SRC]);- const aliceReads = new Set<string>();- const restrictedEvery = share > 0 ? Math.round(1 / share) : 0;+ return engine;+}++/** Page i: its readers at this restricted share (null = open) and whether the member may read it. */+function pageSpec(i: number, share: number): { readers: string | null; memberReads: boolean } {+ const restricted = share > 0 && (share >= 0.5 ? i % 5 !== 0 : i % Math.round(1 / share) === 0);+ return { readers: restricted ? (i % 2 === 0 ? '[alice-example]' : '[group:team-b]') : null, memberReads: !restricted || i % 2 === 0 };+}++async function seed(engine: PGLiteEngine, share: number): Promise<void> {for (let i = 0; i < PAGES; i++) {- const slug = `notes/page-${i}`;- const words = [WORDS[i % WORDS.length], WORDS[(i * 7) % WORDS.length], WORDS[(i * 13 + 3) % WORDS.length]];- const restricted = share > 0 && (share >= 0.5 ? i % 5 !== 0 : i % restrictedEvery === 0);- const readers = restricted ? (i % 2 === 0 ? '[alice-example]' : '[group:team-b]') : null;- if (!restricted || i % 2 === 0) aliceReads.add(slug);+ const { readers } = pageSpec(i, share);+ const words = Array.from({ length: 12 }, (_, j) => wordOf(i, j));const fm = ['---', 'type: note', `title: Page ${i}`, ...(readers ? [`readable_by: ${readers}`] : []), '---'].join('\n');- await importFromContent(engine, slug, `${fm}\n# Page ${i}\n\nNotes about the ${words.join(' ')} for item ${i}.`, { sourceId: SRC });+ await importFromContent(engine, `notes/page-${i}`, `${fm}\n# Page ${i}\n\nNotes about the ${words.join(' ')} for item ${i}.`, { sourceId: SRC });}+}++async function runCell(share: number) {+ const engine = await newEngine();+ const hasMembers = (await engine.executeRaw<{ t: string | null }>("SELECT to_regclass('access_members')::text AS t"))[0]?.t != null;+ if (share > 0 && !hasMembers) { await engine.disconnect(); return null; }+ await seed(engine, share);+if (hasMembers) await engine.executeRaw("INSERT INTO access_members (id) VALUES ('alice-example') ON CONFLICT DO NOTHING");const mint = async (name: string) => mintLegacyToken(engine, { name, scopes: ['read'], takesHolders: ['world'], sourceGrant: [SRC] });const unbound = await mint('bench-unbound');@@ -78,24 +93,31 @@ async function runCell(share: number) {const slugsIn = (text: string) => [...new Set(text.match(/notes\/page-\d+/g) ?? [])];const cells: Record<string, unknown>[] = [];- for (const [who, token, readable] of [['unbound', unbound.token, null], ['member', alice.token, aliceReads]] as const) {- for (const match of ['keyword', 'hybrid'] as const) {- const times: number[] = [];- let hits = 0, possible = 0;- for (let q = 0; q < QUERIES; q++) {- const query = `${WORDS[q % WORDS.length]} ${WORDS[(q * 5 + 1) % WORDS.length]}`;- const params = { query, limit: 10, ...(match === 'keyword' ? { match: 'keyword' } : {}) };- const { ms, text } = await remote(token, params);- if (q >= 5) times.push(ms);- if (readable && share > 0) {- const local = await dispatchToolCall(engine, 'search', { ...params, limit: 200 }, { remote: false, sourceId: SRC });- const oracle = slugsIn(local.content.map(c => c.text).join('\n')).filter(s => readable.has(s)).slice(0, 10);+ for (const match of ['keyword', 'hybrid'] as const) {+ const times: Record<string, number[]> = { unbound: [], member: [] };+ let hits = 0, possible = 0;+ for (let q = 0; q < QUERIES + 5; q++) {+ const page = share > 0 ? [...Array(PAGES).keys()].filter(j => pageSpec(j, share).memberReads)[(q * 7919) % PAGES % Math.max(1, Math.round(PAGES * (share >= 0.5 ? 0.6 : 0.9)))] ?? 0 : (q * 7919) % PAGES;+ const query = `${wordOf(page, q % 12)} ${wordOf(page, (q + 5) % 12)}`;+ const params = { query, limit: 10, ...(match === 'keyword' ? { match: 'keyword' } : {}) };+ const order = q % 2 === 0 ? (['unbound', 'member'] as const) : (['member', 'unbound'] as const);+ for (const who of order) {+ const { ms, text } = await remote(who === 'member' ? alice.token : unbound.token, params);+ if (q >= 5) times[who].push(ms);+ if (who === 'member' && share > 0 && q >= 5) {+ const words = query.split(' ');+ const want = Array.from({ length: PAGES }, (_, j) => j)+ .filter(j => pageSpec(j, share).memberReads && words.every(w => Array.from({ length: 12 }, (_, k) => wordOf(j, k)).includes(w)))+ .slice(0, 10).map(j => `notes/page-${j}`);const got = new Set(slugsIn(text));- hits += oracle.filter(s => got.has(s)).length;- possible += oracle.length;+ hits += want.filter(s => got.has(s)).length;+ possible += want.length;}}- cells.push({ share, who, match, p50: +pct(times, 0.5).toFixed(2), p95: +pct(times, 0.95).toFixed(2), ...(possible ? { hit_at_10: +(hits / possible).toFixed(3) } : {}) });+ }+ for (const who of ['unbound', 'member'] as const) {+ cells.push({ share, who, match, p50: +pct(times[who], 0.5).toFixed(2), p95: +pct(times[who], 0.95).toFixed(2),+ ...(who === 'member' && possible ? { hit_at_10: +(hits / possible).toFixed(3) } : {}) });}}await engine.disconnect();