Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

bench(access): shared-agent audience cells; stop on any degraded search stage

garrytane5bee20d6198
diff --git a/scripts/bench-member-access.ts b/scripts/bench-member-access.ts
index 7c9831a2ae..3ee3961069 100644
--- a/scripts/bench-member-access.ts
+++ b/scripts/bench-member-access.ts
@@ -10,7 +10,10 @@
* runs on two revisions are directly comparable; on a revision without member
* tables only the 0% cell runs. --rules adds that many folder rules (A2) on
* prefixes no page lives under, so the cell measures what the folder layer
- * costs a brain whose pages it does not restrict.
+ * costs a brain whose pages it does not restrict. On a revision with shared agents (GBRA-231), restricted shares
+ * add two cells for a bot token whose harness names an audience: `aud1` (the member alone) and `aud2` (the member and
+ * a second member who reads the same restricted pages, so recall matches the member's). A search that fails, loses its
+ * vector arm or reports any degraded stage (other than the reranker this keyless bench never configures) stops the run.
*
* Usage: bun scripts/bench-member-access.ts [--pages 5000] [--queries 60] [--shares 0,0.2,0.8] [--rules 0] [--json]
*/
@@ -60,7 +63,7 @@ async function newEngine(): Promise<PGLiteEngine> {
/** Page i: its readers at this restricted share (null = open) and whether the member may read it. */
function pageSpec(i: number, share: number): { readers: string | null; memberReads: boolean } {
const restricted = share > 0 && (share >= 0.5 ? i % 5 !== 0 : i % Math.round(1 / share) === 0);
- return { readers: restricted ? (i % 2 === 0 ? '[alice-example]' : '[group:team-b]') : null, memberReads: !restricted || i % 2 === 0 };
+ return { readers: restricted ? (i % 2 === 0 ? '[alice-example, alice-twin]' : '[group:team-b]') : null, memberReads: !restricted || i % 2 === 0 };
}
async function seed(engine: PGLiteEngine, share: number): Promise<void> {
@@ -91,47 +94,57 @@ async function runCell(share: number, rules: number) {
const unbound = await mint('bench-unbound');
const alice = await mint('bench-alice');
if (bindMember) await engine.executeRaw('UPDATE access_tokens SET member_id = $2 WHERE id = $1::uuid', [alice.id, 'alice-example']);
+ const audiences = bindMember && await import('../src/core/access/audience.ts').then(() => true, () => false);
+ const bot = audiences ? await mint('bench-bot') : null;
+ if (bot) {
+ await engine.executeRaw("INSERT INTO access_members (id) VALUES ('alice-twin') ON CONFLICT DO NOTHING");
+ await engine.executeRaw("UPDATE access_tokens SET may_act_for = ARRAY['member:alice-example', 'member:alice-twin'] WHERE id = $1::uuid", [bot.id]);
+ }
const provider = new GBrainOAuthProvider({ sql: sqlQueryForEngine(engine), transaction: fn => engine.transaction(tx => fn(sqlQueryForEngine(tx))) });
- async function remote(token: string, params: Record<string, unknown>) {
+ async function remote(token: string, params: Record<string, unknown>, header?: string) {
const auth = await provider.verifyAccessToken(token) as unknown as AuthInfo;
const sourceId = auth.sourceId ?? 'default';
const localFederated = await noGrantFederatedScope(engine, auth.hasSourceGrant, sourceId);
const t0 = performance.now();
const out = await dispatchToolCall(engine, 'search', params, { remote: true, transport: 'http', auth, sourceId,
- takesHoldersAllowList: auth.takesHoldersAllowList ?? ['world'], ...(localFederated ? { localFederatedSourceIds: localFederated } : {}) });
+ takesHoldersAllowList: auth.takesHoldersAllowList ?? ['world'], ...(localFederated ? { localFederatedSourceIds: localFederated } : {}),
+ ...(header ? { onBehalfOf: { header } } : {}) } as Parameters<typeof dispatchToolCall>[3]);
const text = out.content.map(c => c.text).join('\n');
- if (out.isError || text.includes('vector_arm_failed')) throw new Error(`bench search degraded or failed: ${text.slice(0, 300)}`);
+ const degraded = ((out._meta?.retrieval as { degraded?: Array<{ stage?: string }> } | undefined)?.degraded ?? []).filter(d => d.stage !== 'reranker_skipped');
+ if (out.isError || text.includes('vector_arm_failed') || degraded.length) throw new Error(`bench search degraded or failed: ${JSON.stringify(degraded)} ${text.slice(0, 300)}`);
return { ms: performance.now() - t0, text };
}
const slugsIn = (text: string) => [...new Set(text.match(/notes\/page-\d+/g) ?? [])];
const cells: Record<string, unknown>[] = [];
for (const match of ['keyword', 'hybrid'] as const) {
- const times: Record<string, number[]> = { unbound: [], member: [] };
- let hits = 0, possible = 0;
+ const whos = bot ? ['unbound', 'member', 'aud1', 'aud2'] as const : ['unbound', 'member'] as const;
+ const times: Record<string, number[]> = Object.fromEntries(whos.map(w => [w, []]));
+ const hits: Record<string, number> = {}, possible: Record<string, number> = {};
for (let q = 0; q < QUERIES + 5; q++) {
const page = share > 0 ? [...Array(PAGES).keys()].filter(j => pageSpec(j, share).memberReads)[(q * 7919) % PAGES % Math.max(1, Math.round(PAGES * (share >= 0.5 ? 0.6 : 0.9)))] ?? 0 : (q * 7919) % PAGES;
const query = `${wordOf(page, q % 12)} ${wordOf(page, (q + 5) % 12)}`;
const params = { query, limit: 10, ...(match === 'keyword' ? { match: 'keyword' } : {}) };
- const order = q % 2 === 0 ? (['unbound', 'member'] as const) : (['member', 'unbound'] as const);
+ const order = whos.map((_, k) => whos[(k + q) % whos.length]!);
for (const who of order) {
- const { ms, text } = await remote(who === 'member' ? alice.token : unbound.token, params);
- if (q >= 5) times[who].push(ms);
- if (who === 'member' && share > 0 && q >= 5) {
+ const { ms, text } = await (who === 'aud1' ? remote(bot!.token, params, 'alice-example') : who === 'aud2' ? remote(bot!.token, params, 'alice-example, alice-twin')
+ : remote(who === 'member' ? alice.token : unbound.token, params));
+ if (q >= 5) times[who]!.push(ms);
+ if (who !== 'unbound' && share > 0 && q >= 5) {
const words = query.split(' ');
const want = Array.from({ length: PAGES }, (_, j) => j)
.filter(j => pageSpec(j, share).memberReads && words.every(w => Array.from({ length: 12 }, (_, k) => wordOf(j, k)).includes(w)))
.slice(0, 10).map(j => `notes/page-${j}`);
const got = new Set(slugsIn(text));
- hits += want.filter(s => got.has(s)).length;
- possible += want.length;
+ hits[who] = (hits[who] ?? 0) + want.filter(s => got.has(s)).length;
+ possible[who] = (possible[who] ?? 0) + want.length;
}
}
}
- for (const who of ['unbound', 'member'] as const) {
- cells.push({ share, rules, who, match, p50: +pct(times[who], 0.5).toFixed(2), p95: +pct(times[who], 0.95).toFixed(2),
- ...(who === 'member' && possible ? { hit_at_10: +(hits / possible).toFixed(3) } : {}) });
+ for (const who of whos) {
+ cells.push({ share, rules, who, match, p50: +pct(times[who]!, 0.5).toFixed(2), p95: +pct(times[who]!, 0.95).toFixed(2),
+ ...(possible[who] ? { hit_at_10: +(hits[who]! / possible[who]!).toFixed(3) } : {}) });
}
}
await engine.disconnect();