Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
skills: bound the turn-path freshness sweep; latency bench and module-graph test
garrytana8fbdd4fe0c3
diff --git a/.github/workflows/skill-resolver-bench.yml b/.github/workflows/skill-resolver-bench.ymlnew file mode 100644index 0000000000..285ae90ffc--- /dev/null+++ b/.github/workflows/skill-resolver-bench.yml@@ -0,0 +1,40 @@+name: Skill resolver bench++# GBRA-377: the skill resolver's preregistered latency bars (docs/guides/skill-resolver.md,+# "Latency"): a cold resolveSkills() turn (index opened again, rank, session read and one+# slice of the stat freshness sweep) at p95 <= 10 ms with 1,000 skills and <= 40 ms with+# 7,500 skills, on a 2-vCPU runner. scripts/bench-skill-resolver.ts exits 1 on a miss.+# Runs on pull requests that touch the resolver or the bench, and on dispatch.++on:+ pull_request:+ branches: [master]+ paths:+ - 'src/core/skills/resolver/**'+ - 'scripts/bench-skill-resolver.ts'+ - '.github/workflows/skill-resolver-bench.yml'+ workflow_dispatch:++permissions:+ contents: read++concurrency:+ group: ${{ github.workflow }}-${{ github.ref }}+ cancel-in-progress: true++jobs:+ bench:+ name: Skill resolver bench (1,000 and 7,500 skills, p95 bars)+ runs-on: ubicloud-standard-2-ubuntu-2404+ timeout-minutes: 20+ steps:+ - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262+ - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2+ with:+ bun-version: 1.4.4+ - name: Install dependencies (bounded, retried once)+ timeout-minutes: 8+ run: bun scripts/ci-bun-install.ts --frozen-lockfile --ignore-scripts+ - name: Run the bench+ timeout-minutes: 10+ run: bun scripts/bench-skill-resolver.ts --sizes 1000,7500 --turns 300diff --git a/scripts/bench-skill-resolver.ts b/scripts/bench-skill-resolver.tsnew file mode 100644index 0000000000..782c51419a--- /dev/null+++ b/scripts/bench-skill-resolver.ts@@ -0,0 +1,78 @@+#!/usr/bin/env bun+/**+ * GBRA-377 latency bench for the skill resolver library (the plan's+ * preregistered bars: cold load plus rank p95 at most 10 ms at 1,000+ * skills and at most 40 ms at 7,500 skills).+ *+ * For each size it writes a synthetic SKILL.md directory catalog (made-up+ * words, no real names), builds it, then times resolveSkills() turns. Every+ * timed turn is cold: it opens the index file again, as each CLI or hook+ * invocation does, and runs the turn-path stat freshness check over every+ * skill file (the clock is advanced past the check interval each turn).+ *+ * Usage: bun scripts/bench-skill-resolver.ts [--sizes 1000,7500] [--turns 200] [--json]+ * Exit 1 when a size misses its bar (--no-gate to only report).+ */+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { buildSkillIndex, resolveSkills } from '../src/core/skills/resolver/index.ts';++const BARS_MS: Record<number, number> = { 1000: 10, 7500: 40 };+const args = process.argv.slice(2);+const opt = (f: string, d: string) => { const i = args.indexOf(f); return i >= 0 && args[i + 1] ? args[i + 1]! : d; };+const sizes = opt('--sizes', '1000,7500').split(',').map(Number);+const turns = Number(opt('--turns', '200'));+const json = args.includes('--json');+const gate = !args.includes('--no-gate');++function word(i: number, salt: number): string {+ const C = 'bdfgklmnprtvz';+ const V = 'aeiou';+ let x = i * 7919 + salt * 104729 + 17;+ let w = '';+ for (let s = 0; s < 3; s++) { w += C[x % C.length]! + V[Math.floor(x / C.length) % V.length]!; x = Math.floor(x / (C.length * V.length)) + i + s * 31 + salt; }+ return w + C[i % C.length]!;+}+const VOCAB = Array.from({ length: 3000 }, (_, i) => word(i, 7));+const pick = (seed: number, n: number) => Array.from({ length: n }, (_, k) => VOCAB[(seed * 131 + k * 977) % VOCAB.length]!);++const p95 = (xs: number[]) => { const s = [...xs].sort((a, b) => a - b); return s[Math.min(s.length - 1, Math.ceil(0.95 * s.length) - 1)]!; };+const p50 = (xs: number[]) => { const s = [...xs].sort((a, b) => a - b); return s[Math.floor(s.length / 2)]!; };++const cells: { skills: number; turns: number; build_ms: number; p50_ms: number; p95_ms: number; bar_ms: number | null; pass: boolean }[] = [];+for (const n of sizes) {+ const root = mkdtempSync(join(tmpdir(), 'skres-bench-'));+ try {+ const skillsDir = join(root, 'skills');+ for (let i = 0; i < n; i++) {+ const d = join(skillsDir, `s${String(i).padStart(5, '0')}-${word(i, 3)}`);+ mkdirSync(d, { recursive: true });+ const trig = i % 3 === 0 ? `triggers:\n - "${pick(i, 3).join(' ')}"\n` : '';+ writeFileSync(join(d, 'SKILL.md'), `---\nname: ${word(i, 3)}-${word(i, 4)}\ndescription: "${pick(i + 1, 18).join(' ')}"\n${trig}---\n${pick(i + 2, 220).join(' ')}\n`);+ }+ const home = join(root, 'home');+ const t0 = performance.now();+ await buildSkillIndex({ catalog: 'bench', dirs: [skillsDir] }, { home });+ const buildMs = performance.now() - t0;+ let clock = Date.now();+ const times: number[] = [];+ for (let t = 0; t < turns + 10; t++) {+ clock += 61_000;+ const message = `please help with ${pick(t * 17 + 5, 6).join(' ')} today`;+ const s = performance.now();+ const r = await resolveSkills({ message, session_id: `bench-${t}` }, { home, now: () => clock });+ const e = performance.now() - s;+ if (r.output.error) throw new Error(r.output.error.code);+ if (t >= 10) times.push(e);+ }+ const bar = BARS_MS[n] ?? null;+ const p = p95(times);+ cells.push({ skills: n, turns, build_ms: Math.round(buildMs), p50_ms: Math.round(p50(times) * 100) / 100, p95_ms: Math.round(p * 100) / 100, bar_ms: bar, pass: bar === null || p <= bar });+ } finally {+ rmSync(root, { recursive: true, force: true });+ }+}+if (json) console.log(JSON.stringify({ bench: 'skill-resolver', cpus: navigator.hardwareConcurrency, cells }));+else for (const c of cells) console.log(`${c.skills} skills: build ${c.build_ms} ms, turn p50 ${c.p50_ms} ms, p95 ${c.p95_ms} ms${c.bar_ms !== null ? ` (bar ${c.bar_ms} ms: ${c.pass ? 'pass' : 'FAIL'})` : ''}`);+if (gate && cells.some(c => !c.pass)) process.exit(1);diff --git a/src/core/skills/resolver/build.ts b/src/core/skills/resolver/build.tsindex e0d6ec1109..4695ba1835 100644--- a/src/core/skills/resolver/build.ts+++ b/src/core/skills/resolver/build.ts@@ -283,10 +283,19 @@ export function sessionStatus(sessionId: string, opts: { home?: string; harness?return { session_id: sessionId, harness: opts.harness ?? null, shadow: Boolean(opts.shadow), turn: s.turn, spent: s.spent, budget, remaining: Math.max(0, budget - s.spent), suggested: Object.keys(s.last).length, reset: 'Only a compaction (or reset_session / --reset-session on the next turn) resets a session.' };}-/** The stat-tier check a turn runs (at most once per interval). */-export function dirFingerprintChanged(home: string, name: string): boolean {+/**+ * The stat-tier check a turn runs: up to `limit` of the fingerprint's+ * directory and file entries from `cursor` (the routing file every time).+ * `next` is 0 once the sweep has covered every entry.+ */+export function dirFingerprintSlice(home: string, name: string, cursor: number, limit: number): { changed: boolean; next: number } {try {const f = JSON.parse(readFileSync(catalogFile(home, name, 'fingerprint.json'), 'utf8')) as { fingerprint: DirFingerprint; routing: [string, number, number] | null };- return fingerprintChanged(f.fingerprint, f.routing ?? undefined);- } catch { return true; }+ const { dirs, files } = f.fingerprint;+ const total = dirs.length + files.length;+ const from = cursor > 0 && cursor < total ? cursor : 0;+ const to = Math.min(total, from + limit);+ const part: DirFingerprint = { dirs: dirs.slice(Math.min(from, dirs.length), Math.min(to, dirs.length)), files: files.slice(Math.max(0, from - dirs.length), Math.max(0, to - dirs.length)) };+ return { changed: fingerprintChanged(part, f.routing ?? undefined), next: to >= total ? 0 : to };+ } catch { return { changed: true, next: 0 }; }}diff --git a/src/core/skills/resolver/resolve.ts b/src/core/skills/resolver/resolve.tsindex 1b5be71ac9..bdeaa7e259 100644--- a/src/core/skills/resolver/resolve.ts+++ b/src/core/skills/resolver/resolve.ts@@ -23,7 +23,7 @@ import { createHash } from 'node:crypto';import { join } from 'node:path';import { VERSION } from '../../../version.ts';import { BLOCK_TOKEN_CAP, BLOCK_TOKENIZER, HARNESSES, renderBlock, type BlockLine } from './block.ts';-import { dirFingerprintChanged, listCatalogs, readReceipt, type BuildReceipt } from './build.ts';+import { dirFingerprintSlice, listCatalogs, readReceipt, type BuildReceipt } from './build.ts';import { readResolverSettings, validateSessionTokenBudget } from './config.ts';import { ResolverError } from './errors.ts';import { ensureDir, writeFileDurable } from './fsutil.ts';@@ -39,6 +39,8 @@ export const TOP_K_MAX = 8;export const EXCLUDE_MAX = 64;export const RESUGGEST_AFTER_TURNS = 10;export const STAT_CHECK_INTERVAL_MS = 60_000;+/** Stat calls one turn may spend on the freshness sweep; a larger catalog is swept over several turns. */+export const STAT_CHECK_SLICE = 1024;export interface TurnInput {message: string;@@ -182,12 +184,14 @@ function loadCatalog(home: string, name: string, now: number, autoRebuild: booleif (!stale && receipt.gbrain_version !== VERSION) stale = 'gbrain_version_changed';if (!stale && receipt.catalog.source.kind === 'dir') {const stamp = catalogFile(home, name, 'stat-check');- let cached: string | null = null;- try { if (now - statSync(stamp).mtimeMs < STAT_CHECK_INTERVAL_MS) cached = readFileSync(stamp, 'utf8'); } catch { /* none */ }- if (cached === 'fresh' || cached === 'stale') stale = cached === 'stale' ? 'source_changed' : null;- else {- stale = dirFingerprintChanged(home, name) ? 'source_changed' : null;- try { writeFileDurable(stamp, stale ? 'stale' : 'fresh'); } catch { /* read-only home: check again next turn */ }+ let last: { verdict: 'fresh' | 'stale'; cursor: number } | null = null;+ let age = Infinity;+ try { age = now - statSync(stamp).mtimeMs; last = JSON.parse(readFileSync(stamp, 'utf8')); } catch { /* none or unreadable: sweep from the start */ }+ if (last?.verdict === 'stale') stale = 'source_changed';+ else if (!(last?.verdict === 'fresh' && last.cursor === 0 && age < STAT_CHECK_INTERVAL_MS)) {+ const slice = dirFingerprintSlice(home, name, last?.verdict === 'fresh' ? last.cursor : 0, STAT_CHECK_SLICE);+ stale = slice.changed ? 'source_changed' : null;+ try { writeFileDurable(stamp, JSON.stringify({ verdict: slice.changed ? 'stale' : 'fresh', cursor: slice.changed ? 0 : slice.next })); } catch { /* read-only home: check again next turn */ }}}if (stale && autoRebuild) { try { writeFileDurable(catalogFile(home, name, 'stale.stamp'), `${JSON.stringify({ reason: stale })}\n`); } catch { /* best effort */ } }diff --git a/test/skill-resolver-bench.test.ts b/test/skill-resolver-bench.test.tsnew file mode 100644index 0000000000..df71b42bcd--- /dev/null+++ b/test/skill-resolver-bench.test.ts@@ -0,0 +1,19 @@+/**+ * Smoke run of scripts/bench-skill-resolver.ts on a tiny catalog, so the+ * GBRA-377 latency bench keeps running as the resolver it drives moves. The+ * timed bars run in .github/workflows/skill-resolver-bench.yml on a 2-vCPU+ * runner; this asserts the output shape only, never timings.+ */+import { describe, expect, test } from 'bun:test';+import { join } from 'node:path';++describe('bench-skill-resolver', () => {+ test('a tiny run reports one cell per size', () => {+ const run = Bun.spawnSync([process.execPath, 'scripts/bench-skill-resolver.ts', '--sizes', '40,90', '--turns', '5', '--json', '--no-gate'], { cwd: join(import.meta.dir, '..'), stdout: 'pipe', stderr: 'pipe' });+ expect(run.exitCode, run.stderr.toString()).toBe(0);+ const out = JSON.parse(run.stdout.toString().trim().split('\n').at(-1)!) as { bench: string; cells: { skills: number; turns: number; p95_ms: number; bar_ms: number | null; pass: boolean }[] };+ expect(out.bench).toBe('skill-resolver');+ expect(out.cells.map(c => [c.skills, c.turns, c.bar_ms, c.pass])).toEqual([[40, 5, null, true], [90, 5, null, true]]);+ expect(out.cells.every(c => c.p95_ms > 0)).toBe(true);+ }, 60_000);+});diff --git a/test/skill-resolver-module-graph.test.ts b/test/skill-resolver-module-graph.test.tsnew file mode 100644index 0000000000..da7cd294aa--- /dev/null+++ b/test/skill-resolver-module-graph.test.ts@@ -0,0 +1,55 @@+/**+ * GBRA-377 PR A: the skill resolver stays engine-free.+ *+ * 1. Protects: the turn path (src/core/skills/resolver/index.ts) loads only+ * its own leaf modules, the CJK ranges, the version and js-yaml, both+ * statically and after a bundled build and a turn (dynamic imports); the+ * CLI module loads no engine, database driver, operations table or AI+ * gateway. So a resolve works on a thin client, a brainless install and+ * while `gbrain serve` holds the PGLite lock.+ * 2. Fails when: a resolver module imports skill-catalog.ts, operations.ts,+ * an engine or a provider, statically or dynamically.+ * 3. Seam: none. A child process loads each entry into a fresh registry+ * (pattern: test/cold-read-module-graph.test.ts).+ */+import { describe, expect, test } from 'bun:test';+import { mkdtempSync, rmSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join, resolve } from 'node:path';++const REPO_ROOT = resolve(import.meta.dir, '..');++function loadedBy(script: string): string[] {+ const r = Bun.spawnSync([process.execPath, '-e', `${script}; console.log(JSON.stringify(Object.keys(require.cache)));`], { cwd: REPO_ROOT, stdout: 'pipe', stderr: 'pipe' });+ expect(r.exitCode, r.stderr.toString()).toBe(0);+ return (JSON.parse(r.stdout.toString().trim().split('\n').pop()!) as string[]).map(f => f.slice(REPO_ROOT.length + 1)).sort();+}++const LEAF = new Set(['src/core/cjk.ts', 'src/version.ts', 'package.json', 'src/core/skill-description.ts', 'node_modules/js-yaml/dist/js-yaml.mjs']);+const BUNDLED = new Set(['src/core/skills/bundled.ts', 'src/core/skills/bundled-skills.generated.ts', 'src/core/skill-frontmatter.ts', 'src/core/data-frontmatter.ts', 'src/core/effective-date.ts']);+const FORBIDDEN = /^(src\/core\/(operations\.ts|engine\.ts|engine-(factory|open|connect|construct)\.ts|pglite-engine|postgres-engine\.ts|skill-catalog\.ts|ai\/)|node_modules\/(@electric-sql|postgres|openai|@anthropic-ai))/;++describe('the skill resolver module graph', () => {+ test('static: the library loads only its leaf modules', () => {+ const loaded = loadedBy(`await import(${JSON.stringify(join(REPO_ROOT, 'src/core/skills/resolver/index.ts'))})`);+ expect(loaded).toContain('src/core/skills/resolver/resolve.ts');+ expect(loaded.filter(f => !f.startsWith('src/core/skills/resolver/') && !LEAF.has(f))).toEqual([]);+ }, 60_000);++ test('dynamic: a bundled build and a turn add only the bundled-skill reader and its assets', () => {+ const home = mkdtempSync(join(tmpdir(), 'skres-graph-'));+ try {+ const loaded = loadedBy(`const m = await import(${JSON.stringify(join(REPO_ROOT, 'src/core/skills/resolver/index.ts'))}); await m.buildSkillIndex({ catalog: 'gb', bundled: true }, { home: ${JSON.stringify(home)} }); await m.resolveSkills({ message: 'daily briefing' }, { home: ${JSON.stringify(home)} })`);+ expect(loaded).toContain('src/core/skills/bundled.ts');+ expect(loaded.filter(f => !f.startsWith('src/core/skills/resolver/') && !f.startsWith('skills/') && !LEAF.has(f) && !BUNDLED.has(f))).toEqual([]);+ } finally {+ rmSync(home, { recursive: true, force: true });+ }+ }, 60_000);++ test('the CLI module loads no engine, driver, operations table or provider', () => {+ const loaded = loadedBy(`await import(${JSON.stringify(join(REPO_ROOT, 'src/commands/skill-resolver.ts'))})`);+ expect(loaded).toContain('src/commands/skill-resolver.ts');+ expect(loaded.filter(f => FORBIDDEN.test(f))).toEqual([]);+ }, 60_000);+});diff --git a/test/skill-resolver.test.ts b/test/skill-resolver.test.tsindex 56170981d6..634abf8996 100644--- a/test/skill-resolver.test.ts+++ b/test/skill-resolver.test.ts@@ -28,6 +28,7 @@ import { estimateTokens } from '../src/core/skills/resolver/block.ts';import { acquireLease, holdsLease, releaseLease, tryAcquireLease } from '../src/core/skills/resolver/lock.ts';import { commitSession, gcSessions, loadSession, sessionKey } from '../src/core/skills/resolver/session.ts';import { catalogFile } from '../src/core/skills/resolver/paths.ts';+import { STAT_CHECK_SLICE } from '../src/core/skills/resolver/resolve.ts';const scratch: string[] = [];afterAll(() => { for (const d of scratch) rmSync(d, { recursive: true, force: true }); });@@ -625,6 +626,25 @@ describe('freshness and errors', () => {expect(await turn()).toBe(true);});+ test('a large dir catalog is swept over several turns, at most STAT_CHECK_SLICE stats each, and an edit near the end is still caught', async () => {+ const home = tmp();+ const root = tmp('skres-dir-');+ let last = '';+ for (let i = 0; i < 700; i++) last = skillDir(root, 's' + String(i).padStart(4, '0'), { name: 's' + i, description: 'Synthetic skill' });+ await buildSkillIndex({ catalog: 'c', dirs: [root] }, { home });+ const fp = JSON.parse(readFileSync(catalogFile(home, 'c', 'fingerprint.json'), 'utf8')) as { fingerprint: { dirs: unknown[]; files: unknown[] } };+ const entries = fp.fingerprint.dirs.length + fp.fingerprint.files.length;+ expect(entries).toBeGreaterThan(STAT_CHECK_SLICE);+ const t = Date.now();+ const turn = async () => (await resolveSkills({ message: 'synthetic' }, { home, now: () => t })).output.meta.index_stale;+ writeFileSync(last, readFileSync(last, 'utf8').replace('Synthetic skill', 'Synthetic skill, edited'));+ const verdicts: boolean[] = [];+ for (let i = 0; i < Math.ceil(entries / STAT_CHECK_SLICE); i++) verdicts.push(await turn());+ expect(verdicts.at(-1)).toBe(true);+ expect(verdicts[0]).toBe(false);+ expect(await turn()).toBe(true);+ });+test('GBRAIN_SKILLS_NO_AUTOREBUILD=1 still reports stale but schedules no rebuild', async () => {const home = tmp();const root = tmp('skres-dir-');