Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

skills: bound the turn-path freshness sweep; latency bench and module-graph test

garrytana8fbdd4fe0c3
diff --git a/.github/workflows/skill-resolver-bench.yml b/.github/workflows/skill-resolver-bench.yml
new file mode 100644
index 0000000000..285ae90ffc
--- /dev/null
+++ b/.github/workflows/skill-resolver-bench.yml
@@ -0,0 +1,40 @@
+name: Skill resolver bench
+
+# GBRA-377: the skill resolver's preregistered latency bars (docs/guides/skill-resolver.md,
+# "Latency"): a cold resolveSkills() turn (index opened again, rank, session read and one
+# slice of the stat freshness sweep) at p95 <= 10 ms with 1,000 skills and <= 40 ms with
+# 7,500 skills, on a 2-vCPU runner. scripts/bench-skill-resolver.ts exits 1 on a miss.
+# Runs on pull requests that touch the resolver or the bench, and on dispatch.
+
+on:
+ pull_request:
+ branches: [master]
+ paths:
+ - 'src/core/skills/resolver/**'
+ - 'scripts/bench-skill-resolver.ts'
+ - '.github/workflows/skill-resolver-bench.yml'
+ workflow_dispatch:
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ bench:
+ name: Skill resolver bench (1,000 and 7,500 skills, p95 bars)
+ runs-on: ubicloud-standard-2-ubuntu-2404
+ timeout-minutes: 20
+ steps:
+ - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
+ - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
+ with:
+ bun-version: 1.4.4
+ - name: Install dependencies (bounded, retried once)
+ timeout-minutes: 8
+ run: bun scripts/ci-bun-install.ts --frozen-lockfile --ignore-scripts
+ - name: Run the bench
+ timeout-minutes: 10
+ run: bun scripts/bench-skill-resolver.ts --sizes 1000,7500 --turns 300
diff --git a/scripts/bench-skill-resolver.ts b/scripts/bench-skill-resolver.ts
new file mode 100644
index 0000000000..782c51419a
--- /dev/null
+++ b/scripts/bench-skill-resolver.ts
@@ -0,0 +1,78 @@
+#!/usr/bin/env bun
+/**
+ * GBRA-377 latency bench for the skill resolver library (the plan's
+ * preregistered bars: cold load plus rank p95 at most 10 ms at 1,000
+ * skills and at most 40 ms at 7,500 skills).
+ *
+ * For each size it writes a synthetic SKILL.md directory catalog (made-up
+ * words, no real names), builds it, then times resolveSkills() turns. Every
+ * timed turn is cold: it opens the index file again, as each CLI or hook
+ * invocation does, and runs the turn-path stat freshness check over every
+ * skill file (the clock is advanced past the check interval each turn).
+ *
+ * Usage: bun scripts/bench-skill-resolver.ts [--sizes 1000,7500] [--turns 200] [--json]
+ * Exit 1 when a size misses its bar (--no-gate to only report).
+ */
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { buildSkillIndex, resolveSkills } from '../src/core/skills/resolver/index.ts';
+
+const BARS_MS: Record<number, number> = { 1000: 10, 7500: 40 };
+const args = process.argv.slice(2);
+const opt = (f: string, d: string) => { const i = args.indexOf(f); return i >= 0 && args[i + 1] ? args[i + 1]! : d; };
+const sizes = opt('--sizes', '1000,7500').split(',').map(Number);
+const turns = Number(opt('--turns', '200'));
+const json = args.includes('--json');
+const gate = !args.includes('--no-gate');
+
+function word(i: number, salt: number): string {
+ const C = 'bdfgklmnprtvz';
+ const V = 'aeiou';
+ let x = i * 7919 + salt * 104729 + 17;
+ let w = '';
+ for (let s = 0; s < 3; s++) { w += C[x % C.length]! + V[Math.floor(x / C.length) % V.length]!; x = Math.floor(x / (C.length * V.length)) + i + s * 31 + salt; }
+ return w + C[i % C.length]!;
+}
+const VOCAB = Array.from({ length: 3000 }, (_, i) => word(i, 7));
+const pick = (seed: number, n: number) => Array.from({ length: n }, (_, k) => VOCAB[(seed * 131 + k * 977) % VOCAB.length]!);
+
+const p95 = (xs: number[]) => { const s = [...xs].sort((a, b) => a - b); return s[Math.min(s.length - 1, Math.ceil(0.95 * s.length) - 1)]!; };
+const p50 = (xs: number[]) => { const s = [...xs].sort((a, b) => a - b); return s[Math.floor(s.length / 2)]!; };
+
+const cells: { skills: number; turns: number; build_ms: number; p50_ms: number; p95_ms: number; bar_ms: number | null; pass: boolean }[] = [];
+for (const n of sizes) {
+ const root = mkdtempSync(join(tmpdir(), 'skres-bench-'));
+ try {
+ const skillsDir = join(root, 'skills');
+ for (let i = 0; i < n; i++) {
+ const d = join(skillsDir, `s${String(i).padStart(5, '0')}-${word(i, 3)}`);
+ mkdirSync(d, { recursive: true });
+ const trig = i % 3 === 0 ? `triggers:\n - "${pick(i, 3).join(' ')}"\n` : '';
+ writeFileSync(join(d, 'SKILL.md'), `---\nname: ${word(i, 3)}-${word(i, 4)}\ndescription: "${pick(i + 1, 18).join(' ')}"\n${trig}---\n${pick(i + 2, 220).join(' ')}\n`);
+ }
+ const home = join(root, 'home');
+ const t0 = performance.now();
+ await buildSkillIndex({ catalog: 'bench', dirs: [skillsDir] }, { home });
+ const buildMs = performance.now() - t0;
+ let clock = Date.now();
+ const times: number[] = [];
+ for (let t = 0; t < turns + 10; t++) {
+ clock += 61_000;
+ const message = `please help with ${pick(t * 17 + 5, 6).join(' ')} today`;
+ const s = performance.now();
+ const r = await resolveSkills({ message, session_id: `bench-${t}` }, { home, now: () => clock });
+ const e = performance.now() - s;
+ if (r.output.error) throw new Error(r.output.error.code);
+ if (t >= 10) times.push(e);
+ }
+ const bar = BARS_MS[n] ?? null;
+ const p = p95(times);
+ cells.push({ skills: n, turns, build_ms: Math.round(buildMs), p50_ms: Math.round(p50(times) * 100) / 100, p95_ms: Math.round(p * 100) / 100, bar_ms: bar, pass: bar === null || p <= bar });
+ } finally {
+ rmSync(root, { recursive: true, force: true });
+ }
+}
+if (json) console.log(JSON.stringify({ bench: 'skill-resolver', cpus: navigator.hardwareConcurrency, cells }));
+else for (const c of cells) console.log(`${c.skills} skills: build ${c.build_ms} ms, turn p50 ${c.p50_ms} ms, p95 ${c.p95_ms} ms${c.bar_ms !== null ? ` (bar ${c.bar_ms} ms: ${c.pass ? 'pass' : 'FAIL'})` : ''}`);
+if (gate && cells.some(c => !c.pass)) process.exit(1);
diff --git a/src/core/skills/resolver/build.ts b/src/core/skills/resolver/build.ts
index e0d6ec1109..4695ba1835 100644
--- a/src/core/skills/resolver/build.ts
+++ b/src/core/skills/resolver/build.ts
@@ -283,10 +283,19 @@ export function sessionStatus(sessionId: string, opts: { home?: string; harness?
return { session_id: sessionId, harness: opts.harness ?? null, shadow: Boolean(opts.shadow), turn: s.turn, spent: s.spent, budget, remaining: Math.max(0, budget - s.spent), suggested: Object.keys(s.last).length, reset: 'Only a compaction (or reset_session / --reset-session on the next turn) resets a session.' };
}
-/** The stat-tier check a turn runs (at most once per interval). */
-export function dirFingerprintChanged(home: string, name: string): boolean {
+/**
+ * The stat-tier check a turn runs: up to `limit` of the fingerprint's
+ * directory and file entries from `cursor` (the routing file every time).
+ * `next` is 0 once the sweep has covered every entry.
+ */
+export function dirFingerprintSlice(home: string, name: string, cursor: number, limit: number): { changed: boolean; next: number } {
try {
const f = JSON.parse(readFileSync(catalogFile(home, name, 'fingerprint.json'), 'utf8')) as { fingerprint: DirFingerprint; routing: [string, number, number] | null };
- return fingerprintChanged(f.fingerprint, f.routing ?? undefined);
- } catch { return true; }
+ const { dirs, files } = f.fingerprint;
+ const total = dirs.length + files.length;
+ const from = cursor > 0 && cursor < total ? cursor : 0;
+ const to = Math.min(total, from + limit);
+ const part: DirFingerprint = { dirs: dirs.slice(Math.min(from, dirs.length), Math.min(to, dirs.length)), files: files.slice(Math.max(0, from - dirs.length), Math.max(0, to - dirs.length)) };
+ return { changed: fingerprintChanged(part, f.routing ?? undefined), next: to >= total ? 0 : to };
+ } catch { return { changed: true, next: 0 }; }
}
diff --git a/src/core/skills/resolver/resolve.ts b/src/core/skills/resolver/resolve.ts
index 1b5be71ac9..bdeaa7e259 100644
--- a/src/core/skills/resolver/resolve.ts
+++ b/src/core/skills/resolver/resolve.ts
@@ -23,7 +23,7 @@ import { createHash } from 'node:crypto';
import { join } from 'node:path';
import { VERSION } from '../../../version.ts';
import { BLOCK_TOKEN_CAP, BLOCK_TOKENIZER, HARNESSES, renderBlock, type BlockLine } from './block.ts';
-import { dirFingerprintChanged, listCatalogs, readReceipt, type BuildReceipt } from './build.ts';
+import { dirFingerprintSlice, listCatalogs, readReceipt, type BuildReceipt } from './build.ts';
import { readResolverSettings, validateSessionTokenBudget } from './config.ts';
import { ResolverError } from './errors.ts';
import { ensureDir, writeFileDurable } from './fsutil.ts';
@@ -39,6 +39,8 @@ export const TOP_K_MAX = 8;
export const EXCLUDE_MAX = 64;
export const RESUGGEST_AFTER_TURNS = 10;
export const STAT_CHECK_INTERVAL_MS = 60_000;
+/** Stat calls one turn may spend on the freshness sweep; a larger catalog is swept over several turns. */
+export const STAT_CHECK_SLICE = 1024;
export interface TurnInput {
message: string;
@@ -182,12 +184,14 @@ function loadCatalog(home: string, name: string, now: number, autoRebuild: boole
if (!stale && receipt.gbrain_version !== VERSION) stale = 'gbrain_version_changed';
if (!stale && receipt.catalog.source.kind === 'dir') {
const stamp = catalogFile(home, name, 'stat-check');
- let cached: string | null = null;
- try { if (now - statSync(stamp).mtimeMs < STAT_CHECK_INTERVAL_MS) cached = readFileSync(stamp, 'utf8'); } catch { /* none */ }
- if (cached === 'fresh' || cached === 'stale') stale = cached === 'stale' ? 'source_changed' : null;
- else {
- stale = dirFingerprintChanged(home, name) ? 'source_changed' : null;
- try { writeFileDurable(stamp, stale ? 'stale' : 'fresh'); } catch { /* read-only home: check again next turn */ }
+ let last: { verdict: 'fresh' | 'stale'; cursor: number } | null = null;
+ let age = Infinity;
+ try { age = now - statSync(stamp).mtimeMs; last = JSON.parse(readFileSync(stamp, 'utf8')); } catch { /* none or unreadable: sweep from the start */ }
+ if (last?.verdict === 'stale') stale = 'source_changed';
+ else if (!(last?.verdict === 'fresh' && last.cursor === 0 && age < STAT_CHECK_INTERVAL_MS)) {
+ const slice = dirFingerprintSlice(home, name, last?.verdict === 'fresh' ? last.cursor : 0, STAT_CHECK_SLICE);
+ stale = slice.changed ? 'source_changed' : null;
+ try { writeFileDurable(stamp, JSON.stringify({ verdict: slice.changed ? 'stale' : 'fresh', cursor: slice.changed ? 0 : slice.next })); } catch { /* read-only home: check again next turn */ }
}
}
if (stale && autoRebuild) { try { writeFileDurable(catalogFile(home, name, 'stale.stamp'), `${JSON.stringify({ reason: stale })}\n`); } catch { /* best effort */ } }
diff --git a/test/skill-resolver-bench.test.ts b/test/skill-resolver-bench.test.ts
new file mode 100644
index 0000000000..df71b42bcd
--- /dev/null
+++ b/test/skill-resolver-bench.test.ts
@@ -0,0 +1,19 @@
+/**
+ * Smoke run of scripts/bench-skill-resolver.ts on a tiny catalog, so the
+ * GBRA-377 latency bench keeps running as the resolver it drives moves. The
+ * timed bars run in .github/workflows/skill-resolver-bench.yml on a 2-vCPU
+ * runner; this asserts the output shape only, never timings.
+ */
+import { describe, expect, test } from 'bun:test';
+import { join } from 'node:path';
+
+describe('bench-skill-resolver', () => {
+ test('a tiny run reports one cell per size', () => {
+ const run = Bun.spawnSync([process.execPath, 'scripts/bench-skill-resolver.ts', '--sizes', '40,90', '--turns', '5', '--json', '--no-gate'], { cwd: join(import.meta.dir, '..'), stdout: 'pipe', stderr: 'pipe' });
+ expect(run.exitCode, run.stderr.toString()).toBe(0);
+ const out = JSON.parse(run.stdout.toString().trim().split('\n').at(-1)!) as { bench: string; cells: { skills: number; turns: number; p95_ms: number; bar_ms: number | null; pass: boolean }[] };
+ expect(out.bench).toBe('skill-resolver');
+ expect(out.cells.map(c => [c.skills, c.turns, c.bar_ms, c.pass])).toEqual([[40, 5, null, true], [90, 5, null, true]]);
+ expect(out.cells.every(c => c.p95_ms > 0)).toBe(true);
+ }, 60_000);
+});
diff --git a/test/skill-resolver-module-graph.test.ts b/test/skill-resolver-module-graph.test.ts
new file mode 100644
index 0000000000..da7cd294aa
--- /dev/null
+++ b/test/skill-resolver-module-graph.test.ts
@@ -0,0 +1,55 @@
+/**
+ * GBRA-377 PR A: the skill resolver stays engine-free.
+ *
+ * 1. Protects: the turn path (src/core/skills/resolver/index.ts) loads only
+ * its own leaf modules, the CJK ranges, the version and js-yaml, both
+ * statically and after a bundled build and a turn (dynamic imports); the
+ * CLI module loads no engine, database driver, operations table or AI
+ * gateway. So a resolve works on a thin client, a brainless install and
+ * while `gbrain serve` holds the PGLite lock.
+ * 2. Fails when: a resolver module imports skill-catalog.ts, operations.ts,
+ * an engine or a provider, statically or dynamically.
+ * 3. Seam: none. A child process loads each entry into a fresh registry
+ * (pattern: test/cold-read-module-graph.test.ts).
+ */
+import { describe, expect, test } from 'bun:test';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join, resolve } from 'node:path';
+
+const REPO_ROOT = resolve(import.meta.dir, '..');
+
+function loadedBy(script: string): string[] {
+ const r = Bun.spawnSync([process.execPath, '-e', `${script}; console.log(JSON.stringify(Object.keys(require.cache)));`], { cwd: REPO_ROOT, stdout: 'pipe', stderr: 'pipe' });
+ expect(r.exitCode, r.stderr.toString()).toBe(0);
+ return (JSON.parse(r.stdout.toString().trim().split('\n').pop()!) as string[]).map(f => f.slice(REPO_ROOT.length + 1)).sort();
+}
+
+const LEAF = new Set(['src/core/cjk.ts', 'src/version.ts', 'package.json', 'src/core/skill-description.ts', 'node_modules/js-yaml/dist/js-yaml.mjs']);
+const BUNDLED = new Set(['src/core/skills/bundled.ts', 'src/core/skills/bundled-skills.generated.ts', 'src/core/skill-frontmatter.ts', 'src/core/data-frontmatter.ts', 'src/core/effective-date.ts']);
+const FORBIDDEN = /^(src\/core\/(operations\.ts|engine\.ts|engine-(factory|open|connect|construct)\.ts|pglite-engine|postgres-engine\.ts|skill-catalog\.ts|ai\/)|node_modules\/(@electric-sql|postgres|openai|@anthropic-ai))/;
+
+describe('the skill resolver module graph', () => {
+ test('static: the library loads only its leaf modules', () => {
+ const loaded = loadedBy(`await import(${JSON.stringify(join(REPO_ROOT, 'src/core/skills/resolver/index.ts'))})`);
+ expect(loaded).toContain('src/core/skills/resolver/resolve.ts');
+ expect(loaded.filter(f => !f.startsWith('src/core/skills/resolver/') && !LEAF.has(f))).toEqual([]);
+ }, 60_000);
+
+ test('dynamic: a bundled build and a turn add only the bundled-skill reader and its assets', () => {
+ const home = mkdtempSync(join(tmpdir(), 'skres-graph-'));
+ try {
+ const loaded = loadedBy(`const m = await import(${JSON.stringify(join(REPO_ROOT, 'src/core/skills/resolver/index.ts'))}); await m.buildSkillIndex({ catalog: 'gb', bundled: true }, { home: ${JSON.stringify(home)} }); await m.resolveSkills({ message: 'daily briefing' }, { home: ${JSON.stringify(home)} })`);
+ expect(loaded).toContain('src/core/skills/bundled.ts');
+ expect(loaded.filter(f => !f.startsWith('src/core/skills/resolver/') && !f.startsWith('skills/') && !LEAF.has(f) && !BUNDLED.has(f))).toEqual([]);
+ } finally {
+ rmSync(home, { recursive: true, force: true });
+ }
+ }, 60_000);
+
+ test('the CLI module loads no engine, driver, operations table or provider', () => {
+ const loaded = loadedBy(`await import(${JSON.stringify(join(REPO_ROOT, 'src/commands/skill-resolver.ts'))})`);
+ expect(loaded).toContain('src/commands/skill-resolver.ts');
+ expect(loaded.filter(f => FORBIDDEN.test(f))).toEqual([]);
+ }, 60_000);
+});
diff --git a/test/skill-resolver.test.ts b/test/skill-resolver.test.ts
index 56170981d6..634abf8996 100644
--- a/test/skill-resolver.test.ts
+++ b/test/skill-resolver.test.ts
@@ -28,6 +28,7 @@ import { estimateTokens } from '../src/core/skills/resolver/block.ts';
import { acquireLease, holdsLease, releaseLease, tryAcquireLease } from '../src/core/skills/resolver/lock.ts';
import { commitSession, gcSessions, loadSession, sessionKey } from '../src/core/skills/resolver/session.ts';
import { catalogFile } from '../src/core/skills/resolver/paths.ts';
+import { STAT_CHECK_SLICE } from '../src/core/skills/resolver/resolve.ts';
const scratch: string[] = [];
afterAll(() => { for (const d of scratch) rmSync(d, { recursive: true, force: true }); });
@@ -625,6 +626,25 @@ describe('freshness and errors', () => {
expect(await turn()).toBe(true);
});
+ test('a large dir catalog is swept over several turns, at most STAT_CHECK_SLICE stats each, and an edit near the end is still caught', async () => {
+ const home = tmp();
+ const root = tmp('skres-dir-');
+ let last = '';
+ for (let i = 0; i < 700; i++) last = skillDir(root, 's' + String(i).padStart(4, '0'), { name: 's' + i, description: 'Synthetic skill' });
+ await buildSkillIndex({ catalog: 'c', dirs: [root] }, { home });
+ const fp = JSON.parse(readFileSync(catalogFile(home, 'c', 'fingerprint.json'), 'utf8')) as { fingerprint: { dirs: unknown[]; files: unknown[] } };
+ const entries = fp.fingerprint.dirs.length + fp.fingerprint.files.length;
+ expect(entries).toBeGreaterThan(STAT_CHECK_SLICE);
+ const t = Date.now();
+ const turn = async () => (await resolveSkills({ message: 'synthetic' }, { home, now: () => t })).output.meta.index_stale;
+ writeFileSync(last, readFileSync(last, 'utf8').replace('Synthetic skill', 'Synthetic skill, edited'));
+ const verdicts: boolean[] = [];
+ for (let i = 0; i < Math.ceil(entries / STAT_CHECK_SLICE); i++) verdicts.push(await turn());
+ expect(verdicts.at(-1)).toBe(true);
+ expect(verdicts[0]).toBe(false);
+ expect(await turn()).toBe(true);
+ });
+
test('GBRAIN_SKILLS_NO_AUTOREBUILD=1 still reports stale but schedules no rebuild', async () => {
const home = tmp();
const root = tmp('skres-dir-');