Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

fix(companions): stream Skill queries through a cached disk index

Chrisab44c582f156
diff --git a/apps/desktop/src/main/bot-import/README.md b/apps/desktop/src/main/bot-import/README.md
index cdc51667c9..1485001d3e 100644
--- a/apps/desktop/src/main/bot-import/README.md
+++ b/apps/desktop/src/main/bot-import/README.md
@@ -105,7 +105,7 @@ omitting it retains normal companion defaults, and a rejected request can be cor
catalog. Typed writes invalidate it synchronously; filesystem observation plus
directory identity checks cover hand edits, moves and replacement. Watcher
errors fall back to rebuilding, and missing generated catalogs are recreated.
- The cache retains at most 32 roots, not a maximum number of stored Skills.
+ The cache retains at most 32 runtime/query entries, not a maximum number of stored Skills.
In-flight catalog writers are serialized per root independently of cache
eviction, so an evicted snapshot cannot overwrite its newer replacement.
Failed writers release their successor; idle writer records are removed.
@@ -115,7 +115,16 @@ omitting it retains normal companion defaults, and a rejected request can be cor
previews while the scanner finds the original body offset; ordinary Skill
bodies are not loaded/parsed in full for indexing. Existing legacy migration
remains for small authored files; oversized legacy files retain their original
- bytes and use discovery. Full-text query/authoring APIs still read the originals.
+ bytes and use discovery. Authoring/detail APIs still read the originals.
+ Full-metadata queries reuse a separate disk index of enabled and disabled
+ headers. Construction streams headers (never bodies), sorts byte-bounded runs
+ and merges two records at a time; an oversized header occupies its own run.
+ Warm search and pagination stream that index and retain only the requested
+ response page, without reopening each source file. Full name/description
+ matching, stable name/slug ordering and disabled entries remain available.
+ Mutation invalidation and watcher recovery also cover this query index and
+ disabled metadata. A cold cache builds it once; subsequent queries still scan
+ index bytes for exact matching/counts, rather than maintaining a search database.
Failed directory scans leave the last complete catalog intact for retry.
- Skill discovery follows grouped Hermes directories, configured external roots,
native directory links and disabled lists. OpenClaw discovery covers workspace,
diff --git a/apps/desktop/src/main/maker-ipc/__tests__/botSkillQueryIndex.test.ts b/apps/desktop/src/main/maker-ipc/__tests__/botSkillQueryIndex.test.ts
new file mode 100644
index 0000000000..9a71f9f5f4
--- /dev/null
+++ b/apps/desktop/src/main/maker-ipc/__tests__/botSkillQueryIndex.test.ts
@@ -0,0 +1,143 @@
+import { promises as fs } from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { afterEach, beforeEach, expect, it, vi } from 'vitest';
+import { buildBotSkillQueryIndex, queryBotSkillIndex, searchBotSkillQueryIndex } from '../botSkillQueryIndex';
+import { botSkillRootDir, botSkillsDir, listBotSkills, saveBotSkill, deleteBotSkill } from '../botSkillStore';
+import { listBotSkillsForSession } from '../botSkillService';
+
+const observation = vi.hoisted(() => ({ native: true }));
+vi.mock('node:fs', async importOriginal => {
+ const actual = await importOriginal<typeof import('node:fs')>();
+ return { ...actual, watch: (...args: Parameters<typeof actual.watch>) => observation.native
+ ? actual.watch(...args) : { close() {}, on() { return this; } } };
+});
+
+let home: string;
+const bot = 'bot';
+const root = () => botSkillRootDir(home, bot);
+const deps = () => ({ userDataDir: home, resolveBotId: async () => ({ ok: true as const, botId: bot }) });
+const query = (params = {}) => queryBotSkillIndex(root(), home, bot, params);
+beforeEach(async () => { home = await fs.mkdtemp(path.join(os.tmpdir(), 'cindy-skill-query-')); });
+afterEach(async () => { observation.native = true; vi.restoreAllMocks(); await fs.rm(home, { recursive: true, force: true }); });
+async function write(slug: string, name: string, description = 'Searchable', disabled = false, body = 'Steps') {
+ const dir = path.join(root(), disabled ? 'disabled-skills' : 'skills', slug);
+ await fs.mkdir(dir, { recursive: true });
+ const file = path.join(dir, 'SKILL.md');
+ await fs.writeFile(file, `---\nname: ${name}\ndescription: ${description}\n---\n${body}`);
+ return file;
+}
+
+it('reuses its index for different terms/pages without enumerating or opening source Skills', async () => {
+ // No mutation is signalled in this case. Native notification delivery (which
+ // may include delayed setup writes) is exercised separately below.
+ observation.native = false;
+ await write('alpha', 'Alpha', 'red blue', false, 'Large body\n'.repeat(100_000));
+ await write('beta', 'Beta', 'blue', true);
+ expect(await listBotSkillsForSession({ callerSessionId: 's', limit: 1 }, deps()))
+ .toMatchObject({ ok: true, total: 2, nextOffset: 1, skills: [{ slug: 'alpha' }] });
+ const before = await fs.stat(path.join(root(), '.runtime-skills/query-catalog.jsonl'));
+ const originalOpen = fs.open.bind(fs);
+ const opens = vi.spyOn(fs, 'open').mockImplementation(async (...args) => {
+ if (/[/\\](?:disabled-)?skills[/\\]/.test(String(args[0]))) throw new Error('source reopened');
+ return originalOpen(...args);
+ });
+ const opendir = vi.spyOn(fs, 'opendir').mockRejectedValue(new Error('source enumerated'));
+ const readdir = vi.spyOn(fs, 'readdir').mockRejectedValue(new Error('shelf materialized'));
+ const readFile = vi.spyOn(fs, 'readFile').mockRejectedValue(new Error('full file read'));
+ expect(await listBotSkillsForSession({ callerSessionId: 's', query: 'BLUE', offset: 1 }, deps()))
+ .toMatchObject({ ok: true, total: 2, skills: [{ slug: 'beta', enabled: false }] });
+ expect(await query({ query: 'red blue' })).toMatchObject({ total: 1, skills: [{ slug: 'alpha' }] });
+ expect(await query({ query: 'missing' })).toEqual({ total: 0, skills: [] });
+ expect(opendir).not.toHaveBeenCalled(); expect(readdir).not.toHaveBeenCalled(); expect(readFile).not.toHaveBeenCalled();
+ expect(opens.mock.calls.every(([, mode]) => mode === 'r')).toBe(true);
+ expect((await fs.stat(path.join(root(), '.runtime-skills/query-catalog.jsonl'))).mtimeMs).toBe(before.mtimeMs);
+});
+
+it('preserves full metadata matching, locale ordering, ties, disabled duplicates and byte-bounded pages', async () => {
+ const values = [['z', 'Same'], ['a', 'Same'], ['中', '中文'], ['accent', 'Éclair'], ['alpha', 'Alpha']];
+ for (const [slug, name] of values) await write(slug, name, '界'.repeat(300));
+ await write('a', 'Same', 'Disabled duplicate', true);
+ await write('huge', 'Z'.repeat(600_000) + 'tail-name', 'D'.repeat(600_000) + 'tail-description');
+ const expected = await listBotSkills(home, bot);
+ const all = [];
+ let offset = 0;
+ do {
+ const page = await query({ offset, limit: 2 });
+ all.push(...page.skills.map(item => [item.slug, item.enabled]));
+ offset = page.nextOffset ?? 0;
+ } while (offset);
+ expect(all).toEqual(expected.map(item => [item.slug, item.enabled]));
+ expect(await query({ query: 'tail-name tail-description' })).toMatchObject({ total: 1, skills: [{ slug: 'huge' }] });
+ const result = await query({ query: 'tail-description' });
+ expect(Buffer.byteLength(JSON.stringify(result))).toBeLessThan(16 * 1024);
+});
+
+it('refreshes saved/deleted/disabled/external metadata and missing catalogs without mixing owners', async () => {
+ await write('alpha', 'Alpha', 'Original');
+ await write('off', 'Off', 'Disabled', true);
+ expect((await query()).total).toBe(2);
+ await saveBotSkill(home, bot, { name: 'alpha', description: 'Saved update', body: 'Steps' });
+ expect(await query({ query: 'saved' })).toMatchObject({ total: 1 });
+ await deleteBotSkill(home, bot, 'alpha');
+ expect((await query()).skills.map(item => item.slug)).toEqual(['off']);
+ await write('off', 'Off', 'Hand edited disabled', true);
+ // Native filesystem notifications can be coalesced (not synchronous with writeFile).
+ await vi.waitFor(async () => expect(await query({ query: 'edited' })).toMatchObject({ total: 1 }), { timeout: 5000 });
+ await fs.mkdir(botSkillsDir(home, bot), { recursive: true });
+ await fs.rename(path.join(root(), 'disabled-skills/off'), path.join(root(), 'skills/off'));
+ expect((await query()).skills[0].enabled).toBeUndefined();
+ const catalog = path.join(root(), '.runtime-skills/query-catalog.jsonl');
+ await fs.unlink(catalog);
+ expect((await query()).total).toBe(1);
+ const other = path.join(home, 'other-owner');
+ expect(await queryBotSkillIndex(botSkillRootDir(other, bot), other, bot, {})).toEqual({ total: 0, skills: [] });
+ expect((await query()).total).toBe(1);
+});
+
+it('checks the owner boundary again after streaming a query', async () => {
+ await write('alpha', 'Alpha');
+ let owner = 'before';
+ const originalOpen = fs.open.bind(fs);
+ vi.spyOn(fs, 'open').mockImplementation(async (...args) => {
+ if (String(args[0]).endsWith('query-catalog.jsonl') && args[1] === 'r') owner = 'after';
+ return originalOpen(...args);
+ });
+ expect(await listBotSkillsForSession({ callerSessionId: 's' }, { ...deps(), ownerScopeKey: () => owner }))
+ .toMatchObject({ ok: false, errorCode: 'OWNER_SCOPE_CHANGED' });
+});
+
+it('externally sorts 100,000 entries before enumeration completes and keeps the last page reachable', async () => {
+ const directory = path.join(root(), '.runtime-skills');
+ async function* source() {
+ for (let index = 99_999; index >= 0; index--) {
+ if (index === 95_000) {
+ const staging = (await fs.readdir(directory)).find(name => name.startsWith('query-'))!;
+ const runs = await fs.readdir(path.join(directory, staging));
+ expect(runs.length).toBeGreaterThan(1);
+ expect((await fs.stat(path.join(directory, staging, runs[0]))).size).toBeGreaterThan(0);
+ }
+ const slug = `s-${String(index).padStart(6, '0')}`;
+ yield { slug, name: slug, description: 'Fixture', updatedAt: '', dirPath: `/fixture/${slug}`,
+ filePath: `/fixture/${slug}/SKILL.md`, frontmatterBytes: 60, bodyStartLine: 5 };
+ }
+ }
+ const catalog = await buildBotSkillQueryIndex(root(), source());
+ const text = await fs.readFile(catalog, 'utf8');
+ const rows = text.trim().split('\n');
+ expect(rows).toHaveLength(100_000);
+ expect(JSON.parse(rows[0]).slug).toBe('s-000000');
+ expect(JSON.parse(rows.at(-1)!).slug).toBe('s-099999');
+ expect(await searchBotSkillQueryIndex(catalog, { query: 'Fixture', offset: 99_999, limit: 20 }))
+ .toMatchObject({ total: 100_000, skills: [{ slug: 's-099999' }] });
+ expect(await searchBotSkillQueryIndex(catalog, { query: 's-099999' }))
+ .toMatchObject({ total: 1, skills: [{ slug: 's-099999' }] });
+ async function* failed() {
+ yield { slug: 'partial', name: 'Partial', description: 'x'.repeat(300_000), updatedAt: '',
+ dirPath: '/fixture/partial', filePath: '/fixture/partial/SKILL.md', frontmatterBytes: 60, bodyStartLine: 5 };
+ throw new Error('fixture enumeration failure');
+ }
+ await expect(buildBotSkillQueryIndex(root(), failed())).rejects.toThrow('fixture enumeration failure');
+ expect(await fs.readFile(catalog, 'utf8')).toBe(text);
+ expect(await fs.readdir(directory)).toEqual(['query-catalog.jsonl']);
+}, 30_000);
diff --git a/apps/desktop/src/main/maker-ipc/__tests__/botSkillRuntimeCache.test.ts b/apps/desktop/src/main/maker-ipc/__tests__/botSkillRuntimeCache.test.ts
index e7d5b4093f..af1bca9551 100644
--- a/apps/desktop/src/main/maker-ipc/__tests__/botSkillRuntimeCache.test.ts
+++ b/apps/desktop/src/main/maker-ipc/__tests__/botSkillRuntimeCache.test.ts
@@ -11,6 +11,24 @@ afterEach(async () => { vi.restoreAllMocks(); await fs.rm(root, { recursive: tru
const projection = (name: string) => ({ pluginRoot: root,
skills: [{ name, description: name, path: root, filePath: path.join(root, 'SKILL.md') }] });
+it('keeps query artifacts separate from runtime mounts and invalidates both on a Skill write', async () => {
+ const catalog = path.join(root, 'query-catalog.jsonl');
+ const runtime = vi.fn(async () => projection('runtime'));
+ const query = vi.fn(async () => {
+ await fs.writeFile(catalog, 'fixture');
+ return { pluginRoot: root, skills: [], artifacts: [catalog] };
+ });
+ for (let turn = 0; turn < 2; turn++) {
+ expect((await cachedBotSkillRuntime(root, runtime)).skills[0].name).toBe('runtime');
+ expect((await cachedBotSkillRuntime(root, query, 'query')).artifacts).toEqual([catalog]);
+ }
+ expect(runtime).toHaveBeenCalledTimes(1); expect(query).toHaveBeenCalledTimes(1);
+ invalidateBotSkillRuntime(root);
+ await cachedBotSkillRuntime(root, runtime);
+ await cachedBotSkillRuntime(root, query, 'query');
+ expect(runtime).toHaveBeenCalledTimes(2); expect(query).toHaveBeenCalledTimes(2);
+});
+
it('shares concurrent hydration work and reconciles mutations during the read', async () => {
let release!: () => void;
const held = new Promise<void>(resolve => { release = resolve; });
diff --git a/apps/desktop/src/main/maker-ipc/__tests__/botSkillRuntimeSource.test.ts b/apps/desktop/src/main/maker-ipc/__tests__/botSkillRuntimeSource.test.ts
index 188b70b52f..dcd1df9be2 100644
--- a/apps/desktop/src/main/maker-ipc/__tests__/botSkillRuntimeSource.test.ts
+++ b/apps/desktop/src/main/maker-ipc/__tests__/botSkillRuntimeSource.test.ts
@@ -2,7 +2,7 @@ import { promises as fs } from 'node:fs';
import path from 'node:path';
import os from 'node:os';
import { afterEach, beforeEach, expect, it, vi } from 'vitest';
-import { iterateBotSkillRuntimeSummaries } from '../botSkillRuntimeSource';
+import { iterateBotSkillRuntimeSummaries, iterateBotSkillQuerySummaries } from '../botSkillRuntimeSource';
import { botSkillsDir } from '../botSkillStore';
let home: string;
@@ -15,13 +15,13 @@ async function source(slug: string, text: string) {
await fs.writeFile(file, text);
return file;
}
-async function read() {
+async function read(iterate = iterateBotSkillRuntimeSummaries) {
const items = [];
- for await (const item of iterateBotSkillRuntimeSummaries(home, 'bot')) items.push(item);
+ for await (const item of iterate(home, 'bot')) items.push(item);
return items;
}
-it('uses directory streaming and header-only reads for a large Skill body', async () => {
+it.each([['runtime', iterateBotSkillRuntimeSummaries], ['query', iterateBotSkillQuerySummaries]] as const)('%s uses directory streaming and header-only reads for a large Skill body', async (_kind, iterate) => {
const file = await source('large-body', `---\nname: fixture\ndescription: Preview\n---\n${'Body '.repeat(2 * 1024 * 1024)}`);
const readdir = vi.spyOn(fs, 'readdir').mockRejectedValue(new Error('unbounded enumeration'));
const readFile = vi.spyOn(fs, 'readFile').mockRejectedValue(new Error('full file read'));
@@ -37,7 +37,7 @@ it('uses directory streaming and header-only reads for a large Skill body', asyn
});
return handle;
});
- expect(await read()).toMatchObject([{ name: 'fixture', description: 'Preview', filePath: file, bodyStartLine: 5 }]);
+ expect(await read(iterate)).toMatchObject([{ name: 'fixture', description: 'Preview', filePath: file, bodyStartLine: 5 }]);
expect(bytesRead).toBeLessThanOrEqual(8192);
expect(readdir).not.toHaveBeenCalled(); expect(readFile).not.toHaveBeenCalled();
open.mockRestore();
diff --git a/apps/desktop/src/main/maker-ipc/botSkillQueryIndex.ts b/apps/desktop/src/main/maker-ipc/botSkillQueryIndex.ts
new file mode 100644
index 0000000000..e6f37490b7
--- /dev/null
+++ b/apps/desktop/src/main/maker-ipc/botSkillQueryIndex.ts
@@ -0,0 +1,160 @@
+import { promises as fs } from 'node:fs';
+import path from 'node:path';
+import type { FileHandle } from 'node:fs/promises';
+import type { BotSkillSummary } from './botSkillStore.js';
+import { cachedBotSkillRuntime } from './botSkillRuntimeCache.js';
+import { iterateBotSkillQuerySummaries } from './botSkillRuntimeSource.js';
+import { BOT_SKILL_RUNTIME_INDEX_BYTES, botSkillRuntimeSummary } from './botSkillRuntimeProjection.js';
+
+const SORT_BYTES = 256 * 1024;
+const IO_BYTES = 64 * 1024;
+
+// Match the full-list API's stable name sort: active shelf first, then each
+// shelf's lexical slug order. The original full metadata remains searchable.
+function compare(a: BotSkillSummary, b: BotSkillSummary): number {
+ return a.name.localeCompare(b.name) || Number(a.enabled === false) - Number(b.enabled === false)
+ || (a.slug < b.slug ? -1 : a.slug > b.slug ? 1 : 0);
+}
+
+/** One JSONL record at a time, including metadata larger than the read buffer. */
+async function* records(file: string): AsyncGenerator<BotSkillSummary> {
+ const handle = await fs.open(file, 'r');
+ const buffer = Buffer.alloc(IO_BYTES);
+ let parts: Buffer[] = [];
+ try {
+ while (true) {
+ const { bytesRead } = await handle.read(buffer, 0, buffer.length, null);
+ if (!bytesRead) break;
+ let start = 0;
+ while (start < bytesRead) {
+ const newline = buffer.indexOf(10, start);
+ const end = newline < 0 || newline >= bytesRead ? bytesRead : newline;
+ const segment = buffer.subarray(start, end);
+ if (end === bytesRead) parts.push(Buffer.from(segment));
+ else {
+ const text = (parts.length ? Buffer.concat([...parts, segment]) : segment).toString('utf8');
+ parts = [];
+ yield JSON.parse(text) as BotSkillSummary;
+ }
+ start = end + 1;
+ }
+ }
+ if (parts.length) throw new Error('Incomplete Skill query index');
+ } finally { await handle.close(); }
+}
+
+/** Batch small rows; a large row is written directly, never buffered with a shelf. */
+function bufferedWriter(handle: FileHandle) {
+ let pending: string[] = [];
+ let bytes = 0;
+ const flush = async () => {
+ if (pending.length) await handle.writeFile(pending.join(''));
+ pending = []; bytes = 0;
+ };
+ return {
+ flush,
+ async write(item: BotSkillSummary) {
+ const line = JSON.stringify(item) + '\n';
+ const size = Buffer.byteLength(line);
+ if (bytes + size > IO_BYTES) await flush();
+ if (size >= IO_BYTES) await handle.writeFile(line);
+ else { pending.push(line); bytes += size; }
+ },
+ };
+}
+
+/** Byte-bounded external sort: at most one chunk or two merge records in memory. */
+export async function buildBotSkillQueryIndex(root: string, source: AsyncIterable<BotSkillSummary>): Promise<string> {
+ const directory = path.join(root, '.runtime-skills');
+ await fs.mkdir(directory, { recursive: true });
+ const staging = await fs.mkdtemp(path.join(directory, 'query-'));
+ const catalog = path.join(directory, 'query-catalog.jsonl');
+ let sequence = 0;
+ const nextFile = () => path.join(staging, `${sequence++}.jsonl`);
+ try {
+ let runs: string[] = [];
+ let chunk: BotSkillSummary[] = [];
+ let bytes = 0;
+ const spill = async () => {
+ const file = nextFile();
+ const handle = await fs.open(file, 'wx', 0o600);
+ try {
+ const writer = bufferedWriter(handle);
+ chunk.sort(compare);
+ for (const item of chunk) await writer.write(item);
+ await writer.flush();
+ } finally { await handle.close(); }
+ runs.push(file); chunk = []; bytes = 0;
+ };
+ for await (const item of source) {
+ const size = Buffer.byteLength(JSON.stringify(item));
+ if (chunk.length && bytes + size > SORT_BYTES) await spill();
+ chunk.push(item); bytes += size;
+ if (bytes >= SORT_BYTES) await spill();
+ }
+ if (chunk.length || !runs.length) await spill();
+ while (runs.length > 1) {
+ const merged: string[] = [];
+ for (let index = 0; index < runs.length; index += 2) {
+ if (index + 1 === runs.length) { merged.push(runs[index]); continue; }
+ const output = nextFile();
+ const handle = await fs.open(output, 'wx', 0o600);
+ const left = records(runs[index]);
+ const right = records(runs[index + 1]);
+ try {
+ const writer = bufferedWriter(handle);
+ let a = await left.next(); let b = await right.next();
+ while (!a.done || !b.done) {
+ if (b.done || (!a.done && compare(a.value, b.value) <= 0)) {
+ await writer.write(a.value!); a = await left.next();
+ } else { await writer.write(b.value); b = await right.next(); }
+ }
+ await writer.flush();
+ } finally {
+ await Promise.all([left.return(undefined), right.return(undefined), handle.close()]);
+ }
+ await Promise.all([fs.unlink(runs[index]), fs.unlink(runs[index + 1])]);
+ merged.push(output);
+ }
+ runs = merged;
+ }
+ // Keep a previous complete catalog usable if enumeration or sorting fails.
+ await fs.rename(runs[0], catalog);
+ return catalog;
+ } finally { await fs.rm(staging, { recursive: true, force: true }); }
+}
+
+export async function queryBotSkillIndex(
+ root: string, userDataDir: string, botId: string,
+ params: { query?: string; offset?: number; limit?: number },
+) {
+ const catalog = path.join(root, '.runtime-skills/query-catalog.jsonl');
+ await cachedBotSkillRuntime(root, async () => ({
+ pluginRoot: root, skills: [],
+ artifacts: [await buildBotSkillQueryIndex(root, iterateBotSkillQuerySummaries(userDataDir, botId))],
+ }), 'query');
+ return searchBotSkillQueryIndex(catalog, params);
+}
+
+/** Search a stable catalog stream, retaining only the requested bounded page. */
+export async function searchBotSkillQueryIndex(catalog: string, params: { query?: string; offset?: number; limit?: number }) {
+ const terms = (params.query ?? '').toLocaleLowerCase().split(/\s+/).filter(Boolean);
+ const offset = Math.max(0, Math.floor(params.offset ?? 0));
+ const limit = Math.max(1, Math.min(50, Math.floor(params.limit ?? 20)));
+ const skills: ReturnType<typeof botSkillRuntimeSummary>[] = [];
+ let total = 0; let bytes = 0; let full = false;
+ for await (const item of records(catalog)) {
+ if (terms.length) {
+ const text = `${item.slug}\n${item.name}\n${item.description}`.toLocaleLowerCase();
+ if (!terms.every(term => text.includes(term))) continue;
+ }
+ if (total++ < offset || full) continue;
+ const summary = botSkillRuntimeSummary(item);
+ const size = Buffer.byteLength(JSON.stringify(summary));
+ if (skills.length && bytes + size > BOT_SKILL_RUNTIME_INDEX_BYTES) { full = true; continue; }
+ skills.push(summary); bytes += size;
+ if (skills.length >= limit) full = true;
+ }
+ const next = offset + skills.length;
+ return { skills, total, ...(next < total ? { nextOffset: next } : {}) };
+}
diff --git a/apps/desktop/src/main/maker-ipc/botSkillRuntimeCache.ts b/apps/desktop/src/main/maker-ipc/botSkillRuntimeCache.ts
index 64260201a0..20c58a23d7 100644
--- a/apps/desktop/src/main/maker-ipc/botSkillRuntimeCache.ts
+++ b/apps/desktop/src/main/maker-ipc/botSkillRuntimeCache.ts
@@ -3,6 +3,7 @@ import path from 'node:path';
type Projection = {
pluginRoot: string;
+ artifacts?: string[];
skills: { name: string; description: string; path: string; filePath: string }[];
};
type Entry = {
@@ -23,8 +24,10 @@ const MAX_CACHED_ROOTS = 32;
const writers = new Map<string, Promise<Projection>>();
export function invalidateBotSkillRuntime(root: string): void {
- const entry = entries.get(path.resolve(root));
- if (entry) entry.revision++;
+ for (const key of [path.resolve(root), `${path.resolve(root)}\0query`]) {
+ const entry = entries.get(key);
+ if (entry) entry.revision++;
+ }
}
function discard(root: string, entry: Entry) {
@@ -35,21 +38,22 @@ function discard(root: string, entry: Entry) {
}
/** Constant-size directory identity checks also detect delete/recreate and moves. */
-async function sourceStamp(root: string): Promise<string> {
- const [parent, skills] = await Promise.all([
- fs.stat(root).catch(() => null), fs.stat(path.join(root, 'skills')).catch(() => null),
- ]);
- return `${parent?.dev}:${parent?.ino}/${skills?.dev}:${skills?.ino}:${skills?.mtimeMs}:${skills?.ctimeMs}`;
+async function sourceStamp(root: string, shelves: string[]): Promise<string> {
+ const parent = await fs.stat(root).catch(() => null);
+ const children = await Promise.all(shelves.map(shelf => fs.stat(path.join(root, shelf)).catch(() => null)));
+ return `${parent?.dev}:${parent?.ino}/` + children.map(child => `${child?.dev}:${child?.ino}:${child?.mtimeMs}:${child?.ctimeMs}`).join('/');
}
/** Rebuild once per mutation, share concurrent hydrations, observe external edits. */
-export async function cachedBotSkillRuntime(root: string, build: () => Promise<Projection>): Promise<Projection> {
+export async function cachedBotSkillRuntime(root: string, build: () => Promise<Projection>, kind: 'runtime' | 'query' = 'runtime'): Promise<Projection> {
root = path.resolve(root);
- const stamp = await sourceStamp(root);
- let entry = entries.get(root);
+ const key = kind === 'runtime' ? root : `${root}\0query`;
+ const shelves = kind === 'runtime' ? ['skills'] : ['skills', 'disabled-skills'];
+ const stamp = await sourceStamp(root, shelves);
+ let entry = entries.get(key);
if (!entry) {
entry = { revision: 0, loadedRevision: -1, stamp: '', watchers: [] };
- entries.set(root, entry);
+ entries.set(key, entry);
}
if (entry.stamp !== stamp || !entry.watchers.length) {
// Reattach on directory replacement without forking an in-flight build.
@@ -64,10 +68,10 @@ export async function cachedBotSkillRuntime(root: string, build: () => Promise<P
// watcher covers a missing/replaced skills directory; recursive watching
// catches SKILL.md edits that do not change the directory timestamps.
observed.watchers.push(watch(root, { persistent: false }, (_event, filename) => {
- if (!filename || filename.toString() === 'skills') observed.revision++;
+ if (!filename || shelves.includes(filename.toString())) observed.revision++;
}));
- try {
- observed.watchers.push(watch(path.join(root, 'skills'), { recursive: true, persistent: false }, (_event, filename) => {
+ for (const shelf of shelves) try {
+ observed.watchers.push(watch(path.join(root, shelf), { recursive: true, persistent: false }, (_event, filename) => {
// Script outputs/venv caches do not alter the Skill metadata index.
if (!filename || /^[^/\\]+(?:[/\\]skill\.md)?$/i.test(filename.toString())) observed.revision++;
}));
@@ -86,8 +90,8 @@ export async function cachedBotSkillRuntime(root: string, build: () => Promise<P
}
}
// LRU eviction closes native handles; it does not remove any saved catalog.
- entries.delete(root);
- entries.set(root, entry);
+ entries.delete(key);
+ entries.set(key, entry);
while (entries.size > MAX_CACHED_ROOTS) {
const [oldRoot, oldEntry] = entries.entries().next().value!;
discard(oldRoot, oldEntry);
@@ -96,19 +100,19 @@ export async function cachedBotSkillRuntime(root: string, build: () => Promise<P
if (current.loading) return structuredClone(await current.loading);
if (current.watchers.length && current.value && current.loadedRevision === current.revision) {
// Generated files are disposable: deleting them must rebuild the catalog.
- const present = current.value.pluginRoot === root || await Promise.all([
- fs.access(path.join(current.value.pluginRoot, 'catalog.jsonl')),
- fs.access(current.value.skills[0].filePath),
- ]).then(() => true, () => false);
+ const artifacts = current.value.artifacts ?? (current.value.pluginRoot === root ? [] : [
+ path.join(current.value.pluginRoot, 'catalog.jsonl'), current.value.skills[0].filePath,
+ ]);
+ const present = await Promise.all(artifacts.map(file => fs.access(file))).then(() => true, () => false);
// Eviction may happen while checking generated files. Rejoin the active
// entry rather than enqueueing work from a now-discarded cache entry.
- if (entries.get(root) !== current) return cachedBotSkillRuntime(root, build);
+ if (entries.get(key) !== current) return cachedBotSkillRuntime(root, build, kind);
if (present && current.loadedRevision === current.revision) return structuredClone(current.value);
if (!present) current.revision++;
}
// Another hydration may have started rebuilding during the artifact check.
if (current.loading) return structuredClone(await current.loading);
- const previous = writers.get(root);
+ const previous = writers.get(key);
const loading = (async () => {
// A failed predecessor has already cleaned up its staging files; it must
// neither poison this retry nor race its final writes against this build.
@@ -118,17 +122,17 @@ export async function cachedBotSkillRuntime(root: string, build: () => Promise<P
do {
revision = current.revision;
value = await build();
- } while (revision !== current.revision && entries.get(root) === current);
+ } while (revision !== current.revision && entries.get(key) === current);
current.value = value;
current.loadedRevision = revision;
return structuredClone(value);
})();
current.loading = loading;
- writers.set(root, loading);
+ writers.set(key, loading);
try { return await loading; }
- catch (error) { discard(root, current); throw error; }
+ catch (error) { discard(key, current); throw error; }
finally {
- if (writers.get(root) === loading) writers.delete(root);
+ if (writers.get(key) === loading) writers.delete(key);
current.loading = undefined;
}
}
diff --git a/apps/desktop/src/main/maker-ipc/botSkillRuntimeSource.ts b/apps/desktop/src/main/maker-ipc/botSkillRuntimeSource.ts
index b44fdae6a2..3bd34c71fe 100644
--- a/apps/desktop/src/main/maker-ipc/botSkillRuntimeSource.ts
+++ b/apps/desktop/src/main/maker-ipc/botSkillRuntimeSource.ts
@@ -1,7 +1,7 @@
import { promises as fs } from 'node:fs';
import path from 'node:path';
import {
- BOT_SKILL_MAX_BODY_BYTES, botSkillsDir, readCompatibleBotSkillSource,
+ BOT_SKILL_MAX_BODY_BYTES, botSkillRootDir, botSkillsDir, readCompatibleBotSkillSource,
unescapeFrontmatterValue, type BotSkillSummary,
} from './botSkillStore.js';
@@ -9,7 +9,7 @@ import {
const LINE_PREVIEW_BYTES = 4096;
const READ_BYTES = 8192;
-async function readRuntimeHeader(filePath: string, slug: string, migrate = true): Promise<BotSkillSummary> {
+async function readRuntimeHeader(filePath: string, slug: string, migrate = true, fullMetadata = false): Promise<BotSkillSummary> {
const handle = await fs.open(filePath, 'r');
const fields = new Map<string, string>();
let frontmatterBytes = 0;
@@ -22,12 +22,13 @@ async function readRuntimeHeader(filePath: string, slug: string, migrate = true)
size = (await handle.stat()).size;
const buffer = Buffer.alloc(READ_BYTES);
let line = Buffer.alloc(0);
+ let lineParts: Buffer[] = [];
let lineBytes = 0;
let lineNumber = 0;
let done = false;
const finishLine = (newline: boolean) => {
lineNumber++;
- const text = line.toString('utf8').replace(/\r$/, '');
+ const text = (fullMetadata ? Buffer.concat(lineParts) : line).toString('utf8').replace(/\r$/, '');
const entire = lineBytes === line.length;
frontmatterBytes += lineBytes + (newline ? 1 : 0);
if (lineNumber === 1) {
@@ -45,11 +46,12 @@ async function readRuntimeHeader(filePath: string, slug: string, migrate = true)
if (separator > 0 && ['name', 'displayName', 'description', 'updatedAt'].includes(key)) {
let value = text.slice(separator + 1).trim();
// Close only the preview's quote. The original scalar stays untouched.
- if (!entire && /^["']/.test(value) && !value.endsWith(value[0])) value += value[0];
+ if (!fullMetadata && !entire && /^["']/.test(value) && !value.endsWith(value[0])) value += value[0];
fields.set(key, unescapeFrontmatterValue(value));
}
}
line = Buffer.alloc(0);
+ lineParts = [];
lineBytes = 0;
};
while (!done) {
@@ -64,6 +66,7 @@ async function readRuntimeHeader(filePath: string, slug: string, migrate = true)
const end = found >= 0 && found < bytesRead ? found : bytesRead;
const segment = buffer.subarray(offset, end);
lineBytes += segment.length;
+ if (fullMetadata) lineParts.push(Buffer.from(segment));
if (line.length < LINE_PREVIEW_BYTES) line = Buffer.concat([line, segment.subarray(0, LINE_PREVIEW_BYTES - line.length)]);
if (end < bytesRead) finishLine(true);
offset = end + 1;
@@ -75,7 +78,7 @@ async function readRuntimeHeader(filePath: string, slug: string, migrate = true)
// files. Huge hand-written legacy files remain intact and use body discovery.
if (migrate && legacy && size <= BOT_SKILL_MAX_BODY_BYTES + 16 * 1024) {
await readCompatibleBotSkillSource(filePath, slug);
- return readRuntimeHeader(filePath, slug, false);
+ return readRuntimeHeader(filePath, slug, false, fullMetadata);
}
return {
slug,
@@ -89,9 +92,8 @@ async function readRuntimeHeader(filePath: string, slug: string, migrate = true)
};
}
-/** Runtime-only streaming view; full authoring/query APIs retain original text. */
-export async function* iterateBotSkillRuntimeSummaries(userDataDir: string, botId: string): AsyncGenerator<BotSkillSummary> {
- const root = botSkillsDir(userDataDir, botId);
+async function* iterateHeaders(userDataDir: string, botId: string, enabled: boolean, fullMetadata: boolean): AsyncGenerator<BotSkillSummary> {
+ const root = enabled ? botSkillsDir(userDataDir, botId) : path.join(botSkillRootDir(userDataDir, botId), 'disabled-skills');
let directory;
try { directory = await fs.opendir(root, { bufferSize: 32 }); }
catch (error) {
@@ -104,7 +106,7 @@ export async function* iterateBotSkillRuntimeSummaries(userDataDir: string, botI
const filePath = path.join(root, entry.name, name);
try {
if (!(await fs.stat(filePath)).isFile()) continue;
- yield await readRuntimeHeader(filePath, entry.name);
+ yield { ...await readRuntimeHeader(filePath, entry.name, true, fullMetadata), ...(enabled ? {} : { enabled: false }) };
break;
} catch (error) {
if ((error as NodeJS.ErrnoException).code === 'ENOENT') continue;
@@ -115,3 +117,14 @@ export async function* iterateBotSkillRuntimeSummaries(userDataDir: string, botI
}
}
}
+
+/** Runtime startup retains only short metadata previews. */
+export async function* iterateBotSkillRuntimeSummaries(userDataDir: string, botId: string): AsyncGenerator<BotSkillSummary> {
+ yield* iterateHeaders(userDataDir, botId, true, false);
+}
+
+/** Query-index construction retains one full header at a time, never Skill bodies. */
+export async function* iterateBotSkillQuerySummaries(userDataDir: string, botId: string): AsyncGenerator<BotSkillSummary> {
+ yield* iterateHeaders(userDataDir, botId, true, true);
+ yield* iterateHeaders(userDataDir, botId, false, true);
+}
diff --git a/apps/desktop/src/main/maker-ipc/botSkillService.ts b/apps/desktop/src/main/maker-ipc/botSkillService.ts
index 34e130931f..d726dad2ea 100644
--- a/apps/desktop/src/main/maker-ipc/botSkillService.ts
+++ b/apps/desktop/src/main/maker-ipc/botSkillService.ts
@@ -9,7 +9,8 @@
import { app } from 'electron';
import { collectTeammateGuideMount } from './teammateGuideStore.js';
-import { BOT_SKILL_RUNTIME_INDEX_BYTES, botSkillRuntimeSummary, projectBotSkillMounts } from './botSkillRuntimeProjection.js';
+import { botSkillRuntimeSummary, projectBotSkillMounts } from './botSkillRuntimeProjection.js';
+import { queryBotSkillIndex } from './botSkillQueryIndex.js';
import { cachedBotSkillRuntime } from './botSkillRuntimeCache.js';
import { iterateBotSkillRuntimeSummaries } from './botSkillRuntimeSource.js';
import { requestBotRuntimeEpochRefresh } from './botRuntimeEpochRefreshSignal.js';
@@ -228,27 +229,11 @@ export async function listBotSkillsForSession(
const owner = await (deps.resolveBotId ?? defaultResolveBotId)(params.callerSessionId);
if (!owner.ok) return owner;
assertOwnerBoundary(deps, boundary);
- const all = await listBotSkills(await skillHomeOf(deps, owner.botId, boundary), owner.botId);
+ const home = await skillHomeOf(deps, owner.botId, boundary);
assertOwnerBoundary(deps, boundary);
- const terms = (params.query ?? '').toLocaleLowerCase().split(/\s+/).filter(Boolean);
- const matches = terms.length ? all.filter(item => {
- const text = `${item.slug}\n${item.name}\n${item.description}`.toLocaleLowerCase();
- return terms.every(term => text.includes(term));
- }) : all;
- const offset = Math.max(0, Math.floor(params.offset ?? 0));
- const limit = Math.max(1, Math.min(50, Math.floor(params.limit ?? 20)));
- const skills: ReturnType<typeof botSkillRuntimeSummary>[] = [];
- let bytes = 0;
- for (let index = offset; index < matches.length && skills.length < limit; index++) {
- const summary = botSkillRuntimeSummary(matches[index]);
- const size = Buffer.byteLength(JSON.stringify(summary));
- if (skills.length && bytes + size > BOT_SKILL_RUNTIME_INDEX_BYTES) break;
- skills.push(summary);
- bytes += size;
- }
- const next = offset + skills.length;
- return { ok: true, skills, total: matches.length,
- ...(next < matches.length ? { nextOffset: next } : {}) };
+ const result = await queryBotSkillIndex(botSkillRootDir(home, owner.botId), home, owner.botId, params);
+ assertOwnerBoundary(deps, boundary);
+ return { ok: true, ...result };
} catch (cause) {
return storeError(cause);
}