Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。
TypeScript agentai-agentai-assistantandroidclaude-codecodexelectroniosllmmacosreact-nativetypescriptwindows
fix(companions): bound Skill directory sorting
Chris28ccec392a34
diff --git a/apps/desktop/src/main/bot-import/README.md b/apps/desktop/src/main/bot-import/README.mdindex d17fe5537d..ccf17ff4e3 100644--- a/apps/desktop/src/main/bot-import/README.md+++ b/apps/desktop/src/main/bot-import/README.md@@ -113,6 +113,12 @@ omitting it retains normal companion defaults, and a rejected request can be corfixed quota. Imported SKILL.md files preserve their full body rather than usingthe interactive skill editor's 64 KiB authoring limit. Individual file and totalin-memory snapshot byte bounds remain; this is not an unlimited-byte importer.+ Skill-root discovery charges each streamed directory entry before retaining it.+ Native alphabetical precedence uses 256 KiB sort batches, 64 KiB merge I/O and+ a logarithmic set of runs instead of a whole-directory Dirent array. Large+ roots spill only names/types into a private OS temporary directory, removed+ after success, failure or an early consumer return; small roots stay in memory.+ Skill files, duplicate-name precedence, disabled state and link policy are unchanged.Skills disabled at the source are saved outside the active native skills folderand stay disabled when read or edited.Runtime discovery is separately bounded: small shelves mount directly; headersdiff --git a/apps/desktop/src/main/bot-import/__tests__/skillDirectories.test.ts b/apps/desktop/src/main/bot-import/__tests__/skillDirectories.test.tsnew file mode 100644index 0000000000..c3582753b6--- /dev/null+++ b/apps/desktop/src/main/bot-import/__tests__/skillDirectories.test.ts@@ -0,0 +1,114 @@+import { afterEach, beforeEach, expect, it, vi } from 'vitest';+import { promises as fs } from 'node:fs';+import os from 'node:os';+import path from 'node:path';+import { orderedSkillDirectories } from '../skillDirectories.js';+import { createImportBudget } from '../files.js';++let root: string;+beforeEach(async () => { root = await fs.mkdtemp(path.join(os.tmpdir(), 'cindy-skill-directories-test-')); });+afterEach(async () => { vi.restoreAllMocks(); await fs.rm(root, { recursive: true, force: true }); });++function source(count: number, failAt = -1, nameFor = (index: number) => `${String(index).padStart(6, '0')}-中文-"line\n${'x'.repeat(80)}`) {+ let closed = false; let read = 0;+ const names: string[] = [];+ vi.spyOn(fs, 'opendir').mockImplementation(async () => ({ async *[Symbol.asyncIterator]() {+ try {+ for (let index = count - 1; index >= 0; index--) {+ if (read === failAt) throw Object.assign(new Error('fixture enumeration failure'), { code: 'EIO' });+ const name = nameFor(index);+ names.push(name); read++;+ yield { name, isDirectory: () => index % 2 === 0, isSymbolicLink: () => index % 2 !== 0 };+ }+ } finally { closed = true; }+ } }) as Awaited<ReturnType<typeof fs.opendir>>);+ return { names, get closed() { return closed; }, get read() { return read; } };+}++function trackStaging() {+ const original = fs.mkdtemp.bind(fs); const paths: string[] = [];+ vi.spyOn(fs, 'mkdtemp').mockImplementation(async (...args) => {+ const result = await original(...args);+ paths.push(String(result)); return result;+ });+ return paths;+}++async function expectRemoved(paths: string[]) {+ expect(paths.length).toBeGreaterThan(0);+ for (const directory of paths) await expect(fs.stat(directory)).rejects.toMatchObject({ code: 'ENOENT' });+}++it('sorts a small real directory without creating temporary files and still charges regular files', async () => {+ await fs.mkdir(path.join(root, 'z-last')); await fs.mkdir(path.join(root, 'a-first'));+ await fs.writeFile(path.join(root, 'ordinary.txt'), 'fixture');+ const staging = trackStaging(); const budget = createImportBudget();+ const reserve = vi.spyOn(budget, 'reserve');+ const entries = [];+ for await (const entry of orderedSkillDirectories(root, budget)) entries.push(entry);+ expect(entries).toEqual([{ name: 'a-first', directory: true }, { name: 'z-last', directory: true }]);+ expect(reserve).toHaveBeenCalledTimes(3);+ expect(staging).toEqual([]);+});++it('spills while enumeration is still in progress and preserves every name/type in native sorted order', async () => {+ const input = source(10_003); const staging = trackStaging();+ const open = fs.open.bind(fs); let writesBeforeEnd = 0; let maxWriteBytes = 0;+ vi.spyOn(fs, 'open').mockImplementation(async (...args) => {+ const handle = await open(...args);+ if (args[1] === 'wx') {+ expect(args[2]).toBe(0o600);+ if (input.read < 10_003) writesBeforeEnd++;+ const write = handle.writeFile.bind(handle);+ vi.spyOn(handle, 'writeFile').mockImplementation(async (...values) => {+ maxWriteBytes = Math.max(maxWriteBytes, Buffer.byteLength(String(values[0])));+ return write(...values);+ });+ }+ return handle;+ });+ const entries = [];+ for await (const entry of orderedSkillDirectories(root, createImportBudget())) entries.push(entry);+ expect(entries.map(entry => entry.name)).toEqual([...input.names].sort((a, b) => a.localeCompare(b)));+ expect(entries.every(entry => entry.directory === (Number(entry.name.slice(0, 6)) % 2 === 0))).toBe(true);+ expect(writesBeforeEnd).toBeGreaterThan(0);+ expect(maxWriteBytes).toBeLessThanOrEqual(64 * 1024);+ expect(input.closed).toBe(true);+ expect(staging).toHaveLength(1);+ await expectRemoved(staging);+});++it('retains native stable precedence for equal collations across separate sort runs', async () => {+ expect('tie-é'.localeCompare('tie-e\u0301')).toBe(0);+ const input = source(10_003, -1, index => index === 10_002 ? 'tie-é' : index === 0 ? 'tie-e\u0301' : `z-${index}`);+ const staging = trackStaging(); const names = [];+ for await (const entry of orderedSkillDirectories(root, createImportBudget())) names.push(entry.name);+ expect(names).toEqual([...input.names].sort((a, b) => a.localeCompare(b)));+ expect(names.slice(0, 2)).toEqual(['tie-é', 'tie-e\u0301']);+ await expectRemoved(staging);+});++it.each(['budget', 'enumeration', 'write', 'merge-read', 'consumer'] as const)('closes and cleans up after %s interrupts a spilled directory', async failure => {+ const input = source(10_000, failure === 'enumeration' ? 4_000 : -1);+ const staging = trackStaging();+ const open = fs.open.bind(fs); const handles: Awaited<ReturnType<typeof fs.open>>[] = [];+ let writes = 0;+ vi.spyOn(fs, 'open').mockImplementation(async (...args) => {+ if (failure === 'write' && args[1] === 'wx' && ++writes === 2) throw Object.assign(new Error('fixture disk full'), { code: 'ENOSPC' });+ const handle = await open(...args); handles.push(handle);+ if (failure === 'merge-read' && args[1] === 'r') vi.spyOn(handle, 'read').mockRejectedValue(Object.assign(new Error('fixture read failure'), { code: 'EIO' }));+ return handle;+ });+ const collect = async () => {+ for await (const entry of orderedSkillDirectories(root, createImportBudget(failure === 'budget' ? 1024 * 1024 : undefined))) {+ expect(entry.name).toBeTruthy();+ if (failure === 'consumer') break;+ }+ };+ if (failure === 'consumer') await collect();+ else if (failure === 'budget') await expect(collect()).rejects.toThrow('SOURCE_SNAPSHOT_TOO_LARGE');+ else await expect(collect()).rejects.toMatchObject({ code: failure === 'write' ? 'ENOSPC' : 'EIO' });+ expect(input.closed).toBe(true);+ for (const handle of handles) await expect(handle.stat()).rejects.toMatchObject({ code: 'EBADF' });+ await expectRemoved(staging);+});diff --git a/apps/desktop/src/main/bot-import/__tests__/sources.test.ts b/apps/desktop/src/main/bot-import/__tests__/sources.test.tsindex 01711a270b..49823d3a9f 100644--- a/apps/desktop/src/main/bot-import/__tests__/sources.test.ts+++ b/apps/desktop/src/main/bot-import/__tests__/sources.test.ts@@ -635,6 +635,42 @@ it('preserves sorted skill-name precedence after streaming directory entries', aexpect(items[0]!.files![0]!.bytes.toString()).toContain('First');});+it('does not synchronously sort a whole 100,000-entry Skill root before visiting children', async () => {+ const root = path.join(home, '.hermes'); const directory = path.join(root, 'skills');+ await fs.mkdir(directory, { recursive: true });+ const opendir = fs.opendir.bind(fs);+ let closed = false; let visited = 0; let previous = '';+ vi.spyOn(fs, 'opendir').mockImplementation(async (...args) => {+ if (String(args[0]) !== directory) return opendir(...args);+ return { async *[Symbol.asyncIterator]() {+ try {+ for (let index = 99_999; index >= 0; index--) yield {+ name: `skill-${String(index).padStart(6, '0')}`,+ isDirectory: () => false, isSymbolicLink: () => true,+ };+ } finally { closed = true; }+ } } as Awaited<ReturnType<typeof fs.opendir>>;+ });+ const stat = fs.stat.bind(fs);+ vi.spyOn(fs, 'stat').mockImplementation(async (...args) => {+ if (path.dirname(String(args[0])) !== directory) return stat(...args);+ const name = path.basename(String(args[0]));+ expect(previous.localeCompare(name)).toBeLessThan(0);+ previous = name; visited++;+ throw Object.assign(new Error('fixture dangling link'), { code: 'ENOENT' });+ });+ const sort = Array.prototype.sort;+ vi.spyOn(Array.prototype, 'sort').mockImplementation(function(this: unknown[], compare) {+ if (this.length > 8192) throw new Error('Unbounded synchronous directory sort');+ return sort.call(this, compare);+ });+ const items = await discoverImportSkills({ kind: 'hermes', agentId: 'main', name: 'Ada', root, workspace: root,+ configFile: path.join(root, 'config.yaml') }, {}, home, {}, createImportBudget());+ expect(items).toEqual([]);+ expect(visited).toBe(100_000);+ expect(closed).toBe(true);+}, 30_000);+it.each(['hermes', 'openclaw'] as const)('discovers %s custom archives, hidden TXT corpus and attachments without conflating roots', async kind => {await write(`.${kind}/${kind === 'hermes' ? 'config.yaml' : 'openclaw.json'}`, '{}');const workspace = `.${kind}/${kind === 'hermes' ? '' : 'workspace/'}`;diff --git a/apps/desktop/src/main/bot-import/skillDirectories.ts b/apps/desktop/src/main/bot-import/skillDirectories.tsnew file mode 100644index 0000000000..b0d9bef9b1--- /dev/null+++ b/apps/desktop/src/main/bot-import/skillDirectories.ts@@ -0,0 +1,118 @@+import { promises as fs } from 'node:fs';+import type { FileHandle } from 'node:fs/promises';+import os from 'node:os';+import path from 'node:path';+import { StringDecoder } from 'node:string_decoder';+import type { ImportReadBudget } from './files.js';++type Entry = { name: string; directory: boolean };+const SORT_BYTES = 256 * 1024;+const IO_BYTES = 64 * 1024;+const compare = (a: Entry, b: Entry) => a.name.localeCompare(b.name);++async function* records(file: string): AsyncGenerator<Entry> {+ const handle = await fs.open(file, 'r');+ const buffer = Buffer.alloc(IO_BYTES);+ const decoder = new StringDecoder('utf8');+ let pending = '';+ try {+ while (true) {+ const { bytesRead } = await handle.read(buffer, 0, buffer.length, null);+ if (!bytesRead) break;+ pending += decoder.write(buffer.subarray(0, bytesRead));+ let start = 0; let end: number;+ while ((end = pending.indexOf('\n', start)) >= 0) {+ yield JSON.parse(pending.slice(start, end)) as Entry;+ start = end + 1;+ }+ pending = pending.slice(start);+ }+ if (pending || decoder.end()) throw new Error('Incomplete Skill directory sort');+ } finally { await handle.close(); }+}++function writer(handle: FileHandle) {+ let pending: string[] = []; let bytes = 0;+ const flush = async () => {+ if (pending.length) await handle.writeFile(pending.join(''));+ pending = []; bytes = 0;+ };+ return { flush, async write(entry: Entry) {+ const line = JSON.stringify(entry) + '\n';+ const size = Buffer.byteLength(line);+ if (bytes + size > IO_BYTES) await flush();+ if (size >= IO_BYTES) await handle.writeFile(line);+ else { pending.push(line); bytes += size; }+ } };+}++async function merge(leftFile: string, rightFile: string, output: string): Promise<void> {+ const handle = await fs.open(output, 'wx', 0o600);+ const left = records(leftFile); const right = records(rightFile);+ try {+ const target = writer(handle);+ let a = await left.next(); let b = await right.next();+ while (!a.done || !b.done) {+ // Stable ties retain the original opendir order, matching Array.sort.+ if (b.done || (!a.done && compare(a.value, b.value) <= 0)) {+ await target.write(a.value!); a = await left.next();+ } else { await target.write(b.value); b = await right.next(); }+ }+ await target.flush();+ } finally {+ await Promise.all([left.return(undefined), right.return(undefined), handle.close()]);+ }+ await Promise.all([fs.unlink(leftFile), fs.unlink(rightFile)]);+}++/** Preserve native alphabetical precedence without retaining a directory's Dirents.+ * Large roots use byte-bounded runs and a logarithmic list of merge levels. Only+ * names/types go to a private OS temporary directory, never Skill contents. */+export async function* orderedSkillDirectories(directory: string, budget: ImportReadBudget): AsyncGenerator<Entry> {+ let staging: string | undefined; let sequence = 0;+ let chunk: Entry[] = []; let bytes = 0;+ const levels: Array<string | undefined> = [];+ const nextFile = () => path.join(staging!, `${sequence++}.jsonl`);+ const combine = async (left: string, right: string) => {+ const output = nextFile(); await merge(left, right, output); return output;+ };+ const spill = async () => {+ staging ??= await fs.mkdtemp(path.join(os.tmpdir(), 'cindy-import-skill-sort-'));+ let file = nextFile();+ const handle = await fs.open(file, 'wx', 0o600);+ try {+ const target = writer(handle);+ chunk.sort(compare);+ for (const entry of chunk) await target.write(entry);+ await target.flush();+ } finally { await handle.close(); }+ chunk = []; bytes = 0;+ // Merge while enumerating; even the list of run filenames stays bounded.+ let level = 0;+ while (levels[level]) {+ file = await combine(levels[level]!, file);+ levels[level++] = undefined;+ }+ levels[level] = file;+ };+ try {+ for await (const entry of await fs.opendir(directory)) {+ const size = 128 + Buffer.byteLength(path.join(directory, entry.name));+ budget.reserve(size);+ if (!entry.isDirectory() && !entry.isSymbolicLink()) continue;+ if (chunk.length && bytes + size > SORT_BYTES) await spill();+ chunk.push({ name: entry.name, directory: entry.isDirectory() }); bytes += size;+ if (bytes >= SORT_BYTES) await spill();+ }+ if (!staging) { yield* chunk.sort(compare); return; }+ if (chunk.length) await spill();+ let file: string | undefined;+ // Higher levels contain earlier entries; preserve their precedence on ties.+ for (let level = levels.length - 1; level >= 0; level--) {+ if (levels[level]) file = file ? await combine(file, levels[level]!) : levels[level];+ }+ if (file) yield* records(file);+ } finally {+ if (staging) await fs.rm(staging, { recursive: true, force: true });+ }+}diff --git a/apps/desktop/src/main/bot-import/skills.ts b/apps/desktop/src/main/bot-import/skills.tsindex 114594ba85..9c4071236b 100644--- a/apps/desktop/src/main/bot-import/skills.ts+++ b/apps/desktop/src/main/bot-import/skills.ts@@ -1,8 +1,9 @@-import { constants, promises as fs, type Dirent } from 'node:fs';+import { constants, promises as fs } from 'node:fs';import path from 'node:path';import yaml from 'js-yaml';import { fingerprint, inside, readImportFile, readImportTree, type ImportReadBudget } from './files.js';import { CompanionImportError, object, string, type ImportItem, type ImportSource } from './types.js';+import { orderedSkillDirectories } from './skillDirectories.js';/** Discovery roots are derived on the host from the selected source, never from IPC paths. */// `any` preserves native Hermes personal/external and OpenClaw managed/personal@@ -14,16 +15,6 @@ const ignored = new Set(['.git', '.github', '.hub', '.archive', '_archive', '.veconst strings = (value: unknown): string[] => Array.isArray(value) ? value.filter((v): v is string => typeof v === 'string' && !!v.trim()) : [];const disabledSkills = (config: Record<string, unknown>) => new Set([...strings(config.disabled), ...strings(object(config.platform_disabled).cli)]);-/** Charge entries before retaining them; sorting preserves native name precedence. */-async function directoryEntries(directory: string, budget: ImportReadBudget): Promise<Dirent[]> {- const entries: Dirent[] = [];- for await (const entry of await fs.opendir(directory)) {- budget.reserve(128 + Buffer.byteLength(path.join(directory, entry.name)));- if (entry.isDirectory() || entry.isSymbolicLink()) entries.push(entry);- }- return entries.sort((a, b) => a.name.localeCompare(b.name));-}-/** Resolve only path configuration; never expand credentials into public names. */function sourcePath(value: string, base: string, home: string, env: NodeJS.ProcessEnv) {const expanded = value.replace(/\$\{([A-Za-z_][A-Za-z0-9_]*)\}|\$([A-Za-z_][A-Za-z0-9_]*)/g, (match, a, b) => env[a ?? b] ?? match);@@ -76,7 +67,7 @@ async function rootsFor(source: ImportSource, values: Record<string, unknown>, hconst locations = [...strings(object(plugins.load).paths).map(resolve),...Object.values(object(plugins.installs)).flatMap(value => string(object(value).installPath) ? [resolve(string(object(value).installPath))] : [])];for (const parent of [path.join(source.root, 'extensions'), ...(install ? [path.join(install, 'extensions')] : [])]) {- try { for (const entry of await directoryEntries(parent, budget)) if (entry.isDirectory()) locations.push(path.join(parent, entry.name)); }+ try { for await (const entry of orderedSkillDirectories(parent, budget)) if (entry.directory) locations.push(path.join(parent, entry.name)); }catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; }}for (const location of new Set(locations)) {@@ -202,11 +193,11 @@ export async function discoverImportSkills(source: ImportSource, values: Record<items.push(item); return;}if (source.kind === 'openclaw' && depth >= 6) return;- for (const entry of await directoryEntries(directory, budget)) {+ for await (const entry of orderedSkillDirectories(directory, budget)) {if (ignored.has(entry.name) || entry.name.startsWith('.')) continue;const child = path.join(directory, entry.name);- if (entry.isDirectory()) await visit(child, depth + 1);- else if (entry.isSymbolicLink()) {+ if (entry.directory) await visit(child, depth + 1);+ else {try { if ((await fs.stat(child)).isDirectory()) await visit(child, depth + 1); }catch (error) { if (!['ENOENT', 'ELOOP'].includes((error as NodeJS.ErrnoException).code ?? '')) throw error; }}