Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

Keep server startup responsive with large skill bundles (PRD-9085) (#5890)

Andrew Mikofalvya17e202e892d
diff --git a/.changeset/prd-9085-responsive-startup-with-large-skills.md b/.changeset/prd-9085-responsive-startup-with-large-skills.md
new file mode 100644
index 000000000..3e6e7ba43
--- /dev/null
+++ b/.changeset/prd-9085-responsive-startup-with-large-skills.md
@@ -0,0 +1,5 @@
+---
+"@inkeep/open-knowledge": patch
+---
+
+The server no longer stops responding for a minute or more after startup while it reads large installed skills, which made the first MCP call time out.
diff --git a/packages/core/src/skills-catalog/acquire/acquire.test.ts b/packages/core/src/skills-catalog/acquire/acquire.test.ts
index a1cc653df..17556b9bf 100644
--- a/packages/core/src/skills-catalog/acquire/acquire.test.ts
+++ b/packages/core/src/skills-catalog/acquire/acquire.test.ts
@@ -13,7 +13,7 @@ import {
SkillsLockSchema,
upsertLockEntry,
} from './lockfile.ts';
-import { discoverSkillDirs, parseSkillDir } from './parse.ts';
+import { discoverSkillDirs, parseSkillDir, parseSkillDirAsync } from './parse.ts';
import { resolveSkillsShImportSource } from './skills-sh.ts';
import { discoverWellKnownSkills } from './well-known.ts';
@@ -116,6 +116,27 @@ describe('parseSkillDir', () => {
mkdirSync(join(root, 'empty'), { recursive: true });
expect(parseSkillDir(join(root, 'empty'))).toBeNull();
});
+
+ test('the async parser returns exactly what the sync parser returns', async () => {
+ const dir = join(root, 'async-parity');
+ writeSkill(dir, 'name: parity\ndescription: Same either way');
+ mkdirSync(join(dir, 'references', 'deep'), { recursive: true });
+ writeFileSync(join(dir, 'references', 'deep', 'notes.md'), '# Notes\n', 'utf-8');
+ mkdirSync(join(dir, 'assets'), { recursive: true });
+ writeFileSync(join(dir, 'assets', 'blob.bin'), Buffer.from([0, 255, 1, 0, 7]));
+ mkdirSync(join(dir, 'node_modules', 'dep'), { recursive: true });
+ writeFileSync(join(dir, 'node_modules', 'dep', 'index.js'), 'ignored\n', 'utf-8');
+ writeFileSync(join(dir, 'z.txt'), 'last\n', 'utf-8');
+
+ const sync = parseSkillDir(dir);
+ expect(sync?.files.map((f) => f.relPath)).toEqual([
+ 'assets/blob.bin',
+ 'references/deep/notes.md',
+ 'z.txt',
+ ]);
+ expect(await parseSkillDirAsync(dir)).toEqual(sync);
+ expect(await parseSkillDirAsync(join(root, 'empty-async'))).toBeNull();
+ });
});
describe('discoverSkillDirs', () => {
diff --git a/packages/core/src/skills-catalog/acquire/parse.ts b/packages/core/src/skills-catalog/acquire/parse.ts
index 8da5b1a45..5622c7b95 100644
--- a/packages/core/src/skills-catalog/acquire/parse.ts
+++ b/packages/core/src/skills-catalog/acquire/parse.ts
@@ -1,5 +1,6 @@
import { createHash } from 'node:crypto';
import { type Dirent, existsSync, readdirSync, readFileSync, statSync } from 'node:fs';
+import { readdir, readFile } from 'node:fs/promises';
import { basename, join, relative } from 'node:path';
import {
SKILL_IMPORT_MAX_BUNDLE_FILES,
@@ -8,7 +9,7 @@ import {
} from '../import-limits.ts';
import { readSkillManifestMeta } from '../manifest-meta.ts';
-const BUNDLE_WALK_IGNORE: ReadonlySet<string> = new Set(['.git', 'node_modules']);
+export const BUNDLE_WALK_IGNORE: ReadonlySet<string> = new Set(['.git', 'node_modules']);
const DISCOVER_WALK_IGNORE: ReadonlySet<string> = new Set([
...BUNDLE_WALK_IGNORE,
@@ -30,6 +31,21 @@ export interface AcquiredSkill {
readonly contentHash: string;
}
+function acquiredFile(relPath: string, buf: Buffer): AcquiredFile {
+ return buf.includes(0)
+ ? { relPath, content: null, bytes: new Uint8Array(buf) }
+ : { relPath, content: buf.toString('utf-8') };
+}
+
+function bundleRelPath(dir: string, abs: string): string | null {
+ const relPath = relative(dir, abs).split('\\').join('/');
+ return relPath === 'SKILL.md' ? null : relPath;
+}
+
+function byRelPath(a: AcquiredFile, b: AcquiredFile): number {
+ return a.relPath.localeCompare(b.relPath);
+}
+
function readBundleFiles(dir: string): AcquiredFile[] {
const out: AcquiredFile[] = [];
const walk = (d: string): void => {
@@ -40,18 +56,30 @@ function readBundleFiles(dir: string): AcquiredFile[] {
continue;
}
if (!e.isFile()) continue;
- const relPath = relative(dir, abs).split('\\').join('/');
- if (relPath === 'SKILL.md') continue;
- const buf = readFileSync(abs);
- out.push(
- buf.includes(0)
- ? { relPath, content: null, bytes: new Uint8Array(buf) }
- : { relPath, content: buf.toString('utf-8') },
- );
+ const relPath = bundleRelPath(dir, abs);
+ if (relPath !== null) out.push(acquiredFile(relPath, readFileSync(abs)));
}
};
walk(dir);
- return out.sort((a, b) => a.relPath.localeCompare(b.relPath));
+ return out.sort(byRelPath);
+}
+
+async function readBundleFilesAsync(dir: string): Promise<AcquiredFile[]> {
+ const out: AcquiredFile[] = [];
+ const walk = async (d: string): Promise<void> => {
+ for (const e of await readdir(d, { withFileTypes: true })) {
+ const abs = join(d, e.name);
+ if (e.isDirectory()) {
+ if (!BUNDLE_WALK_IGNORE.has(e.name)) await walk(abs);
+ continue;
+ }
+ if (!e.isFile()) continue;
+ const relPath = bundleRelPath(dir, abs);
+ if (relPath !== null) out.push(acquiredFile(relPath, await readFile(abs)));
+ }
+ };
+ await walk(dir);
+ return out.sort(byRelPath);
}
export function readSkillDirMeta(dir: string): { name: string; description: string } | null {
@@ -133,10 +161,23 @@ export function parseSkillDir(dir: string): AcquiredSkill | null {
const skillMdPath = join(dir, 'SKILL.md');
if (!existsSync(skillMdPath)) return null;
const skillMd = readFileSync(skillMdPath, 'utf-8');
+ return acquiredSkill(dir, skillMd, readBundleFiles(dir));
+}
- const { name, description } = readSkillManifestMeta(skillMd, basename(dir));
+export async function parseSkillDirAsync(dir: string): Promise<AcquiredSkill | null> {
+ let skillMd: string;
+ try {
+ skillMd = await readFile(join(dir, 'SKILL.md'), 'utf-8');
+ } catch (err) {
+ const code = (err as NodeJS.ErrnoException).code;
+ if (code === 'ENOENT' || code === 'ENOTDIR') return null;
+ throw err;
+ }
+ return acquiredSkill(dir, skillMd, await readBundleFilesAsync(dir));
+}
- const files = readBundleFiles(dir);
+function acquiredSkill(dir: string, skillMd: string, files: AcquiredFile[]): AcquiredSkill {
+ const { name, description } = readSkillManifestMeta(skillMd, basename(dir));
const hash = createHash('sha256');
hash.update(`SKILL.md\n${skillMd}\n`);
diff --git a/packages/core/src/skills-catalog/enumerate.ts b/packages/core/src/skills-catalog/enumerate.ts
index 4bfe5e64e..0038eea34 100644
--- a/packages/core/src/skills-catalog/enumerate.ts
+++ b/packages/core/src/skills-catalog/enumerate.ts
@@ -13,6 +13,7 @@ import { catalogRawScopeToOkScope, isDetectedSkillInProject } from './scope.ts';
export interface EnumerateOptions {
home?: string;
projectDir?: string;
+ hashOf?: (skillDir: string) => string | undefined;
}
const OK_OWNED_SKILL_PREFIX = 'open-knowledge';
@@ -51,20 +52,27 @@ function toInstalledSkill(winner: RawSkill, harnesses: string[]): CatalogSkill {
};
}
-function contentIdentity(s: RawSkill): string {
+function parsedContentHash(skillDir: string): string | undefined {
+ return parseSkillDir(skillDir)?.contentHash;
+}
+
+function contentIdentity(s: RawSkill, hashOf: (skillDir: string) => string | undefined): string {
try {
- return parseSkillDir(s.home)?.contentHash ?? `unreadable:${s.home}`;
+ return hashOf(s.home) ?? `unreadable:${s.home}`;
} catch {
return `unreadable:${s.home}`;
}
}
-function dedupeSkills(raw: RawSkill[]): CatalogSkill[] {
+function dedupeSkills(
+ raw: RawSkill[],
+ hashOf: (skillDir: string) => string | undefined,
+): CatalogSkill[] {
const byIdentity = new Map<string, { winner: RawSkill; harnesses: Set<string> }>();
for (const s of raw) {
const scope = catalogRawScopeToOkScope(s.provenance.scope);
const projectPath = scope === 'project' ? (s.provenance.projectPath ?? '') : '';
- const key = `${scope}\0${projectPath}\0${s.name}\0${contentIdentity(s)}`;
+ const key = `${scope}\0${projectPath}\0${s.name}\0${contentIdentity(s, hashOf)}`;
const cur = byIdentity.get(key);
if (!cur) {
byIdentity.set(key, { winner: s, harnesses: new Set([s.harness]) });
@@ -164,7 +172,10 @@ export function enumerateInstalledSkills(opts: EnumerateOptions = {}): Installed
}))
.filter((bundle) => bundle.skills.length > 0);
const raw = localBundles.flatMap((b) => b.skills);
- return { skills: dedupeSkills(raw), packs: toPacks(localBundles) };
+ return {
+ skills: dedupeSkills(raw, opts.hashOf ?? parsedContentHash),
+ packs: toPacks(localBundles),
+ };
}
function enumerateProjectHarnessSkills(projectDir: string): SkillBundle[] {
diff --git a/packages/core/src/skills-catalog/index.ts b/packages/core/src/skills-catalog/index.ts
index c4ee21cc7..cf4d8f5f0 100644
--- a/packages/core/src/skills-catalog/index.ts
+++ b/packages/core/src/skills-catalog/index.ts
@@ -27,8 +27,10 @@ export {
type AcquiredFile,
type AcquiredSkill,
acquiredBundleTooLarge,
+ BUNDLE_WALK_IGNORE,
discoverSkillDirs,
parseSkillDir,
+ parseSkillDirAsync,
readSkillDirMeta,
} from './acquire/parse.ts';
export {
diff --git a/packages/server/src/boot-skill-read-stall.test.ts b/packages/server/src/boot-skill-read-stall.test.ts
new file mode 100644
index 000000000..74bf174f0
--- /dev/null
+++ b/packages/server/src/boot-skill-read-stall.test.ts
@@ -0,0 +1,172 @@
+import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { performance } from 'node:perf_hooks';
+import { setTimeout as sleep } from 'node:timers/promises';
+import { expect, test, vi } from 'vitest';
+import { bootServer } from './boot.ts';
+import { ConfigSchema } from './config/schema.ts';
+import { scanGlobalInPlaceSkillsAsync } from './in-place-skills.ts';
+
+const SLOW_READ_MS = 3_000;
+
+const fixture = vi.hoisted(() => ({
+ home: undefined as string | undefined,
+ slowFile: undefined as string | undefined,
+ slowReads: 0,
+ lockedFile: undefined as string | undefined,
+ lockedReads: 0,
+}));
+
+vi.mock('node:os', async (importOriginal) => {
+ const os = await importOriginal<typeof import('node:os')>();
+ return { ...os, homedir: () => fixture.home ?? os.homedir() };
+});
+
+vi.mock('node:fs', async (importOriginal) => {
+ const fs = await importOriginal<typeof import('node:fs')>();
+ const readFileSync = ((...args: Parameters<typeof fs.readFileSync>) => {
+ if (fixture.lockedFile !== undefined && String(args[0]) === fixture.lockedFile) {
+ throw Object.assign(new Error('EPERM: operation not permitted'), { code: 'EPERM' });
+ }
+ if (fixture.slowFile !== undefined && String(args[0]) === fixture.slowFile) {
+ fixture.slowReads += 1;
+ const until = performance.now() + SLOW_READ_MS;
+ while (performance.now() < until) {}
+ }
+ return fs.readFileSync(...args);
+ }) as typeof fs.readFileSync;
+ return { ...fs, default: { ...fs, readFileSync }, readFileSync };
+});
+
+vi.mock('node:fs/promises', async (importOriginal) => {
+ const fsp = await importOriginal<typeof import('node:fs/promises')>();
+ const readFile = (async (...args: Parameters<typeof fsp.readFile>) => {
+ if (fixture.lockedFile !== undefined && String(args[0]) === fixture.lockedFile) {
+ fixture.lockedReads += 1;
+ throw Object.assign(new Error('EPERM: operation not permitted'), { code: 'EPERM' });
+ }
+ if (fixture.slowFile !== undefined && String(args[0]) === fixture.slowFile) {
+ fixture.slowReads += 1;
+ await sleep(SLOW_READ_MS);
+ }
+ return fsp.readFile(...args);
+ }) as typeof fsp.readFile;
+ return { ...fsp, default: { ...fsp, readFile }, readFile };
+});
+
+test('a slow-to-read global skill bundle stalls neither boot nor a skills request made while it is read', async () => {
+ const home = await mkdtemp(join(tmpdir(), 'ok-boot-skill-home-'));
+ const projectDir = await mkdtemp(join(tmpdir(), 'ok-boot-skill-project-'));
+ const skillDir = join(home, '.codex', 'skills', 'heavy-skill');
+ await mkdir(join(skillDir, 'assets'), { recursive: true });
+ await writeFile(
+ join(skillDir, 'SKILL.md'),
+ '---\nname: heavy-skill\ndescription: A skill with a large bundle.\n---\n# Heavy\n',
+ );
+ const slowFile = join(skillDir, 'assets', 'model.bin');
+ await writeFile(slowFile, Buffer.from([0, 1, 2, 3]));
+ await mkdir(join(skillDir, 'node_modules', 'dep'), { recursive: true });
+ await writeFile(join(skillDir, 'node_modules', 'dep', 'index.js'), 'export {};\n');
+ await mkdir(join(projectDir, '.ok'), { recursive: true });
+ await writeFile(join(projectDir, '.ok', 'config.yml'), '');
+ await writeFile(join(projectDir, 'a.md'), '# a\n');
+ fixture.home = home;
+ fixture.slowFile = slowFile;
+
+ let worstStallMs = 0;
+ let lastTick = performance.now();
+ const heartbeat = setInterval(() => {
+ const now = performance.now();
+ worstStallMs = Math.max(worstStallMs, now - lastTick);
+ lastTick = now;
+ }, 10);
+
+ let booted: Awaited<ReturnType<typeof bootServer>> | undefined;
+ try {
+ booted = await bootServer({
+ host: '127.0.0.1',
+ config: ConfigSchema.parse({}),
+ contentDir: projectDir,
+ configHomedirOverride: home,
+ port: 0,
+ quiet: true,
+ gitEnabled: false,
+ idleShutdownMs: null,
+ ephemeral: true,
+ });
+ await booted.ready;
+ const skills = await fetch(`http://127.0.0.1:${booted.port}/api/skills`);
+ expect(skills.status).toBe(200);
+ await scanGlobalInPlaceSkillsAsync(home);
+ } finally {
+ clearInterval(heartbeat);
+ await booted?.destroy('test');
+ fixture.home = undefined;
+ fixture.slowFile = undefined;
+ await rm(home, { recursive: true, force: true });
+ await rm(projectDir, { recursive: true, force: true });
+ }
+
+ expect(fixture.slowReads).toBe(1);
+ expect(worstStallMs).toBeLessThan(SLOW_READ_MS / 2);
+});
+
+test('a folder that becomes a skill after its first read-ahead is still read off the event loop', async () => {
+ const home = await mkdtemp(join(tmpdir(), 'ok-late-skill-home-'));
+ const skillDir = join(home, '.codex', 'skills', 'late-skill');
+ await mkdir(join(skillDir, 'assets'), { recursive: true });
+ fixture.slowReads = 0;
+ let worstStallMs = 0;
+ let heartbeat: ReturnType<typeof setInterval> | undefined;
+ try {
+ expect(await scanGlobalInPlaceSkillsAsync(home)).toEqual([]);
+ await writeFile(
+ join(skillDir, 'SKILL.md'),
+ '---\nname: late-skill\ndescription: Written after its folder.\n---\n# Late\n',
+ );
+ const slowFile = join(skillDir, 'assets', 'model.bin');
+ await writeFile(slowFile, Buffer.from([0, 1, 2, 3]));
+ fixture.slowFile = slowFile;
+
+ let lastTick = performance.now();
+ heartbeat = setInterval(() => {
+ const now = performance.now();
+ worstStallMs = Math.max(worstStallMs, now - lastTick);
+ lastTick = now;
+ }, 10);
+ const skills = await scanGlobalInPlaceSkillsAsync(home);
+ expect(skills.map((s) => s.name)).toEqual(['late-skill']);
+ await sleep(50);
+ } finally {
+ clearInterval(heartbeat);
+ fixture.slowFile = undefined;
+ await rm(home, { recursive: true, force: true });
+ }
+
+ expect(fixture.slowReads).toBe(1);
+ expect(worstStallMs).toBeLessThan(SLOW_READ_MS / 2);
+});
+
+test('a skill whose bundle cannot be read is read ahead once, not on every scan', async () => {
+ const home = await mkdtemp(join(tmpdir(), 'ok-locked-skill-home-'));
+ const skillDir = join(home, '.codex', 'skills', 'locked-skill');
+ await mkdir(join(skillDir, 'assets'), { recursive: true });
+ await writeFile(
+ join(skillDir, 'SKILL.md'),
+ '---\nname: locked-skill\ndescription: One file is locked.\n---\n# Locked\n',
+ );
+ const lockedFile = join(skillDir, 'assets', 'locked.bin');
+ await writeFile(lockedFile, Buffer.from([0, 1]));
+ fixture.lockedFile = lockedFile;
+ fixture.lockedReads = 0;
+ try {
+ await scanGlobalInPlaceSkillsAsync(home);
+ await scanGlobalInPlaceSkillsAsync(home);
+ await scanGlobalInPlaceSkillsAsync(home);
+ } finally {
+ fixture.lockedFile = undefined;
+ await rm(home, { recursive: true, force: true });
+ }
+ expect(fixture.lockedReads).toBe(1);
+});
diff --git a/packages/server/src/boot.ts b/packages/server/src/boot.ts
index 13e60a582..85b2ced58 100644
--- a/packages/server/src/boot.ts
+++ b/packages/server/src/boot.ts
@@ -46,7 +46,7 @@ import {
type CollabClientCounter,
type IdleShutdownHandle,
} from './idle-shutdown.ts';
-import { scanGlobalInPlaceSkills, scanInPlaceSkills } from './in-place-skills.ts';
+import { scanGlobalInPlaceSkillsAsync, scanInPlaceSkills } from './in-place-skills.ts';
import { buildIngressPolicy, ExposureConsentError } from './ingress-policy.ts';
import { resolveLocalSinkConfig } from './local-sink-resolver.ts';
import { getLogger, loggerFactory, type PinoLogger } from './logger.ts';
@@ -786,7 +786,8 @@ async function bootServerInner(opts: BootServerOptions): Promise<BootedServer> {
const m = await migrateStoreSkillsInPlace({
projectDir,
skillsRoot: resolve(opts.contentDir, OK_DIR, SKILLS_STORE_DIRNAME),
- inPlaceNames: genuineInPlaceNames(opts.contentDir, scanInPlaceSkills(opts.contentDir)),
+ inPlaceNames: async () =>
+ genuineInPlaceNames(opts.contentDir, scanInPlaceSkills(opts.contentDir)),
});
if (m.migrated.length + m.skipped.length > 0) {
log.info?.(
@@ -811,7 +812,7 @@ async function bootServerInner(opts: BootServerOptions): Promise<BootedServer> {
projectDir: home,
skillsRoot: resolve(home, OK_DIR, SKILLS_STORE_DIRNAME),
hostRoots: USER_HOST_ROOTS_BY_PRECEDENCE,
- inPlaceNames: genuineInPlaceNames(home, scanGlobalInPlaceSkills(home)),
+ inPlaceNames: async () => genuineInPlaceNames(home, await scanGlobalInPlaceSkillsAsync(home)),
});
if (gm.migrated.length + gm.skipped.length > 0) {
log.info?.(
diff --git a/packages/server/src/http/skills-list-routes.ts b/packages/server/src/http/skills-list-routes.ts
index fc54e006d..b1676cfb6 100644
--- a/packages/server/src/http/skills-list-routes.ts
+++ b/packages/server/src/http/skills-list-routes.ts
@@ -19,7 +19,7 @@ import {
aliasedSourceRoots,
isActivatedSkillRoot,
resolveGlobalNativeSkillDir,
- scanGlobalInPlaceSkills,
+ scanGlobalInPlaceSkillsAsync,
scanHostRootAliases,
scanInPlaceSkills,
standardSkillRoots,
@@ -143,6 +143,7 @@ export function createSkillsListRoutes(deps: SkillsListRouteDeps): ApiRouteGroup
EmptyRequestSchema,
catchErrors(
async (_req, res) => {
+ const globalInPlaceSkills = await scanGlobalInPlaceSkillsAsync(skillsHome);
const inPlaceFp =
(contentFilter?.peekFreshInPlaceSkillDirsFingerprint() ?? '') +
'\u0001' +
@@ -152,7 +153,7 @@ export function createSkillsListRoutes(deps: SkillsListRouteDeps): ApiRouteGroup
.map((h) => h.editorId)
.join(',') +
'\u0001' +
- scanGlobalInPlaceSkills(skillsHome)
+ globalInPlaceSkills
.map((s) => s.dir)
.sort()
.join(',');
@@ -342,10 +343,10 @@ export function createSkillsListRoutes(deps: SkillsListRouteDeps): ApiRouteGroup
const globalLock = readSkillsLock(join(skillsHome, ...SKILLS_LOCK_REL));
const globalAliases = scanHostRootAliases(skillsHome, 'global');
const globalAliasRoots = aliasedSourceRoots(globalAliases, 'global');
- const globalInPlaceNames = new Set(scanGlobalInPlaceSkills(skillsHome).map((s) => s.name));
+ const globalInPlaceNames = new Set(globalInPlaceSkills.map((s) => s.name));
globalSkills.skills = globalSkills.skills.filter((s) => !globalInPlaceNames.has(s.name));
const globalNameSeen = new Set<string>();
- const globalInPlace = scanGlobalInPlaceSkills(skillsHome).map((s) => {
+ const globalInPlace = globalInPlaceSkills.map((s) => {
const tracked = !globalNameSeen.has(s.name);
globalNameSeen.add(s.name);
const placementsForRow = tracked ? globalPlacements[s.name] : undefined;
diff --git a/packages/server/src/http/skills-read-routes.ts b/packages/server/src/http/skills-read-routes.ts
index 9279e71e1..27319efe6 100644
--- a/packages/server/src/http/skills-read-routes.ts
+++ b/packages/server/src/http/skills-read-routes.ts
@@ -14,7 +14,7 @@ import { resolveProjectIdentity } from '@inkeep/open-knowledge-core/shadow-repo-
import type { enumerateInstalledSkills } from '@inkeep/open-knowledge-core/skills-catalog';
import type { CommentService } from '../comments/comment-service.ts';
import { resolveProjectTemplates } from '../content/templates-resolver.ts';
-import { scanGlobalInPlaceSkills, scanInPlaceSkills } from '../in-place-skills.ts';
+import { scanGlobalInPlaceSkillsAsync, scanInPlaceSkills } from '../in-place-skills.ts';
import { readSkillInstallStateSnapshot } from '../skill-state.ts';
import type { ApiRouteTable } from './api-pipeline.ts';
import { catchErrors } from './catch-errors.ts';
@@ -62,12 +62,13 @@ export function createSkillsReadRoutes(deps: SkillsReadRouteDeps): SkillsReadRou
try {
const projectIdentity = resolveProjectIdentity(projectDir ?? contentDir);
const physicalProjectDir = resolve(projectDir ?? contentDir);
+ const globalInPlaceSkills = await scanGlobalInPlaceSkillsAsync(skillsHome);
const catalog = enumerateInstalledSkillsCached({
projectDir: projectIdentity,
...(homeDirOverride !== undefined ? { home: homeDirOverride } : {}),
});
const inPlaceNames = new Set(scanInPlaceSkills(contentDir).map((s) => s.name));
- const globalInPlaceNames = new Set(scanGlobalInPlaceSkills(skillsHome).map((s) => s.name));
+ const globalInPlaceNames = new Set(globalInPlaceSkills.map((s) => s.name));
const result = {
...catalog,
skills: catalog.skills
diff --git a/packages/server/src/in-place-skills.test.ts b/packages/server/src/in-place-skills.test.ts
index 8ff5e1106..ea2ebf750 100644
--- a/packages/server/src/in-place-skills.test.ts
+++ b/packages/server/src/in-place-skills.test.ts
@@ -317,6 +317,24 @@ describe('parse cache (bundle stamp invalidation)', () => {
const second = scanInPlaceSkills(contentDir).find((s) => s.name === 'foo');
expect(second?.contentHash).not.toBe(first?.contentHash);
});
+
+ test('files the content hash ignores never invalidate the cache', () => {
+ writeSkill(contentDir, '.claude/skills/foo', '# body one');
+ utimesSync(skillMdOf('foo'), PINNED, PINNED);
+ const first = scanInPlaceSkills(contentDir).find((s) => s.name === 'foo');
+
+ const swapped = readFileSync(skillMdOf('foo'), 'utf8').replace('# body one', '# body two');
+ writeFileSync(skillMdOf('foo'), swapped);
+ utimesSync(skillMdOf('foo'), PINNED, PINNED);
+ for (const ignored of ['node_modules/dep', '.git/objects']) {
+ const dir = join(contentDir, '.claude/skills/foo', ignored);
+ mkdirSync(dir, { recursive: true });
+ writeFileSync(join(dir, 'file'), 'not part of the bundle\n');
+ }
+
+ const second = scanInPlaceSkills(contentDir).find((s) => s.name === 'foo');
+ expect(second?.contentHash).toBe(first?.contentHash);
+ });
});
describe('global tier (R12): scanGlobalInPlaceSkills + resolveGlobalNativeSkillDir', () => {
diff --git a/packages/server/src/in-place-skills.ts b/packages/server/src/in-place-skills.ts
index 98270a093..d801e2574 100644
--- a/packages/server/src/in-place-skills.ts
+++ b/packages/server/src/in-place-skills.ts
@@ -1,4 +1,5 @@
import { existsSync, lstatSync, readdirSync, realpathSync, statSync } from 'node:fs';
+import { readdir, stat } from 'node:fs/promises';
import { isAbsolute, join, relative, sep } from 'node:path';
import {
AGENTS_SKILLS_ROOT,
@@ -16,14 +17,18 @@ import {
skillRootActivationPath,
} from '@inkeep/open-knowledge-core';
import {
+ type AcquiredSkill,
+ BUNDLE_WALK_IGNORE,
buildSkillRegistry,
groupSkillsByIdentity,
type LocatedSkillOccurrence,
packMarkerOf,
parseSkillDir,
+ parseSkillDirAsync,
SKILL_CANONICAL_PRECEDENCE,
type SkillHostId,
} from '@inkeep/open-knowledge-core/skills-catalog';
+import { getLogger } from './logger.ts';
import { isInternalBundleSkillName, isUserGlobalBundleSkillName } from './skill-bundles.ts';
import {
observeSkillPlacementInputsForScan,
@@ -233,8 +238,9 @@ function bundleStamp(absDir: string): string | null {
for (const e of readdirSync(dir, { withFileTypes: true })) {
const p = join(dir, e.name);
const r = rel === '' ? e.name : `${rel}/${e.name}`;
- if (e.isDirectory()) walk(p, r);
- else if (e.isFile()) {
+ if (e.isDirectory()) {
+ if (!BUNDLE_WALK_IGNORE.has(e.name)) walk(p, r);
+ } else if (e.isFile()) {
const st = statSync(p);
parts.push(`${r}:${st.size}:${st.mtimeMs}`);
}
@@ -244,50 +250,144 @@ function bundleStamp(absDir: string): string | null {
return parts.sort().join('|');
}
-const parseCache = new Map<
- string,
- { stamp: string; contentHash: string; description: string; size: SkillCostTiers }
->();
+async function bundleStampAsync(absDir: string): Promise<string | null> {
+ try {
+ if (!(await stat(join(absDir, 'SKILL.md'))).isFile()) return null;
+ } catch (error) {
+ if (isAbsentPath(error)) return null;
+ throw error;
+ }
+ const parts: string[] = [];
+ const walk = async (dir: string, rel: string): Promise<void> => {
+ for (const e of await readdir(dir, { withFileTypes: true })) {
+ const p = join(dir, e.name);
+ const r = rel === '' ? e.name : `${rel}/${e.name}`;
+ if (e.isDirectory()) {
+ if (!BUNDLE_WALK_IGNORE.has(e.name)) await walk(p, r);
+ } else if (e.isFile()) {
+ const st = await stat(p);
+ parts.push(`${r}:${st.size}:${st.mtimeMs}`);
+ }
+ }
+ };
+ await walk(absDir, '');
+ return parts.sort().join('|');
+}
-function parseSkillDirCached(
- absDir: string,
-): { contentHash: string; description: string; size: SkillCostTiers; pack?: string } | null {
- const stamp = bundleStamp(absDir);
- if (stamp === null) return null;
- const hit = parseCache.get(absDir);
- if (hit !== undefined && hit.stamp === stamp) return hit;
- const parsed = parseSkillDir(absDir);
- if (!parsed) return null;
+interface ParseCacheEntry {
+ stamp: string;
+ contentHash: string;
+ description: string;
+ size: SkillCostTiers;
+ pack?: string;
+}
+
+const parseCache = new Map<string, ParseCacheEntry>();
+
+function parseCacheEntry(stamp: string, parsed: AcquiredSkill): ParseCacheEntry {
const pack = packMarkerOf(parseFrontmatterRecord(parsed.skillMd) ?? {});
- const entry = {
+ return {
stamp,
contentHash: parsed.contentHash,
description: parsed.description,
size: estimateSkillCost(parsed),
...(pack !== undefined ? { pack } : {}),
};
+}
+
+function parseSkillDirCached(absDir: string): ParseCacheEntry | null {
+ const stamp = bundleStamp(absDir);
+ if (stamp === null) return null;
+ const hit = parseCache.get(absDir);
+ if (hit !== undefined && hit.stamp === stamp) return hit;
+ const parsed = parseSkillDir(absDir);
+ if (!parsed) return null;
+ const entry = parseCacheEntry(stamp, parsed);
parseCache.set(absDir, entry);
return entry;
}
-function scanBase(base: string, scope: SkillScope): SkillScanObservation {
- const occurrences: ScanOccurrence[] = [];
+export function cachedSkillContentHash(skillDir: string): string | undefined {
+ return parseSkillDirCached(skillDir)?.contentHash;
+}
+
+const parseCacheWarming = new Map<string, Promise<void>>();
+const parseCacheAwaitingSkillMd = new Map<string, boolean>();
+
+function warmParseCache(absDir: string, root: string): Promise<void> {
+ const inFlight = parseCacheWarming.get(absDir);
+ if (inFlight !== undefined) return inFlight;
+ const warming = (async () => {
+ const stamp = await bundleStampAsync(absDir);
+ if (stamp === null) return true;
+ if (parseCache.get(absDir)?.stamp === stamp) return false;
+ const parsed = await parseSkillDirAsync(absDir);
+ if (parsed) parseCache.set(absDir, parseCacheEntry(stamp, parsed));
+ return false;
+ })()
+ .catch((error) => {
+ logWarmFailure(root, error);
+ return false;
+ })
+ .then((awaitingSkillMd) => {
+ parseCacheAwaitingSkillMd.set(absDir, awaitingSkillMd);
+ })
+ .finally(() => parseCacheWarming.delete(absDir));
+ parseCacheWarming.set(absDir, warming);
+ return warming;
+}
+
+function logWarmFailure(root: string, error: unknown): void {
+ if (isAbsentPath(error)) return;
+ getLogger('in-place-skills').warn(
+ { root, err: error },
+ '[in-place-skills] skills not read ahead of the scan; the scan reads them synchronously',
+ );
+}
+
+async function warmGlobalScanCache(home: string): Promise<void> {
+ for (const { root } of observeScanInputs(home, 'global').roots) {
+ const absRoot = join(home, root);
+ let entries: string[];
+ try {
+ entries = await readdir(absRoot);
+ } catch (error) {
+ logWarmFailure(root, error);
+ continue;
+ }
+ for (const entry of entries) await warmParseCache(join(absRoot, entry), root);
+ }
+}
+
+function observeScanInputs(
+ base: string,
+ scope: SkillScope,
+): {
+ sourcePrefs: Record<string, string>;
+ roots: ReturnType<typeof skillRootsWithCustom>;
+ failures: SkillScanFailure[];
+} {
const failures: SkillScanFailure[] = [];
- const recordFailure = (error: unknown): void => {
- failures.push({ scope: 'scan', error });
- };
let sourcePrefs: Record<string, string> = {};
let customRoots: string[] = [];
try {
const inputs = observeSkillPlacementInputsForScan(base);
sourcePrefs = inputs.sources;
customRoots = inputs.roots;
- if (inputs.kind === 'incomplete') recordFailure(inputs.error);
+ if (inputs.kind === 'incomplete') failures.push({ scope: 'scan', error: inputs.error });
if (inputs.kind === 'unparsable') failures.push({ scope: 'ledger-parse', error: inputs.error });
} catch (error) {
- recordFailure(error);
+ failures.push({ scope: 'scan', error });
}
- const allRoots = skillRootsWithCustom(scope, customRoots);
+ return { sourcePrefs, roots: skillRootsWithCustom(scope, customRoots), failures };
+}
+
+function scanBase(base: string, scope: SkillScope): SkillScanObservation {
+ const occurrences: ScanOccurrence[] = [];
+ const { sourcePrefs, roots: allRoots, failures } = observeScanInputs(base, scope);
+ const recordFailure = (error: unknown): void => {
+ failures.push({ scope: 'scan', error });
+ };
let baseReal: string | null = null;
try {
@@ -411,6 +511,36 @@ export function scanGlobalInPlaceSkills(home: string): InPlaceSkill[] {
return scanBase(home, 'global').skills;
}
+export async function scanGlobalInPlaceSkillsAsync(home: string): Promise<InPlaceSkill[]> {
+ if (globalScanHasUnwarmedDir(home)) await warmGlobalScanCache(home);
+ return scanGlobalInPlaceSkills(home);
+}
+
+function needsWarm(absDir: string): boolean {
+ const awaitingSkillMd = parseCacheAwaitingSkillMd.get(absDir);
+ if (awaitingSkillMd === undefined) return true;
+ if (!awaitingSkillMd) return false;
+ try {
+ return statSync(join(absDir, 'SKILL.md')).isFile();
+ } catch {
+ return false;
+ }
+}
+
+function globalScanHasUnwarmedDir(home: string): boolean {
+ for (const { root } of observeScanInputs(home, 'global').roots) {
+ const absRoot = join(home, root);
+ let entries: string[];
+ try {
+ entries = readdirSync(absRoot);
+ } catch {
+ continue;
+ }
+ if (entries.some((entry) => needsWarm(join(absRoot, entry)))) return true;
+ }
+ return false;
+}
+
const USER_ROOTS_BY_PRECEDENCE = [...USER_SKILL_ROOTS].sort((a, b) => {
const ra = SKILL_CANONICAL_PRECEDENCE.indexOf(a.editor);
const rb = SKILL_CANONICAL_PRECEDENCE.indexOf(b.editor);
diff --git a/packages/server/src/server-factory.ts b/packages/server/src/server-factory.ts
index 5b48e0da1..38e4cee32 100644
--- a/packages/server/src/server-factory.ts
+++ b/packages/server/src/server-factory.ts
@@ -182,7 +182,7 @@ import type { NativeApiHandle } from './http/http-app.ts';
import type { LocalApiDispatch } from './http/local-api-dispatch.ts';
import type { GeneratedIndexSettingsStatus } from './http/workspace-tools-routes.ts';
import {
- scanGlobalInPlaceSkills,
+ scanGlobalInPlaceSkillsAsync,
scanInPlaceSkillDirs,
scanInPlaceSkills,
skillRootPathsFor,
@@ -1969,7 +1969,8 @@ export function createServer(options: ServerOptions): ServerInstance {
globalCopyResyncTimer = setTimeout(() => {
globalCopyResyncTimer = null;
const home = configHomedirOverride ?? homedir();
- void resyncRecordedSkillCopies(home, home, scanGlobalInPlaceSkills(home))
+ void scanGlobalInPlaceSkillsAsync(home)
+ .then((inPlace) => resyncRecordedSkillCopies(home, home, inPlace))
.then((n) => {
if (n > 0) log.info({ refreshed: n }, '[in-place-skills] post-edit copy re-sync');
})
@@ -4286,7 +4287,8 @@ export function createServer(options: ServerOptions): ServerInstance {
.catch((err) => log.warn({ err }, '[in-place-skills] boot copy re-sync failed'));
{
const home = configHomedirOverride ?? homedir();
- void resyncRecordedSkillCopies(home, home, scanGlobalInPlaceSkills(home))
+ void scanGlobalInPlaceSkillsAsync(home)
+ .then((inPlace) => resyncRecordedSkillCopies(home, home, inPlace))
.then((n) => {
if (n > 0) log.info({ refreshed: n }, '[in-place-skills] global boot copy re-sync');
})
diff --git a/packages/server/src/skill-migrate.test.ts b/packages/server/src/skill-migrate.test.ts
index 1cd1a2342..54d481162 100644
--- a/packages/server/src/skill-migrate.test.ts
+++ b/packages/server/src/skill-migrate.test.ts
@@ -205,7 +205,7 @@ describe('migrateStoreSkillsInPlace', () => {
const r = await migrateStoreSkillsInPlace({
projectDir: root,
skillsRoot,
- inPlaceNames: new Set(['placed']),
+ inPlaceNames: async () => new Set(['placed']),
});
expect(r.migrated).toEqual([]);
expect(existsSync(join(skillsRoot, 'placed', 'SKILL.md'))).toBe(true);
@@ -216,6 +216,41 @@ describe('migrateStoreSkillsInPlace', () => {
expect(r.migrated).toEqual([]);
expect(r.skipped).toEqual([]);
});
+
+ test('scans in-place skills only when the store holds a skill to classify', async () => {
+ let scans = 0;
+ const inPlaceNames = async () => {
+ scans += 1;
+ return new Set<string>(['placed']);
+ };
+ await migrateStoreSkillsInPlace({ projectDir: root, skillsRoot, inPlaceNames });
+ rmSync(skillsRoot, { recursive: true, force: true });
+ await migrateStoreSkillsInPlace({ projectDir: root, skillsRoot, inPlaceNames });
+ expect(scans).toBe(0);
+
+ makeStore('placed', '# P');
+ makeStore('other', '# O');
+ const r = await migrateStoreSkillsInPlace({ projectDir: root, skillsRoot, inPlaceNames });
+ expect(scans).toBe(1);
+ expect(r.skipped).toContainEqual({ name: 'placed', reason: 'placement-of-in-place' });
+ });
+
+ test('a failed in-place scan migrates nothing', async () => {
+ makeStore('placed', '# P');
+ makeStore('other', '# O');
+ mkdirSync(join(root, '.claude', 'skills'), { recursive: true });
+ const r = await migrateStoreSkillsInPlace({
+ projectDir: root,
+ skillsRoot,
+ inPlaceNames: async () => {
+ throw new Error('scan failed');
+ },
+ });
+ expect(r.migrated).toEqual([]);
+ expect(r.skipped.map((s) => s.reason)).toEqual(['error', 'error']);
+ expect(existsSync(join(skillsRoot, 'placed', 'SKILL.md'))).toBe(true);
+ expect(existsSync(join(skillsRoot, 'other', 'SKILL.md'))).toBe(true);
+ });
});
describe('genuineInPlaceNames', () => {
diff --git a/packages/server/src/skill-migrate.ts b/packages/server/src/skill-migrate.ts
index bb66f8a89..c7b412929 100644
--- a/packages/server/src/skill-migrate.ts
+++ b/packages/server/src/skill-migrate.ts
@@ -119,11 +119,15 @@ export function genuineInPlaceNames(
export async function migrateStoreSkillsInPlace(opts: {
projectDir: string;
skillsRoot: string;
- inPlaceNames?: ReadonlySet<string>;
+ inPlaceNames?: () => Promise<ReadonlySet<string>>;
hostRoots?: ReadonlyArray<{ host: SkillHostId; root: string }>;
}): Promise<StoreMigrationResult> {
const { projectDir, skillsRoot } = opts;
- const inPlaceNames = opts.inPlaceNames ?? new Set<string>();
+ let inPlaceNames: Promise<ReadonlySet<string>> | undefined;
+ const isInPlaceName = async (name: string): Promise<boolean> => {
+ inPlaceNames ??= opts.inPlaceNames?.() ?? Promise.resolve(new Set<string>());
+ return (await inPlaceNames).has(name);
+ };
const hostRootsByPrecedence = opts.hostRoots ?? HOST_ROOTS_BY_PRECEDENCE;
const result: StoreMigrationResult = { migrated: [], skipped: [] };
if (!existsSync(skillsRoot)) return result;
@@ -147,7 +151,7 @@ export async function migrateStoreSkillsInPlace(opts: {
for (const name of entries) {
try {
if (!SKILL_NAME_REGEX.test(name) || INTERNAL_BUNDLE_SKILL_NAMES.has(name)) continue;
- if (inPlaceNames.has(name)) {
+ if (await isInPlaceName(name)) {
result.skipped.push({ name, reason: 'placement-of-in-place' });
continue;
}
diff --git a/packages/server/src/skills-catalog-cache.test.ts b/packages/server/src/skills-catalog-cache.test.ts
index 65988086e..00bd218fc 100644
--- a/packages/server/src/skills-catalog-cache.test.ts
+++ b/packages/server/src/skills-catalog-cache.test.ts
@@ -127,7 +127,7 @@ test('plugin content refreshes only on identity change or its independent thirty
clock.mockReturnValue(130_000);
const expired = c.pluginSkillsByName('project-a');
expect(expired.get('alpha')?.contentHash).not.toBe(hash);
- installPlugin('third');
+ installPlugin('third revision');
c.bumpSkillsCatalogGen();
const changedIdentity = c.pluginSkillsByName('project-b');
expect(changedIdentity.get('alpha')?.contentHash).not.toBe(expired.get('alpha')?.contentHash);
diff --git a/packages/server/src/skills-catalog-cache.ts b/packages/server/src/skills-catalog-cache.ts
index 5c1b520a9..2f3693215 100644
--- a/packages/server/src/skills-catalog-cache.ts
+++ b/packages/server/src/skills-catalog-cache.ts
@@ -1,9 +1,9 @@
import {
enumerateInstalledSkills,
type PluginUpstream,
- parseSkillDir,
pluginUpstreamsByName,
} from '@inkeep/open-knowledge-core/skills-catalog';
+import { cachedSkillContentHash } from './in-place-skills.ts';
import type { PinoLogger } from './logger.ts';
export type SkillsCatalogCache = ReturnType<typeof createSkillsCatalogCache>;
@@ -41,7 +41,7 @@ export function createSkillsCatalogCache({
) {
return { freshBuild: false, key, value: installedCatalogCache.value };
}
- const value = enumerateInstalledSkills(opts);
+ const value = enumerateInstalledSkills({ ...opts, hashOf: cachedSkillContentHash });
installedCatalogCache = { at: Date.now(), gen: skillsCatalogGen, key, value };
return { freshBuild: true, key, value };
}
@@ -96,10 +96,7 @@ export function createSkillsCatalogCache({
? { home: homeDirOverride, projectDir: identity }
: { projectDir: identity },
);
- byName = pluginUpstreamsByName(
- catalogLookup.value.skills,
- (home) => parseSkillDir(home)?.contentHash,
- );
+ byName = pluginUpstreamsByName(catalogLookup.value.skills, cachedSkillContentHash);
} catch (err) {
log.warn({ err }, 'plugin upstream index failed; origins will be omitted');
}