Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
Keep server startup responsive with large skill bundles (PRD-9085) (#5890)
Andrew Mikofalvya17e202e892d
diff --git a/.changeset/prd-9085-responsive-startup-with-large-skills.md b/.changeset/prd-9085-responsive-startup-with-large-skills.mdnew file mode 100644index 000000000..3e6e7ba43--- /dev/null+++ b/.changeset/prd-9085-responsive-startup-with-large-skills.md@@ -0,0 +1,5 @@+---+"@inkeep/open-knowledge": patch+---++The server no longer stops responding for a minute or more after startup while it reads large installed skills, which made the first MCP call time out.diff --git a/packages/core/src/skills-catalog/acquire/acquire.test.ts b/packages/core/src/skills-catalog/acquire/acquire.test.tsindex a1cc653df..17556b9bf 100644--- a/packages/core/src/skills-catalog/acquire/acquire.test.ts+++ b/packages/core/src/skills-catalog/acquire/acquire.test.ts@@ -13,7 +13,7 @@ import {SkillsLockSchema,upsertLockEntry,} from './lockfile.ts';-import { discoverSkillDirs, parseSkillDir } from './parse.ts';+import { discoverSkillDirs, parseSkillDir, parseSkillDirAsync } from './parse.ts';import { resolveSkillsShImportSource } from './skills-sh.ts';import { discoverWellKnownSkills } from './well-known.ts';@@ -116,6 +116,27 @@ describe('parseSkillDir', () => {mkdirSync(join(root, 'empty'), { recursive: true });expect(parseSkillDir(join(root, 'empty'))).toBeNull();});++ test('the async parser returns exactly what the sync parser returns', async () => {+ const dir = join(root, 'async-parity');+ writeSkill(dir, 'name: parity\ndescription: Same either way');+ mkdirSync(join(dir, 'references', 'deep'), { recursive: true });+ writeFileSync(join(dir, 'references', 'deep', 'notes.md'), '# Notes\n', 'utf-8');+ mkdirSync(join(dir, 'assets'), { recursive: true });+ writeFileSync(join(dir, 'assets', 'blob.bin'), Buffer.from([0, 255, 1, 0, 7]));+ mkdirSync(join(dir, 'node_modules', 'dep'), { recursive: true });+ writeFileSync(join(dir, 'node_modules', 'dep', 'index.js'), 'ignored\n', 'utf-8');+ writeFileSync(join(dir, 'z.txt'), 'last\n', 'utf-8');++ const sync = parseSkillDir(dir);+ expect(sync?.files.map((f) => f.relPath)).toEqual([+ 'assets/blob.bin',+ 'references/deep/notes.md',+ 'z.txt',+ ]);+ expect(await parseSkillDirAsync(dir)).toEqual(sync);+ expect(await parseSkillDirAsync(join(root, 'empty-async'))).toBeNull();+ });});describe('discoverSkillDirs', () => {diff --git a/packages/core/src/skills-catalog/acquire/parse.ts b/packages/core/src/skills-catalog/acquire/parse.tsindex 8da5b1a45..5622c7b95 100644--- a/packages/core/src/skills-catalog/acquire/parse.ts+++ b/packages/core/src/skills-catalog/acquire/parse.ts@@ -1,5 +1,6 @@import { createHash } from 'node:crypto';import { type Dirent, existsSync, readdirSync, readFileSync, statSync } from 'node:fs';+import { readdir, readFile } from 'node:fs/promises';import { basename, join, relative } from 'node:path';import {SKILL_IMPORT_MAX_BUNDLE_FILES,@@ -8,7 +9,7 @@ import {} from '../import-limits.ts';import { readSkillManifestMeta } from '../manifest-meta.ts';-const BUNDLE_WALK_IGNORE: ReadonlySet<string> = new Set(['.git', 'node_modules']);+export const BUNDLE_WALK_IGNORE: ReadonlySet<string> = new Set(['.git', 'node_modules']);const DISCOVER_WALK_IGNORE: ReadonlySet<string> = new Set([...BUNDLE_WALK_IGNORE,@@ -30,6 +31,21 @@ export interface AcquiredSkill {readonly contentHash: string;}+function acquiredFile(relPath: string, buf: Buffer): AcquiredFile {+ return buf.includes(0)+ ? { relPath, content: null, bytes: new Uint8Array(buf) }+ : { relPath, content: buf.toString('utf-8') };+}++function bundleRelPath(dir: string, abs: string): string | null {+ const relPath = relative(dir, abs).split('\\').join('/');+ return relPath === 'SKILL.md' ? null : relPath;+}++function byRelPath(a: AcquiredFile, b: AcquiredFile): number {+ return a.relPath.localeCompare(b.relPath);+}+function readBundleFiles(dir: string): AcquiredFile[] {const out: AcquiredFile[] = [];const walk = (d: string): void => {@@ -40,18 +56,30 @@ function readBundleFiles(dir: string): AcquiredFile[] {continue;}if (!e.isFile()) continue;- const relPath = relative(dir, abs).split('\\').join('/');- if (relPath === 'SKILL.md') continue;- const buf = readFileSync(abs);- out.push(- buf.includes(0)- ? { relPath, content: null, bytes: new Uint8Array(buf) }- : { relPath, content: buf.toString('utf-8') },- );+ const relPath = bundleRelPath(dir, abs);+ if (relPath !== null) out.push(acquiredFile(relPath, readFileSync(abs)));}};walk(dir);- return out.sort((a, b) => a.relPath.localeCompare(b.relPath));+ return out.sort(byRelPath);+}++async function readBundleFilesAsync(dir: string): Promise<AcquiredFile[]> {+ const out: AcquiredFile[] = [];+ const walk = async (d: string): Promise<void> => {+ for (const e of await readdir(d, { withFileTypes: true })) {+ const abs = join(d, e.name);+ if (e.isDirectory()) {+ if (!BUNDLE_WALK_IGNORE.has(e.name)) await walk(abs);+ continue;+ }+ if (!e.isFile()) continue;+ const relPath = bundleRelPath(dir, abs);+ if (relPath !== null) out.push(acquiredFile(relPath, await readFile(abs)));+ }+ };+ await walk(dir);+ return out.sort(byRelPath);}export function readSkillDirMeta(dir: string): { name: string; description: string } | null {@@ -133,10 +161,23 @@ export function parseSkillDir(dir: string): AcquiredSkill | null {const skillMdPath = join(dir, 'SKILL.md');if (!existsSync(skillMdPath)) return null;const skillMd = readFileSync(skillMdPath, 'utf-8');+ return acquiredSkill(dir, skillMd, readBundleFiles(dir));+}- const { name, description } = readSkillManifestMeta(skillMd, basename(dir));+export async function parseSkillDirAsync(dir: string): Promise<AcquiredSkill | null> {+ let skillMd: string;+ try {+ skillMd = await readFile(join(dir, 'SKILL.md'), 'utf-8');+ } catch (err) {+ const code = (err as NodeJS.ErrnoException).code;+ if (code === 'ENOENT' || code === 'ENOTDIR') return null;+ throw err;+ }+ return acquiredSkill(dir, skillMd, await readBundleFilesAsync(dir));+}- const files = readBundleFiles(dir);+function acquiredSkill(dir: string, skillMd: string, files: AcquiredFile[]): AcquiredSkill {+ const { name, description } = readSkillManifestMeta(skillMd, basename(dir));const hash = createHash('sha256');hash.update(`SKILL.md\n${skillMd}\n`);diff --git a/packages/core/src/skills-catalog/enumerate.ts b/packages/core/src/skills-catalog/enumerate.tsindex 4bfe5e64e..0038eea34 100644--- a/packages/core/src/skills-catalog/enumerate.ts+++ b/packages/core/src/skills-catalog/enumerate.ts@@ -13,6 +13,7 @@ import { catalogRawScopeToOkScope, isDetectedSkillInProject } from './scope.ts';export interface EnumerateOptions {home?: string;projectDir?: string;+ hashOf?: (skillDir: string) => string | undefined;}const OK_OWNED_SKILL_PREFIX = 'open-knowledge';@@ -51,20 +52,27 @@ function toInstalledSkill(winner: RawSkill, harnesses: string[]): CatalogSkill {};}-function contentIdentity(s: RawSkill): string {+function parsedContentHash(skillDir: string): string | undefined {+ return parseSkillDir(skillDir)?.contentHash;+}++function contentIdentity(s: RawSkill, hashOf: (skillDir: string) => string | undefined): string {try {- return parseSkillDir(s.home)?.contentHash ?? `unreadable:${s.home}`;+ return hashOf(s.home) ?? `unreadable:${s.home}`;} catch {return `unreadable:${s.home}`;}}-function dedupeSkills(raw: RawSkill[]): CatalogSkill[] {+function dedupeSkills(+ raw: RawSkill[],+ hashOf: (skillDir: string) => string | undefined,+): CatalogSkill[] {const byIdentity = new Map<string, { winner: RawSkill; harnesses: Set<string> }>();for (const s of raw) {const scope = catalogRawScopeToOkScope(s.provenance.scope);const projectPath = scope === 'project' ? (s.provenance.projectPath ?? '') : '';- const key = `${scope}\0${projectPath}\0${s.name}\0${contentIdentity(s)}`;+ const key = `${scope}\0${projectPath}\0${s.name}\0${contentIdentity(s, hashOf)}`;const cur = byIdentity.get(key);if (!cur) {byIdentity.set(key, { winner: s, harnesses: new Set([s.harness]) });@@ -164,7 +172,10 @@ export function enumerateInstalledSkills(opts: EnumerateOptions = {}): Installed})).filter((bundle) => bundle.skills.length > 0);const raw = localBundles.flatMap((b) => b.skills);- return { skills: dedupeSkills(raw), packs: toPacks(localBundles) };+ return {+ skills: dedupeSkills(raw, opts.hashOf ?? parsedContentHash),+ packs: toPacks(localBundles),+ };}function enumerateProjectHarnessSkills(projectDir: string): SkillBundle[] {diff --git a/packages/core/src/skills-catalog/index.ts b/packages/core/src/skills-catalog/index.tsindex c4ee21cc7..cf4d8f5f0 100644--- a/packages/core/src/skills-catalog/index.ts+++ b/packages/core/src/skills-catalog/index.ts@@ -27,8 +27,10 @@ export {type AcquiredFile,type AcquiredSkill,acquiredBundleTooLarge,+ BUNDLE_WALK_IGNORE,discoverSkillDirs,parseSkillDir,+ parseSkillDirAsync,readSkillDirMeta,} from './acquire/parse.ts';export {diff --git a/packages/server/src/boot-skill-read-stall.test.ts b/packages/server/src/boot-skill-read-stall.test.tsnew file mode 100644index 000000000..74bf174f0--- /dev/null+++ b/packages/server/src/boot-skill-read-stall.test.ts@@ -0,0 +1,172 @@+import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { performance } from 'node:perf_hooks';+import { setTimeout as sleep } from 'node:timers/promises';+import { expect, test, vi } from 'vitest';+import { bootServer } from './boot.ts';+import { ConfigSchema } from './config/schema.ts';+import { scanGlobalInPlaceSkillsAsync } from './in-place-skills.ts';++const SLOW_READ_MS = 3_000;++const fixture = vi.hoisted(() => ({+ home: undefined as string | undefined,+ slowFile: undefined as string | undefined,+ slowReads: 0,+ lockedFile: undefined as string | undefined,+ lockedReads: 0,+}));++vi.mock('node:os', async (importOriginal) => {+ const os = await importOriginal<typeof import('node:os')>();+ return { ...os, homedir: () => fixture.home ?? os.homedir() };+});++vi.mock('node:fs', async (importOriginal) => {+ const fs = await importOriginal<typeof import('node:fs')>();+ const readFileSync = ((...args: Parameters<typeof fs.readFileSync>) => {+ if (fixture.lockedFile !== undefined && String(args[0]) === fixture.lockedFile) {+ throw Object.assign(new Error('EPERM: operation not permitted'), { code: 'EPERM' });+ }+ if (fixture.slowFile !== undefined && String(args[0]) === fixture.slowFile) {+ fixture.slowReads += 1;+ const until = performance.now() + SLOW_READ_MS;+ while (performance.now() < until) {}+ }+ return fs.readFileSync(...args);+ }) as typeof fs.readFileSync;+ return { ...fs, default: { ...fs, readFileSync }, readFileSync };+});++vi.mock('node:fs/promises', async (importOriginal) => {+ const fsp = await importOriginal<typeof import('node:fs/promises')>();+ const readFile = (async (...args: Parameters<typeof fsp.readFile>) => {+ if (fixture.lockedFile !== undefined && String(args[0]) === fixture.lockedFile) {+ fixture.lockedReads += 1;+ throw Object.assign(new Error('EPERM: operation not permitted'), { code: 'EPERM' });+ }+ if (fixture.slowFile !== undefined && String(args[0]) === fixture.slowFile) {+ fixture.slowReads += 1;+ await sleep(SLOW_READ_MS);+ }+ return fsp.readFile(...args);+ }) as typeof fsp.readFile;+ return { ...fsp, default: { ...fsp, readFile }, readFile };+});++test('a slow-to-read global skill bundle stalls neither boot nor a skills request made while it is read', async () => {+ const home = await mkdtemp(join(tmpdir(), 'ok-boot-skill-home-'));+ const projectDir = await mkdtemp(join(tmpdir(), 'ok-boot-skill-project-'));+ const skillDir = join(home, '.codex', 'skills', 'heavy-skill');+ await mkdir(join(skillDir, 'assets'), { recursive: true });+ await writeFile(+ join(skillDir, 'SKILL.md'),+ '---\nname: heavy-skill\ndescription: A skill with a large bundle.\n---\n# Heavy\n',+ );+ const slowFile = join(skillDir, 'assets', 'model.bin');+ await writeFile(slowFile, Buffer.from([0, 1, 2, 3]));+ await mkdir(join(skillDir, 'node_modules', 'dep'), { recursive: true });+ await writeFile(join(skillDir, 'node_modules', 'dep', 'index.js'), 'export {};\n');+ await mkdir(join(projectDir, '.ok'), { recursive: true });+ await writeFile(join(projectDir, '.ok', 'config.yml'), '');+ await writeFile(join(projectDir, 'a.md'), '# a\n');+ fixture.home = home;+ fixture.slowFile = slowFile;++ let worstStallMs = 0;+ let lastTick = performance.now();+ const heartbeat = setInterval(() => {+ const now = performance.now();+ worstStallMs = Math.max(worstStallMs, now - lastTick);+ lastTick = now;+ }, 10);++ let booted: Awaited<ReturnType<typeof bootServer>> | undefined;+ try {+ booted = await bootServer({+ host: '127.0.0.1',+ config: ConfigSchema.parse({}),+ contentDir: projectDir,+ configHomedirOverride: home,+ port: 0,+ quiet: true,+ gitEnabled: false,+ idleShutdownMs: null,+ ephemeral: true,+ });+ await booted.ready;+ const skills = await fetch(`http://127.0.0.1:${booted.port}/api/skills`);+ expect(skills.status).toBe(200);+ await scanGlobalInPlaceSkillsAsync(home);+ } finally {+ clearInterval(heartbeat);+ await booted?.destroy('test');+ fixture.home = undefined;+ fixture.slowFile = undefined;+ await rm(home, { recursive: true, force: true });+ await rm(projectDir, { recursive: true, force: true });+ }++ expect(fixture.slowReads).toBe(1);+ expect(worstStallMs).toBeLessThan(SLOW_READ_MS / 2);+});++test('a folder that becomes a skill after its first read-ahead is still read off the event loop', async () => {+ const home = await mkdtemp(join(tmpdir(), 'ok-late-skill-home-'));+ const skillDir = join(home, '.codex', 'skills', 'late-skill');+ await mkdir(join(skillDir, 'assets'), { recursive: true });+ fixture.slowReads = 0;+ let worstStallMs = 0;+ let heartbeat: ReturnType<typeof setInterval> | undefined;+ try {+ expect(await scanGlobalInPlaceSkillsAsync(home)).toEqual([]);+ await writeFile(+ join(skillDir, 'SKILL.md'),+ '---\nname: late-skill\ndescription: Written after its folder.\n---\n# Late\n',+ );+ const slowFile = join(skillDir, 'assets', 'model.bin');+ await writeFile(slowFile, Buffer.from([0, 1, 2, 3]));+ fixture.slowFile = slowFile;++ let lastTick = performance.now();+ heartbeat = setInterval(() => {+ const now = performance.now();+ worstStallMs = Math.max(worstStallMs, now - lastTick);+ lastTick = now;+ }, 10);+ const skills = await scanGlobalInPlaceSkillsAsync(home);+ expect(skills.map((s) => s.name)).toEqual(['late-skill']);+ await sleep(50);+ } finally {+ clearInterval(heartbeat);+ fixture.slowFile = undefined;+ await rm(home, { recursive: true, force: true });+ }++ expect(fixture.slowReads).toBe(1);+ expect(worstStallMs).toBeLessThan(SLOW_READ_MS / 2);+});++test('a skill whose bundle cannot be read is read ahead once, not on every scan', async () => {+ const home = await mkdtemp(join(tmpdir(), 'ok-locked-skill-home-'));+ const skillDir = join(home, '.codex', 'skills', 'locked-skill');+ await mkdir(join(skillDir, 'assets'), { recursive: true });+ await writeFile(+ join(skillDir, 'SKILL.md'),+ '---\nname: locked-skill\ndescription: One file is locked.\n---\n# Locked\n',+ );+ const lockedFile = join(skillDir, 'assets', 'locked.bin');+ await writeFile(lockedFile, Buffer.from([0, 1]));+ fixture.lockedFile = lockedFile;+ fixture.lockedReads = 0;+ try {+ await scanGlobalInPlaceSkillsAsync(home);+ await scanGlobalInPlaceSkillsAsync(home);+ await scanGlobalInPlaceSkillsAsync(home);+ } finally {+ fixture.lockedFile = undefined;+ await rm(home, { recursive: true, force: true });+ }+ expect(fixture.lockedReads).toBe(1);+});diff --git a/packages/server/src/boot.ts b/packages/server/src/boot.tsindex 13e60a582..85b2ced58 100644--- a/packages/server/src/boot.ts+++ b/packages/server/src/boot.ts@@ -46,7 +46,7 @@ import {type CollabClientCounter,type IdleShutdownHandle,} from './idle-shutdown.ts';-import { scanGlobalInPlaceSkills, scanInPlaceSkills } from './in-place-skills.ts';+import { scanGlobalInPlaceSkillsAsync, scanInPlaceSkills } from './in-place-skills.ts';import { buildIngressPolicy, ExposureConsentError } from './ingress-policy.ts';import { resolveLocalSinkConfig } from './local-sink-resolver.ts';import { getLogger, loggerFactory, type PinoLogger } from './logger.ts';@@ -786,7 +786,8 @@ async function bootServerInner(opts: BootServerOptions): Promise<BootedServer> {const m = await migrateStoreSkillsInPlace({projectDir,skillsRoot: resolve(opts.contentDir, OK_DIR, SKILLS_STORE_DIRNAME),- inPlaceNames: genuineInPlaceNames(opts.contentDir, scanInPlaceSkills(opts.contentDir)),+ inPlaceNames: async () =>+ genuineInPlaceNames(opts.contentDir, scanInPlaceSkills(opts.contentDir)),});if (m.migrated.length + m.skipped.length > 0) {log.info?.(@@ -811,7 +812,7 @@ async function bootServerInner(opts: BootServerOptions): Promise<BootedServer> {projectDir: home,skillsRoot: resolve(home, OK_DIR, SKILLS_STORE_DIRNAME),hostRoots: USER_HOST_ROOTS_BY_PRECEDENCE,- inPlaceNames: genuineInPlaceNames(home, scanGlobalInPlaceSkills(home)),+ inPlaceNames: async () => genuineInPlaceNames(home, await scanGlobalInPlaceSkillsAsync(home)),});if (gm.migrated.length + gm.skipped.length > 0) {log.info?.(diff --git a/packages/server/src/http/skills-list-routes.ts b/packages/server/src/http/skills-list-routes.tsindex fc54e006d..b1676cfb6 100644--- a/packages/server/src/http/skills-list-routes.ts+++ b/packages/server/src/http/skills-list-routes.ts@@ -19,7 +19,7 @@ import {aliasedSourceRoots,isActivatedSkillRoot,resolveGlobalNativeSkillDir,- scanGlobalInPlaceSkills,+ scanGlobalInPlaceSkillsAsync,scanHostRootAliases,scanInPlaceSkills,standardSkillRoots,@@ -143,6 +143,7 @@ export function createSkillsListRoutes(deps: SkillsListRouteDeps): ApiRouteGroupEmptyRequestSchema,catchErrors(async (_req, res) => {+ const globalInPlaceSkills = await scanGlobalInPlaceSkillsAsync(skillsHome);const inPlaceFp =(contentFilter?.peekFreshInPlaceSkillDirsFingerprint() ?? '') +'\u0001' +@@ -152,7 +153,7 @@ export function createSkillsListRoutes(deps: SkillsListRouteDeps): ApiRouteGroup.map((h) => h.editorId).join(',') +'\u0001' +- scanGlobalInPlaceSkills(skillsHome)+ globalInPlaceSkills.map((s) => s.dir).sort().join(',');@@ -342,10 +343,10 @@ export function createSkillsListRoutes(deps: SkillsListRouteDeps): ApiRouteGroupconst globalLock = readSkillsLock(join(skillsHome, ...SKILLS_LOCK_REL));const globalAliases = scanHostRootAliases(skillsHome, 'global');const globalAliasRoots = aliasedSourceRoots(globalAliases, 'global');- const globalInPlaceNames = new Set(scanGlobalInPlaceSkills(skillsHome).map((s) => s.name));+ const globalInPlaceNames = new Set(globalInPlaceSkills.map((s) => s.name));globalSkills.skills = globalSkills.skills.filter((s) => !globalInPlaceNames.has(s.name));const globalNameSeen = new Set<string>();- const globalInPlace = scanGlobalInPlaceSkills(skillsHome).map((s) => {+ const globalInPlace = globalInPlaceSkills.map((s) => {const tracked = !globalNameSeen.has(s.name);globalNameSeen.add(s.name);const placementsForRow = tracked ? globalPlacements[s.name] : undefined;diff --git a/packages/server/src/http/skills-read-routes.ts b/packages/server/src/http/skills-read-routes.tsindex 9279e71e1..27319efe6 100644--- a/packages/server/src/http/skills-read-routes.ts+++ b/packages/server/src/http/skills-read-routes.ts@@ -14,7 +14,7 @@ import { resolveProjectIdentity } from '@inkeep/open-knowledge-core/shadow-repo-import type { enumerateInstalledSkills } from '@inkeep/open-knowledge-core/skills-catalog';import type { CommentService } from '../comments/comment-service.ts';import { resolveProjectTemplates } from '../content/templates-resolver.ts';-import { scanGlobalInPlaceSkills, scanInPlaceSkills } from '../in-place-skills.ts';+import { scanGlobalInPlaceSkillsAsync, scanInPlaceSkills } from '../in-place-skills.ts';import { readSkillInstallStateSnapshot } from '../skill-state.ts';import type { ApiRouteTable } from './api-pipeline.ts';import { catchErrors } from './catch-errors.ts';@@ -62,12 +62,13 @@ export function createSkillsReadRoutes(deps: SkillsReadRouteDeps): SkillsReadRoutry {const projectIdentity = resolveProjectIdentity(projectDir ?? contentDir);const physicalProjectDir = resolve(projectDir ?? contentDir);+ const globalInPlaceSkills = await scanGlobalInPlaceSkillsAsync(skillsHome);const catalog = enumerateInstalledSkillsCached({projectDir: projectIdentity,...(homeDirOverride !== undefined ? { home: homeDirOverride } : {}),});const inPlaceNames = new Set(scanInPlaceSkills(contentDir).map((s) => s.name));- const globalInPlaceNames = new Set(scanGlobalInPlaceSkills(skillsHome).map((s) => s.name));+ const globalInPlaceNames = new Set(globalInPlaceSkills.map((s) => s.name));const result = {...catalog,skills: catalog.skillsdiff --git a/packages/server/src/in-place-skills.test.ts b/packages/server/src/in-place-skills.test.tsindex 8ff5e1106..ea2ebf750 100644--- a/packages/server/src/in-place-skills.test.ts+++ b/packages/server/src/in-place-skills.test.ts@@ -317,6 +317,24 @@ describe('parse cache (bundle stamp invalidation)', () => {const second = scanInPlaceSkills(contentDir).find((s) => s.name === 'foo');expect(second?.contentHash).not.toBe(first?.contentHash);});++ test('files the content hash ignores never invalidate the cache', () => {+ writeSkill(contentDir, '.claude/skills/foo', '# body one');+ utimesSync(skillMdOf('foo'), PINNED, PINNED);+ const first = scanInPlaceSkills(contentDir).find((s) => s.name === 'foo');++ const swapped = readFileSync(skillMdOf('foo'), 'utf8').replace('# body one', '# body two');+ writeFileSync(skillMdOf('foo'), swapped);+ utimesSync(skillMdOf('foo'), PINNED, PINNED);+ for (const ignored of ['node_modules/dep', '.git/objects']) {+ const dir = join(contentDir, '.claude/skills/foo', ignored);+ mkdirSync(dir, { recursive: true });+ writeFileSync(join(dir, 'file'), 'not part of the bundle\n');+ }++ const second = scanInPlaceSkills(contentDir).find((s) => s.name === 'foo');+ expect(second?.contentHash).toBe(first?.contentHash);+ });});describe('global tier (R12): scanGlobalInPlaceSkills + resolveGlobalNativeSkillDir', () => {diff --git a/packages/server/src/in-place-skills.ts b/packages/server/src/in-place-skills.tsindex 98270a093..d801e2574 100644--- a/packages/server/src/in-place-skills.ts+++ b/packages/server/src/in-place-skills.ts@@ -1,4 +1,5 @@import { existsSync, lstatSync, readdirSync, realpathSync, statSync } from 'node:fs';+import { readdir, stat } from 'node:fs/promises';import { isAbsolute, join, relative, sep } from 'node:path';import {AGENTS_SKILLS_ROOT,@@ -16,14 +17,18 @@ import {skillRootActivationPath,} from '@inkeep/open-knowledge-core';import {+ type AcquiredSkill,+ BUNDLE_WALK_IGNORE,buildSkillRegistry,groupSkillsByIdentity,type LocatedSkillOccurrence,packMarkerOf,parseSkillDir,+ parseSkillDirAsync,SKILL_CANONICAL_PRECEDENCE,type SkillHostId,} from '@inkeep/open-knowledge-core/skills-catalog';+import { getLogger } from './logger.ts';import { isInternalBundleSkillName, isUserGlobalBundleSkillName } from './skill-bundles.ts';import {observeSkillPlacementInputsForScan,@@ -233,8 +238,9 @@ function bundleStamp(absDir: string): string | null {for (const e of readdirSync(dir, { withFileTypes: true })) {const p = join(dir, e.name);const r = rel === '' ? e.name : `${rel}/${e.name}`;- if (e.isDirectory()) walk(p, r);- else if (e.isFile()) {+ if (e.isDirectory()) {+ if (!BUNDLE_WALK_IGNORE.has(e.name)) walk(p, r);+ } else if (e.isFile()) {const st = statSync(p);parts.push(`${r}:${st.size}:${st.mtimeMs}`);}@@ -244,50 +250,144 @@ function bundleStamp(absDir: string): string | null {return parts.sort().join('|');}-const parseCache = new Map<- string,- { stamp: string; contentHash: string; description: string; size: SkillCostTiers }->();+async function bundleStampAsync(absDir: string): Promise<string | null> {+ try {+ if (!(await stat(join(absDir, 'SKILL.md'))).isFile()) return null;+ } catch (error) {+ if (isAbsentPath(error)) return null;+ throw error;+ }+ const parts: string[] = [];+ const walk = async (dir: string, rel: string): Promise<void> => {+ for (const e of await readdir(dir, { withFileTypes: true })) {+ const p = join(dir, e.name);+ const r = rel === '' ? e.name : `${rel}/${e.name}`;+ if (e.isDirectory()) {+ if (!BUNDLE_WALK_IGNORE.has(e.name)) await walk(p, r);+ } else if (e.isFile()) {+ const st = await stat(p);+ parts.push(`${r}:${st.size}:${st.mtimeMs}`);+ }+ }+ };+ await walk(absDir, '');+ return parts.sort().join('|');+}-function parseSkillDirCached(- absDir: string,-): { contentHash: string; description: string; size: SkillCostTiers; pack?: string } | null {- const stamp = bundleStamp(absDir);- if (stamp === null) return null;- const hit = parseCache.get(absDir);- if (hit !== undefined && hit.stamp === stamp) return hit;- const parsed = parseSkillDir(absDir);- if (!parsed) return null;+interface ParseCacheEntry {+ stamp: string;+ contentHash: string;+ description: string;+ size: SkillCostTiers;+ pack?: string;+}++const parseCache = new Map<string, ParseCacheEntry>();++function parseCacheEntry(stamp: string, parsed: AcquiredSkill): ParseCacheEntry {const pack = packMarkerOf(parseFrontmatterRecord(parsed.skillMd) ?? {});- const entry = {+ return {stamp,contentHash: parsed.contentHash,description: parsed.description,size: estimateSkillCost(parsed),...(pack !== undefined ? { pack } : {}),};+}++function parseSkillDirCached(absDir: string): ParseCacheEntry | null {+ const stamp = bundleStamp(absDir);+ if (stamp === null) return null;+ const hit = parseCache.get(absDir);+ if (hit !== undefined && hit.stamp === stamp) return hit;+ const parsed = parseSkillDir(absDir);+ if (!parsed) return null;+ const entry = parseCacheEntry(stamp, parsed);parseCache.set(absDir, entry);return entry;}-function scanBase(base: string, scope: SkillScope): SkillScanObservation {- const occurrences: ScanOccurrence[] = [];+export function cachedSkillContentHash(skillDir: string): string | undefined {+ return parseSkillDirCached(skillDir)?.contentHash;+}++const parseCacheWarming = new Map<string, Promise<void>>();+const parseCacheAwaitingSkillMd = new Map<string, boolean>();++function warmParseCache(absDir: string, root: string): Promise<void> {+ const inFlight = parseCacheWarming.get(absDir);+ if (inFlight !== undefined) return inFlight;+ const warming = (async () => {+ const stamp = await bundleStampAsync(absDir);+ if (stamp === null) return true;+ if (parseCache.get(absDir)?.stamp === stamp) return false;+ const parsed = await parseSkillDirAsync(absDir);+ if (parsed) parseCache.set(absDir, parseCacheEntry(stamp, parsed));+ return false;+ })()+ .catch((error) => {+ logWarmFailure(root, error);+ return false;+ })+ .then((awaitingSkillMd) => {+ parseCacheAwaitingSkillMd.set(absDir, awaitingSkillMd);+ })+ .finally(() => parseCacheWarming.delete(absDir));+ parseCacheWarming.set(absDir, warming);+ return warming;+}++function logWarmFailure(root: string, error: unknown): void {+ if (isAbsentPath(error)) return;+ getLogger('in-place-skills').warn(+ { root, err: error },+ '[in-place-skills] skills not read ahead of the scan; the scan reads them synchronously',+ );+}++async function warmGlobalScanCache(home: string): Promise<void> {+ for (const { root } of observeScanInputs(home, 'global').roots) {+ const absRoot = join(home, root);+ let entries: string[];+ try {+ entries = await readdir(absRoot);+ } catch (error) {+ logWarmFailure(root, error);+ continue;+ }+ for (const entry of entries) await warmParseCache(join(absRoot, entry), root);+ }+}++function observeScanInputs(+ base: string,+ scope: SkillScope,+): {+ sourcePrefs: Record<string, string>;+ roots: ReturnType<typeof skillRootsWithCustom>;+ failures: SkillScanFailure[];+} {const failures: SkillScanFailure[] = [];- const recordFailure = (error: unknown): void => {- failures.push({ scope: 'scan', error });- };let sourcePrefs: Record<string, string> = {};let customRoots: string[] = [];try {const inputs = observeSkillPlacementInputsForScan(base);sourcePrefs = inputs.sources;customRoots = inputs.roots;- if (inputs.kind === 'incomplete') recordFailure(inputs.error);+ if (inputs.kind === 'incomplete') failures.push({ scope: 'scan', error: inputs.error });if (inputs.kind === 'unparsable') failures.push({ scope: 'ledger-parse', error: inputs.error });} catch (error) {- recordFailure(error);+ failures.push({ scope: 'scan', error });}- const allRoots = skillRootsWithCustom(scope, customRoots);+ return { sourcePrefs, roots: skillRootsWithCustom(scope, customRoots), failures };+}++function scanBase(base: string, scope: SkillScope): SkillScanObservation {+ const occurrences: ScanOccurrence[] = [];+ const { sourcePrefs, roots: allRoots, failures } = observeScanInputs(base, scope);+ const recordFailure = (error: unknown): void => {+ failures.push({ scope: 'scan', error });+ };let baseReal: string | null = null;try {@@ -411,6 +511,36 @@ export function scanGlobalInPlaceSkills(home: string): InPlaceSkill[] {return scanBase(home, 'global').skills;}+export async function scanGlobalInPlaceSkillsAsync(home: string): Promise<InPlaceSkill[]> {+ if (globalScanHasUnwarmedDir(home)) await warmGlobalScanCache(home);+ return scanGlobalInPlaceSkills(home);+}++function needsWarm(absDir: string): boolean {+ const awaitingSkillMd = parseCacheAwaitingSkillMd.get(absDir);+ if (awaitingSkillMd === undefined) return true;+ if (!awaitingSkillMd) return false;+ try {+ return statSync(join(absDir, 'SKILL.md')).isFile();+ } catch {+ return false;+ }+}++function globalScanHasUnwarmedDir(home: string): boolean {+ for (const { root } of observeScanInputs(home, 'global').roots) {+ const absRoot = join(home, root);+ let entries: string[];+ try {+ entries = readdirSync(absRoot);+ } catch {+ continue;+ }+ if (entries.some((entry) => needsWarm(join(absRoot, entry)))) return true;+ }+ return false;+}+const USER_ROOTS_BY_PRECEDENCE = [...USER_SKILL_ROOTS].sort((a, b) => {const ra = SKILL_CANONICAL_PRECEDENCE.indexOf(a.editor);const rb = SKILL_CANONICAL_PRECEDENCE.indexOf(b.editor);diff --git a/packages/server/src/server-factory.ts b/packages/server/src/server-factory.tsindex 5b48e0da1..38e4cee32 100644--- a/packages/server/src/server-factory.ts+++ b/packages/server/src/server-factory.ts@@ -182,7 +182,7 @@ import type { NativeApiHandle } from './http/http-app.ts';import type { LocalApiDispatch } from './http/local-api-dispatch.ts';import type { GeneratedIndexSettingsStatus } from './http/workspace-tools-routes.ts';import {- scanGlobalInPlaceSkills,+ scanGlobalInPlaceSkillsAsync,scanInPlaceSkillDirs,scanInPlaceSkills,skillRootPathsFor,@@ -1969,7 +1969,8 @@ export function createServer(options: ServerOptions): ServerInstance {globalCopyResyncTimer = setTimeout(() => {globalCopyResyncTimer = null;const home = configHomedirOverride ?? homedir();- void resyncRecordedSkillCopies(home, home, scanGlobalInPlaceSkills(home))+ void scanGlobalInPlaceSkillsAsync(home)+ .then((inPlace) => resyncRecordedSkillCopies(home, home, inPlace)).then((n) => {if (n > 0) log.info({ refreshed: n }, '[in-place-skills] post-edit copy re-sync');})@@ -4286,7 +4287,8 @@ export function createServer(options: ServerOptions): ServerInstance {.catch((err) => log.warn({ err }, '[in-place-skills] boot copy re-sync failed'));{const home = configHomedirOverride ?? homedir();- void resyncRecordedSkillCopies(home, home, scanGlobalInPlaceSkills(home))+ void scanGlobalInPlaceSkillsAsync(home)+ .then((inPlace) => resyncRecordedSkillCopies(home, home, inPlace)).then((n) => {if (n > 0) log.info({ refreshed: n }, '[in-place-skills] global boot copy re-sync');})diff --git a/packages/server/src/skill-migrate.test.ts b/packages/server/src/skill-migrate.test.tsindex 1cd1a2342..54d481162 100644--- a/packages/server/src/skill-migrate.test.ts+++ b/packages/server/src/skill-migrate.test.ts@@ -205,7 +205,7 @@ describe('migrateStoreSkillsInPlace', () => {const r = await migrateStoreSkillsInPlace({projectDir: root,skillsRoot,- inPlaceNames: new Set(['placed']),+ inPlaceNames: async () => new Set(['placed']),});expect(r.migrated).toEqual([]);expect(existsSync(join(skillsRoot, 'placed', 'SKILL.md'))).toBe(true);@@ -216,6 +216,41 @@ describe('migrateStoreSkillsInPlace', () => {expect(r.migrated).toEqual([]);expect(r.skipped).toEqual([]);});++ test('scans in-place skills only when the store holds a skill to classify', async () => {+ let scans = 0;+ const inPlaceNames = async () => {+ scans += 1;+ return new Set<string>(['placed']);+ };+ await migrateStoreSkillsInPlace({ projectDir: root, skillsRoot, inPlaceNames });+ rmSync(skillsRoot, { recursive: true, force: true });+ await migrateStoreSkillsInPlace({ projectDir: root, skillsRoot, inPlaceNames });+ expect(scans).toBe(0);++ makeStore('placed', '# P');+ makeStore('other', '# O');+ const r = await migrateStoreSkillsInPlace({ projectDir: root, skillsRoot, inPlaceNames });+ expect(scans).toBe(1);+ expect(r.skipped).toContainEqual({ name: 'placed', reason: 'placement-of-in-place' });+ });++ test('a failed in-place scan migrates nothing', async () => {+ makeStore('placed', '# P');+ makeStore('other', '# O');+ mkdirSync(join(root, '.claude', 'skills'), { recursive: true });+ const r = await migrateStoreSkillsInPlace({+ projectDir: root,+ skillsRoot,+ inPlaceNames: async () => {+ throw new Error('scan failed');+ },+ });+ expect(r.migrated).toEqual([]);+ expect(r.skipped.map((s) => s.reason)).toEqual(['error', 'error']);+ expect(existsSync(join(skillsRoot, 'placed', 'SKILL.md'))).toBe(true);+ expect(existsSync(join(skillsRoot, 'other', 'SKILL.md'))).toBe(true);+ });});describe('genuineInPlaceNames', () => {diff --git a/packages/server/src/skill-migrate.ts b/packages/server/src/skill-migrate.tsindex bb66f8a89..c7b412929 100644--- a/packages/server/src/skill-migrate.ts+++ b/packages/server/src/skill-migrate.ts@@ -119,11 +119,15 @@ export function genuineInPlaceNames(export async function migrateStoreSkillsInPlace(opts: {projectDir: string;skillsRoot: string;- inPlaceNames?: ReadonlySet<string>;+ inPlaceNames?: () => Promise<ReadonlySet<string>>;hostRoots?: ReadonlyArray<{ host: SkillHostId; root: string }>;}): Promise<StoreMigrationResult> {const { projectDir, skillsRoot } = opts;- const inPlaceNames = opts.inPlaceNames ?? new Set<string>();+ let inPlaceNames: Promise<ReadonlySet<string>> | undefined;+ const isInPlaceName = async (name: string): Promise<boolean> => {+ inPlaceNames ??= opts.inPlaceNames?.() ?? Promise.resolve(new Set<string>());+ return (await inPlaceNames).has(name);+ };const hostRootsByPrecedence = opts.hostRoots ?? HOST_ROOTS_BY_PRECEDENCE;const result: StoreMigrationResult = { migrated: [], skipped: [] };if (!existsSync(skillsRoot)) return result;@@ -147,7 +151,7 @@ export async function migrateStoreSkillsInPlace(opts: {for (const name of entries) {try {if (!SKILL_NAME_REGEX.test(name) || INTERNAL_BUNDLE_SKILL_NAMES.has(name)) continue;- if (inPlaceNames.has(name)) {+ if (await isInPlaceName(name)) {result.skipped.push({ name, reason: 'placement-of-in-place' });continue;}diff --git a/packages/server/src/skills-catalog-cache.test.ts b/packages/server/src/skills-catalog-cache.test.tsindex 65988086e..00bd218fc 100644--- a/packages/server/src/skills-catalog-cache.test.ts+++ b/packages/server/src/skills-catalog-cache.test.ts@@ -127,7 +127,7 @@ test('plugin content refreshes only on identity change or its independent thirtyclock.mockReturnValue(130_000);const expired = c.pluginSkillsByName('project-a');expect(expired.get('alpha')?.contentHash).not.toBe(hash);- installPlugin('third');+ installPlugin('third revision');c.bumpSkillsCatalogGen();const changedIdentity = c.pluginSkillsByName('project-b');expect(changedIdentity.get('alpha')?.contentHash).not.toBe(expired.get('alpha')?.contentHash);diff --git a/packages/server/src/skills-catalog-cache.ts b/packages/server/src/skills-catalog-cache.tsindex 5c1b520a9..2f3693215 100644--- a/packages/server/src/skills-catalog-cache.ts+++ b/packages/server/src/skills-catalog-cache.ts@@ -1,9 +1,9 @@import {enumerateInstalledSkills,type PluginUpstream,- parseSkillDir,pluginUpstreamsByName,} from '@inkeep/open-knowledge-core/skills-catalog';+import { cachedSkillContentHash } from './in-place-skills.ts';import type { PinoLogger } from './logger.ts';export type SkillsCatalogCache = ReturnType<typeof createSkillsCatalogCache>;@@ -41,7 +41,7 @@ export function createSkillsCatalogCache({) {return { freshBuild: false, key, value: installedCatalogCache.value };}- const value = enumerateInstalledSkills(opts);+ const value = enumerateInstalledSkills({ ...opts, hashOf: cachedSkillContentHash });installedCatalogCache = { at: Date.now(), gen: skillsCatalogGen, key, value };return { freshBuild: true, key, value };}@@ -96,10 +96,7 @@ export function createSkillsCatalogCache({? { home: homeDirOverride, projectDir: identity }: { projectDir: identity },);- byName = pluginUpstreamsByName(- catalogLookup.value.skills,- (home) => parseSkillDir(home)?.contentHash,- );+ byName = pluginUpstreamsByName(catalogLookup.value.skills, cachedSkillContentHash);} catch (err) {log.warn({ err }, 'plugin upstream index failed; origins will be omitted');}