Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

perf(ok): take the template scan off the CRDT event loop (PRD-7765) (#3166)

Andrew Mikofalvy50f9489f7f80
diff --git a/.changeset/template-scan-off-the-event-loop.md b/.changeset/template-scan-off-the-event-loop.md
new file mode 100644
index 000000000..c0c856e1c
--- /dev/null
+++ b/.changeset/template-scan-off-the-event-loop.md
@@ -0,0 +1,5 @@
+---
+"@inkeep/open-knowledge": patch
+---
+
+Typing in a large knowledge base no longer stutters while the template list is being gathered. Building the list of templates you can create from means walking the project's folders, and on a big repository that walk is long enough to hit its internal limit. It ran start to finish without pausing, on the same loop that carries your keystrokes between the editor and the server, so a burst of requests left the editor unable to keep up with typing for a noticeable stretch. The walk now pauses between folders (the longest uninterrupted pause it causes drops from roughly 84 ms to about 1 ms) and reads each folder with a single system call instead of one per entry. Symlinked folders are still followed.
diff --git a/packages/server/src/api-extension.ts b/packages/server/src/api-extension.ts
index 83ec07328..9d5cbb156 100644
--- a/packages/server/src/api-extension.ts
+++ b/packages/server/src/api-extension.ts
@@ -3869,7 +3869,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
admitted.add(`${MANAGED_ARTIFACT_PREFIX_SKILL}${scope}/${skill.name}`);
}
}
- for (const tpl of resolveProjectTemplates(resolve(contentDir)).templates) {
+ for (const tpl of (await resolveProjectTemplates(resolve(contentDir))).templates) {
admitted.add(templateDocNameFor(tpl.source_folder, tpl.name));
}
} catch (err) {
@@ -15331,7 +15331,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
catchErrors(
async (_req, res) => {
const resolvedContentDir = resolve(contentDir);
- const result = resolveProjectTemplates(resolvedContentDir);
+ const result = await resolveProjectTemplates(resolvedContentDir);
// Drop `scope` from each entry — every flat-enumeration entry is
// implicitly `scope: 'local'` to its own `source_folder`, so the
// field carries no information here. `TemplatesListEntrySchema` is
diff --git a/packages/server/src/content/templates-resolver.test.ts b/packages/server/src/content/templates-resolver.test.ts
index d4b88a485..290919952 100644
--- a/packages/server/src/content/templates-resolver.test.ts
+++ b/packages/server/src/content/templates-resolver.test.ts
@@ -1,4 +1,4 @@
-import { mkdirSync, writeFileSync } from 'node:fs';
+import { mkdirSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
import { mkdtemp, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
@@ -242,13 +242,13 @@ describe('resolveProjectTemplates', () => {
return `---\ntitle: ${title}\ndescription: ${description}\n---\nbody\n`;
}
- test('returns flat list of every template across the project', () => {
+ test('returns flat list of every template across the project', async () => {
writeTemplate('', 'daily-note', withFm('Daily note', 'Date-stamped log for today'));
writeTemplate('meetings', 'meeting-notes', withFm('Meeting notes', 'Attendees, agenda, items'));
writeTemplate('research', 'research-log', withFm('Research log', 'Working notes'));
writeTemplate('specs', 'spec', withFm('Spec / RFC', 'Problem · proposal · decision'));
- const result = resolveProjectTemplates(projectDir);
+ const result = await resolveProjectTemplates(projectDir);
const byName = Object.fromEntries(result.templates.map((t) => [t.name, t]));
expect(Object.keys(byName).sort()).toEqual([
'daily-note',
@@ -263,27 +263,27 @@ describe('resolveProjectTemplates', () => {
expect(result.truncated).toBe(false);
});
- test('templates in nested subfolders surface with their source_folder', () => {
+ test('templates in nested subfolders surface with their source_folder', async () => {
writeTemplate('a/b/c', 'deep', withFm('Deep template', 'Buried in a/b/c'));
- const result = resolveProjectTemplates(projectDir);
+ const result = await resolveProjectTemplates(projectDir);
expect(result.templates).toHaveLength(1);
expect(result.templates[0]?.source_folder).toBe('a/b/c');
expect(result.templates[0]?.path).toBe('a/b/c/.ok/templates/deep.md');
});
- test('skips node_modules, dist, build, and dot-prefixed dirs', () => {
+ test('skips node_modules, dist, build, and dot-prefixed dirs', async () => {
writeTemplate('keep', 'visible', withFm('Visible', 'Should appear'));
writeTemplate('node_modules/dep', 'hidden', withFm('Hidden', 'Should NOT appear'));
writeTemplate('dist/output', 'hidden', withFm('Hidden', 'Should NOT appear'));
writeTemplate('build/out', 'hidden', withFm('Hidden', 'Should NOT appear'));
writeTemplate('.archive', 'hidden', withFm('Hidden', 'Dot-prefix excluded'));
- const result = resolveProjectTemplates(projectDir);
+ const result = await resolveProjectTemplates(projectDir);
const folders = result.templates.map((t) => t.source_folder).sort();
expect(folders).toEqual(['keep']);
});
- test('walker terminates within PROJECT_TEMPLATE_SCAN_CAP — guards against pathological trees', () => {
+ test('walker terminates within PROJECT_TEMPLATE_SCAN_CAP — guards against pathological trees', async () => {
// Build a wide tree exceeding the 2000-dir cap to verify the walker
// bails out cleanly rather than hanging or stack-overflowing AND
// collects templates queued before the cap. The `aa-` prefix makes
@@ -296,25 +296,113 @@ describe('resolveProjectTemplates', () => {
for (let i = 0; i < 2100; i++) {
mkdirSync(join(projectDir, `bulk-${i}`), { recursive: true });
}
- const result = resolveProjectTemplates(projectDir);
+ const result = await resolveProjectTemplates(projectDir);
expect(result.templates.some((t) => t.name === 'visible-root')).toBe(true);
expect(result.templates.some((t) => t.name === 'visible-early')).toBe(true);
// Cap was exceeded — flag must be set so the UI / response can signal it.
expect(result.truncated).toBe(true);
});
- test('returns empty array when no templates exist anywhere', () => {
+ test('returns empty array when no templates exist anywhere', async () => {
mkdirSync(join(projectDir, 'docs'), { recursive: true });
- expect(resolveProjectTemplates(projectDir)).toEqual({ templates: [], truncated: false });
+ await expect(resolveProjectTemplates(projectDir)).resolves.toEqual({
+ templates: [],
+ truncated: false,
+ });
});
- test('every entry carries scope: local (no inheritance context in flat enumeration)', () => {
+ test('every entry carries scope: local (no inheritance context in flat enumeration)', async () => {
writeTemplate('', 'root-tpl', withFm('Root', 'At project root'));
writeTemplate('subfolder', 'sub-tpl', withFm('Sub', 'In a subfolder'));
- const result = resolveProjectTemplates(projectDir);
+ const result = await resolveProjectTemplates(projectDir);
expect(result.templates).toHaveLength(2);
for (const t of result.templates) {
expect(t.scope).toBe('local');
}
});
+
+ test('descends through a symlinked directory', async () => {
+ // A Dirent reports the link's own type, so the walker needs an explicit
+ // following stat for symlinks or symlinked folders silently stop being
+ // enumerated.
+ writeTemplate('real', 'linked-tpl', withFm('Linked', 'Behind a symlink'));
+ symlinkSync(join(projectDir, 'real'), join(projectDir, 'alias'), 'dir');
+
+ const result = await resolveProjectTemplates(projectDir);
+ const folders = result.templates.map((t) => t.source_folder).sort();
+ expect(folders).toEqual(['alias', 'real']);
+ // `path` is how a caller reaches the file, and it stays relative to the
+ // link rather than resolving through it — both spellings are valid on disk.
+ expect(result.templates.map((t) => t.path).sort()).toEqual([
+ 'alias/.ok/templates/linked-tpl.md',
+ 'real/.ok/templates/linked-tpl.md',
+ ]);
+ });
+
+ test('a dangling symlink is skipped without aborting the walk', async () => {
+ // The following stat throws ENOENT on a broken link. Swallowing it is what
+ // keeps one stale link from cutting the scan short and hiding every
+ // template the walker had not reached yet.
+ writeTemplate('real', 'survivor', withFm('Survivor', 'Must still surface'));
+ symlinkSync(join(projectDir, 'no-such-target'), join(projectDir, 'dangling'), 'dir');
+
+ const result = await resolveProjectTemplates(projectDir);
+ expect(result.templates.map((t) => t.source_folder)).toEqual(['real']);
+ expect(result.truncated).toBe(false);
+ });
+
+ test('every call re-walks, so a template written between scans surfaces', async () => {
+ // Templates reach disk through the filesystem writers, the CRDT
+ // managed-artifact path, and plain external edits. Nothing here may
+ // serve a snapshot that predates the caller: a memo would have to be
+ // invalidated from all three, and a missed one hides a template the
+ // user just created.
+ writeTemplate('', 'first', withFm('First', 'Present at first scan'));
+ const before = await resolveProjectTemplates(projectDir);
+ expect(before.templates.map((t) => t.name)).toEqual(['first']);
+
+ writeTemplate('', 'second', withFm('Second', 'Added after the first scan'));
+ const after = await resolveProjectTemplates(projectDir);
+ expect(after.templates.map((t) => t.name).sort()).toEqual(['first', 'second']);
+ });
+
+ test('yields to the event loop while walking', async () => {
+ // The reason this resolver is async at all: the walk must not hold the
+ // loop that services CRDT sync, or users feel it as keystroke lag. The
+ // other tests here all still pass against a synchronous walk wrapped in
+ // an async function, because awaiting a non-promise is a no-op — so
+ // without this one, a refactor back to sync I/O lands green.
+ //
+ // A check-phase callback is the discriminator. A synchronous walk leaves
+ // the promise already settled, so the await continuation runs as a
+ // microtask, and microtasks drain before the loop ever reaches its check
+ // phase. Real async I/O suspends at the first readdir, so the loop turns
+ // and the callback runs.
+ //
+ // `setImmediate`, not `setTimeout(…, 0)`: a 0 ms timeout is clamped to
+ // 1 ms, and a walk this small can finish inside that window — the loop
+ // turned, but no timer was due to witness it. That reads as a failure.
+ for (let i = 0; i < 20; i++) {
+ mkdirSync(join(projectDir, `dir-${i}`), { recursive: true });
+ }
+ let ticked = false;
+ const pending = setImmediate(() => {
+ ticked = true;
+ });
+
+ await resolveProjectTemplates(projectDir);
+ clearImmediate(pending);
+
+ expect(ticked).toBe(true);
+ });
+
+ test('a deleted template stops surfacing on the next call', async () => {
+ writeTemplate('', 'doomed', withFm('Doomed', 'About to be removed'));
+ const before = await resolveProjectTemplates(projectDir);
+ expect(before.templates.map((t) => t.name)).toEqual(['doomed']);
+
+ rmSync(join(projectDir, '.ok', 'templates', 'doomed.md'));
+ const after = await resolveProjectTemplates(projectDir);
+ expect(after.templates).toEqual([]);
+ });
});
diff --git a/packages/server/src/content/templates-resolver.ts b/packages/server/src/content/templates-resolver.ts
index de2d48d77..ca95952a6 100644
--- a/packages/server/src/content/templates-resolver.ts
+++ b/packages/server/src/content/templates-resolver.ts
@@ -17,10 +17,16 @@
* frontmatter. `title` is required at template-write time; a stored
* template always has one. `description` is optional.
*
- * Synchronous I/O; matches the pattern in `nested-folder-rules.ts`.
+ * `resolveTemplatesAvailable` uses synchronous I/O; matches the pattern in
+ * `nested-folder-rules.ts`. Its walk is bounded by the target folder's path
+ * depth, so it stays cheap regardless of project size.
+ *
+ * `resolveProjectTemplates` walks the WHOLE project and is async for that
+ * reason — see its doc comment.
*/
-import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs';
+import { type Dirent, existsSync, readdirSync, readFileSync, statSync } from 'node:fs';
+import { readdir, stat } from 'node:fs/promises';
import { join, posix } from 'node:path';
import { parseTemplateFile } from '@inkeep/open-knowledge-core';
import { errnoCode } from '../http/handler-utils.ts';
@@ -113,8 +119,22 @@ export interface ProjectTemplatesResult {
*
* Scope is always `'local'` here. Bounded by `PROJECT_TEMPLATE_SCAN_CAP`
* directories visited; `truncated: true` in the result signals the cap hit.
+ *
+ * Async because this walks the entire project: on a large repo the walk
+ * reaches the cap, and doing that synchronously stalls the same event loop
+ * that services CRDT sync — which readers feel as keystroke lag. Awaiting
+ * per directory lets sync messages interleave.
+ *
+ * Every call re-walks, deliberately. Templates reach disk through two
+ * unrelated substrates (the filesystem writers in `templates-write.ts` and
+ * the CRDT managed-artifact persistence path behind `PUT /api/template`) as
+ * well as plain external edits, so any memo here has to be invalidated from
+ * each of them — a completeness obligation that is easy to violate silently,
+ * and whose failure mode is a template the user just created not appearing.
+ * The async walk is what removes the latency harm; caching on top bought
+ * little and risked that.
*/
-export function resolveProjectTemplates(projectDir: string): ProjectTemplatesResult {
+export async function resolveProjectTemplates(projectDir: string): Promise<ProjectTemplatesResult> {
const out: TemplateEntry[] = [];
const seenPerFolder = new Map<string, Set<string>>();
@@ -145,14 +165,11 @@ export function resolveProjectTemplates(projectDir: string): ProjectTemplatesRes
collectFromFolder(projectDir, folderRel, 'local', seen, out);
const absDir = folderRel ? join(projectDir, folderRel) : projectDir;
- let entries: string[];
+ let entries: Dirent[];
try {
- // readdirSync order is filesystem-dependent (ext4 htree hash order vs
- // APFS), so an unsorted BFS makes which folders fall inside
- // PROJECT_TEMPLATE_SCAN_CAP nondeterministic — a folder that survives the
- // cap on one run can be dropped on the next. Sort so the dequeue order,
- // and thus the cap truncation boundary, is stable across runs/platforms.
- entries = readdirSync(absDir).sort();
+ // `withFileTypes` carries the entry kind in the readdir result, so the
+ // common case needs no per-entry stat at all.
+ entries = await readdir(absDir, { withFileTypes: true });
} catch (err) {
// Non-ENOENT failures (EPERM, EACCES, ENOTDIR, symlink loop) indicate
// a real problem worth a once-per-path log so an operator can trace
@@ -171,21 +188,46 @@ export function resolveProjectTemplates(projectDir: string): ProjectTemplatesRes
}
continue;
}
- for (const name of entries) {
- if (PROJECT_TEMPLATE_DIR_SKIP.has(name)) continue;
+ // readdir order is filesystem-dependent (ext4 htree hash order vs APFS),
+ // so an unsorted BFS makes which folders fall inside
+ // PROJECT_TEMPLATE_SCAN_CAP nondeterministic — a folder that survives the
+ // cap on one run can be dropped on the next. Sort so the dequeue order,
+ // and thus the cap truncation boundary, is stable across runs/platforms.
+ entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
+ for (const entry of entries) {
+ if (PROJECT_TEMPLATE_DIR_SKIP.has(entry.name)) continue;
// Dot-prefixed dirs (other than `.ok`, already skipped) are user-
// hidden — `.archive/`, `.private/`, etc. — and follow the same
// visibility rule the sidebar's filterVisibleEntries uses.
- if (name.startsWith('.')) continue;
- const childAbs = join(absDir, name);
- let s: ReturnType<typeof statSync>;
- try {
- s = statSync(childAbs);
- } catch {
- continue;
+ if (entry.name.startsWith('.')) continue;
+ let isDirectory = entry.isDirectory();
+ // A Dirent reports the LINK's own type, so a symlink pointing at a
+ // directory reads as not-a-directory. Resolve just those with a
+ // following stat — symlinked folders stay walkable and non-links
+ // still skip the syscall.
+ if (!isDirectory && entry.isSymbolicLink()) {
+ const linkPath = join(absDir, entry.name);
+ try {
+ isDirectory = (await stat(linkPath)).isDirectory();
+ } catch (err) {
+ // A dangling link (ENOENT) is ordinary and stays silent. Anything
+ // else — EACCES, ELOOP, EPERM — drops a whole subtree out of the
+ // list, so it earns the same once-per-path warn the readdir failure
+ // above does, sharing its dedupe set.
+ const code = errnoCode(err);
+ if (code !== 'ENOENT' && !templateMetaWarnedPaths.has(linkPath)) {
+ templateMetaWarnedPaths.add(linkPath);
+ const reason = err instanceof Error ? err.message : String(err);
+ getLogger('templates').warn(
+ { link: linkPath, reason },
+ `failed to resolve symlink ${linkPath} during project scan — skipped. Reason: ${reason}`,
+ );
+ }
+ continue;
+ }
}
- if (!s.isDirectory()) continue;
- const childRel = folderRel ? posix.join(folderRel, name) : name;
+ if (!isDirectory) continue;
+ const childRel = folderRel ? posix.join(folderRel, entry.name) : entry.name;
queue.push(childRel);
}
}
diff --git a/packages/server/src/server-factory.ts b/packages/server/src/server-factory.ts
index 303259d04..8ea18aa26 100644
--- a/packages/server/src/server-factory.ts
+++ b/packages/server/src/server-factory.ts
@@ -3517,7 +3517,7 @@ export function createServer(options: ServerOptions): ServerInstance {
try {
const templateFolders = new Set<string>(['']);
try {
- for (const t of resolveProjectTemplates(projectDir).templates) {
+ for (const t of (await resolveProjectTemplates(projectDir)).templates) {
templateFolders.add(t.source_folder);
}
} catch (err) {