Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
fix(open-knowledge): make boot-time index walks yield to the event loop (#1731)
mike-inkeepb6a5d02b49a3
diff --git a/.changeset/async-boot-index-walks.md b/.changeset/async-boot-index-walks.mdnew file mode 100644index 000000000..4456f2231--- /dev/null+++ b/.changeset/async-boot-index-walks.md@@ -0,0 +1,5 @@+---+"@inkeep/open-knowledge": patch+---++`ok start` stays responsive while indexing large content directories. The boot-time index scans (backlink graph, tags, asset basenames, and the file-watcher's seed walk) previously used synchronous recursive directory reads, so on a vault with thousands of files the server's event loop was blocked for several seconds — Ctrl+C produced no feedback and editor/API connections couldn't be served until indexing finished. The walks now use async filesystem calls that yield to the event loop, so shutdown signals and incoming connections are handled while indexing proceeds. The same fix shortens the window where the desktop app's editor is slow to connect on large projects, since the desktop spawns the same server boot path.diff --git a/.changeset/cli-fast-exit-log-sink-race.md b/.changeset/cli-fast-exit-log-sink-race.mdnew file mode 100644index 000000000..52ba5bb4f--- /dev/null+++ b/.changeset/cli-fast-exit-log-sink-race.md@@ -0,0 +1,5 @@+---+"@inkeep/open-knowledge": patch+---++Fast-exiting CLI commands no longer dump a wall of minified bundle source to the terminal. Commands that exit within the same tick as startup — most visibly `ok start` in a directory that hasn't been `ok init`'d — raced the asynchronous open of the CLI's diagnostics log file: pino's exit-time flush hook then threw `sonic boom is not ready yet` on the never-opened file descriptor, burying the real one-line message under the error and a giant minified code frame. The log-file destination now opens synchronously, so the exit-time flush always finds a ready file descriptor and the clean message (e.g. "Run `ok init` first") is all you see.diff --git a/packages/server/src/api-extension.ts b/packages/server/src/api-extension.tsindex c5386a68b..3a1861eab 100644--- a/packages/server/src/api-extension.ts+++ b/packages/server/src/api-extension.ts@@ -1547,7 +1547,7 @@ export interface ApiExtensionOptions {getFolderIndex?: () => ReadonlyMap<string, FolderIndexEntry>;onReferencedAssetsCacheInvalidator?: (invalidate: () => void) => void;getAliasMap?: () => ReadonlyMap<string, string>;- rescanFiles?: () => void;+ rescanFiles?: () => void | Promise<void>;enableTestRoutes?: boolean;shadowRef?: ShadowRef;flushGitCommit?: () => Promise<void>;@@ -5057,7 +5057,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {);return;}- rescanFiles();+ await rescanFiles();signalChannel?.('files');successResponse(res,diff --git a/packages/server/src/api-tags.test.ts b/packages/server/src/api-tags.test.tsindex 1a12fbe6c..a098a4b18 100644--- a/packages/server/src/api-tags.test.ts+++ b/packages/server/src/api-tags.test.ts@@ -81,7 +81,7 @@ describe('tag endpoints', () => {writeFileSync(join(dir, 'alpha.md'), '#proj/team #standalone\n', 'utf-8');writeFileSync(join(dir, 'beta.md'), '#proj\n', 'utf-8');const idx = new TagIndex({ contentDir: dir });- idx.init();+ await idx.init();const result = await callRoute(dir, '/api/tags', new Map(), idx);expect(result.status).toBe(200);@@ -109,7 +109,7 @@ describe('tag endpoints', () => {writeFileSync(join(dir, 'gamma.md'), '#unrelated\n', 'utf-8');const idx = new TagIndex({ contentDir: dir });- idx.init();+ await idx.init();const result = await callRoute(dir, '/api/tags/proj', new Map(), idx);expect(result.status).toBe(200);@@ -138,7 +138,7 @@ describe('tag endpoints', () => {try {writeFileSync(join(dir, 'alpha.md'), '#proj/team/2026\n', 'utf-8');const idx = new TagIndex({ contentDir: dir });- idx.init();+ await idx.init();const result = await callRoute(dir, '/api/tags/proj%2Fteam', new Map(), idx);expect(result.status).toBe(200);@@ -154,7 +154,7 @@ describe('tag endpoints', () => {const dir = mkdtempSync(join(tmpdir(), 'ok-tags-unknown-'));try {const idx = new TagIndex({ contentDir: dir });- idx.init();+ await idx.init();const result = await callRoute(dir, '/api/tags/nope', new Map(), idx);expect(result.status).toBe(200);@@ -179,7 +179,7 @@ describe('tag endpoints', () => {'utf-8',);const idx = new TagIndex({ contentDir: dir });- idx.init();+ await idx.init();const result = await callRoute(dir, '/api/tags/showcase', new Map(), idx);const body = JSON.parse(result.body) as { docs: Array<{ docName: string }> };@@ -212,7 +212,7 @@ describe('tag endpoint independence from fileIndex content', () => {mkdirSync(dir, { recursive: true });writeFileSync(join(dir, 'alpha.md'), '#x\n', 'utf-8');const idx = new TagIndex({ contentDir: dir });- idx.init();+ await idx.init();const result = await callRoute(dir, '/api/tags/x', new Map(), idx);expect(result.status).toBe(200);const body = JSON.parse(result.body) as { docs: Array<{ docName: string }> };diff --git a/packages/server/src/asset-walk.test.ts b/packages/server/src/asset-walk.test.tsindex d3e2c4f1c..42e7afe82 100644--- a/packages/server/src/asset-walk.test.ts+++ b/packages/server/src/asset-walk.test.ts@@ -26,7 +26,7 @@ afterEach(() => {});describe('seedBasenameIndex — initial walk (no filter)', () => {- test('admits asset extensions; ignores markdown and unknown', () => {+ test('admits asset extensions; ignores markdown and unknown', async () => {write('docs/meeting.md');write('docs/photo.png');write('docs/diagram.svg');@@ -34,7 +34,7 @@ describe('seedBasenameIndex — initial walk (no filter)', () => {write('archive/old.png');const idx = createBasenameIndex();- seedBasenameIndex({ contentDir, basenameIndex: idx });+ await seedBasenameIndex({ contentDir, basenameIndex: idx });expect(idx.resolveEmbed('photo.png', 'docs/meeting.md')).toBe('docs/photo.png');expect(idx.resolveEmbed('diagram.svg', 'docs/meeting.md')).toBe('docs/diagram.svg');@@ -43,27 +43,27 @@ describe('seedBasenameIndex — initial walk (no filter)', () => {expect(idx.resolveEmbed('arbitrary.xyz', 'docs/meeting.md')).toBeNull();});- test('admits .base and .canvas files into the basename index', () => {+ test('admits .base and .canvas files into the basename index', async () => {write('vault/note.md');write('vault/Characters.base', 'fields:\n - name\n');write('vault/Board.canvas', '{"nodes":[],"edges":[]}\n');const idx = createBasenameIndex();- seedBasenameIndex({ contentDir, basenameIndex: idx });+ await seedBasenameIndex({ contentDir, basenameIndex: idx });expect(idx.resolveEmbed('Characters.base', 'vault/note.md')).toBe('vault/Characters.base');expect(idx.resolveEmbed('Board.canvas', 'vault/note.md')).toBe('vault/Board.canvas');});- test('empty contentDir produces empty index without throwing', () => {+ test('empty contentDir produces empty index without throwing', async () => {const idx = createBasenameIndex();- seedBasenameIndex({ contentDir, basenameIndex: idx });+ await seedBasenameIndex({ contentDir, basenameIndex: idx });expect(idx.size()).toBe(0);});});describe('seedBasenameIndex — initial walk (with ContentFilter sibling-asset admission)', () => {- test('admits assets only in markdown-neighbored directories', () => {+ test('admits assets only in markdown-neighbored directories', async () => {write('docs/meeting.md');write('docs/photo.png');write('no-md-here/orphan.png'); // no sibling .md → excluded@@ -74,13 +74,13 @@ describe('seedBasenameIndex — initial walk (with ContentFilter sibling-asset acontentDir,});contentFilter.incrementMdDir('docs');- seedBasenameIndex({ contentDir, contentFilter, basenameIndex: idx });+ await seedBasenameIndex({ contentDir, contentFilter, basenameIndex: idx });expect(idx.resolveEmbed('photo.png', 'docs/meeting.md')).toBe('docs/photo.png');expect(idx.resolveEmbed('orphan.png', 'docs/meeting.md')).toBeNull();});- test('respects .okignore exclusion patterns', () => {+ test('respects .okignore exclusion patterns', async () => {write('docs/meeting.md');write('docs/photo.png');write('secret/hidden.png');@@ -92,7 +92,7 @@ describe('seedBasenameIndex — initial walk (with ContentFilter sibling-asset acontentDir,});contentFilter.incrementMdDir('docs');- seedBasenameIndex({ contentDir, contentFilter, basenameIndex: idx });+ await seedBasenameIndex({ contentDir, contentFilter, basenameIndex: idx });expect(idx.resolveEmbed('photo.png', 'docs/meeting.md')).toBe('docs/photo.png');expect(idx.resolveEmbed('hidden.png', 'docs/meeting.md')).toBeNull();@@ -100,7 +100,7 @@ describe('seedBasenameIndex — initial walk (with ContentFilter sibling-asset a});describe('seedBasenameIndex — symlink safety', () => {- test('follows symlinks inside contentDir but rejects escapes', () => {+ test('follows symlinks inside contentDir but rejects escapes', async () => {write('docs/meeting.md');write('docs/real.png');const outside = join(baseDir, 'outside');@@ -112,7 +112,7 @@ describe('seedBasenameIndex — symlink safety', () => {symlinkSync(join(contentDir, 'alias-target'), join(contentDir, 'docs', 'alias'));const idx = createBasenameIndex();- seedBasenameIndex({ contentDir, basenameIndex: idx });+ await seedBasenameIndex({ contentDir, basenameIndex: idx });expect(idx.resolveEmbed('real.png', 'docs/meeting.md')).toBe('docs/real.png');expect(idx.resolveEmbed('aliased.png', 'docs/meeting.md')).not.toBeNull();diff --git a/packages/server/src/asset-walk.ts b/packages/server/src/asset-walk.tsindex c3fe3376a..6683a177a 100644--- a/packages/server/src/asset-walk.ts+++ b/packages/server/src/asset-walk.ts@@ -1,5 +1,6 @@-import type { Dirent } from 'node:fs';-import { lstatSync, readdirSync, realpathSync, statSync } from 'node:fs';+import type { Dirent, Stats } from 'node:fs';+import { readdirSync } from 'node:fs';+import { lstat, readdir, realpath, stat } from 'node:fs/promises';import { join, relative, sep } from 'node:path';import {ASSET_EXTENSIONS,@@ -53,14 +54,14 @@ export function seedSingleDirBasenameIndex(opts: {}}-export function seedBasenameIndex(opts: SeedOptions): void {+export async function seedBasenameIndex(opts: SeedOptions): Promise<void> {const root = opts.contentDir;const visited = new Set<number>();- function walk(dir: string): void {+ async function walk(dir: string): Promise<void> {let entries: Dirent[];try {- entries = readdirSync(dir, { withFileTypes: true }) as Dirent[];+ entries = (await readdir(dir, { withFileTypes: true })) as Dirent[];} catch (err) {const code = errnoCode(err);if (code !== 'ENOENT') opts.onSkip?.('read-failed', code, dir);@@ -72,19 +73,19 @@ export function seedBasenameIndex(opts: SeedOptions): void {if (rel.startsWith('..')) continue;if (opts.contentFilter?.isDirExcluded(rel) && entry.isDirectory()) continue;- let stat: ReturnType<typeof statSync>;+ let entryStat: Stats;try {- stat = lstatSync(full);+ entryStat = await lstat(full);} catch (err) {const code = errnoCode(err);if (code !== 'ENOENT') opts.onSkip?.('lstat-failed', code, full);continue;}- if (stat.isSymbolicLink()) {+ if (entryStat.isSymbolicLink()) {let canonical: string;try {- canonical = realpathSync(full);+ canonical = await realpath(full);} catch (err) {const code = errnoCode(err);if (code !== 'ENOENT') opts.onSkip?.('realpath-failed', code, full);@@ -94,9 +95,9 @@ export function seedBasenameIndex(opts: SeedOptions): void {opts.onSkip?.('symlink-escape', undefined, full);continue;}- let realStat: ReturnType<typeof statSync>;+ let realStat: Stats;try {- realStat = statSync(canonical);+ realStat = await stat(canonical);} catch (err) {const code = errnoCode(err);if (code !== 'ENOENT') opts.onSkip?.('symlink-stat-failed', code, canonical);@@ -104,7 +105,7 @@ export function seedBasenameIndex(opts: SeedOptions): void {}if (visited.has(realStat.ino)) continue;visited.add(realStat.ino);- if (realStat.isDirectory()) walk(canonical);+ if (realStat.isDirectory()) await walk(canonical);else if (realStat.isFile() &&isSupportedAssetFile(full, LINKABLE_ASSET_EXTENSIONS) &&@@ -115,14 +116,14 @@ export function seedBasenameIndex(opts: SeedOptions): void {continue;}- if (stat.isDirectory()) {- if (visited.has(stat.ino)) continue;- visited.add(stat.ino);- walk(full);+ if (entryStat.isDirectory()) {+ if (visited.has(entryStat.ino)) continue;+ visited.add(entryStat.ino);+ await walk(full);continue;}if (- stat.isFile() &&+ entryStat.isFile() &&isSupportedAssetFile(full, LINKABLE_ASSET_EXTENSIONS) &&!opts.contentFilter?.isExcluded(rel)) {@@ -131,5 +132,5 @@ export function seedBasenameIndex(opts: SeedOptions): void {}}- walk(root);+ await walk(root);}diff --git a/packages/server/src/backlink-index.ts b/packages/server/src/backlink-index.tsindex 22b4dfa36..d2b26a706 100644--- a/packages/server/src/backlink-index.ts+++ b/packages/server/src/backlink-index.ts@@ -1,5 +1,5 @@-import { type Dirent, existsSync, mkdirSync, readdirSync } from 'node:fs';-import { readFile, stat, writeFile } from 'node:fs/promises';+import { type Dirent, existsSync, mkdirSync } from 'node:fs';+import { readdir, readFile, stat, writeFile } from 'node:fs/promises';import { dirname, join, relative, resolve } from 'node:path';import {classifyMarkdownHref,@@ -1141,7 +1141,7 @@ export class BacklinkIndex {const state = createEmptyState();const mtimes = new Map<string, number>();const rawDocs: Array<{ docName: string; filePath: string }> = [];- this.walkForPaths(this.contentDir, rawDocs);+ await this.walkForPaths(this.contentDir, rawDocs);const seen = new Set<string>();const allDocs = rawDocs.filter(({ docName }) => {@@ -1216,10 +1216,13 @@ export class BacklinkIndex {this.mtimesByBranch.set(branch, mtimes);}- private walkForPaths(dir: string, results: Array<{ docName: string; filePath: string }>): void {+ private async walkForPaths(+ dir: string,+ results: Array<{ docName: string; filePath: string }>,+ ): Promise<void> {let entries: Dirent[];try {- entries = readdirSync(dir, { withFileTypes: true });+ entries = await readdir(dir, { withFileTypes: true });} catch (err) {console.warn(`[backlinks] Failed to read directory ${dir}:`, err);return;@@ -1229,7 +1232,7 @@ export class BacklinkIndex {if (entry.isDirectory()) {const relDir = relative(this.contentDir, fullPath);if (this.contentFilter && relDir && this.contentFilter.isDirExcluded(relDir)) continue;- this.walkForPaths(fullPath, results);+ await this.walkForPaths(fullPath, results);} else if (entry.isFile() && isSupportedDocFile(entry.name)) {const relPath = relative(this.contentDir, fullPath);if (this.contentFilter?.isExcluded(relPath)) continue;@@ -1247,7 +1250,7 @@ export class BacklinkIndex {const storedMtimes = this.mtimesByBranch.get(branch) ?? new Map<string, number>();const rawDocs: Array<{ docName: string; filePath: string }> = [];- this.walkForPaths(this.contentDir, rawDocs);+ await this.walkForPaths(this.contentDir, rawDocs);const seen = new Set<string>();const docs = rawDocs.filter(({ docName }) => {diff --git a/packages/server/src/boot-walk-responsiveness.test.ts b/packages/server/src/boot-walk-responsiveness.test.tsnew file mode 100644index 000000000..77138ccaa--- /dev/null+++ b/packages/server/src/boot-walk-responsiveness.test.ts@@ -0,0 +1,100 @@+import { afterEach, beforeEach, describe, expect, test } from 'bun:test';+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { createBasenameIndex } from '@inkeep/open-knowledge-core';+import { seedBasenameIndex } from './asset-walk.ts';+import { BacklinkIndex } from './backlink-index.ts';+import { startWatcher } from './file-watcher.ts';+import { TagIndex } from './tag-index.ts';++const DIR_COUNT = 100;+const FILES_PER_DIR = 2;++let baseDir: string;+let contentDir: string;+let expectedDocs: Set<string>;+let expectedAssets: Set<string>;++beforeEach(() => {+ baseDir = mkdtempSync(join(tmpdir(), 'ok-boot-walk-'));+ contentDir = join(baseDir, 'content');+ mkdirSync(contentDir, { recursive: true });+ expectedDocs = new Set();+ expectedAssets = new Set();+ for (let d = 0; d < DIR_COUNT; d++) {+ const dir = join(contentDir, `dir-${d}`);+ mkdirSync(dir, { recursive: true });+ for (let f = 0; f < FILES_PER_DIR; f++) {+ writeFileSync(join(dir, `note-${f}.md`), `# note ${d}/${f}\n#tag-${d}\n`, 'utf-8');+ expectedDocs.add(`dir-${d}/note-${f}`);+ }+ writeFileSync(join(dir, `pic-${d}.png`), 'bytes', 'utf-8');+ expectedAssets.add(`dir-${d}/pic-${d}.png`);+ }+});++afterEach(() => {+ rmSync(baseDir, { recursive: true, force: true });+});++async function timerFiresDuring<T>(+ walk: () => Promise<T>,+): Promise<{ timerWon: boolean; result: T }> {+ let timerFired = false;+ const timer = setTimeout(() => {+ timerFired = true;+ }, 0);+ try {+ const result = await walk();+ return { timerWon: timerFired, result };+ } finally {+ clearTimeout(timer);+ }+}++describe('boot-time walks yield to the event loop', () => {+ test('BacklinkIndex.rebuildFromDisk yields and collects every doc', async () => {+ const index = new BacklinkIndex({ projectDir: baseDir, contentDir });+ const { timerWon } = await timerFiresDuring(() => index.rebuildFromDisk());+ expect(timerWon).toBe(true);+ const nodeIds = new Set(index.getLinkGraph().nodes.map((n) => n.id.replace(/\\/g, '/')));+ expect(nodeIds).toEqual(expectedDocs);+ });++ test('TagIndex.init yields and indexes every doc', async () => {+ const index = new TagIndex({ contentDir });+ const { timerWon } = await timerFiresDuring(() => index.init());+ expect(timerWon).toBe(true);+ const indexed = new Set<string>();+ for (let d = 0; d < DIR_COUNT; d++) {+ for (const doc of index.getDocsForTag(`tag-${d}`)) {+ indexed.add(doc.replace(/\\/g, '/'));+ }+ }+ expect(indexed).toEqual(expectedDocs);+ });++ test('seedBasenameIndex yields and collects every asset', async () => {+ const idx = createBasenameIndex();+ const { timerWon } = await timerFiresDuring(() =>+ seedBasenameIndex({ contentDir, basenameIndex: idx }),+ );+ expect(timerWon).toBe(true);+ expect(idx.size()).toBe(expectedAssets.size);+ for (let d = 0; d < DIR_COUNT; d++) {+ expect(idx.resolveEmbed(`pic-${d}.png`, `dir-${d}/note-0.md`)).toBe(`dir-${d}/pic-${d}.png`);+ }+ });++ test('file-watcher seed walk yields and indexes every doc', async () => {+ const handle = await startWatcher(contentDir, async () => {});+ try {+ expect(new Set(handle.getFileIndex().keys())).toEqual(expectedDocs);+ const { timerWon } = await timerFiresDuring(() => handle.rescanFromDisk());+ expect(timerWon).toBe(true);+ } finally {+ await handle.unsubscribe();+ }+ });+});diff --git a/packages/server/src/file-logger.test.ts b/packages/server/src/file-logger.test.tsindex 70abc5148..b239ab1ce 100644--- a/packages/server/src/file-logger.test.ts+++ b/packages/server/src/file-logger.test.ts@@ -76,6 +76,18 @@ describe('file logger', () => {expect(files).toContain('big.log.1');});+ test('opens the destination synchronously so same-tick process.exit cannot race the open', () => {+ mkdirSync(TEST_DIR, { recursive: true });+ const filePath = join(TEST_DIR, 'sync-open.log');+ const { createFileLogger } = require('./file-logger.ts');+ const logger = createFileLogger({ name: 'sync-open', filePath });+ const stream = (logger as unknown as Record<symbol, { fd: number; flushSync: () => void }>)[+ pino.symbols.streamSym+ ];+ expect(stream.fd).toBeGreaterThanOrEqual(0);+ expect(() => stream.flushSync()).not.toThrow();+ });+test('createFileLogger unrefs the deferred prune timer so it never blocks process exit', () => {mkdirSync(TEST_DIR, { recursive: true });const { createFileLogger } = require('./file-logger.ts');diff --git a/packages/server/src/file-logger.ts b/packages/server/src/file-logger.tsindex dadda7429..6669ad054 100644--- a/packages/server/src/file-logger.ts+++ b/packages/server/src/file-logger.ts@@ -121,7 +121,7 @@ export function createFileLogger(opts: FileLoggerOptions): PinoLoggerInstance {const scheduleTimer = opts._setTimeout ?? setTimeout;scheduleTimer(() => pruneLogsDir(OK_LOGS_DIR), 5000).unref();- const dest = pino.destination({ dest: filePath, append: true, sync: false });+ const dest = pino.destination({ dest: filePath, append: true, sync: true });const level = resolveLogLevel();diff --git a/packages/server/src/file-watcher.test.ts b/packages/server/src/file-watcher.test.tsindex 830d4f1af..add33bed7 100644--- a/packages/server/src/file-watcher.test.ts+++ b/packages/server/src/file-watcher.test.ts@@ -719,7 +719,7 @@ describe('reconcileFileIndexAfterFilterRebuild', () => {const result = await filter.rebuildIgnorePatterns();expect(result.ok).toBe(true);- const { prunedFiles, prunedFolders } = reconcileFileIndexAfterFilterRebuild(handle);+ const { prunedFiles, prunedFolders } = await reconcileFileIndexAfterFilterRebuild(handle);expect(prunedFiles).toBe(0);expect(prunedFolders).toBe(0);expect(handle.getFileIndex().has('hide-me')).toBe(true);@@ -744,7 +744,7 @@ describe('reconcileFileIndexAfterFilterRebuild', () => {const result = await filter.rebuildIgnorePatterns();expect(result.ok).toBe(true);- const { prunedFiles, prunedFolders } = reconcileFileIndexAfterFilterRebuild(handle);+ const { prunedFiles, prunedFolders } = await reconcileFileIndexAfterFilterRebuild(handle);expect(prunedFiles).toBe(1);expect(prunedFolders).toBe(0);expect(handle.getFileIndex().has('will-hide')).toBe(false);@@ -772,7 +772,7 @@ describe('reconcileFileIndexAfterFilterRebuild', () => {const result = await filter.rebuildIgnorePatterns();expect(result.ok).toBe(true);- reconcileFileIndexAfterFilterRebuild(handle);+ await reconcileFileIndexAfterFilterRebuild(handle);expect(handle.getFolderIndex().has('archive')).toBe(true);expect(handle.getFolderIndex().has('archive/sub')).toBe(true);expect(handle.getFileIndex().has('archive/sub/old')).toBe(true);@@ -790,7 +790,7 @@ describe('reconcileFileIndexAfterFilterRebuild', () => {try {writeFileSync(resolve(tmpDir, '.okignore'), 'unrelated.md\n');await filter.rebuildIgnorePatterns();- const { prunedFiles, prunedFolders } = reconcileFileIndexAfterFilterRebuild(handle);+ const { prunedFiles, prunedFolders } = await reconcileFileIndexAfterFilterRebuild(handle);expect(prunedFiles).toBe(0);expect(prunedFolders).toBe(0);expect(handle.getFileIndex().has('keep')).toBe(true);@@ -814,7 +814,7 @@ describe('reconcileFileIndexAfterFilterRebuild', () => {const result = await filter.rebuildIgnorePatterns();expect(result.ok).toBe(true);- const { prunedFiles } = reconcileFileIndexAfterFilterRebuild(handle);+ const { prunedFiles } = await reconcileFileIndexAfterFilterRebuild(handle);expect(prunedFiles).toBe(1);expect(handle.getFileIndex().has('will-hide')).toBe(false);expect(handle.getFileIndex().has('was-hidden')).toBe(true);@@ -823,8 +823,8 @@ describe('reconcileFileIndexAfterFilterRebuild', () => {}});- test('returns zero counts when watcher is undefined (defensive guard)', () => {- const { prunedFiles, prunedFolders } = reconcileFileIndexAfterFilterRebuild(undefined);+ test('returns zero counts when watcher is undefined (defensive guard)', async () => {+ const { prunedFiles, prunedFolders } = await reconcileFileIndexAfterFilterRebuild(undefined);expect(prunedFiles).toBe(0);expect(prunedFolders).toBe(0);});diff --git a/packages/server/src/file-watcher.ts b/packages/server/src/file-watcher.tsindex 753a5c5a4..02752d5ea 100644--- a/packages/server/src/file-watcher.ts+++ b/packages/server/src/file-watcher.ts@@ -1,6 +1,6 @@import { createHash } from 'node:crypto';-import { type Dirent, lstatSync, readdirSync, readFileSync, realpathSync, statSync } from 'node:fs';-import { readFile } from 'node:fs/promises';+import { type Dirent, lstatSync, readdirSync, realpathSync, type Stats, statSync } from 'node:fs';+import { lstat, readdir, readFile, realpath, stat } from 'node:fs/promises';import { dirname, extname, join, relative } from 'node:path';import { LINKABLE_ASSET_EXTENSIONS } from '@inkeep/open-knowledge-core';import { isConfigDoc, isSystemDoc } from './cc1-broadcast.ts';@@ -74,7 +74,7 @@ export interface WatcherHandle {getAliasMap: () => ReadonlyMap<string, string>;pruneFileIndexNowExcluded: () => number;pruneFolderIndexNowExcluded: () => number;- rescanFromDisk: () => void;+ rescanFromDisk: () => Promise<void>;}export const writeTracker = new Map<string, Array<{ hash: string; timestamp: number }>>();@@ -401,7 +401,7 @@ export function isSelfWrite(filePath: string, hash: string): boolean {return true;}-function seedLastKnownHashes(+async function seedLastKnownHashes(dir: string,contentDir: string,contentFilter: ContentFilter | undefined,@@ -409,15 +409,15 @@ function seedLastKnownHashes(folderIndex: Map<string, FolderIndexEntry>,aliasMap: Map<string, string>,visitedInodes?: Set<number>,-): void {+): Promise<void> {const visited = visitedInodes ?? new Set<number>();try {- const entries = readdirSync(dir, { withFileTypes: true });+ const entries = await readdir(dir, { withFileTypes: true });for (const entry of entries) {const fullPath = join(dir, entry.name);- let lst: ReturnType<typeof lstatSync>;+ let lst: Stats;try {- lst = lstatSync(fullPath);+ lst = await lstat(fullPath);} catch (e) {const code = (e as NodeJS.ErrnoException).code;if (code !== 'ENOENT') {@@ -429,7 +429,7 @@ function seedLastKnownHashes(if (lst.isSymbolicLink()) {let canonical: string;try {- canonical = realpathSync(fullPath);+ canonical = await realpath(fullPath);} catch (e) {const code = (e as NodeJS.ErrnoException).code;if (code === 'ENOENT' || code === 'ELOOP') {@@ -446,7 +446,7 @@ function seedLastKnownHashes(}try {- const canonStat = statSync(canonical);+ const canonStat = await stat(canonical);if (visited.has(canonStat.ino)) {if (canonStat.isFile() && isSupportedDocFile(entry.name)) {const aliasDocName = pathToDocName(fullPath, contentDir);@@ -467,7 +467,7 @@ function seedLastKnownHashes(if (!relPath || contentFilter.isDirExcluded(relPath)) continue;}upsertFolderIndexEntry(folderIndex, contentDir, fullPath, canonStat, canonical);- seedLastKnownHashes(+ await seedLastKnownHashes(canonical,contentDir,contentFilter,@@ -486,7 +486,7 @@ function seedLastKnownHashes(aliasMap.set(aliasDocName, canonicalDocName);try {- const content = readFileSync(canonical, 'utf-8');+ const content = await readFile(canonical, 'utf-8');const hash = contentHash(content);lastKnownHash.set(canonical, hash);const ext = extractDocExtension(canonical);@@ -522,7 +522,7 @@ function seedLastKnownHashes(if (!relPath || contentFilter.isDirExcluded(relPath)) continue;}upsertFolderIndexEntry(folderIndex, contentDir, fullPath, lst);- seedLastKnownHashes(+ await seedLastKnownHashes(fullPath,contentDir,contentFilter,@@ -540,7 +540,7 @@ function seedLastKnownHashes(if (contentFilter.isExcluded(relPath)) continue;}try {- const content = readFileSync(fullPath, 'utf-8');+ const content = await readFile(fullPath, 'utf-8');lastKnownHash.set(fullPath, contentHash(content));const docName = pathToDocName(fullPath, contentDir);@@ -1059,7 +1059,14 @@ export async function startWatcher(const folderIndex = new Map<string, FolderIndexEntry>();const aliasMap = new Map<string, string>();- seedLastKnownHashes(contentDir, contentDir, contentFilter, fileIndex, folderIndex, aliasMap);+ await seedLastKnownHashes(+ contentDir,+ contentDir,+ contentFilter,+ fileIndex,+ folderIndex,+ aliasMap,+ );const evictionInterval = setInterval(evictStaleTrackerEntries, WRITE_TRACKER_TTL_MS);@@ -1137,18 +1144,27 @@ export async function startWatcher(return pruned;},rescanFromDisk() {- seedLastKnownHashes(contentDir, contentDir, contentFilter, fileIndex, folderIndex, aliasMap);+ return seedLastKnownHashes(+ contentDir,+ contentDir,+ contentFilter,+ fileIndex,+ folderIndex,+ aliasMap,+ );},};}-export function reconcileFileIndexAfterFilterRebuild(watcher: WatcherHandle | null | undefined): {+export async function reconcileFileIndexAfterFilterRebuild(+ watcher: WatcherHandle | null | undefined,+): Promise<{prunedFiles: number;prunedFolders: number;-} {+}> {if (!watcher) return { prunedFiles: 0, prunedFolders: 0 };const prunedFiles = watcher.pruneFileIndexNowExcluded();const prunedFolders = watcher.pruneFolderIndexNowExcluded();- watcher.rescanFromDisk();+ await watcher.rescanFromDisk();return { prunedFiles, prunedFolders };}diff --git a/packages/server/src/server-factory.ts b/packages/server/src/server-factory.tsindex 09f761286..029921abe 100644--- a/packages/server/src/server-factory.ts+++ b/packages/server/src/server-factory.ts@@ -401,43 +401,43 @@ export function createServer(options: ServerOptions): ServerInstance {'[content-filter] backlink-index rebuild failed after onAfterRebuild',);});- try {- tagIndex.init();- } catch (err) {+ void tagIndex.init().catch((err) => {getLogger('server-factory').warn({ err },'[content-filter] tag-index rebuild failed after onAfterRebuild',);- }- try {- const { prunedFiles, prunedFolders } = reconcileFileIndexAfterFilterRebuild(watcher);- const pruned = prunedFiles + prunedFolders;- if (pruned > 0) {- getLogger('server-factory').info(- { pruned, prunedFiles, prunedFolders },- '[content-filter] reconciled file indexes after onAfterRebuild',- );- } else {- getLogger('server-factory').debug(- { prunedFiles, prunedFolders },- '[content-filter] file index reconcile completed after onAfterRebuild (no entries pruned; rescan may have added entries)',+ });+ void reconcileFileIndexAfterFilterRebuild(watcher)+ .then(({ prunedFiles, prunedFolders }) => {+ const pruned = prunedFiles + prunedFolders;+ if (pruned > 0) {+ getLogger('server-factory').info(+ { pruned, prunedFiles, prunedFolders },+ '[content-filter] reconciled file indexes after onAfterRebuild',+ );+ } else {+ getLogger('server-factory').debug(+ { prunedFiles, prunedFolders },+ '[content-filter] file index reconcile completed after onAfterRebuild (no entries pruned; rescan may have added entries)',+ );+ }+ })+ .catch((err) => {+ getLogger('server-factory').warn(+ { err },+ '[content-filter] file index reconcile failed after onAfterRebuild',);- }- } catch (err) {- getLogger('server-factory').warn(- { err },- '[content-filter] file index reconcile failed after onAfterRebuild',- );- }+ });},});backlinkIndex = new BacklinkIndex({ projectDir, contentDir, contentFilter });tagIndex = new TagIndex({ contentDir, contentFilter });- try {- tagIndex.init();- } catch (err) {- console.warn('[server-factory] tag-index init failed; continuing with empty index:', err);- }+ void tagIndex.init().catch((err) => {+ getLogger('server-factory').warn(+ { err },+ '[server-factory] tag-index init failed; continuing with empty index',+ );+ });shadowRef = { current: shadowRepo };@@ -1828,7 +1828,15 @@ export function createServer(options: ServerOptions): ServerInstance {}}watcher = await startWatcher(contentDir, onDiskEvent, contentFilter);- tagIndex.init();+ try {+ await tagIndex.init();+ } catch (err) {+ log.error(+ { err },+ '[tag-index] startup re-init failed; tag index updates incrementally via watcher events',+ );+ degraded.push('tag-index');+ }let seedSkipCount = 0;try {if (singleDocRelPath !== undefined) {@@ -1844,7 +1852,7 @@ export function createServer(options: ServerOptions): ServerInstance {},});} else {- seedBasenameIndex({+ await seedBasenameIndex({contentDir,contentFilter,basenameIndex,@@ -1964,7 +1972,7 @@ export function createServer(options: ServerOptions): ServerInstance {try {let reseedSkipCount = 0;basenameIndex.clear();- seedBasenameIndex({+ await seedBasenameIndex({contentDir,contentFilter,basenameIndex,@@ -2066,7 +2074,7 @@ export function createServer(options: ServerOptions): ServerInstance {'[backlinks] branch-switch rebuild failed; backlinks may be stale',);}- tagIndex.init();+ await tagIndex.init();if (shadowRef.current && info.batchKind === 'cross-branch') {let restoredCount = 0;diff --git a/packages/server/src/tag-index.test.ts b/packages/server/src/tag-index.test.tsindex 4041abe6d..3e345cd1b 100644--- a/packages/server/src/tag-index.test.ts+++ b/packages/server/src/tag-index.test.ts@@ -200,7 +200,7 @@ describe('TagIndex', () => {}});- test('init walks contentDir and indexes every md/mdx file', () => {+ test('init walks contentDir and indexes every md/mdx file', async () => {const { contentDir, cleanup } = tempContentDir();try {writeFileSync(join(contentDir, 'alpha.md'), '#typescript\n', 'utf-8');@@ -209,7 +209,7 @@ describe('TagIndex', () => {writeFileSync(join(contentDir, 'sub', 'gamma.md'), '#proj/team\n', 'utf-8');const idx = new TagIndex({ contentDir });- idx.init();+ await idx.init();expect(idx.getDocsForTag('typescript')).toEqual(['alpha']);expect(idx.getDocsForTag('react')).toEqual(['beta']);diff --git a/packages/server/src/tag-index.ts b/packages/server/src/tag-index.tsindex e7944617f..f7f7bc5a9 100644--- a/packages/server/src/tag-index.ts+++ b/packages/server/src/tag-index.ts@@ -1,4 +1,5 @@-import { type Dirent, existsSync, readdirSync, readFileSync } from 'node:fs';+import { type Dirent, existsSync } from 'node:fs';+import { readdir, readFile } from 'node:fs/promises';import { join, relative } from 'node:path';import {createTagInTextRegex,@@ -87,6 +88,7 @@ export class TagIndex {private readonly contentDir: string;private readonly contentFilter?: ContentFilter;private state: TagIndexState = createEmptyState();+ private initChain: Promise<void> = Promise.resolve();constructor(options: TagIndexOptions) {this.contentDir = options.contentDir;@@ -180,16 +182,38 @@ export class TagIndex {.sort((a, b) => a.name.localeCompare(b.name));}- init(): void {+ init(): Promise<void> {+ const run = this.initChain.then(() => this.initOnce());+ this.initChain = run.catch((err) => {+ console.warn('[tag-index] init failed (chain cleared for next init):', err);+ });+ return run;+ }++ private async initOnce(): Promise<void> {this.state = createEmptyState();if (!existsSync(this.contentDir)) return;- const entries = this.listDocsWithPaths();- for (const { docName, filePath } of entries) {- try {- const markdown = readFileSync(filePath, 'utf-8');- this.updateDocumentFromMarkdown(docName, markdown);- } catch (err) {- console.warn(`[tag-index] Failed to read ${docName} during init:`, err);+ const entries = await this.listDocsWithPaths();+ const BATCH_SIZE = 50;+ for (let i = 0; i < entries.length; i += BATCH_SIZE) {+ const batch = entries.slice(i, i + BATCH_SIZE);+ const results = await Promise.all(+ batch.map(async ({ docName, filePath }) => {+ try {+ return { docName, markdown: await readFile(filePath, 'utf-8') };+ } catch (err) {+ console.warn(`[tag-index] Failed to read ${docName} during init:`, err);+ return null;+ }+ }),+ );+ for (const result of results) {+ if (!result) continue;+ try {+ this.updateDocumentFromMarkdown(result.docName, result.markdown);+ } catch (err) {+ console.warn(`[tag-index] Failed to index ${result.docName} during init:`, err);+ }}}}@@ -227,9 +251,9 @@ export class TagIndex {}}- private listDocsWithPaths(): Array<{ docName: string; filePath: string }> {+ private async listDocsWithPaths(): Promise<Array<{ docName: string; filePath: string }>> {const out: Array<{ docName: string; filePath: string }> = [];- this.walkContentDir(this.contentDir, out);+ await this.walkContentDir(this.contentDir, out);out.sort((a, b) => {if (a.docName !== b.docName) return a.docName.localeCompare(b.docName);return b.filePath.localeCompare(a.filePath);@@ -242,10 +266,13 @@ export class TagIndex {});}- private walkContentDir(dir: string, out: Array<{ docName: string; filePath: string }>): void {+ private async walkContentDir(+ dir: string,+ out: Array<{ docName: string; filePath: string }>,+ ): Promise<void> {let entries: Dirent[];try {- entries = readdirSync(dir, { withFileTypes: true });+ entries = await readdir(dir, { withFileTypes: true });} catch (err) {console.warn(`[tag-index] Failed to read directory ${dir}:`, err);return;@@ -255,7 +282,7 @@ export class TagIndex {if (entry.isDirectory()) {const relDir = relative(this.contentDir, fullPath);if (this.contentFilter && relDir && this.contentFilter.isDirExcluded(relDir)) continue;- this.walkContentDir(fullPath, out);+ await this.walkContentDir(fullPath, out);continue;}if (!entry.isFile() || !isSupportedDocFile(entry.name)) continue;