Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

perf(open-knowledge): serve /api/pages from in-memory index (#2277)

Andrew Mikofalvy9136f72388e9
diff --git a/.changeset/pages-list-from-index-cache.md b/.changeset/pages-list-from-index-cache.md
new file mode 100644
index 000000000..ab7f2a8da
--- /dev/null
+++ b/.changeset/pages-list-from-index-cache.md
@@ -0,0 +1,5 @@
+---
+"@inkeep/open-knowledge": patch
+---
+
+Speed up project load for knowledge bases with many files. `GET /api/pages` previously re-read and re-parsed every markdown file from disk on every request β€” including the redundant full-directory pass that ran concurrently with the watcher's seed walk on cold load, and a fresh full re-read on every window focus / file-change refresh. Page titles and icons are now cached on the in-memory file index (derived from the content the watcher already reads for its change-detection hash, so no extra disk reads), and `/api/pages` serves them straight from memory. Titles/icons stay current through create, edit, and rename events. Behavior is unchanged; only the cost of listing pages drops from O(files) disk reads per request to an in-memory scan.
diff --git a/packages/server/src/api-extension.ts b/packages/server/src/api-extension.ts
index ff2708efe..ede6e5e22 100644
--- a/packages/server/src/api-extension.ts
+++ b/packages/server/src/api-extension.ts
@@ -8060,16 +8060,22 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
icon?: string;
}[] = [];
for (const [docName, entry] of index) {
- let title = docName;
- let icon: string | undefined;
const docExt = getDocExtension(docName);
- try {
- const filePath = resolve(contentDir, `${docName}${docExt}`);
- const content = readFileSync(filePath, 'utf-8');
- title = extractPageTitle(content, docName);
- icon = extractPageIcon(content);
- } catch (err) {
- console.warn(`[pages] Failed to read title for ${docName}:`, err);
+ let title: string;
+ let icon: string | undefined;
+ if (entry.title !== undefined) {
+ title = entry.title;
+ icon = entry.icon;
+ } else {
+ title = docName;
+ try {
+ const filePath = resolve(contentDir, `${docName}${docExt}`);
+ const content = readFileSync(filePath, 'utf-8');
+ title = extractPageTitle(content, docName);
+ icon = extractPageIcon(content);
+ } catch (err) {
+ console.warn(`[pages] Failed to read title for ${docName}:`, err);
+ }
}
pages.push({ docName, title, docExt, size: entry.size, modified: entry.modified, icon });
}
diff --git a/packages/server/src/api-pages.test.ts b/packages/server/src/api-pages.test.ts
index f7beb4944..d3226448a 100644
--- a/packages/server/src/api-pages.test.ts
+++ b/packages/server/src/api-pages.test.ts
@@ -230,8 +230,11 @@ function buildFileIndex(dir: string, base = ''): ReadonlyMap<string, FileIndexEn
return index;
}
-async function callPages(contentDir: string, method = 'GET'): Promise<CapturedResponse> {
- const fileIndex = buildFileIndex(contentDir);
+async function callPagesWithIndex(
+ contentDir: string,
+ fileIndex: ReadonlyMap<string, FileIndexEntry>,
+ method = 'GET',
+): Promise<CapturedResponse> {
const ext = createApiExtension({
hocuspocus: {} as unknown as Parameters<typeof createApiExtension>[0]['hocuspocus'],
sessionManager: {} as unknown as Parameters<typeof createApiExtension>[0]['sessionManager'],
@@ -248,6 +251,10 @@ async function callPages(contentDir: string, method = 'GET'): Promise<CapturedRe
return captured;
}
+async function callPages(contentDir: string, method = 'GET'): Promise<CapturedResponse> {
+ return callPagesWithIndex(contentDir, buildFileIndex(contentDir), method);
+}
+
describe('GET /api/pages', () => {
test('returns flat { pages } success body and lists markdown files recursively', async () => {
const dir = mkdtempSync(join(tmpdir(), 'ok-pages-'));
@@ -300,6 +307,68 @@ describe('GET /api/pages', () => {
}
});
+ test('serves cached title/icon from the index without reading disk', async () => {
+ const dir = mkdtempSync(join(tmpdir(), 'ok-pages-cache-'));
+ try {
+ const index = new Map<string, FileIndexEntry>([
+ [
+ 'ghost',
+ {
+ size: 123,
+ modified: new Date(0).toISOString(),
+ canonicalPath: join(dir, 'ghost.md'),
+ inode: 1,
+ aliases: [],
+ kind: 'markdown',
+ title: 'Cached Title',
+ icon: '🎯',
+ },
+ ],
+ ]);
+
+ const result = await callPagesWithIndex(dir, index);
+ expect(result.status).toBe(200);
+ const body = JSON.parse(result.body) as {
+ pages?: Array<{ docName: string; title: string; icon?: string }>;
+ };
+ const entry = (body.pages ?? []).find((p) => p.docName === 'ghost');
+ expect(entry?.title).toBe('Cached Title');
+ expect(entry?.icon).toBe('🎯');
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+ });
+
+ test('bare entry whose file is missing falls back to the docName title (ENOENT path)', async () => {
+ const dir = mkdtempSync(join(tmpdir(), 'ok-pages-bare-'));
+ try {
+ const index = new Map<string, FileIndexEntry>([
+ [
+ 'orphan',
+ {
+ size: 0,
+ modified: new Date(0).toISOString(),
+ canonicalPath: join(dir, 'orphan.md'),
+ inode: 1,
+ aliases: [],
+ kind: 'markdown',
+ },
+ ],
+ ]);
+
+ const result = await callPagesWithIndex(dir, index);
+ expect(result.status).toBe(200);
+ const body = JSON.parse(result.body) as {
+ pages?: Array<{ docName: string; title: string; icon?: string }>;
+ };
+ const entry = (body.pages ?? []).find((p) => p.docName === 'orphan');
+ expect(entry?.title).toBe('orphan');
+ expect(entry?.icon ?? undefined).toBeUndefined();
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+ });
+
test('returns RFC 9457 problem+json 405 envelope for unsupported methods', async () => {
const dir = mkdtempSync(join(tmpdir(), 'ok-pages-'));
try {
diff --git a/packages/server/src/file-watcher.test.ts b/packages/server/src/file-watcher.test.ts
index 868bd6123..4658dcad6 100644
--- a/packages/server/src/file-watcher.test.ts
+++ b/packages/server/src/file-watcher.test.ts
@@ -391,6 +391,27 @@ describe('startWatcher file index', () => {
}
});
+ test('initial scan caches page title + icon on each markdown entry', async () => {
+ writeFileSync(
+ resolve(contentDir, 'with-meta.md'),
+ '---\ntitle: Meta Title\nicon: πŸ“\n---\n\n# Heading Ignored\n',
+ );
+ writeFileSync(resolve(contentDir, 'heading-only.md'), '# Heading Title\n');
+ writeFileSync(resolve(contentDir, 'plain.md'), 'just body text, no heading\n');
+
+ const handle = await startWatcher(contentDir, async () => {});
+ try {
+ const index = handle.getFileIndex();
+ expect(index.get('with-meta')?.title).toBe('Meta Title');
+ expect(index.get('with-meta')?.icon).toBe('πŸ“');
+ expect(index.get('heading-only')?.title).toBe('Heading Title');
+ expect(index.get('heading-only')?.icon).toBeUndefined();
+ expect(index.get('plain')?.title).toBe('plain');
+ } finally {
+ await handle.unsubscribe();
+ }
+ });
+
test('initial scan preserves uppercase .MD/.MDX extension casing', async () => {
writeFileSync(resolve(contentDir, 'Upper.MD'), '# Upper\n');
writeFileSync(resolve(contentDir, 'Mixed.MdX'), '# Mixed\n');
@@ -519,6 +540,61 @@ describe('startWatcher file index', () => {
expect(index.get('new-name')?.size).toBe(Buffer.byteLength('# Renamed\n', 'utf-8'));
});
+ test('file index caches title + icon on create/update/rename/conflict events', () => {
+ const { updateFileIndex } = require('./file-watcher.ts');
+ const index = new Map();
+
+ updateFileIndex(
+ {
+ kind: 'create' as const,
+ path: resolve(contentDir, 'doc.md'),
+ docName: 'doc',
+ content: '---\ntitle: Created\nicon: πŸš€\n---\n\nBody\n',
+ },
+ index,
+ );
+ expect(index.get('doc')?.title).toBe('Created');
+ expect(index.get('doc')?.icon).toBe('πŸš€');
+
+ updateFileIndex(
+ {
+ kind: 'update' as const,
+ path: resolve(contentDir, 'doc.md'),
+ docName: 'doc',
+ content: '# Edited Title\n',
+ },
+ index,
+ );
+ expect(index.get('doc')?.title).toBe('Edited Title');
+ expect(index.get('doc')?.icon).toBeUndefined();
+
+ updateFileIndex(
+ {
+ kind: 'rename' as const,
+ oldPath: resolve(contentDir, 'doc.md'),
+ newPath: resolve(contentDir, 'renamed.md'),
+ oldDocName: 'doc',
+ newDocName: 'renamed',
+ content: '---\ntitle: Renamed Title\nicon: πŸ”–\n---\n\nBody\n',
+ },
+ index,
+ );
+ expect(index.get('renamed')?.title).toBe('Renamed Title');
+ expect(index.get('renamed')?.icon).toBe('πŸ”–');
+
+ updateFileIndex(
+ {
+ kind: 'conflict' as const,
+ path: resolve(contentDir, 'renamed.md'),
+ docName: 'renamed',
+ content: '---\ntitle: Conflicted\nicon: ⚠️\n---\n<<<<<<< HEAD\na\n=======\nb\n>>>>>>> x\n',
+ },
+ index,
+ );
+ expect(index.get('renamed')?.title).toBe('Conflicted');
+ expect(index.get('renamed')?.icon).toBe('⚠️');
+ });
+
test('getFileIndex returns empty map when no .md files exist', async () => {
const handle = await startWatcher(contentDir, async () => {});
try {
diff --git a/packages/server/src/file-watcher.ts b/packages/server/src/file-watcher.ts
index 6b85dbafa..cee6539a9 100644
--- a/packages/server/src/file-watcher.ts
+++ b/packages/server/src/file-watcher.ts
@@ -14,6 +14,7 @@ import {
} from './doc-extensions.ts';
import { classifyFsPath, normalizeFsPath } from './fs-traced.ts';
import { getLogger } from './logger.ts';
+import { extractPageIcon, extractPageTitle } from './page-identity.ts';
import { toPosix } from './path-utils.ts';
import { isWithinContentDir } from './persistence.ts';
import { containsConflictMarkers } from './reconciliation.ts';
@@ -79,6 +80,8 @@ export interface FileIndexEntry {
inode: number;
aliases: string[];
kind: 'markdown' | 'file';
+ title?: string;
+ icon?: string;
}
export interface FolderIndexEntry {
@@ -88,6 +91,13 @@ export interface FolderIndexEntry {
inode: number;
}
+function derivePageMeta(
+ content: string,
+ docName: string,
+): { title: string; icon: string | undefined } {
+ return { title: extractPageTitle(content, docName), icon: extractPageIcon(content) };
+}
+
function markdownIndexView(
inner: ReadonlyMap<string, FileIndexEntry>,
): ReadonlyMap<string, FileIndexEntry> {
@@ -554,6 +564,7 @@ async function seedLastKnownHashes(
inode: canonStat.ino,
aliases: [aliasDocName],
kind: 'markdown',
+ ...derivePageMeta(content, canonicalDocName),
});
} catch (err) {
const code = (err as NodeJS.ErrnoException).code;
@@ -626,6 +637,7 @@ async function seedLastKnownHashes(
inode: lst.ino,
aliases: [],
kind: 'markdown',
+ ...derivePageMeta(content, docName),
});
} catch (err) {
const code = (err as NodeJS.ErrnoException).code;
@@ -717,6 +729,7 @@ export function updateFileIndex(event: DiskEvent, fileIndex: Map<string, FileInd
inode: existing?.inode ?? 0,
aliases: existing?.aliases ?? [],
kind: 'markdown',
+ ...derivePageMeta(event.content, docName),
});
break;
}
@@ -748,6 +761,7 @@ export function updateFileIndex(event: DiskEvent, fileIndex: Map<string, FileInd
inode: existing?.inode ?? 0,
aliases: existing?.aliases ?? [],
kind: 'markdown',
+ ...derivePageMeta(event.content, event.newDocName),
});
break;
}