Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
perf(open-knowledge): serve /api/pages from in-memory index (#2277)
Andrew Mikofalvy9136f72388e9
diff --git a/.changeset/pages-list-from-index-cache.md b/.changeset/pages-list-from-index-cache.mdnew file mode 100644index 000000000..ab7f2a8da--- /dev/null+++ b/.changeset/pages-list-from-index-cache.md@@ -0,0 +1,5 @@+---+"@inkeep/open-knowledge": patch+---++Speed up project load for knowledge bases with many files. `GET /api/pages` previously re-read and re-parsed every markdown file from disk on every request β including the redundant full-directory pass that ran concurrently with the watcher's seed walk on cold load, and a fresh full re-read on every window focus / file-change refresh. Page titles and icons are now cached on the in-memory file index (derived from the content the watcher already reads for its change-detection hash, so no extra disk reads), and `/api/pages` serves them straight from memory. Titles/icons stay current through create, edit, and rename events. Behavior is unchanged; only the cost of listing pages drops from O(files) disk reads per request to an in-memory scan.diff --git a/packages/server/src/api-extension.ts b/packages/server/src/api-extension.tsindex ff2708efe..ede6e5e22 100644--- a/packages/server/src/api-extension.ts+++ b/packages/server/src/api-extension.ts@@ -8060,16 +8060,22 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {icon?: string;}[] = [];for (const [docName, entry] of index) {- let title = docName;- let icon: string | undefined;const docExt = getDocExtension(docName);- try {- const filePath = resolve(contentDir, `${docName}${docExt}`);- const content = readFileSync(filePath, 'utf-8');- title = extractPageTitle(content, docName);- icon = extractPageIcon(content);- } catch (err) {- console.warn(`[pages] Failed to read title for ${docName}:`, err);+ let title: string;+ let icon: string | undefined;+ if (entry.title !== undefined) {+ title = entry.title;+ icon = entry.icon;+ } else {+ title = docName;+ try {+ const filePath = resolve(contentDir, `${docName}${docExt}`);+ const content = readFileSync(filePath, 'utf-8');+ title = extractPageTitle(content, docName);+ icon = extractPageIcon(content);+ } catch (err) {+ console.warn(`[pages] Failed to read title for ${docName}:`, err);+ }}pages.push({ docName, title, docExt, size: entry.size, modified: entry.modified, icon });}diff --git a/packages/server/src/api-pages.test.ts b/packages/server/src/api-pages.test.tsindex f7beb4944..d3226448a 100644--- a/packages/server/src/api-pages.test.ts+++ b/packages/server/src/api-pages.test.ts@@ -230,8 +230,11 @@ function buildFileIndex(dir: string, base = ''): ReadonlyMap<string, FileIndexEnreturn index;}-async function callPages(contentDir: string, method = 'GET'): Promise<CapturedResponse> {- const fileIndex = buildFileIndex(contentDir);+async function callPagesWithIndex(+ contentDir: string,+ fileIndex: ReadonlyMap<string, FileIndexEntry>,+ method = 'GET',+): Promise<CapturedResponse> {const ext = createApiExtension({hocuspocus: {} as unknown as Parameters<typeof createApiExtension>[0]['hocuspocus'],sessionManager: {} as unknown as Parameters<typeof createApiExtension>[0]['sessionManager'],@@ -248,6 +251,10 @@ async function callPages(contentDir: string, method = 'GET'): Promise<CapturedRereturn captured;}+async function callPages(contentDir: string, method = 'GET'): Promise<CapturedResponse> {+ return callPagesWithIndex(contentDir, buildFileIndex(contentDir), method);+}+describe('GET /api/pages', () => {test('returns flat { pages } success body and lists markdown files recursively', async () => {const dir = mkdtempSync(join(tmpdir(), 'ok-pages-'));@@ -300,6 +307,68 @@ describe('GET /api/pages', () => {}});+ test('serves cached title/icon from the index without reading disk', async () => {+ const dir = mkdtempSync(join(tmpdir(), 'ok-pages-cache-'));+ try {+ const index = new Map<string, FileIndexEntry>([+ [+ 'ghost',+ {+ size: 123,+ modified: new Date(0).toISOString(),+ canonicalPath: join(dir, 'ghost.md'),+ inode: 1,+ aliases: [],+ kind: 'markdown',+ title: 'Cached Title',+ icon: 'π―',+ },+ ],+ ]);++ const result = await callPagesWithIndex(dir, index);+ expect(result.status).toBe(200);+ const body = JSON.parse(result.body) as {+ pages?: Array<{ docName: string; title: string; icon?: string }>;+ };+ const entry = (body.pages ?? []).find((p) => p.docName === 'ghost');+ expect(entry?.title).toBe('Cached Title');+ expect(entry?.icon).toBe('π―');+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+ });++ test('bare entry whose file is missing falls back to the docName title (ENOENT path)', async () => {+ const dir = mkdtempSync(join(tmpdir(), 'ok-pages-bare-'));+ try {+ const index = new Map<string, FileIndexEntry>([+ [+ 'orphan',+ {+ size: 0,+ modified: new Date(0).toISOString(),+ canonicalPath: join(dir, 'orphan.md'),+ inode: 1,+ aliases: [],+ kind: 'markdown',+ },+ ],+ ]);++ const result = await callPagesWithIndex(dir, index);+ expect(result.status).toBe(200);+ const body = JSON.parse(result.body) as {+ pages?: Array<{ docName: string; title: string; icon?: string }>;+ };+ const entry = (body.pages ?? []).find((p) => p.docName === 'orphan');+ expect(entry?.title).toBe('orphan');+ expect(entry?.icon ?? undefined).toBeUndefined();+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+ });+test('returns RFC 9457 problem+json 405 envelope for unsupported methods', async () => {const dir = mkdtempSync(join(tmpdir(), 'ok-pages-'));try {diff --git a/packages/server/src/file-watcher.test.ts b/packages/server/src/file-watcher.test.tsindex 868bd6123..4658dcad6 100644--- a/packages/server/src/file-watcher.test.ts+++ b/packages/server/src/file-watcher.test.ts@@ -391,6 +391,27 @@ describe('startWatcher file index', () => {}});+ test('initial scan caches page title + icon on each markdown entry', async () => {+ writeFileSync(+ resolve(contentDir, 'with-meta.md'),+ '---\ntitle: Meta Title\nicon: π\n---\n\n# Heading Ignored\n',+ );+ writeFileSync(resolve(contentDir, 'heading-only.md'), '# Heading Title\n');+ writeFileSync(resolve(contentDir, 'plain.md'), 'just body text, no heading\n');++ const handle = await startWatcher(contentDir, async () => {});+ try {+ const index = handle.getFileIndex();+ expect(index.get('with-meta')?.title).toBe('Meta Title');+ expect(index.get('with-meta')?.icon).toBe('π');+ expect(index.get('heading-only')?.title).toBe('Heading Title');+ expect(index.get('heading-only')?.icon).toBeUndefined();+ expect(index.get('plain')?.title).toBe('plain');+ } finally {+ await handle.unsubscribe();+ }+ });+test('initial scan preserves uppercase .MD/.MDX extension casing', async () => {writeFileSync(resolve(contentDir, 'Upper.MD'), '# Upper\n');writeFileSync(resolve(contentDir, 'Mixed.MdX'), '# Mixed\n');@@ -519,6 +540,61 @@ describe('startWatcher file index', () => {expect(index.get('new-name')?.size).toBe(Buffer.byteLength('# Renamed\n', 'utf-8'));});+ test('file index caches title + icon on create/update/rename/conflict events', () => {+ const { updateFileIndex } = require('./file-watcher.ts');+ const index = new Map();++ updateFileIndex(+ {+ kind: 'create' as const,+ path: resolve(contentDir, 'doc.md'),+ docName: 'doc',+ content: '---\ntitle: Created\nicon: π\n---\n\nBody\n',+ },+ index,+ );+ expect(index.get('doc')?.title).toBe('Created');+ expect(index.get('doc')?.icon).toBe('π');++ updateFileIndex(+ {+ kind: 'update' as const,+ path: resolve(contentDir, 'doc.md'),+ docName: 'doc',+ content: '# Edited Title\n',+ },+ index,+ );+ expect(index.get('doc')?.title).toBe('Edited Title');+ expect(index.get('doc')?.icon).toBeUndefined();++ updateFileIndex(+ {+ kind: 'rename' as const,+ oldPath: resolve(contentDir, 'doc.md'),+ newPath: resolve(contentDir, 'renamed.md'),+ oldDocName: 'doc',+ newDocName: 'renamed',+ content: '---\ntitle: Renamed Title\nicon: π\n---\n\nBody\n',+ },+ index,+ );+ expect(index.get('renamed')?.title).toBe('Renamed Title');+ expect(index.get('renamed')?.icon).toBe('π');++ updateFileIndex(+ {+ kind: 'conflict' as const,+ path: resolve(contentDir, 'renamed.md'),+ docName: 'renamed',+ content: '---\ntitle: Conflicted\nicon: β οΈ\n---\n<<<<<<< HEAD\na\n=======\nb\n>>>>>>> x\n',+ },+ index,+ );+ expect(index.get('renamed')?.title).toBe('Conflicted');+ expect(index.get('renamed')?.icon).toBe('β οΈ');+ });+test('getFileIndex returns empty map when no .md files exist', async () => {const handle = await startWatcher(contentDir, async () => {});try {diff --git a/packages/server/src/file-watcher.ts b/packages/server/src/file-watcher.tsindex 6b85dbafa..cee6539a9 100644--- a/packages/server/src/file-watcher.ts+++ b/packages/server/src/file-watcher.ts@@ -14,6 +14,7 @@ import {} from './doc-extensions.ts';import { classifyFsPath, normalizeFsPath } from './fs-traced.ts';import { getLogger } from './logger.ts';+import { extractPageIcon, extractPageTitle } from './page-identity.ts';import { toPosix } from './path-utils.ts';import { isWithinContentDir } from './persistence.ts';import { containsConflictMarkers } from './reconciliation.ts';@@ -79,6 +80,8 @@ export interface FileIndexEntry {inode: number;aliases: string[];kind: 'markdown' | 'file';+ title?: string;+ icon?: string;}export interface FolderIndexEntry {@@ -88,6 +91,13 @@ export interface FolderIndexEntry {inode: number;}+function derivePageMeta(+ content: string,+ docName: string,+): { title: string; icon: string | undefined } {+ return { title: extractPageTitle(content, docName), icon: extractPageIcon(content) };+}+function markdownIndexView(inner: ReadonlyMap<string, FileIndexEntry>,): ReadonlyMap<string, FileIndexEntry> {@@ -554,6 +564,7 @@ async function seedLastKnownHashes(inode: canonStat.ino,aliases: [aliasDocName],kind: 'markdown',+ ...derivePageMeta(content, canonicalDocName),});} catch (err) {const code = (err as NodeJS.ErrnoException).code;@@ -626,6 +637,7 @@ async function seedLastKnownHashes(inode: lst.ino,aliases: [],kind: 'markdown',+ ...derivePageMeta(content, docName),});} catch (err) {const code = (err as NodeJS.ErrnoException).code;@@ -717,6 +729,7 @@ export function updateFileIndex(event: DiskEvent, fileIndex: Map<string, FileIndinode: existing?.inode ?? 0,aliases: existing?.aliases ?? [],kind: 'markdown',+ ...derivePageMeta(event.content, docName),});break;}@@ -748,6 +761,7 @@ export function updateFileIndex(event: DiskEvent, fileIndex: Map<string, FileIndinode: existing?.inode ?? 0,aliases: existing?.aliases ?? [],kind: 'markdown',+ ...derivePageMeta(event.content, event.newDocName),});break;}