Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
Rebuild the search index incrementally instead of re-reading every file (#2022)
Andrew Mikofalvya37db9c60f93
diff --git a/.changeset/search-incremental-corpus-cache.md b/.changeset/search-incremental-corpus-cache.mdnew file mode 100644index 000000000..f8973bb5a--- /dev/null+++ b/.changeset/search-incremental-corpus-cache.md@@ -0,0 +1,7 @@+---+"@inkeep/open-knowledge": patch+---++Make the first search after an edit faster on large workspaces.++The workspace search index is rebuilt whenever the file index changes, and a rebuild previously re-read and re-parsed every markdown file on disk — so on a large workspace, the first search after any edit paid the cost of ingesting the entire corpus again. Each page's parsed search document is now reused across rebuilds when the watcher reports its file unchanged (by size, modified time, or inode), so a rebuild re-reads only the files that actually changed. Deleted and renamed pages drop out, and a failed read is retried rather than cached, so results stay current for typical edits.diff --git a/packages/server/src/api-extension.ts b/packages/server/src/api-extension.tsindex 6a19eed21..fe3832286 100644--- a/packages/server/src/api-extension.ts+++ b/packages/server/src/api-extension.ts@@ -10488,12 +10488,19 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {};}+ function entrySearchKey(entry: FileIndexEntry): string {+ return `${entry.modified}\0${entry.size}\0${entry.canonicalPath}\0${entry.inode}\0${entry.aliases.join('\0')}`;+ }++ const pageDocCache = new Map<string, { key: string; doc: WorkspaceSearchDocument }>();+async function buildWorkspaceSearchDocumentsFromIndex(): Promise<{documents: WorkspaceSearchDocument[];truncated: boolean;}> {const pages: WorkspaceSearchDocument[] = [];const files: WorkspaceSearchDocument[] = [];+ const seenPages: Set<string> = new Set();for (const [docName, entry] of getAllFilesIndex()) {if (isSystemDoc(docName) || isConfigDoc(docName)) continue;if (entry.kind === 'file') {@@ -10507,24 +10514,42 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {);continue;}+ seenPages.add(docName);+ const entryKey = entrySearchKey(entry);+ const cached = pageDocCache.get(docName);+ if (cached && cached.key === entryKey) {+ pages.push(cached.doc);+ continue;+ }let content = '';let title = docName;+ let readFailed = false;try {content = await readFile(entry.canonicalPath, 'utf-8');- title = extractPageTitle(content, docName);} catch (err) {- console.warn(`[search] Failed to index ${docName}:`, err);- }- pages.push(- createWorkspaceSearchDocument({- kind: 'page',- path: docName,- title,- content,- modifiedTs: Date.parse(entry.modified),- aliases: entry.aliases,- }),- );+ readFailed = true;+ console.warn(`[search] Failed to read ${docName}:`, err);+ }+ if (!readFailed) {+ try {+ title = extractPageTitle(content, docName);+ } catch (err) {+ console.warn(`[search] Failed to extract title for ${docName}:`, err);+ }+ }+ const doc = createWorkspaceSearchDocument({+ kind: 'page',+ path: docName,+ title,+ content,+ modifiedTs: Date.parse(entry.modified),+ aliases: entry.aliases,+ });+ if (!readFailed) pageDocCache.set(docName, { key: entryKey, doc });+ pages.push(doc);+ }+ for (const docName of pageDocCache.keys()) {+ if (!seenPages.has(docName)) pageDocCache.delete(docName);}const maxFiles = getSearchMaxEntries();let admittedFiles = files;@@ -10563,10 +10588,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {return [...getAllFilesIndex()].filter(([docName]) => !isSystemDoc(docName) && !isConfigDoc(docName)).sort(([a], [b]) => a.localeCompare(b))- .map(- ([docName, entry]) =>- `${docName} ${entry.modified} ${entry.size} ${entry.canonicalPath} ${entry.inode} ${entry.aliases.join(',')}`,- )+ .map(([docName, entry]) => `${docName}\0${entrySearchKey(entry)}`).join('');}diff --git a/packages/server/src/api-search.test.ts b/packages/server/src/api-search.test.tsindex 70a4775d9..47f86cd97 100644--- a/packages/server/src/api-search.test.ts+++ b/packages/server/src/api-search.test.ts@@ -816,3 +816,95 @@ describe('GET /api/search — cold-start readiness', () => {}});});++describe('GET /api/search — incremental corpus (per-page document cache)', () => {+ async function onRequest(ext: ReturnType<typeof createApiExtension>, url: string) {+ const req = makeReq('GET', url);+ const { res, captured } = makeRes();+ await (+ ext as {+ onRequest: (ctx: { request: IncomingMessage; response: ServerResponse }) => Promise<void>;+ }+ ).onRequest({ request: req, response: res });+ return captured;+ }++ test('reuses an unchanged page across rebuilds, re-reads a changed entry, and prunes deletes', async () => {+ const dir = mkdtempSync(join(tmpdir(), 'ok-search-incremental-'));+ try {+ const alphaPath = join(dir, 'alpha.md');+ const betaPath = join(dir, 'beta.md');+ writeFileSync(alphaPath, '# Alpha\n\nalphaversionone\n', 'utf-8');+ const alphaV1 = indexEntry(alphaPath, 'markdown');+ const index = new Map<string, FileIndexEntry>([['alpha', alphaV1]]);+ const ext = createApiExtension({+ hocuspocus: {} as unknown as Parameters<typeof createApiExtension>[0]['hocuspocus'],+ sessionManager: {} as unknown as Parameters<typeof createApiExtension>[0]['sessionManager'],+ contentDir: dir,+ serverInstanceId: 'test-server',+ getFileIndex: () => index,+ getAllFilesIndex: () => index,+ });+ const run = (q: string) => onRequest(ext, `/api/search?query=${q}&intent=full_text`);+ const paths = (c: CapturedResponse) =>+ (JSON.parse(c.body) as { results?: Array<{ path: string }> }).results?.map((r) => r.path) ??+ [];++ expect(paths(await run('alphaversionone'))).toContain('alpha');++ writeFileSync(alphaPath, '# Alpha\n\nalphaversiontwo-now-longer\n', 'utf-8');+ writeFileSync(betaPath, '# Beta\n\nbetafreshbody\n', 'utf-8');+ index.set('beta', indexEntry(betaPath, 'markdown'));++ expect(paths(await run('alphaversiontwo'))).not.toContain('alpha');+ expect(paths(await run('alphaversionone'))).toContain('alpha');+ expect(paths(await run('betafreshbody'))).toContain('beta');++ index.set('alpha', indexEntry(alphaPath, 'markdown'));+ expect(paths(await run('alphaversiontwo'))).toContain('alpha');++ index.delete('beta');+ expect(paths(await run('betafreshbody'))).not.toContain('beta');+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+ });++ test('a failed read is NOT cached — the page self-heals on the next rebuild', async () => {+ const dir = mkdtempSync(join(tmpdir(), 'ok-search-readfail-'));+ try {+ const alphaPath = join(dir, 'alpha.md');+ const alphaEntry: FileIndexEntry = {+ size: 0,+ modified: new Date(0).toISOString(),+ canonicalPath: alphaPath,+ inode: 424242,+ aliases: [],+ kind: 'markdown',+ };+ const index = new Map<string, FileIndexEntry>([['alpha', alphaEntry]]);+ const ext = createApiExtension({+ hocuspocus: {} as unknown as Parameters<typeof createApiExtension>[0]['hocuspocus'],+ sessionManager: {} as unknown as Parameters<typeof createApiExtension>[0]['sessionManager'],+ contentDir: dir,+ serverInstanceId: 'test-server',+ getFileIndex: () => index,+ getAllFilesIndex: () => index,+ });+ const run = (q: string) => onRequest(ext, `/api/search?query=${q}&intent=full_text`);+ const paths = (c: CapturedResponse) =>+ (JSON.parse(c.body) as { results?: Array<{ path: string }> }).results?.map((r) => r.path) ??+ [];++ expect(paths(await run('healedtoken'))).not.toContain('alpha');++ writeFileSync(alphaPath, '# Alpha\n\nhealedtoken body\n', 'utf-8');+ writeFileSync(join(dir, 'beta.md'), '# Beta\n\nbetabody\n', 'utf-8');+ index.set('beta', indexEntry(join(dir, 'beta.md'), 'markdown'));++ expect(paths(await run('healedtoken'))).toContain('alpha');+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+ });+});