Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
perf(server): incremental workspace search corpus maintenance (#2815)
Andrew Mikofalvy3e6f451dc449
diff --git a/.changeset/incremental-search-corpus.md b/.changeset/incremental-search-corpus.mdnew file mode 100644index 000000000..9af5d4a11--- /dev/null+++ b/.changeset/incremental-search-corpus.md@@ -0,0 +1,5 @@+---+"@inkeep/open-knowledge": patch+---++Workspace search now maintains its full-text index incrementally. Previously any file change invalidated the whole search corpus and the next search rebuilt the entire index from scratch on the server's event loop — on a large workspace, a write burst with interleaved searches meant repeated full re-indexing. The server now diffs the document set against the live index and applies per-document insert/update/remove, so one write re-indexes one document. A from-scratch rebuild remains as the cold-start path and as an automatic fallback whenever the incremental patch cannot be proven consistent.diff --git a/packages/core/src/index.ts b/packages/core/src/index.tsindex 5d72bbe86..3b9fee46c 100644--- a/packages/core/src/index.ts+++ b/packages/core/src/index.ts@@ -1260,12 +1260,15 @@ export {MAX_WORKSPACE_SEARCH_LIMIT,searchWorkspaceCorpus,searchWorkspaceDocuments,+ updateWorkspaceSearchCorpus,type WorkspaceSearchCorpus,+ type WorkspaceSearchCorpusUpdate,type WorkspaceSearchDocument,type WorkspaceSearchIntent,type WorkspaceSearchKind,type WorkspaceSearchOptions,type WorkspaceSearchRanking,+ type WorkspaceSearchRebuildReason,type WorkspaceSearchResult,type WorkspaceSearchScope,type WorkspaceSemanticInput,diff --git a/packages/core/src/search/workspace-search.incremental.test.ts b/packages/core/src/search/workspace-search.incremental.test.tsnew file mode 100644index 000000000..f1b25056a--- /dev/null+++ b/packages/core/src/search/workspace-search.incremental.test.ts@@ -0,0 +1,246 @@+import { describe, expect, test } from 'vitest';+import {+ createWorkspaceSearchCorpus,+ createWorkspaceSearchDocument,+ searchWorkspaceCorpus,+ updateWorkspaceSearchCorpus,+ type WorkspaceSearchDocument,+ type WorkspaceSearchOptions,+} from './workspace-search.ts';++function page(+ path: string,+ content: string,+ modifiedTs = 10,+ title?: string,+): WorkspaceSearchDocument {+ return createWorkspaceSearchDocument({ kind: 'page', path, title, content, modifiedTs });+}++function folder(path: string, modifiedTs = 0): WorkspaceSearchDocument {+ return createWorkspaceSearchDocument({ kind: 'folder', path, modifiedTs });+}++function file(path: string, modifiedTs = 5): WorkspaceSearchDocument {+ return createWorkspaceSearchDocument({ kind: 'file', path, modifiedTs });+}++const baseDocuments: WorkspaceSearchDocument[] = [+ page('docs/api', 'HTTP endpoint contracts', 10, 'API Reference'),+ page('architecture/overview', 'Observer bridge and CRDT topology', 30, 'Architecture Overview'),+ page('notes/graphing', 'Visual explorer notes', 20, 'Graphing Notes'),+ folder('docs'),+ folder('architecture'),+ folder('notes'),+ file('assets/diagram.png'),+];++/**+ * The correctness bar for the incremental path: for any query/options pair, an+ * incrementally-maintained corpus must be indistinguishable from a corpus+ * built from scratch over the same documents — paths, scores, and signals.+ */+function expectSearchEquivalence(+ incremental: ReturnType<typeof createWorkspaceSearchCorpus>,+ documents: readonly WorkspaceSearchDocument[],+ queries: readonly string[],+ options: WorkspaceSearchOptions = { intent: 'full_text' },+) {+ const fresh = createWorkspaceSearchCorpus(documents);+ for (const query of queries) {+ const incrementalResults = searchWorkspaceCorpus(incremental, query, options);+ const freshResults = searchWorkspaceCorpus(fresh, query, options);+ expect(+ incrementalResults.map((r) => ({+ path: r.document.path,+ kind: r.document.kind,+ score: r.score,+ signals: r.signals,+ })),+ ).toEqual(+ freshResults.map((r) => ({+ path: r.document.path,+ kind: r.document.kind,+ score: r.score,+ signals: r.signals,+ })),+ );+ }+}++describe('updateWorkspaceSearchCorpus', () => {+ test('inserting a document makes it searchable without a rebuild', () => {+ const corpus = createWorkspaceSearchCorpus(baseDocuments);+ const next = [...baseDocuments, page('guides/tutorial', 'Zebra migration walkthrough', 40)];++ const update = updateWorkspaceSearchCorpus(corpus, next);++ expect(update.rebuilt).toBe(false);+ expect(update.inserted).toBe(1);+ expect(update.updated).toBe(0);+ expect(update.removed).toBe(0);+ const results = searchWorkspaceCorpus(update.corpus, 'zebra', { intent: 'full_text' });+ expect(results.map((r) => r.document.path)).toContain('guides/tutorial');+ expectSearchEquivalence(update.corpus, next, ['zebra', 'tutorial', 'topology', 'arch']);+ });++ test('updating a document swaps its indexed content', () => {+ const corpus = createWorkspaceSearchCorpus(baseDocuments);+ const next = baseDocuments.map((doc) =>+ doc.path === 'notes/graphing'+ ? page('notes/graphing', 'Quokka habitat research', 21, 'Graphing Notes')+ : doc,+ );++ const update = updateWorkspaceSearchCorpus(corpus, next);++ expect(update.rebuilt).toBe(false);+ expect(update.inserted).toBe(0);+ expect(update.updated).toBe(1);+ expect(update.removed).toBe(0);+ const newHits = searchWorkspaceCorpus(update.corpus, 'quokka', { intent: 'full_text' });+ expect(newHits.map((r) => r.document.path)).toContain('notes/graphing');+ const oldHits = searchWorkspaceCorpus(update.corpus, 'explorer', { intent: 'full_text' });+ expect(oldHits.map((r) => r.document.path)).not.toContain('notes/graphing');+ expectSearchEquivalence(update.corpus, next, ['quokka', 'explorer', 'notes']);+ });++ test('removing a document drops it from every search tier', () => {+ const corpus = createWorkspaceSearchCorpus(baseDocuments);+ const next = baseDocuments.filter((doc) => doc.path !== 'docs/api');++ const update = updateWorkspaceSearchCorpus(corpus, next);++ expect(update.rebuilt).toBe(false);+ expect(update.inserted).toBe(0);+ expect(update.updated).toBe(0);+ expect(update.removed).toBe(1);+ for (const query of ['endpoint', 'api']) {+ const hits = searchWorkspaceCorpus(update.corpus, query, { intent: 'full_text' });+ expect(hits.map((r) => r.document.path)).not.toContain('docs/api');+ }+ expectSearchEquivalence(update.corpus, next, ['endpoint', 'api', 'contracts']);+ });++ test('unchanged document set patches nothing and stays searchable', () => {+ const corpus = createWorkspaceSearchCorpus(baseDocuments);+ // New array, mixed identity: some reused objects, some re-created equal ones+ // (the server reuses page objects via its cache but re-creates metadata-only+ // folder/file documents every build).+ const next = baseDocuments.map((doc) =>+ doc.kind === 'page' ? doc : createWorkspaceSearchDocument({ ...doc }),+ );++ const update = updateWorkspaceSearchCorpus(corpus, next);++ expect(update.rebuilt).toBe(false);+ expect(update.inserted).toBe(0);+ expect(update.updated).toBe(0);+ expect(update.removed).toBe(0);+ expectSearchEquivalence(update.corpus, next, ['topology', 'arch', 'diagram']);+ });++ test('a removed id can be re-added across successive updates', () => {+ const corpus = createWorkspaceSearchCorpus(baseDocuments);+ const without = baseDocuments.filter((doc) => doc.path !== 'docs/api');+ const first = updateWorkspaceSearchCorpus(corpus, without);+ expect(first.rebuilt).toBe(false);++ const readded = [...without, page('docs/api', 'Fresh endpoint catalog', 50, 'API Reference')];+ const second = updateWorkspaceSearchCorpus(first.corpus, readded);++ expect(second.rebuilt).toBe(false);+ const hits = searchWorkspaceCorpus(second.corpus, 'catalog', { intent: 'full_text' });+ expect(hits.map((r) => r.document.path)).toContain('docs/api');+ expectSearchEquivalence(second.corpus, readded, ['catalog', 'endpoint']);+ });++ test('emptying the corpus rebuilds electively and leaves a searchable empty index', () => {+ const corpus = createWorkspaceSearchCorpus(baseDocuments);+ const update = updateWorkspaceSearchCorpus(corpus, []);+ // Removing everything mutates more entries than an (empty) rebuild inserts,+ // so the bulk-change gate takes the cheap from-scratch path.+ expect(update.rebuilt).toBe(true);+ expect(update.rebuildReason).toBe('bulk-change');+ expect(searchWorkspaceCorpus(update.corpus, 'topology', { intent: 'full_text' })).toEqual([]);+ });++ test('duplicate ids fail loudly, matching the from-scratch build', () => {+ const corpus = createWorkspaceSearchCorpus(baseDocuments);+ const duplicated = [...baseDocuments, page('docs/api', 'Shadow copy', 60)];+ expect(() => createWorkspaceSearchCorpus(duplicated)).toThrow();+ expect(() => updateWorkspaceSearchCorpus(corpus, duplicated)).toThrow();+ });++ test('reusing a consumed base falls back to a from-scratch rebuild', () => {+ const corpus = createWorkspaceSearchCorpus(baseDocuments);+ const first = updateWorkspaceSearchCorpus(corpus, [+ ...baseDocuments,+ page('guides/alpha', 'Alpha content', 40),+ ]);+ expect(first.rebuilt).toBe(false);++ // Diffing against `corpus` again would patch an index that has already+ // moved past it; the consumed-base gate must force a rebuild instead.+ const next = [...baseDocuments, page('guides/beta', 'Beta content', 41)];+ const second = updateWorkspaceSearchCorpus(corpus, next);++ expect(second.rebuilt).toBe(true);+ expect(second.rebuildReason).toBe('stale-base');+ expectSearchEquivalence(second.corpus, next, ['beta', 'alpha', 'topology']);+ });++ test('a diff touching more documents than a rebuild would insert rebuilds electively', () => {+ const corpus = createWorkspaceSearchCorpus(baseDocuments);+ // Replace everything: every old id removed, every new id inserted.+ const next = [page('brand/new', 'Entirely new workspace', 70)];++ const update = updateWorkspaceSearchCorpus(corpus, next);++ expect(update.rebuilt).toBe(true);+ expect(update.rebuildReason).toBe('bulk-change');+ expectSearchEquivalence(update.corpus, next, ['workspace', 'topology']);+ });++ test('mixed burst of inserts, updates, and removes matches a from-scratch build', () => {+ // Deterministic pseudo-random burst: enough documents and churn to exercise+ // the diff paths together, seeded so a failure replays exactly.+ let seed = 0xc0ffee;+ const rand = () => {+ // LCG (Numerical Recipes constants) — deterministic across runs/platforms.+ seed = (seed * 1664525 + 1013904223) >>> 0;+ return seed / 0x100000000;+ };+ const words = ['alpha', 'bravo', 'crdt', 'delta', 'editor', 'fjord', 'graph', 'hocus'];+ const makeContent = () =>+ Array.from({ length: 5 }, () => words[Math.trunc(rand() * words.length)]).join(' ');++ let documents: WorkspaceSearchDocument[] = Array.from({ length: 40 }, (_, i) =>+ page(`corpus/doc-${i}`, makeContent(), i),+ );+ let corpus = createWorkspaceSearchCorpus(documents);++ for (let step = 0; step < 8; step++) {+ const next = [...documents];+ // Remove one, update two, insert one per step — small deltas, like a+ // write burst with interleaved searches.+ next.splice(Math.trunc(rand() * next.length), 1);+ for (let u = 0; u < 2; u++) {+ const i = Math.trunc(rand() * next.length);+ const target = next[i];+ if (target) next[i] = page(target.path, makeContent(), target.modifiedTs + 100);+ }+ next.push(page(`corpus/new-${step}`, makeContent(), 1000 + step));++ const update = updateWorkspaceSearchCorpus(corpus, next);+ expect(update.rebuilt).toBe(false);+ corpus = update.corpus;+ documents = next;++ expectSearchEquivalence(corpus, documents, words, { intent: 'full_text' });+ expectSearchEquivalence(corpus, documents, ['corpus', 'doc-3', 'new-'], {+ intent: 'omnibar',+ });+ }+ });+});diff --git a/packages/core/src/search/workspace-search.ts b/packages/core/src/search/workspace-search.tsindex 02a6f90d3..f03433a16 100644--- a/packages/core/src/search/workspace-search.ts+++ b/packages/core/src/search/workspace-search.ts@@ -1,4 +1,4 @@-import { type AnyOrama, create, insertMultiple, search } from '@orama/orama';+import { type AnyOrama, count, create, insertMultiple, removeMultiple, search } from '@orama/orama';import { isHiddenDocName } from '../util/doc-name.ts';export type WorkspaceSearchKind = 'page' | 'folder' | 'file';@@ -281,6 +281,154 @@ function getWorkspaceSearchIndex(corpus: WorkspaceSearchCorpus): WorkspaceSearchreturn index;}+/**+ * Corpora already consumed as the base of an incremental update. A diff is only+ * valid against the exact document set the shared index currently reflects, and+ * that is the corpus most recently RETURNED by `createWorkspaceSearchCorpus` /+ * `updateWorkspaceSearchCorpus` — reusing an older base would apply a diff the+ * index has already moved past, silently desynchronizing BM25 from+ * `corpus.documents`. Membership here makes that misuse deterministic: a+ * consumed base forces the from-scratch rebuild path instead.+ */+const consumedUpdateBases = new WeakSet<WorkspaceSearchCorpus>();++/** Why an incremental corpus update fell back to a from-scratch index build. */+export type WorkspaceSearchRebuildReason =+ | 'no-index'+ | 'stale-base'+ | 'duplicate-id'+ | 'bulk-change'+ | 'mutation-failed';++export interface WorkspaceSearchCorpusUpdate {+ corpus: WorkspaceSearchCorpus;+ /** Documents newly added to the index (includes the insert half of updates). */+ inserted: number;+ /** Documents whose content/metadata changed (counted once, not as insert+remove). */+ updated: number;+ /** Documents removed from the index (excludes the remove half of updates). */+ removed: number;+ /** True when the index was rebuilt from scratch instead of patched in place. */+ rebuilt: boolean;+ rebuildReason?: WorkspaceSearchRebuildReason;+}++/**+ * Field-level equality for diffing. Object identity short-circuits the common+ * case (callers reuse unchanged document objects across builds); the field+ * compare covers metadata-only documents that are re-created each build with+ * identical values. `id` is the diff key, so it is compared by the caller.+ */+function sameSearchDocument(a: WorkspaceSearchDocument, b: WorkspaceSearchDocument): boolean {+ return (+ a === b ||+ (a.kind === b.kind &&+ a.path === b.path &&+ a.title === b.title &&+ a.name === b.name &&+ a.pathSegments === b.pathSegments &&+ a.modifiedTs === b.modifiedTs &&+ a.content === b.content)+ );+}++/**+ * Advance a corpus to a new document set by patching the existing Orama index+ * in place — per-document remove/insert for only the changed entries — instead+ * of re-tokenizing the entire workspace. On a large corpus the from-scratch+ * build is the dominant cost of search-cache invalidation, and it runs on the+ * event loop; this keeps steady-state maintenance proportional to the change.+ *+ * Falls back to a from-scratch build (`rebuilt: true`) whenever the+ * incremental path cannot be proven safe: `previous` has no live index or was+ * already consumed as an update base, the new set carries a duplicate id (the+ * rebuild then throws exactly like `createWorkspaceSearchCorpus`), the diff+ * touches more documents than a rebuild would insert, or an index mutation+ * fails / leaves the index count inconsistent.+ *+ * The returned corpus shares the (now-patched) index with `previous`. A search+ * still holding `previous` keeps a coherent `documents` snapshot for the+ * lexical/recency tiers but reads the patched index for BM25 — a bounded,+ * one-step freshness skew, accepted over the alternatives (an index copy per+ * update, or re-building the old snapshot's index on demand).+ */+export function updateWorkspaceSearchCorpus(+ previous: WorkspaceSearchCorpus,+ documents: readonly WorkspaceSearchDocument[],+): WorkspaceSearchCorpusUpdate {+ const rebuild = (rebuildReason: WorkspaceSearchRebuildReason): WorkspaceSearchCorpusUpdate => ({+ corpus: createWorkspaceSearchCorpus(documents),+ inserted: 0,+ updated: 0,+ removed: 0,+ rebuilt: true,+ rebuildReason,+ });++ const index = workspaceSearchIndexes.get(previous);+ if (!index) return rebuild('no-index');+ if (consumedUpdateBases.has(previous)) return rebuild('stale-base');++ const nextById = new Map(documents.map((document) => [document.id, document] as const));+ if (nextById.size !== documents.length) return rebuild('duplicate-id');++ const removedIds: string[] = [];+ const insertedDocuments: WorkspaceSearchDocument[] = [];+ let updated = 0;+ for (const previousDocument of previous.documents) {+ const nextDocument = nextById.get(previousDocument.id);+ if (!nextDocument) {+ removedIds.push(previousDocument.id);+ } else if (!sameSearchDocument(previousDocument, nextDocument)) {+ // Changed document: remove the stale tokens, insert the new ones. Orama's+ // `update` is the same remove+insert internally; batching both halves+ // keeps the mutation count observable for the bulk-change gate below.+ removedIds.push(previousDocument.id);+ insertedDocuments.push(nextDocument);+ updated += 1;+ }+ }+ const previousIds = new Set(previous.documents.map((document) => document.id));+ for (const document of documents) {+ if (!previousIds.has(document.id)) insertedDocuments.push(document);+ }++ // When the diff would mutate more index entries than a rebuild would insert+ // (e.g. a branch switch replacing most of the workspace), the from-scratch+ // build is the cheaper operation — take it electively.+ if (removedIds.length + insertedDocuments.length > documents.length) {+ return rebuild('bulk-change');+ }++ consumedUpdateBases.add(previous);+ try {+ if (removedIds.length > 0) removeMultiple(index, removedIds);+ if (insertedDocuments.length > 0) {+ insertMultiple(index, insertedDocuments as WorkspaceSearchDocument[]);+ }+ // A count mismatch means a removal targeted an id the index did not hold+ // (or an insert landed twice) — the index no longer mirrors `documents`,+ // so discard it rather than serve drifted BM25 results.+ if (count(index) !== documents.length) {+ throw new Error(+ `Search index count ${count(index)} diverged from document count ${documents.length} after incremental update`,+ );+ }+ } catch {+ return rebuild('mutation-failed');+ }++ const corpus: WorkspaceSearchCorpus = { documents };+ workspaceSearchIndexes.set(corpus, index);+ return {+ corpus,+ inserted: insertedDocuments.length - updated,+ updated,+ removed: removedIds.length - updated,+ rebuilt: false,+ };+}+function defaultScopes(intent: WorkspaceSearchIntent): readonly WorkspaceSearchScope[] {// Autocomplete stays page-only: it backs the `[[wikilink]]` target picker,// whose resolution model is markdown-scoped (the all-files autocomplete targetdiff --git a/packages/server/src/api-extension.ts b/packages/server/src/api-extension.tsindex ee3fe2ed4..acb33394f 100644--- a/packages/server/src/api-extension.ts+++ b/packages/server/src/api-extension.ts@@ -241,6 +241,7 @@ import {UploadAssetSuccessSchema,UploadRequestSchema,unwrapFrontmatterFences,+ updateWorkspaceSearchCorpus,type WorkspaceSearchCorpus,type WorkspaceSearchDocument,type WorkspaceSearchIntent,@@ -742,6 +743,21 @@ function searchCorpusTruncatedCounter(): ReturnType<ReturnType<typeof getMeter>[return _searchCorpusTruncatedCounter;}+// Counter for search-corpus builds by mode: `cold` (no prior corpus),+// `incremental` (in-place index patch of only the changed documents), or+// `rebuild` (incremental path fell back to a from-scratch build; the bounded+// `reason` attribute says why). A steady stream of `rebuild` where+// `incremental` is expected is the drift signal worth alerting on.+let _searchCorpusUpdateCounter: ReturnType<ReturnType<typeof getMeter>['createCounter']> | null =+ null;+function searchCorpusUpdateCounter(): ReturnType<ReturnType<typeof getMeter>['createCounter']> {+ _searchCorpusUpdateCounter ||= getMeter().createCounter('ok.search.corpus_update_total', {+ description:+ 'Count of workspace search corpus builds, by mode (cold | incremental | rebuild) and, for rebuilds, the bounded fallback reason.',+ });+ return _searchCorpusUpdateCounter;+}+/** Bounded handler label for the content-divergence counters. */type DivergenceHandler = 'agent-write-md' | 'agent-patch' | 'rollback';@@ -17235,10 +17251,48 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {return workspaceSearchCache.pending;}- const pending = buildWorkspaceSearchDocumentsFromIndex().then(({ documents, truncated }) => ({- corpus: createWorkspaceSearchCorpus(documents),- truncated,- }));+ // Stale-but-live corpus (or the build about to produce one): the base for+ // an incremental index patch, so one write re-indexes one document instead+ // of re-tokenizing the whole workspace on the event loop.+ const priorCorpus = workspaceSearchCache?.corpus;+ const priorPending = workspaceSearchCache?.pending;+ const pending = (async () => {+ // Serialize behind any in-flight build: an incremental diff is only valid+ // against the corpus it was computed from, and the in-flight build owns+ // the shared index right now. Chaining keeps updates linear (each build+ // bases on its predecessor's output) without coalescing away freshness —+ // the document snapshot below is read AFTER this fingerprint was seen.+ const base = priorPending+ ? await priorPending.then(+ (result) => result.corpus,+ () => undefined,+ )+ : priorCorpus;+ const { documents, truncated } = await buildWorkspaceSearchDocumentsFromIndex();+ if (!base) {+ searchCorpusUpdateCounter().add(1, { mode: 'cold' });+ return { corpus: createWorkspaceSearchCorpus(documents), truncated };+ }+ const update = updateWorkspaceSearchCorpus(base, documents);+ if (update.rebuilt) {+ searchCorpusUpdateCounter().add(1, { mode: 'rebuild', reason: update.rebuildReason });+ // `mutation-failed` means the patched index diverged from the document+ // set (or a mutation threw) — recovered by the rebuild, but worth an+ // operator-visible signal; the elective reasons are routine.+ const logLevel = update.rebuildReason === 'mutation-failed' ? 'warn' : 'debug';+ getLogger('search')[logLevel](+ { reason: update.rebuildReason, documents: documents.length },+ '[search] corpus update fell back to a full index rebuild',+ );+ } else {+ searchCorpusUpdateCounter().add(1, { mode: 'incremental' });+ getLogger('search').debug(+ { inserted: update.inserted, updated: update.updated, removed: update.removed },+ '[search] corpus updated incrementally',+ );+ }+ return { corpus: update.corpus, truncated };+ })();workspaceSearchCaches.set(cacheKey, { fingerprint, pending });try {const result = await pending;diff --git a/packages/server/src/api-search-incremental.test.ts b/packages/server/src/api-search-incremental.test.tsnew file mode 100644index 000000000..c37404db3--- /dev/null+++ b/packages/server/src/api-search-incremental.test.ts@@ -0,0 +1,267 @@+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';+import type { IncomingMessage, ServerResponse } from 'node:http';+import { tmpdir } from 'node:os';+import { dirname, join } from 'node:path';+import { Readable } from 'node:stream';+import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';+import { createApiExtension } from './api-extension.ts';+import type { FileIndexEntry } from './file-watcher.ts';++// Count how the server maintains the search corpus: `create` = from-scratch+// builds requested directly by the server (cold start), `update` = incremental+// maintenance. `updateWorkspaceSearchCorpus`'s internal fallback rebuild calls+// the module-local `createWorkspaceSearchCorpus`, which this mock does NOT+// intercept — so `create` counts exactly the server's own cold builds.+const corpusCalls = vi.hoisted(() => ({ create: 0, update: 0 }));+vi.mock('@inkeep/open-knowledge-core', async (importOriginal) => {+ const actual = await importOriginal<typeof import('@inkeep/open-knowledge-core')>();+ return {+ ...actual,+ createWorkspaceSearchCorpus: (+ ...args: Parameters<typeof actual.createWorkspaceSearchCorpus>+ ) => {+ corpusCalls.create += 1;+ return actual.createWorkspaceSearchCorpus(...args);+ },+ updateWorkspaceSearchCorpus: (+ ...args: Parameters<typeof actual.updateWorkspaceSearchCorpus>+ ) => {+ corpusCalls.update += 1;+ return actual.updateWorkspaceSearchCorpus(...args);+ },+ };+});++interface CapturedResponse {+ status: number;+ headers: Record<string, string>;+ body: string;+}++interface SearchResultEntry {+ kind: string;+ path: string;+ title: string;+ score: number;+ signals: Record<string, number>;+ snippet?: string;+}++function makeReq(method: string, url: string): IncomingMessage {+ const readable = Readable.from(Buffer.from('')) as unknown as IncomingMessage;+ readable.method = method;+ readable.url = url;+ readable.headers = { host: 'localhost' };+ return readable;+}++function makeRes(): { res: ServerResponse; captured: CapturedResponse } {+ const captured: CapturedResponse = { status: 0, headers: {}, body: '' };+ const res = {+ writeHead(status: number, headers?: Record<string, string>) {+ captured.status = status;+ if (headers) Object.assign(captured.headers, headers);+ },+ end(body?: string) {+ captured.body = body ?? '';+ },+ } as unknown as ServerResponse;+ return { res, captured };+}++/**+ * A persistent extension over a mutable in-memory file index — the shape the+ * production server has (one long-lived extension whose index the file watcher+ * mutates), unlike the per-call throwaway extension in `api-search.test.ts`.+ * `modified` stamps are assigned from a deterministic sequence so every content+ * change also changes the entry fingerprint, mirroring the watcher's behavior.+ */+function createHarness(contentDir: string, options: { withGeneration: boolean }) {+ const index = new Map<string, FileIndexEntry>();+ let generation = 0;+ let stampSeq = 0;+ const nextStamp = () => new Date(1700000000000 + ++stampSeq * 1000).toISOString();++ const ext = createApiExtension({+ hocuspocus: {} as unknown as Parameters<typeof createApiExtension>[0]['hocuspocus'],+ sessionManager: {} as unknown as Parameters<typeof createApiExtension>[0]['sessionManager'],+ contentDir,+ serverInstanceId: 'test-server',+ getFileIndex: () => index,+ getAllFilesIndex: () => index,+ ...(options.withGeneration ? { getFileIndexGeneration: () => generation } : {}),+ });++ const setDoc = (docName: string, content: string, modified = nextStamp()) => {+ const abs = join(contentDir, `${docName}.md`);+ mkdirSync(dirname(abs), { recursive: true });+ writeFileSync(abs, content, 'utf-8');+ index.set(docName, {+ size: Buffer.byteLength(content),+ modified,+ canonicalPath: abs,+ inode: ++stampSeq,+ aliases: [],+ kind: 'markdown',+ });+ generation += 1;+ return modified;+ };++ const setRawEntry = (docName: string, entry: Partial<FileIndexEntry>) => {+ index.set(docName, {+ size: 0,+ modified: nextStamp(),+ canonicalPath: join(contentDir, docName),+ inode: ++stampSeq,+ aliases: [],+ kind: 'markdown',+ ...entry,+ });+ generation += 1;+ };++ const removeDoc = (docName: string) => {+ index.delete(docName);+ generation += 1;+ };++ const search = async (query: string, intent = 'full_text') => {+ const req = makeReq('GET', `/api/search?query=${encodeURIComponent(query)}&intent=${intent}`);+ const { res, captured } = makeRes();+ await (+ ext as {+ onRequest: (ctx: { request: IncomingMessage; response: ServerResponse }) => Promise<void>;+ }+ ).onRequest({ request: req, response: res });+ expect(captured.status).toBe(200);+ return JSON.parse(captured.body) as { results: SearchResultEntry[]; ready: boolean };+ };++ return { setDoc, setRawEntry, removeDoc, search };+}++describe('incremental workspace search corpus maintenance', () => {+ let contentDir: string;++ beforeEach(() => {+ contentDir = mkdtempSync(join(tmpdir(), 'ok-search-incr-'));+ corpusCalls.create = 0;+ corpusCalls.update = 0;+ });++ afterEach(() => {+ rmSync(contentDir, { recursive: true, force: true });+ });++ test('one cold build, then per-write incremental updates (generation fingerprint)', async () => {+ const harness = createHarness(contentDir, { withGeneration: true });+ harness.setDoc('docs/api', '# API Reference\n\nHTTP endpoint contracts\n');+ harness.setDoc('notes/graphing', '# Graphing Notes\n\nVisual explorer notes\n');++ const cold = await harness.search('endpoint');+ expect(cold.results.map((r) => r.path)).toContain('docs/api');+ expect(corpusCalls).toEqual({ create: 1, update: 0 });++ // Insert: a new doc is searchable without another from-scratch build.+ harness.setDoc('guides/tutorial', '# Tutorial\n\nZebra migration walkthrough\n');+ const afterInsert = await harness.search('zebra');+ expect(afterInsert.results.map((r) => r.path)).toContain('guides/tutorial');+ expect(corpusCalls).toEqual({ create: 1, update: 1 });++ // Update: changed body content is re-indexed; the old tokens are gone.+ harness.setDoc('notes/graphing', '# Graphing Notes\n\nQuokka habitat research\n');+ const afterUpdate = await harness.search('quokka');+ expect(afterUpdate.results.map((r) => r.path)).toContain('notes/graphing');+ // The second search after the same write hits the fingerprint cache — no+ // further corpus work of either kind.+ const oldTokens = await harness.search('explorer');+ expect(oldTokens.results.map((r) => r.path)).not.toContain('notes/graphing');+ expect(corpusCalls).toEqual({ create: 1, update: 2 });++ // Remove: the doc disappears from name and body tiers.+ harness.removeDoc('docs/api');+ const afterRemove = await harness.search('endpoint');+ expect(afterRemove.results.map((r) => r.path)).not.toContain('docs/api');+ expect(corpusCalls).toEqual({ create: 1, update: 3 });+ });++ test('incremental maintenance also runs on the fallback (no-generation) fingerprint', async () => {+ const harness = createHarness(contentDir, { withGeneration: false });+ harness.setDoc('docs/api', '# API\n\nendpoint contracts\n');++ await harness.search('endpoint');+ expect(corpusCalls).toEqual({ create: 1, update: 0 });++ harness.setDoc('guides/new-page', '# New Page\n\nxylophone content\n');+ const results = await harness.search('xylophone');+ expect(results.results.map((r) => r.path)).toContain('guides/new-page');+ expect(corpusCalls).toEqual({ create: 1, update: 1 });+ });++ test('system and config docs added after the cold build never enter the corpus', async () => {+ const harness = createHarness(contentDir, { withGeneration: true });+ harness.setDoc('docs/real-page', '# Real\n\nsystem architecture content\n');+ await harness.search('system');++ // The build-time predicate skips these before any disk read, so the raw+ // entries need no backing file.+ harness.setRawEntry('__system__', {});+ harness.setRawEntry('__config__/project', {});+ // full_text searches name AND body, so this asserts the synthetic names are+ // absent from every tier while the real page still matches on content.+ const results = await harness.search('system', 'full_text');+ const paths = results.results.map((r) => r.path);+ expect(paths).not.toContain('__system__');+ expect(paths).not.toContain('__config__/project');+ expect(paths).toContain('docs/real-page');+ });++ test('after a mixed write burst, results match a from-scratch build of the same workspace', async () => {+ const harness = createHarness(contentDir, { withGeneration: true });+ const stamps = new Map<string, string>();+ stamps.set('docs/api', harness.setDoc('docs/api', '# API\n\nendpoint contracts\n'));+ stamps.set('notes/one', harness.setDoc('notes/one', '# One\n\nalpha bravo crdt\n'));+ stamps.set('notes/two', harness.setDoc('notes/two', '# Two\n\ndelta editor fjord\n'));+ await harness.search('alpha');++ // Burst: update, insert, remove, insert — each interleaved with a search so+ // every step goes through the incremental path rather than one big diff.+ stamps.set('notes/one', harness.setDoc('notes/one', '# One\n\nalpha graph hocus\n'));+ await harness.search('graph');+ stamps.set('guides/four', harness.setDoc('guides/four', '# Four\n\nbravo delta alpha\n'));+ await harness.search('bravo');+ harness.removeDoc('notes/two');+ stamps.delete('notes/two');+ await harness.search('delta');+ stamps.set('guides/five', harness.setDoc('guides/five', '# Five\n\nhocus fjord\n'));+ expect(corpusCalls.create).toBe(1);++ // Reference: an identical workspace (same docNames, contents, and modified+ // stamps) in a fresh contentDir, indexed from scratch by its own harness.+ const referenceDir = mkdtempSync(join(tmpdir(), 'ok-search-ref-'));+ try {+ const reference = createHarness(referenceDir, { withGeneration: true });+ const finalDocs: Array<[string, string]> = [+ ['docs/api', '# API\n\nendpoint contracts\n'],+ ['notes/one', '# One\n\nalpha graph hocus\n'],+ ['guides/four', '# Four\n\nbravo delta alpha\n'],+ ['guides/five', '# Five\n\nhocus fjord\n'],+ ];+ for (const [docName, content] of finalDocs) {+ const stamp = stamps.get(docName);+ if (!stamp) throw new Error(`missing stamp for ${docName}`);+ reference.setDoc(docName, content, stamp);+ }+ for (const query of ['alpha', 'bravo', 'delta', 'fjord', 'graph', 'hocus', 'guides']) {+ for (const intent of ['full_text', 'omnibar'] as const) {+ const incremental = await harness.search(query, intent);+ const fresh = await reference.search(query, intent);+ expect(incremental.results).toEqual(fresh.results);+ }+ }+ } finally {+ rmSync(referenceDir, { recursive: true, force: true });+ }+ });+});