Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

perf(server): incremental workspace search corpus maintenance (#2815)

Andrew Mikofalvy3e6f451dc449
diff --git a/.changeset/incremental-search-corpus.md b/.changeset/incremental-search-corpus.md
new file mode 100644
index 000000000..9af5d4a11
--- /dev/null
+++ b/.changeset/incremental-search-corpus.md
@@ -0,0 +1,5 @@
+---
+"@inkeep/open-knowledge": patch
+---
+
+Workspace search now maintains its full-text index incrementally. Previously any file change invalidated the whole search corpus and the next search rebuilt the entire index from scratch on the server's event loop — on a large workspace, a write burst with interleaved searches meant repeated full re-indexing. The server now diffs the document set against the live index and applies per-document insert/update/remove, so one write re-indexes one document. A from-scratch rebuild remains as the cold-start path and as an automatic fallback whenever the incremental patch cannot be proven consistent.
diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts
index 5d72bbe86..3b9fee46c 100644
--- a/packages/core/src/index.ts
+++ b/packages/core/src/index.ts
@@ -1260,12 +1260,15 @@ export {
MAX_WORKSPACE_SEARCH_LIMIT,
searchWorkspaceCorpus,
searchWorkspaceDocuments,
+ updateWorkspaceSearchCorpus,
type WorkspaceSearchCorpus,
+ type WorkspaceSearchCorpusUpdate,
type WorkspaceSearchDocument,
type WorkspaceSearchIntent,
type WorkspaceSearchKind,
type WorkspaceSearchOptions,
type WorkspaceSearchRanking,
+ type WorkspaceSearchRebuildReason,
type WorkspaceSearchResult,
type WorkspaceSearchScope,
type WorkspaceSemanticInput,
diff --git a/packages/core/src/search/workspace-search.incremental.test.ts b/packages/core/src/search/workspace-search.incremental.test.ts
new file mode 100644
index 000000000..f1b25056a
--- /dev/null
+++ b/packages/core/src/search/workspace-search.incremental.test.ts
@@ -0,0 +1,246 @@
+import { describe, expect, test } from 'vitest';
+import {
+ createWorkspaceSearchCorpus,
+ createWorkspaceSearchDocument,
+ searchWorkspaceCorpus,
+ updateWorkspaceSearchCorpus,
+ type WorkspaceSearchDocument,
+ type WorkspaceSearchOptions,
+} from './workspace-search.ts';
+
+function page(
+ path: string,
+ content: string,
+ modifiedTs = 10,
+ title?: string,
+): WorkspaceSearchDocument {
+ return createWorkspaceSearchDocument({ kind: 'page', path, title, content, modifiedTs });
+}
+
+function folder(path: string, modifiedTs = 0): WorkspaceSearchDocument {
+ return createWorkspaceSearchDocument({ kind: 'folder', path, modifiedTs });
+}
+
+function file(path: string, modifiedTs = 5): WorkspaceSearchDocument {
+ return createWorkspaceSearchDocument({ kind: 'file', path, modifiedTs });
+}
+
+const baseDocuments: WorkspaceSearchDocument[] = [
+ page('docs/api', 'HTTP endpoint contracts', 10, 'API Reference'),
+ page('architecture/overview', 'Observer bridge and CRDT topology', 30, 'Architecture Overview'),
+ page('notes/graphing', 'Visual explorer notes', 20, 'Graphing Notes'),
+ folder('docs'),
+ folder('architecture'),
+ folder('notes'),
+ file('assets/diagram.png'),
+];
+
+/**
+ * The correctness bar for the incremental path: for any query/options pair, an
+ * incrementally-maintained corpus must be indistinguishable from a corpus
+ * built from scratch over the same documents — paths, scores, and signals.
+ */
+function expectSearchEquivalence(
+ incremental: ReturnType<typeof createWorkspaceSearchCorpus>,
+ documents: readonly WorkspaceSearchDocument[],
+ queries: readonly string[],
+ options: WorkspaceSearchOptions = { intent: 'full_text' },
+) {
+ const fresh = createWorkspaceSearchCorpus(documents);
+ for (const query of queries) {
+ const incrementalResults = searchWorkspaceCorpus(incremental, query, options);
+ const freshResults = searchWorkspaceCorpus(fresh, query, options);
+ expect(
+ incrementalResults.map((r) => ({
+ path: r.document.path,
+ kind: r.document.kind,
+ score: r.score,
+ signals: r.signals,
+ })),
+ ).toEqual(
+ freshResults.map((r) => ({
+ path: r.document.path,
+ kind: r.document.kind,
+ score: r.score,
+ signals: r.signals,
+ })),
+ );
+ }
+}
+
+describe('updateWorkspaceSearchCorpus', () => {
+ test('inserting a document makes it searchable without a rebuild', () => {
+ const corpus = createWorkspaceSearchCorpus(baseDocuments);
+ const next = [...baseDocuments, page('guides/tutorial', 'Zebra migration walkthrough', 40)];
+
+ const update = updateWorkspaceSearchCorpus(corpus, next);
+
+ expect(update.rebuilt).toBe(false);
+ expect(update.inserted).toBe(1);
+ expect(update.updated).toBe(0);
+ expect(update.removed).toBe(0);
+ const results = searchWorkspaceCorpus(update.corpus, 'zebra', { intent: 'full_text' });
+ expect(results.map((r) => r.document.path)).toContain('guides/tutorial');
+ expectSearchEquivalence(update.corpus, next, ['zebra', 'tutorial', 'topology', 'arch']);
+ });
+
+ test('updating a document swaps its indexed content', () => {
+ const corpus = createWorkspaceSearchCorpus(baseDocuments);
+ const next = baseDocuments.map((doc) =>
+ doc.path === 'notes/graphing'
+ ? page('notes/graphing', 'Quokka habitat research', 21, 'Graphing Notes')
+ : doc,
+ );
+
+ const update = updateWorkspaceSearchCorpus(corpus, next);
+
+ expect(update.rebuilt).toBe(false);
+ expect(update.inserted).toBe(0);
+ expect(update.updated).toBe(1);
+ expect(update.removed).toBe(0);
+ const newHits = searchWorkspaceCorpus(update.corpus, 'quokka', { intent: 'full_text' });
+ expect(newHits.map((r) => r.document.path)).toContain('notes/graphing');
+ const oldHits = searchWorkspaceCorpus(update.corpus, 'explorer', { intent: 'full_text' });
+ expect(oldHits.map((r) => r.document.path)).not.toContain('notes/graphing');
+ expectSearchEquivalence(update.corpus, next, ['quokka', 'explorer', 'notes']);
+ });
+
+ test('removing a document drops it from every search tier', () => {
+ const corpus = createWorkspaceSearchCorpus(baseDocuments);
+ const next = baseDocuments.filter((doc) => doc.path !== 'docs/api');
+
+ const update = updateWorkspaceSearchCorpus(corpus, next);
+
+ expect(update.rebuilt).toBe(false);
+ expect(update.inserted).toBe(0);
+ expect(update.updated).toBe(0);
+ expect(update.removed).toBe(1);
+ for (const query of ['endpoint', 'api']) {
+ const hits = searchWorkspaceCorpus(update.corpus, query, { intent: 'full_text' });
+ expect(hits.map((r) => r.document.path)).not.toContain('docs/api');
+ }
+ expectSearchEquivalence(update.corpus, next, ['endpoint', 'api', 'contracts']);
+ });
+
+ test('unchanged document set patches nothing and stays searchable', () => {
+ const corpus = createWorkspaceSearchCorpus(baseDocuments);
+ // New array, mixed identity: some reused objects, some re-created equal ones
+ // (the server reuses page objects via its cache but re-creates metadata-only
+ // folder/file documents every build).
+ const next = baseDocuments.map((doc) =>
+ doc.kind === 'page' ? doc : createWorkspaceSearchDocument({ ...doc }),
+ );
+
+ const update = updateWorkspaceSearchCorpus(corpus, next);
+
+ expect(update.rebuilt).toBe(false);
+ expect(update.inserted).toBe(0);
+ expect(update.updated).toBe(0);
+ expect(update.removed).toBe(0);
+ expectSearchEquivalence(update.corpus, next, ['topology', 'arch', 'diagram']);
+ });
+
+ test('a removed id can be re-added across successive updates', () => {
+ const corpus = createWorkspaceSearchCorpus(baseDocuments);
+ const without = baseDocuments.filter((doc) => doc.path !== 'docs/api');
+ const first = updateWorkspaceSearchCorpus(corpus, without);
+ expect(first.rebuilt).toBe(false);
+
+ const readded = [...without, page('docs/api', 'Fresh endpoint catalog', 50, 'API Reference')];
+ const second = updateWorkspaceSearchCorpus(first.corpus, readded);
+
+ expect(second.rebuilt).toBe(false);
+ const hits = searchWorkspaceCorpus(second.corpus, 'catalog', { intent: 'full_text' });
+ expect(hits.map((r) => r.document.path)).toContain('docs/api');
+ expectSearchEquivalence(second.corpus, readded, ['catalog', 'endpoint']);
+ });
+
+ test('emptying the corpus rebuilds electively and leaves a searchable empty index', () => {
+ const corpus = createWorkspaceSearchCorpus(baseDocuments);
+ const update = updateWorkspaceSearchCorpus(corpus, []);
+ // Removing everything mutates more entries than an (empty) rebuild inserts,
+ // so the bulk-change gate takes the cheap from-scratch path.
+ expect(update.rebuilt).toBe(true);
+ expect(update.rebuildReason).toBe('bulk-change');
+ expect(searchWorkspaceCorpus(update.corpus, 'topology', { intent: 'full_text' })).toEqual([]);
+ });
+
+ test('duplicate ids fail loudly, matching the from-scratch build', () => {
+ const corpus = createWorkspaceSearchCorpus(baseDocuments);
+ const duplicated = [...baseDocuments, page('docs/api', 'Shadow copy', 60)];
+ expect(() => createWorkspaceSearchCorpus(duplicated)).toThrow();
+ expect(() => updateWorkspaceSearchCorpus(corpus, duplicated)).toThrow();
+ });
+
+ test('reusing a consumed base falls back to a from-scratch rebuild', () => {
+ const corpus = createWorkspaceSearchCorpus(baseDocuments);
+ const first = updateWorkspaceSearchCorpus(corpus, [
+ ...baseDocuments,
+ page('guides/alpha', 'Alpha content', 40),
+ ]);
+ expect(first.rebuilt).toBe(false);
+
+ // Diffing against `corpus` again would patch an index that has already
+ // moved past it; the consumed-base gate must force a rebuild instead.
+ const next = [...baseDocuments, page('guides/beta', 'Beta content', 41)];
+ const second = updateWorkspaceSearchCorpus(corpus, next);
+
+ expect(second.rebuilt).toBe(true);
+ expect(second.rebuildReason).toBe('stale-base');
+ expectSearchEquivalence(second.corpus, next, ['beta', 'alpha', 'topology']);
+ });
+
+ test('a diff touching more documents than a rebuild would insert rebuilds electively', () => {
+ const corpus = createWorkspaceSearchCorpus(baseDocuments);
+ // Replace everything: every old id removed, every new id inserted.
+ const next = [page('brand/new', 'Entirely new workspace', 70)];
+
+ const update = updateWorkspaceSearchCorpus(corpus, next);
+
+ expect(update.rebuilt).toBe(true);
+ expect(update.rebuildReason).toBe('bulk-change');
+ expectSearchEquivalence(update.corpus, next, ['workspace', 'topology']);
+ });
+
+ test('mixed burst of inserts, updates, and removes matches a from-scratch build', () => {
+ // Deterministic pseudo-random burst: enough documents and churn to exercise
+ // the diff paths together, seeded so a failure replays exactly.
+ let seed = 0xc0ffee;
+ const rand = () => {
+ // LCG (Numerical Recipes constants) — deterministic across runs/platforms.
+ seed = (seed * 1664525 + 1013904223) >>> 0;
+ return seed / 0x100000000;
+ };
+ const words = ['alpha', 'bravo', 'crdt', 'delta', 'editor', 'fjord', 'graph', 'hocus'];
+ const makeContent = () =>
+ Array.from({ length: 5 }, () => words[Math.trunc(rand() * words.length)]).join(' ');
+
+ let documents: WorkspaceSearchDocument[] = Array.from({ length: 40 }, (_, i) =>
+ page(`corpus/doc-${i}`, makeContent(), i),
+ );
+ let corpus = createWorkspaceSearchCorpus(documents);
+
+ for (let step = 0; step < 8; step++) {
+ const next = [...documents];
+ // Remove one, update two, insert one per step — small deltas, like a
+ // write burst with interleaved searches.
+ next.splice(Math.trunc(rand() * next.length), 1);
+ for (let u = 0; u < 2; u++) {
+ const i = Math.trunc(rand() * next.length);
+ const target = next[i];
+ if (target) next[i] = page(target.path, makeContent(), target.modifiedTs + 100);
+ }
+ next.push(page(`corpus/new-${step}`, makeContent(), 1000 + step));
+
+ const update = updateWorkspaceSearchCorpus(corpus, next);
+ expect(update.rebuilt).toBe(false);
+ corpus = update.corpus;
+ documents = next;
+
+ expectSearchEquivalence(corpus, documents, words, { intent: 'full_text' });
+ expectSearchEquivalence(corpus, documents, ['corpus', 'doc-3', 'new-'], {
+ intent: 'omnibar',
+ });
+ }
+ });
+});
diff --git a/packages/core/src/search/workspace-search.ts b/packages/core/src/search/workspace-search.ts
index 02a6f90d3..f03433a16 100644
--- a/packages/core/src/search/workspace-search.ts
+++ b/packages/core/src/search/workspace-search.ts
@@ -1,4 +1,4 @@
-import { type AnyOrama, create, insertMultiple, search } from '@orama/orama';
+import { type AnyOrama, count, create, insertMultiple, removeMultiple, search } from '@orama/orama';
import { isHiddenDocName } from '../util/doc-name.ts';
export type WorkspaceSearchKind = 'page' | 'folder' | 'file';
@@ -281,6 +281,154 @@ function getWorkspaceSearchIndex(corpus: WorkspaceSearchCorpus): WorkspaceSearch
return index;
}
+/**
+ * Corpora already consumed as the base of an incremental update. A diff is only
+ * valid against the exact document set the shared index currently reflects, and
+ * that is the corpus most recently RETURNED by `createWorkspaceSearchCorpus` /
+ * `updateWorkspaceSearchCorpus` — reusing an older base would apply a diff the
+ * index has already moved past, silently desynchronizing BM25 from
+ * `corpus.documents`. Membership here makes that misuse deterministic: a
+ * consumed base forces the from-scratch rebuild path instead.
+ */
+const consumedUpdateBases = new WeakSet<WorkspaceSearchCorpus>();
+
+/** Why an incremental corpus update fell back to a from-scratch index build. */
+export type WorkspaceSearchRebuildReason =
+ | 'no-index'
+ | 'stale-base'
+ | 'duplicate-id'
+ | 'bulk-change'
+ | 'mutation-failed';
+
+export interface WorkspaceSearchCorpusUpdate {
+ corpus: WorkspaceSearchCorpus;
+ /** Documents newly added to the index (includes the insert half of updates). */
+ inserted: number;
+ /** Documents whose content/metadata changed (counted once, not as insert+remove). */
+ updated: number;
+ /** Documents removed from the index (excludes the remove half of updates). */
+ removed: number;
+ /** True when the index was rebuilt from scratch instead of patched in place. */
+ rebuilt: boolean;
+ rebuildReason?: WorkspaceSearchRebuildReason;
+}
+
+/**
+ * Field-level equality for diffing. Object identity short-circuits the common
+ * case (callers reuse unchanged document objects across builds); the field
+ * compare covers metadata-only documents that are re-created each build with
+ * identical values. `id` is the diff key, so it is compared by the caller.
+ */
+function sameSearchDocument(a: WorkspaceSearchDocument, b: WorkspaceSearchDocument): boolean {
+ return (
+ a === b ||
+ (a.kind === b.kind &&
+ a.path === b.path &&
+ a.title === b.title &&
+ a.name === b.name &&
+ a.pathSegments === b.pathSegments &&
+ a.modifiedTs === b.modifiedTs &&
+ a.content === b.content)
+ );
+}
+
+/**
+ * Advance a corpus to a new document set by patching the existing Orama index
+ * in place — per-document remove/insert for only the changed entries — instead
+ * of re-tokenizing the entire workspace. On a large corpus the from-scratch
+ * build is the dominant cost of search-cache invalidation, and it runs on the
+ * event loop; this keeps steady-state maintenance proportional to the change.
+ *
+ * Falls back to a from-scratch build (`rebuilt: true`) whenever the
+ * incremental path cannot be proven safe: `previous` has no live index or was
+ * already consumed as an update base, the new set carries a duplicate id (the
+ * rebuild then throws exactly like `createWorkspaceSearchCorpus`), the diff
+ * touches more documents than a rebuild would insert, or an index mutation
+ * fails / leaves the index count inconsistent.
+ *
+ * The returned corpus shares the (now-patched) index with `previous`. A search
+ * still holding `previous` keeps a coherent `documents` snapshot for the
+ * lexical/recency tiers but reads the patched index for BM25 — a bounded,
+ * one-step freshness skew, accepted over the alternatives (an index copy per
+ * update, or re-building the old snapshot's index on demand).
+ */
+export function updateWorkspaceSearchCorpus(
+ previous: WorkspaceSearchCorpus,
+ documents: readonly WorkspaceSearchDocument[],
+): WorkspaceSearchCorpusUpdate {
+ const rebuild = (rebuildReason: WorkspaceSearchRebuildReason): WorkspaceSearchCorpusUpdate => ({
+ corpus: createWorkspaceSearchCorpus(documents),
+ inserted: 0,
+ updated: 0,
+ removed: 0,
+ rebuilt: true,
+ rebuildReason,
+ });
+
+ const index = workspaceSearchIndexes.get(previous);
+ if (!index) return rebuild('no-index');
+ if (consumedUpdateBases.has(previous)) return rebuild('stale-base');
+
+ const nextById = new Map(documents.map((document) => [document.id, document] as const));
+ if (nextById.size !== documents.length) return rebuild('duplicate-id');
+
+ const removedIds: string[] = [];
+ const insertedDocuments: WorkspaceSearchDocument[] = [];
+ let updated = 0;
+ for (const previousDocument of previous.documents) {
+ const nextDocument = nextById.get(previousDocument.id);
+ if (!nextDocument) {
+ removedIds.push(previousDocument.id);
+ } else if (!sameSearchDocument(previousDocument, nextDocument)) {
+ // Changed document: remove the stale tokens, insert the new ones. Orama's
+ // `update` is the same remove+insert internally; batching both halves
+ // keeps the mutation count observable for the bulk-change gate below.
+ removedIds.push(previousDocument.id);
+ insertedDocuments.push(nextDocument);
+ updated += 1;
+ }
+ }
+ const previousIds = new Set(previous.documents.map((document) => document.id));
+ for (const document of documents) {
+ if (!previousIds.has(document.id)) insertedDocuments.push(document);
+ }
+
+ // When the diff would mutate more index entries than a rebuild would insert
+ // (e.g. a branch switch replacing most of the workspace), the from-scratch
+ // build is the cheaper operation — take it electively.
+ if (removedIds.length + insertedDocuments.length > documents.length) {
+ return rebuild('bulk-change');
+ }
+
+ consumedUpdateBases.add(previous);
+ try {
+ if (removedIds.length > 0) removeMultiple(index, removedIds);
+ if (insertedDocuments.length > 0) {
+ insertMultiple(index, insertedDocuments as WorkspaceSearchDocument[]);
+ }
+ // A count mismatch means a removal targeted an id the index did not hold
+ // (or an insert landed twice) — the index no longer mirrors `documents`,
+ // so discard it rather than serve drifted BM25 results.
+ if (count(index) !== documents.length) {
+ throw new Error(
+ `Search index count ${count(index)} diverged from document count ${documents.length} after incremental update`,
+ );
+ }
+ } catch {
+ return rebuild('mutation-failed');
+ }
+
+ const corpus: WorkspaceSearchCorpus = { documents };
+ workspaceSearchIndexes.set(corpus, index);
+ return {
+ corpus,
+ inserted: insertedDocuments.length - updated,
+ updated,
+ removed: removedIds.length - updated,
+ rebuilt: false,
+ };
+}
+
function defaultScopes(intent: WorkspaceSearchIntent): readonly WorkspaceSearchScope[] {
// Autocomplete stays page-only: it backs the `[[wikilink]]` target picker,
// whose resolution model is markdown-scoped (the all-files autocomplete target
diff --git a/packages/server/src/api-extension.ts b/packages/server/src/api-extension.ts
index ee3fe2ed4..acb33394f 100644
--- a/packages/server/src/api-extension.ts
+++ b/packages/server/src/api-extension.ts
@@ -241,6 +241,7 @@ import {
UploadAssetSuccessSchema,
UploadRequestSchema,
unwrapFrontmatterFences,
+ updateWorkspaceSearchCorpus,
type WorkspaceSearchCorpus,
type WorkspaceSearchDocument,
type WorkspaceSearchIntent,
@@ -742,6 +743,21 @@ function searchCorpusTruncatedCounter(): ReturnType<ReturnType<typeof getMeter>[
return _searchCorpusTruncatedCounter;
}
+// Counter for search-corpus builds by mode: `cold` (no prior corpus),
+// `incremental` (in-place index patch of only the changed documents), or
+// `rebuild` (incremental path fell back to a from-scratch build; the bounded
+// `reason` attribute says why). A steady stream of `rebuild` where
+// `incremental` is expected is the drift signal worth alerting on.
+let _searchCorpusUpdateCounter: ReturnType<ReturnType<typeof getMeter>['createCounter']> | null =
+ null;
+function searchCorpusUpdateCounter(): ReturnType<ReturnType<typeof getMeter>['createCounter']> {
+ _searchCorpusUpdateCounter ||= getMeter().createCounter('ok.search.corpus_update_total', {
+ description:
+ 'Count of workspace search corpus builds, by mode (cold | incremental | rebuild) and, for rebuilds, the bounded fallback reason.',
+ });
+ return _searchCorpusUpdateCounter;
+}
+
/** Bounded handler label for the content-divergence counters. */
type DivergenceHandler = 'agent-write-md' | 'agent-patch' | 'rollback';
@@ -17235,10 +17251,48 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
return workspaceSearchCache.pending;
}
- const pending = buildWorkspaceSearchDocumentsFromIndex().then(({ documents, truncated }) => ({
- corpus: createWorkspaceSearchCorpus(documents),
- truncated,
- }));
+ // Stale-but-live corpus (or the build about to produce one): the base for
+ // an incremental index patch, so one write re-indexes one document instead
+ // of re-tokenizing the whole workspace on the event loop.
+ const priorCorpus = workspaceSearchCache?.corpus;
+ const priorPending = workspaceSearchCache?.pending;
+ const pending = (async () => {
+ // Serialize behind any in-flight build: an incremental diff is only valid
+ // against the corpus it was computed from, and the in-flight build owns
+ // the shared index right now. Chaining keeps updates linear (each build
+ // bases on its predecessor's output) without coalescing away freshness —
+ // the document snapshot below is read AFTER this fingerprint was seen.
+ const base = priorPending
+ ? await priorPending.then(
+ (result) => result.corpus,
+ () => undefined,
+ )
+ : priorCorpus;
+ const { documents, truncated } = await buildWorkspaceSearchDocumentsFromIndex();
+ if (!base) {
+ searchCorpusUpdateCounter().add(1, { mode: 'cold' });
+ return { corpus: createWorkspaceSearchCorpus(documents), truncated };
+ }
+ const update = updateWorkspaceSearchCorpus(base, documents);
+ if (update.rebuilt) {
+ searchCorpusUpdateCounter().add(1, { mode: 'rebuild', reason: update.rebuildReason });
+ // `mutation-failed` means the patched index diverged from the document
+ // set (or a mutation threw) — recovered by the rebuild, but worth an
+ // operator-visible signal; the elective reasons are routine.
+ const logLevel = update.rebuildReason === 'mutation-failed' ? 'warn' : 'debug';
+ getLogger('search')[logLevel](
+ { reason: update.rebuildReason, documents: documents.length },
+ '[search] corpus update fell back to a full index rebuild',
+ );
+ } else {
+ searchCorpusUpdateCounter().add(1, { mode: 'incremental' });
+ getLogger('search').debug(
+ { inserted: update.inserted, updated: update.updated, removed: update.removed },
+ '[search] corpus updated incrementally',
+ );
+ }
+ return { corpus: update.corpus, truncated };
+ })();
workspaceSearchCaches.set(cacheKey, { fingerprint, pending });
try {
const result = await pending;
diff --git a/packages/server/src/api-search-incremental.test.ts b/packages/server/src/api-search-incremental.test.ts
new file mode 100644
index 000000000..c37404db3
--- /dev/null
+++ b/packages/server/src/api-search-incremental.test.ts
@@ -0,0 +1,267 @@
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import type { IncomingMessage, ServerResponse } from 'node:http';
+import { tmpdir } from 'node:os';
+import { dirname, join } from 'node:path';
+import { Readable } from 'node:stream';
+import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';
+import { createApiExtension } from './api-extension.ts';
+import type { FileIndexEntry } from './file-watcher.ts';
+
+// Count how the server maintains the search corpus: `create` = from-scratch
+// builds requested directly by the server (cold start), `update` = incremental
+// maintenance. `updateWorkspaceSearchCorpus`'s internal fallback rebuild calls
+// the module-local `createWorkspaceSearchCorpus`, which this mock does NOT
+// intercept — so `create` counts exactly the server's own cold builds.
+const corpusCalls = vi.hoisted(() => ({ create: 0, update: 0 }));
+vi.mock('@inkeep/open-knowledge-core', async (importOriginal) => {
+ const actual = await importOriginal<typeof import('@inkeep/open-knowledge-core')>();
+ return {
+ ...actual,
+ createWorkspaceSearchCorpus: (
+ ...args: Parameters<typeof actual.createWorkspaceSearchCorpus>
+ ) => {
+ corpusCalls.create += 1;
+ return actual.createWorkspaceSearchCorpus(...args);
+ },
+ updateWorkspaceSearchCorpus: (
+ ...args: Parameters<typeof actual.updateWorkspaceSearchCorpus>
+ ) => {
+ corpusCalls.update += 1;
+ return actual.updateWorkspaceSearchCorpus(...args);
+ },
+ };
+});
+
+interface CapturedResponse {
+ status: number;
+ headers: Record<string, string>;
+ body: string;
+}
+
+interface SearchResultEntry {
+ kind: string;
+ path: string;
+ title: string;
+ score: number;
+ signals: Record<string, number>;
+ snippet?: string;
+}
+
+function makeReq(method: string, url: string): IncomingMessage {
+ const readable = Readable.from(Buffer.from('')) as unknown as IncomingMessage;
+ readable.method = method;
+ readable.url = url;
+ readable.headers = { host: 'localhost' };
+ return readable;
+}
+
+function makeRes(): { res: ServerResponse; captured: CapturedResponse } {
+ const captured: CapturedResponse = { status: 0, headers: {}, body: '' };
+ const res = {
+ writeHead(status: number, headers?: Record<string, string>) {
+ captured.status = status;
+ if (headers) Object.assign(captured.headers, headers);
+ },
+ end(body?: string) {
+ captured.body = body ?? '';
+ },
+ } as unknown as ServerResponse;
+ return { res, captured };
+}
+
+/**
+ * A persistent extension over a mutable in-memory file index — the shape the
+ * production server has (one long-lived extension whose index the file watcher
+ * mutates), unlike the per-call throwaway extension in `api-search.test.ts`.
+ * `modified` stamps are assigned from a deterministic sequence so every content
+ * change also changes the entry fingerprint, mirroring the watcher's behavior.
+ */
+function createHarness(contentDir: string, options: { withGeneration: boolean }) {
+ const index = new Map<string, FileIndexEntry>();
+ let generation = 0;
+ let stampSeq = 0;
+ const nextStamp = () => new Date(1700000000000 + ++stampSeq * 1000).toISOString();
+
+ const ext = createApiExtension({
+ hocuspocus: {} as unknown as Parameters<typeof createApiExtension>[0]['hocuspocus'],
+ sessionManager: {} as unknown as Parameters<typeof createApiExtension>[0]['sessionManager'],
+ contentDir,
+ serverInstanceId: 'test-server',
+ getFileIndex: () => index,
+ getAllFilesIndex: () => index,
+ ...(options.withGeneration ? { getFileIndexGeneration: () => generation } : {}),
+ });
+
+ const setDoc = (docName: string, content: string, modified = nextStamp()) => {
+ const abs = join(contentDir, `${docName}.md`);
+ mkdirSync(dirname(abs), { recursive: true });
+ writeFileSync(abs, content, 'utf-8');
+ index.set(docName, {
+ size: Buffer.byteLength(content),
+ modified,
+ canonicalPath: abs,
+ inode: ++stampSeq,
+ aliases: [],
+ kind: 'markdown',
+ });
+ generation += 1;
+ return modified;
+ };
+
+ const setRawEntry = (docName: string, entry: Partial<FileIndexEntry>) => {
+ index.set(docName, {
+ size: 0,
+ modified: nextStamp(),
+ canonicalPath: join(contentDir, docName),
+ inode: ++stampSeq,
+ aliases: [],
+ kind: 'markdown',
+ ...entry,
+ });
+ generation += 1;
+ };
+
+ const removeDoc = (docName: string) => {
+ index.delete(docName);
+ generation += 1;
+ };
+
+ const search = async (query: string, intent = 'full_text') => {
+ const req = makeReq('GET', `/api/search?query=${encodeURIComponent(query)}&intent=${intent}`);
+ const { res, captured } = makeRes();
+ await (
+ ext as {
+ onRequest: (ctx: { request: IncomingMessage; response: ServerResponse }) => Promise<void>;
+ }
+ ).onRequest({ request: req, response: res });
+ expect(captured.status).toBe(200);
+ return JSON.parse(captured.body) as { results: SearchResultEntry[]; ready: boolean };
+ };
+
+ return { setDoc, setRawEntry, removeDoc, search };
+}
+
+describe('incremental workspace search corpus maintenance', () => {
+ let contentDir: string;
+
+ beforeEach(() => {
+ contentDir = mkdtempSync(join(tmpdir(), 'ok-search-incr-'));
+ corpusCalls.create = 0;
+ corpusCalls.update = 0;
+ });
+
+ afterEach(() => {
+ rmSync(contentDir, { recursive: true, force: true });
+ });
+
+ test('one cold build, then per-write incremental updates (generation fingerprint)', async () => {
+ const harness = createHarness(contentDir, { withGeneration: true });
+ harness.setDoc('docs/api', '# API Reference\n\nHTTP endpoint contracts\n');
+ harness.setDoc('notes/graphing', '# Graphing Notes\n\nVisual explorer notes\n');
+
+ const cold = await harness.search('endpoint');
+ expect(cold.results.map((r) => r.path)).toContain('docs/api');
+ expect(corpusCalls).toEqual({ create: 1, update: 0 });
+
+ // Insert: a new doc is searchable without another from-scratch build.
+ harness.setDoc('guides/tutorial', '# Tutorial\n\nZebra migration walkthrough\n');
+ const afterInsert = await harness.search('zebra');
+ expect(afterInsert.results.map((r) => r.path)).toContain('guides/tutorial');
+ expect(corpusCalls).toEqual({ create: 1, update: 1 });
+
+ // Update: changed body content is re-indexed; the old tokens are gone.
+ harness.setDoc('notes/graphing', '# Graphing Notes\n\nQuokka habitat research\n');
+ const afterUpdate = await harness.search('quokka');
+ expect(afterUpdate.results.map((r) => r.path)).toContain('notes/graphing');
+ // The second search after the same write hits the fingerprint cache — no
+ // further corpus work of either kind.
+ const oldTokens = await harness.search('explorer');
+ expect(oldTokens.results.map((r) => r.path)).not.toContain('notes/graphing');
+ expect(corpusCalls).toEqual({ create: 1, update: 2 });
+
+ // Remove: the doc disappears from name and body tiers.
+ harness.removeDoc('docs/api');
+ const afterRemove = await harness.search('endpoint');
+ expect(afterRemove.results.map((r) => r.path)).not.toContain('docs/api');
+ expect(corpusCalls).toEqual({ create: 1, update: 3 });
+ });
+
+ test('incremental maintenance also runs on the fallback (no-generation) fingerprint', async () => {
+ const harness = createHarness(contentDir, { withGeneration: false });
+ harness.setDoc('docs/api', '# API\n\nendpoint contracts\n');
+
+ await harness.search('endpoint');
+ expect(corpusCalls).toEqual({ create: 1, update: 0 });
+
+ harness.setDoc('guides/new-page', '# New Page\n\nxylophone content\n');
+ const results = await harness.search('xylophone');
+ expect(results.results.map((r) => r.path)).toContain('guides/new-page');
+ expect(corpusCalls).toEqual({ create: 1, update: 1 });
+ });
+
+ test('system and config docs added after the cold build never enter the corpus', async () => {
+ const harness = createHarness(contentDir, { withGeneration: true });
+ harness.setDoc('docs/real-page', '# Real\n\nsystem architecture content\n');
+ await harness.search('system');
+
+ // The build-time predicate skips these before any disk read, so the raw
+ // entries need no backing file.
+ harness.setRawEntry('__system__', {});
+ harness.setRawEntry('__config__/project', {});
+ // full_text searches name AND body, so this asserts the synthetic names are
+ // absent from every tier while the real page still matches on content.
+ const results = await harness.search('system', 'full_text');
+ const paths = results.results.map((r) => r.path);
+ expect(paths).not.toContain('__system__');
+ expect(paths).not.toContain('__config__/project');
+ expect(paths).toContain('docs/real-page');
+ });
+
+ test('after a mixed write burst, results match a from-scratch build of the same workspace', async () => {
+ const harness = createHarness(contentDir, { withGeneration: true });
+ const stamps = new Map<string, string>();
+ stamps.set('docs/api', harness.setDoc('docs/api', '# API\n\nendpoint contracts\n'));
+ stamps.set('notes/one', harness.setDoc('notes/one', '# One\n\nalpha bravo crdt\n'));
+ stamps.set('notes/two', harness.setDoc('notes/two', '# Two\n\ndelta editor fjord\n'));
+ await harness.search('alpha');
+
+ // Burst: update, insert, remove, insert — each interleaved with a search so
+ // every step goes through the incremental path rather than one big diff.
+ stamps.set('notes/one', harness.setDoc('notes/one', '# One\n\nalpha graph hocus\n'));
+ await harness.search('graph');
+ stamps.set('guides/four', harness.setDoc('guides/four', '# Four\n\nbravo delta alpha\n'));
+ await harness.search('bravo');
+ harness.removeDoc('notes/two');
+ stamps.delete('notes/two');
+ await harness.search('delta');
+ stamps.set('guides/five', harness.setDoc('guides/five', '# Five\n\nhocus fjord\n'));
+ expect(corpusCalls.create).toBe(1);
+
+ // Reference: an identical workspace (same docNames, contents, and modified
+ // stamps) in a fresh contentDir, indexed from scratch by its own harness.
+ const referenceDir = mkdtempSync(join(tmpdir(), 'ok-search-ref-'));
+ try {
+ const reference = createHarness(referenceDir, { withGeneration: true });
+ const finalDocs: Array<[string, string]> = [
+ ['docs/api', '# API\n\nendpoint contracts\n'],
+ ['notes/one', '# One\n\nalpha graph hocus\n'],
+ ['guides/four', '# Four\n\nbravo delta alpha\n'],
+ ['guides/five', '# Five\n\nhocus fjord\n'],
+ ];
+ for (const [docName, content] of finalDocs) {
+ const stamp = stamps.get(docName);
+ if (!stamp) throw new Error(`missing stamp for ${docName}`);
+ reference.setDoc(docName, content, stamp);
+ }
+ for (const query of ['alpha', 'bravo', 'delta', 'fjord', 'graph', 'hocus', 'guides']) {
+ for (const intent of ['full_text', 'omnibar'] as const) {
+ const incremental = await harness.search(query, intent);
+ const fresh = await reference.search(query, intent);
+ expect(incremental.results).toEqual(fresh.results);
+ }
+ }
+ } finally {
+ rmSync(referenceDir, { recursive: true, force: true });
+ }
+ });
+});