Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

Centralize the derived-document index lifecycle (#2881)

Dimitri POSTOLOV59dd42269f0b
diff --git a/.changeset/deepen-derived-index-lifecycle.md b/.changeset/deepen-derived-index-lifecycle.md
new file mode 100644
index 000000000..83280a174
--- /dev/null
+++ b/.changeset/deepen-derived-index-lifecycle.md
@@ -0,0 +1,10 @@
+---
+"@inkeep/open-knowledge": patch
+"@inkeep/open-knowledge-server": patch
+---
+
+Keep backlinks, graph data, and tags consistent across every document lifecycle path. Direct creates and duplicates, disk reconciliation, deletes, folder renames, project-skill moves, ignore-file changes, startup, and branch switches now pass through one ordered derived-index coordinator with shared admission, readiness, persistence, and invalidation rules.
+
+Derived-view queries wait for startup and branch replacement to settle instead of observing a partially rebuilt index. Rapid consecutive branch switches are serialized so one transition cannot release another transition's barrier.
+
+Durable creates, duplicates, renames, deletes, and persistence stores now return according to authoritative disk state. Derived projection and cache persistence remain best-effort, so coordinator shutdown or a delayed cache save cannot turn an already successful write into a failure.
diff --git a/packages/app/tests/integration/derived-document-index-lifecycle.test.ts b/packages/app/tests/integration/derived-document-index-lifecycle.test.ts
new file mode 100644
index 000000000..49741ecc8
--- /dev/null
+++ b/packages/app/tests/integration/derived-document-index-lifecycle.test.ts
@@ -0,0 +1,254 @@
+import { mkdirSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { setTimeout as wait } from 'node:timers/promises';
+import { HocuspocusProvider } from '@hocuspocus/provider';
+import {
+ type CC1DerivedViewPayload,
+ CC1DerivedViewPayloadSchema,
+ SYSTEM_DOC_NAME,
+} from '@inkeep/open-knowledge-core';
+import { expect, test } from 'vitest';
+import * as Y from 'yjs';
+import { createTestServer, pollUntil, waitForSync } from './test-harness.ts';
+
+type RelationChannel = 'backlinks' | 'graph' | 'tags';
+const RELATION_CHANNELS: readonly RelationChannel[] = ['backlinks', 'graph', 'tags'];
+
+function connectRelationSignals(port: number): {
+ provider: HocuspocusProvider;
+ signals: CC1DerivedViewPayload[];
+ destroy: () => void;
+} {
+ const document = new Y.Doc();
+ const signals: CC1DerivedViewPayload[] = [];
+ const provider = new HocuspocusProvider({
+ url: `ws://127.0.0.1:${port}/collab`,
+ name: SYSTEM_DOC_NAME,
+ document,
+ connect: true,
+ onStateless: ({ payload }) => {
+ let raw: unknown;
+ try {
+ raw = JSON.parse(payload);
+ } catch {
+ return;
+ }
+ const parsed = CC1DerivedViewPayloadSchema.safeParse(raw);
+ if (parsed.success && RELATION_CHANNELS.includes(parsed.data.ch as RelationChannel)) {
+ signals.push(parsed.data);
+ }
+ },
+ });
+ return {
+ provider,
+ signals,
+ destroy: () => {
+ provider.destroy();
+ document.destroy();
+ },
+ };
+}
+
+async function expectRelationSignalsAfter<T>(
+ signals: readonly CC1DerivedViewPayload[],
+ action: () => Promise<T>,
+): Promise<T> {
+ const baseline = new Map(
+ RELATION_CHANNELS.map((channel) => [
+ channel,
+ signals.filter((signal) => signal.ch === channel).length,
+ ]),
+ );
+ const result = await action();
+ await pollUntil(
+ () =>
+ RELATION_CHANNELS.every(
+ (channel) =>
+ signals.filter((signal) => signal.ch === channel).length > (baseline.get(channel) ?? 0),
+ ),
+ 5000,
+ 20,
+ );
+ return result;
+}
+
+async function postJson(
+ baseUrl: string,
+ path: string,
+ body: Record<string, unknown>,
+ method = 'POST',
+): Promise<{ status: number; body: Record<string, unknown> }> {
+ const response = await fetch(`${baseUrl}${path}`, {
+ method,
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify(body),
+ });
+ return {
+ status: response.status,
+ body: (await response.json()) as Record<string, unknown>,
+ };
+}
+
+async function tagDocuments(baseUrl: string, tag: string): Promise<string[]> {
+ const response = await fetch(`${baseUrl}/api/tags/${encodeURIComponent(tag)}`);
+ expect(response.status).toBe(200);
+ const body = (await response.json()) as { docs: Array<{ docName: string }> };
+ return body.docs.map((entry) => entry.docName).sort();
+}
+
+async function backlinkSources(baseUrl: string, docName: string): Promise<string[]> {
+ const response = await fetch(`${baseUrl}/api/backlinks?docName=${encodeURIComponent(docName)}`);
+ expect(response.status).toBe(200);
+ const body = (await response.json()) as { backlinks: Array<{ source: string }> };
+ return body.backlinks.map((entry) => entry.source).sort();
+}
+
+test('direct create, duplicate, rename, and delete keep links and tags in lockstep', async () => {
+ const server = await createTestServer();
+ const relationSignals = connectRelationSignals(server.port);
+ const templateDir = join(server.contentDir, '.ok', 'templates');
+ mkdirSync(templateDir, { recursive: true });
+ writeFileSync(
+ join(templateDir, 'lifecycle.md'),
+ '---\ntemplate:\n title: Lifecycle\ntags: [lifecycle]\n---\n\nSee [[target]].\n',
+ 'utf-8',
+ );
+
+ try {
+ await waitForSync(relationSignals.provider);
+ await wait(300);
+
+ const created = await expectRelationSignalsAfter(relationSignals.signals, () =>
+ postJson(server.baseUrl, '/api/create-page', {
+ path: 'folder/item.md',
+ template: 'lifecycle',
+ }),
+ );
+ expect(created.status).toBe(200);
+ expect(await tagDocuments(server.baseUrl, 'lifecycle')).toEqual(['folder/item']);
+ expect(await backlinkSources(server.baseUrl, 'target')).toEqual(['folder/item']);
+
+ const fileDuplicate = await expectRelationSignalsAfter(relationSignals.signals, () =>
+ postJson(server.baseUrl, '/api/duplicate-path', {
+ kind: 'file',
+ path: 'folder/item',
+ }),
+ );
+ expect(fileDuplicate.status).toBe(200);
+ expect(await tagDocuments(server.baseUrl, 'lifecycle')).toEqual([
+ 'folder/item',
+ 'folder/item copy',
+ ]);
+
+ const folderDuplicate = await expectRelationSignalsAfter(relationSignals.signals, () =>
+ postJson(server.baseUrl, '/api/duplicate-path', {
+ kind: 'folder',
+ path: 'folder',
+ }),
+ );
+ expect(folderDuplicate.status).toBe(200);
+ expect(await tagDocuments(server.baseUrl, 'lifecycle')).toEqual([
+ 'folder copy/item',
+ 'folder copy/item copy',
+ 'folder/item',
+ 'folder/item copy',
+ ]);
+
+ const renamed = await expectRelationSignalsAfter(relationSignals.signals, () =>
+ postJson(server.baseUrl, '/api/rename-path', {
+ kind: 'folder',
+ fromPath: 'folder copy',
+ toPath: 'archive',
+ }),
+ );
+ expect(renamed.status).toBe(200);
+ expect(await tagDocuments(server.baseUrl, 'lifecycle')).toEqual([
+ 'archive/item',
+ 'archive/item copy',
+ 'folder/item',
+ 'folder/item copy',
+ ]);
+ expect(await backlinkSources(server.baseUrl, 'target')).toEqual([
+ 'archive/item',
+ 'archive/item copy',
+ 'folder/item',
+ 'folder/item copy',
+ ]);
+
+ const deleted = await expectRelationSignalsAfter(relationSignals.signals, () =>
+ postJson(server.baseUrl, '/api/delete-path', {
+ kind: 'folder',
+ path: 'archive',
+ }),
+ );
+ expect(deleted.status).toBe(200);
+ expect(await tagDocuments(server.baseUrl, 'lifecycle')).toEqual([
+ 'folder/item',
+ 'folder/item copy',
+ ]);
+ expect(await backlinkSources(server.baseUrl, 'target')).toEqual([
+ 'folder/item',
+ 'folder/item copy',
+ ]);
+ } finally {
+ relationSignals.destroy();
+ await server.cleanup();
+ }
+});
+
+test('project skill moves refresh backlinks and tags at the relocated content doc name', async () => {
+ const server = await createTestServer();
+ const relationSignals = connectRelationSignals(server.port);
+
+ try {
+ await waitForSync(relationSignals.provider);
+ await wait(300);
+
+ const created = await expectRelationSignalsAfter(relationSignals.signals, () =>
+ postJson(
+ server.baseUrl,
+ '/api/skill',
+ {
+ scope: 'project',
+ name: 'lifecycle-skill',
+ frontmatter: {
+ name: 'lifecycle-skill',
+ description: 'Exercises derived relationship views.',
+ },
+ body: 'See [[skill-target]]. #skill-lifecycle\n',
+ },
+ 'PUT',
+ ),
+ );
+ expect(created.status).toBe(200);
+
+ await pollUntil(async () => {
+ const tags = await tagDocuments(server.baseUrl, 'skill-lifecycle');
+ const backlinks = await backlinkSources(server.baseUrl, 'skill-target');
+ return (
+ tags.includes('.ok/skills/lifecycle-skill/SKILL') &&
+ backlinks.includes('.ok/skills/lifecycle-skill/SKILL')
+ );
+ });
+
+ const moved = await expectRelationSignalsAfter(relationSignals.signals, () =>
+ postJson(server.baseUrl, '/api/skill', {
+ scope: 'project',
+ fromName: 'lifecycle-skill',
+ toName: 'relocated-skill',
+ }),
+ );
+ expect(moved.status).toBe(200);
+
+ await pollUntil(async () => {
+ const expected = '.ok/skills/relocated-skill/SKILL';
+ const old = '.ok/skills/lifecycle-skill/SKILL';
+ const tags = await tagDocuments(server.baseUrl, 'skill-lifecycle');
+ const backlinks = await backlinkSources(server.baseUrl, 'skill-target');
+ return tags.includes(expected) && !tags.includes(old) && backlinks.includes(expected);
+ });
+ } finally {
+ relationSignals.destroy();
+ await server.cleanup();
+ }
+});
diff --git a/packages/app/tests/integration/managed-artifact-reconcile-anchor.test.ts b/packages/app/tests/integration/managed-artifact-reconcile-anchor.test.ts
index d674072c1..e7a8b6ae4 100644
--- a/packages/app/tests/integration/managed-artifact-reconcile-anchor.test.ts
+++ b/packages/app/tests/integration/managed-artifact-reconcile-anchor.test.ts
@@ -22,9 +22,9 @@
*
* Fidelity: real boot, real WS client, real `onStoreDocument` managed-artifact
* branch, real file lock, real shadow repo. The divergent write is a plain
- * `writeFileSync` — a second writer's edit needs no seam, because `.ok/templates`
- * is not on the content watcher, so the divergence survives until the next store
- * exactly as it does in production.
+ * `writeFileSync`. The test invokes the real store hook immediately afterward
+ * so the managed-artifact watcher cannot consume the same change first and turn
+ * this into coverage of its separate disk-intake path.
*/
import { execFileSync } from 'node:child_process';
@@ -108,6 +108,20 @@ async function pollUntil(
throw new Error(`pollUntil timed out after ${timeoutMs}ms`);
}
+async function storeImmediately(rig: TestServer, docName: string): Promise<void> {
+ const document = rig.instance.hocuspocus.documents.get(docName);
+ if (!document) throw new Error(`document not loaded: ${docName}`);
+
+ await rig.instance.hocuspocus.hooks('onStoreDocument', {
+ clientsCount: document.getConnectionsCount(),
+ document,
+ lastContext: {},
+ lastTransactionOrigin: undefined,
+ documentName: docName,
+ instance: rig.instance.hocuspocus,
+ });
+}
+
describe('managed-artifact reconcile-on-divergence — content-loss conventions', () => {
test(
'the reconcile checkpoints the live artifact and reports the loss before discarding it',
@@ -124,14 +138,7 @@ describe('managed-artifact reconcile-on-divergence — content-loss conventions'
client.doc.transact(() => client.ytext.insert(0, v1));
await pollUntil(() => existsSync(tplFile) && readFileSync(tplFile, 'utf-8') === v1);
- // A second writer changes the file underneath us. `.ok/templates` is not on
- // the content watcher, so this stays invisible until the next store — the
- // production shape of a concurrent OK window or a hand edit.
- mkdirSync(resolve(tplFile, '..'), { recursive: true });
- const foreign = `---\ntitle: T\ndescription: d\n---\n\n# Template\n\n${FOREIGN_LINE}\n`;
- writeFileSync(tplFile, foreign, 'utf-8');
-
- // Meanwhile the author keeps typing. These bytes are live and unstored.
+ // The author keeps typing. These bytes are live and unstored.
const live = client.ytext.toString();
client.doc.transact(() => client.ytext.insert(live.length, `\n${AUTHOR_PENDING_LINE}\n`));
await pollUntil(() =>
@@ -144,6 +151,15 @@ describe('managed-artifact reconcile-on-divergence — content-loss conventions'
),
);
+ // A second writer changes the file underneath us. Invoke the real store
+ // hook in the same turn so this test deterministically reaches the
+ // concurrent-writer reconcile before the separate template watcher can
+ // import the file.
+ mkdirSync(resolve(tplFile, '..'), { recursive: true });
+ const foreign = `---\ntitle: T\ndescription: d\n---\n\n# Template\n\n${FOREIGN_LINE}\n`;
+ writeFileSync(tplFile, foreign, 'utf-8');
+ await storeImmediately(server, docName);
+
// The next store reads disk, sees `disk !== lkg && disk !== content`, and
// reconciles: the foreign bytes win and the author's sentence is discarded.
// This premise assertion PASSES on the unfixed tree — it documents the harm.
diff --git a/packages/cli/src/commands/init.test.ts b/packages/cli/src/commands/init.test.ts
index fa637fc26..7115921c9 100644
--- a/packages/cli/src/commands/init.test.ts
+++ b/packages/cli/src/commands/init.test.ts
@@ -8,6 +8,7 @@ import {
realpathSync,
rmSync,
symlinkSync,
+ unlinkSync,
writeFileSync,
} from 'node:fs';
import { tmpdir } from 'node:os';
@@ -2094,7 +2095,7 @@ describe('resolveRequestedContentDir', () => {
expect(resolveRequestedContentDir('.', root, linkParent)).toBe('.');
expect(resolveRequestedContentDir('sub', root, linkParent)).toBe('sub');
} finally {
- rmSync(linkParent, { force: true });
+ unlinkSync(linkParent);
}
});
});
diff --git a/packages/core/src/handoff/urn-ipc-registry.ts b/packages/core/src/handoff/urn-ipc-registry.ts
index e2dab6a79..c71c8aeb6 100644
--- a/packages/core/src/handoff/urn-ipc-registry.ts
+++ b/packages/core/src/handoff/urn-ipc-registry.ts
@@ -206,6 +206,7 @@ export const URN_HTTP_ONLY: ReadonlySet<ProblemType> = new Set<ProblemType>([
'urn:ok:error:payload-too-large',
'urn:ok:error:request-timeout',
'urn:ok:error:internal-server-error',
+ 'urn:ok:error:derived-index-unavailable',
// Local-op security gates (fire before IPC; bypassed by IPC channels by design)
'urn:ok:error:loopback-required',
'urn:ok:error:invalid-origin',
diff --git a/packages/core/src/schemas/api/_envelope.test.ts b/packages/core/src/schemas/api/_envelope.test.ts
index 5458f3d61..405198c4f 100644
--- a/packages/core/src/schemas/api/_envelope.test.ts
+++ b/packages/core/src/schemas/api/_envelope.test.ts
@@ -141,6 +141,7 @@ describe('ProblemTypeSchema', () => {
'urn:ok:error:method-not-allowed',
'urn:ok:error:invalid-request',
'urn:ok:error:internal-server-error',
+ 'urn:ok:error:derived-index-unavailable',
];
for (const t of tokens) {
const result = ProblemTypeSchema.safeParse(t);
diff --git a/packages/core/src/schemas/api/_envelope.ts b/packages/core/src/schemas/api/_envelope.ts
index 72ff7128a..06ff6a588 100644
--- a/packages/core/src/schemas/api/_envelope.ts
+++ b/packages/core/src/schemas/api/_envelope.ts
@@ -204,6 +204,10 @@ export const ProblemTypeSchema = z.enum([
// size-class (413 → reduce payload).
'urn:ok:error:request-timeout',
'urn:ok:error:internal-server-error',
+ // Derived-index queries reject with this retryable lifecycle state once
+ // coordinator shutdown begins. Configuration absence retains the
+ // index-specific `*-not-configured` tokens below.
+ 'urn:ok:error:derived-index-unavailable',
// /api/local-op/* security gate (shared by all local-op endpoints)
'urn:ok:error:loopback-required',
'urn:ok:error:invalid-origin',
diff --git a/packages/server/src/api-backlinks.test.ts b/packages/server/src/api-backlinks.test.ts
index 6041270b4..5c237dff3 100644
--- a/packages/server/src/api-backlinks.test.ts
+++ b/packages/server/src/api-backlinks.test.ts
@@ -3,11 +3,19 @@ import type { IncomingMessage, ServerResponse } from 'node:http';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { Readable } from 'node:stream';
-import { describe, expect, test } from 'vitest';
-import { createApiExtension } from './api-extension.test-helper.ts';
+import { describe, expect, test, vi } from 'vitest';
+import {
+ createApiExtension,
+ createDerivedDocumentIndexApiPortStub,
+} from './api-extension.test-helper.ts';
import { BacklinkIndex } from './backlink-index.ts';
import { type ContentFilter, createContentFilter } from './content-filter.ts';
+import {
+ DerivedDocumentIndex,
+ type DerivedDocumentIndexApiPort,
+} from './derived-document-index.ts';
import type { FileIndexEntry } from './file-watcher.ts';
+import { getLogger } from './logger.ts';
interface CapturedResponse {
status: number;
@@ -42,7 +50,12 @@ async function callRoute(
url: string,
fileIndex: ReadonlyMap<string, FileIndexEntry>,
backlinkIndex?: BacklinkIndex,
- options?: { method?: string; enableTestRoutes?: boolean; contentFilter?: ContentFilter },
+ options?: {
+ method?: string;
+ enableTestRoutes?: boolean;
+ contentFilter?: ContentFilter;
+ derivedDocumentIndex?: DerivedDocumentIndexApiPort;
+ },
): Promise<CapturedResponse> {
const ext = createApiExtension({
hocuspocus: {} as never,
@@ -50,6 +63,7 @@ async function callRoute(
contentDir,
getFileIndex: () => fileIndex,
backlinkIndex,
+ derivedDocumentIndex: options?.derivedDocumentIndex,
enableTestRoutes: options?.enableTestRoutes,
...(options?.contentFilter ? { contentFilter: options.contentFilter } : {}),
});
@@ -464,6 +478,91 @@ describe('graph endpoints', () => {
}
});
+ test('returns a quiet 503 when derived-index query routes race shutdown', async () => {
+ const projectDir = mkdtempSync(join(tmpdir(), 'ok-derived-query-shutdown-'));
+ const contentDir = join(projectDir, 'content');
+ mkdirSync(contentDir, { recursive: true });
+ const contentFilter = createContentFilter({ projectDir, contentDir });
+ const derivedDocumentIndex = new DerivedDocumentIndex({
+ projectDir,
+ contentDir,
+ contentFilter,
+ getGlobalSkillRoots: () => [],
+ signalChannel: () => {},
+ });
+ await derivedDocumentIndex.close();
+ const httpLogger = getLogger('http');
+ const errorLog = vi.spyOn(httpLogger, 'error');
+ const debugLog = vi.spyOn(httpLogger, 'debug');
+
+ const routes = [
+ '/api/backlinks?docName=doc',
+ '/api/backlink-counts?docNames=doc',
+ '/api/forward-links?docName=doc',
+ '/api/link-graph',
+ '/api/orphans',
+ '/api/hubs',
+ '/api/dead-links',
+ '/api/tags',
+ '/api/tags/example',
+ ] as const;
+
+ try {
+ for (const route of routes) {
+ const response = await callRoute(contentDir, route, new Map(), undefined, {
+ derivedDocumentIndex,
+ });
+ expect(response.status).toBe(503);
+ expect(JSON.parse(response.body)).toMatchObject({
+ type: 'urn:ok:error:derived-index-unavailable',
+ title: 'Derived index is shutting down.',
+ status: 503,
+ });
+ }
+ expect(errorLog).not.toHaveBeenCalled();
+ expect(debugLog).toHaveBeenCalledTimes(routes.length);
+ } finally {
+ errorLog.mockRestore();
+ debugLog.mockRestore();
+ rmSync(projectDir, { recursive: true, force: true });
+ }
+ });
+
+ test('keeps unexpected derived-index query failures as 500 faults', async () => {
+ const projectDir = mkdtempSync(join(tmpdir(), 'ok-derived-query-failure-'));
+ const contentDir = join(projectDir, 'content');
+ mkdirSync(contentDir, { recursive: true });
+ const failure = new Error('unexpected query failure');
+ const derivedDocumentIndex = createDerivedDocumentIndexApiPortStub({
+ async getBacklinks() {
+ throw failure;
+ },
+ });
+ const errorLog = vi.spyOn(getLogger('http'), 'error');
+
+ try {
+ const response = await callRoute(
+ contentDir,
+ '/api/backlinks?docName=doc',
+ new Map(),
+ undefined,
+ { derivedDocumentIndex },
+ );
+ expect(response.status).toBe(500);
+ expect(JSON.parse(response.body)).toMatchObject({
+ type: 'urn:ok:error:internal-server-error',
+ status: 500,
+ });
+ expect(errorLog).toHaveBeenCalledWith(
+ expect.objectContaining({ event: 'api.error', err: failure }),
+ 'Failed to read backlinks.',
+ );
+ } finally {
+ errorLog.mockRestore();
+ rmSync(projectDir, { recursive: true, force: true });
+ }
+ });
+
test('forward-links / hubs / link-graph fall back to docName for excluded targets', async () => {
// Wiki-link targets parsed from indexed docs may name a doc that is
// itself excluded from the content scope (e.g. by `.gitignore` /
diff --git a/packages/server/src/api-create-page.test.ts b/packages/server/src/api-create-page.test.ts
index 7999222f6..7131fa643 100644
--- a/packages/server/src/api-create-page.test.ts
+++ b/packages/server/src/api-create-page.test.ts
@@ -13,9 +13,13 @@ import { join } from 'node:path';
import { Readable } from 'node:stream';
import type { Principal } from '@inkeep/open-knowledge-core';
import { afterEach, beforeEach, describe, expect, test } from 'vitest';
-import { createApiExtension } from './api-extension.test-helper.ts';
+import {
+ createApiExtension,
+ createDerivedDocumentIndexApiPortStub,
+} from './api-extension.test-helper.ts';
import { BacklinkIndex } from './backlink-index.ts';
import { contributorCount, hasContributor, swapContributors } from './contributor-tracker.ts';
+import type { DerivedDocumentIndexApiPort } from './derived-document-index.ts';
import type { FileIndexEntry } from './file-watcher.ts';
// ---------------------------------------------------------------------------
@@ -58,6 +62,7 @@ async function callCreatePage(
options?: {
fileIndex?: Map<string, FileIndexEntry>;
backlinkIndex?: BacklinkIndex;
+ derivedDocumentIndex?: DerivedDocumentIndexApiPort;
getPrincipal?: () => Principal | null;
},
): Promise<CapturedResponse> {
@@ -67,6 +72,7 @@ async function callCreatePage(
contentDir,
getFileIndex: () => options?.fileIndex ?? new Map<string, FileIndexEntry>(),
backlinkIndex: options?.backlinkIndex,
+ derivedDocumentIndex: options?.derivedDocumentIndex,
getPrincipal: options?.getPrincipal,
});
const req = makeReq(method, body);
@@ -123,6 +129,28 @@ describe('POST /api/create-page', () => {
expect(existsSync(join(dir, 'my-page.md'))).toBe(true);
});
+ test('keeps a created file successful when the derived index is closed', async () => {
+ const dir = setupTmpDir();
+ const derivedDocumentIndex = createDerivedDocumentIndexApiPortStub({
+ async recordDirectDocument() {
+ const error = new Error('Derived document index is closed');
+ error.name = 'DerivedDocumentIndexClosedError';
+ throw error;
+ },
+ });
+
+ const result = await callCreatePage(
+ dir,
+ 'POST',
+ { path: 'survives-shutdown.md' },
+ { derivedDocumentIndex },
+ );
+
+ expect(result.status).toBe(200);
+ expect(existsSync(join(dir, 'survives-shutdown.md'))).toBe(true);
+ expect(JSON.parse(result.body)).toMatchObject({ docName: 'survives-shutdown' });
+ });
+
test('creates a .mdx file and returns the extension-less docName', async () => {
const dir = setupTmpDir();
const result = await callCreatePage(dir, 'POST', { path: 'component.mdx' });
diff --git a/packages/server/src/api-derived-index-ready-gate.test.ts b/packages/server/src/api-derived-index-ready-gate.test.ts
new file mode 100644
index 000000000..421664b1f
--- /dev/null
+++ b/packages/server/src/api-derived-index-ready-gate.test.ts
@@ -0,0 +1,204 @@
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import type { IncomingMessage, ServerResponse } from 'node:http';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { Readable } from 'node:stream';
+import { describe, expect, test, vi } from 'vitest';
+import {
+ createApiExtension,
+ createDerivedDocumentIndexApiPortStub,
+} from './api-extension.test-helper.ts';
+import { BacklinkIndex } from './backlink-index.ts';
+import { createContentFilter } from './content-filter.ts';
+import { DerivedDocumentIndex } from './derived-document-index.ts';
+
+interface PendingRoute {
+ captured: { status: number; body: string };
+ done: Promise<void>;
+}
+
+function startRoute(
+ extension: ReturnType<typeof createApiExtension>,
+ url: string,
+ method = 'GET',
+ body?: unknown,
+): PendingRoute {
+ const request = Readable.from(
+ Buffer.from(body === undefined ? '' : JSON.stringify(body)),
+ ) as unknown as IncomingMessage;
+ request.method = method;
+ request.url = url;
+ request.headers = { host: 'localhost' };
+ const captured = { status: 0, body: '' };
+ const response = {
+ writeHead(status: number) {
+ captured.status = status;
+ },
+ setHeader() {},
+ end(body?: string) {
+ captured.body = body ?? '';
+ },
+ } as unknown as ServerResponse;
+ const done = (
+ extension as {
+ onRequest: (ctx: { request: IncomingMessage; response: ServerResponse }) => Promise<void>;
+ }
+ ).onRequest({ request, response });
+ return { captured, done };
+}
+
+describe('derived-index API readiness gate', () => {
+ test('backlink and tag queries wait for post-watcher startup settlement', async () => {
+ const projectDir = mkdtempSync(join(tmpdir(), 'ok-derived-ready-'));
+ const contentDir = join(projectDir, 'content');
+ mkdirSync(contentDir, { recursive: true });
+ writeFileSync(join(contentDir, 'source.md'), 'See [[target]]. #startup\n');
+ const index = new DerivedDocumentIndex({
+ projectDir,
+ contentDir,
+ contentFilter: createContentFilter({ projectDir, contentDir }),
+ getGlobalSkillRoots: () => [],
+ signalChannel: () => {},
+ });
+
+ try {
+ const startup = index.beginStartup('main');
+ const extension = createApiExtension({
+ hocuspocus: {} as never,
+ sessionManager: {} as never,
+ contentDir,
+ getFileIndex: () => new Map(),
+ derivedDocumentIndex: index,
+ });
+ const backlinksRoute = startRoute(extension, '/api/backlinks?docName=target');
+ const tagsRoute = startRoute(extension, '/api/tags/startup');
+
+ await startup.backlinksReady;
+ await Promise.resolve();
+ expect(backlinksRoute.captured.status).toBe(0);
+ expect(tagsRoute.captured.status).toBe(0);
+
+ await index.settleStartupAfterWatcherSeed();
+ await Promise.all([backlinksRoute.done, tagsRoute.done]);
+
+ expect(backlinksRoute.captured.status).toBe(200);
+ expect(JSON.parse(backlinksRoute.captured.body)).toMatchObject({
+ docName: 'target',
+ backlinks: [{ source: 'source' }],
+ });
+ expect(tagsRoute.captured.status).toBe(200);
+ expect(JSON.parse(tagsRoute.captured.body)).toMatchObject({
+ name: 'startup',
+ docs: [{ docName: 'source', matchingTags: ['startup'] }],
+ });
+ } finally {
+ await index.close();
+ rmSync(projectDir, { recursive: true, force: true });
+ }
+ });
+
+ test('test rescan retains the route-level 500 boundary for coordinator failures', async () => {
+ const projectDir = mkdtempSync(join(tmpdir(), 'ok-derived-rescan-error-'));
+ const contentDir = join(projectDir, 'content');
+ mkdirSync(contentDir, { recursive: true });
+ const index = new DerivedDocumentIndex({
+ projectDir,
+ contentDir,
+ contentFilter: createContentFilter({ projectDir, contentDir }),
+ getGlobalSkillRoots: () => [],
+ signalChannel: () => {},
+ });
+
+ try {
+ const startup = index.beginStartup('main');
+ await startup.backlinksReady;
+ await index.settleStartupAfterWatcherSeed();
+ vi.spyOn(BacklinkIndex.prototype, 'ingestGlobalSkillBundles').mockRejectedValueOnce(
+ new Error('global rescan failure'),
+ );
+ const extension = createApiExtension({
+ hocuspocus: {} as never,
+ sessionManager: {} as never,
+ contentDir,
+ getFileIndex: () => new Map(),
+ derivedDocumentIndex: index,
+ enableTestRoutes: true,
+ });
+
+ const route = startRoute(extension, '/api/test-rescan-backlinks', 'POST');
+ await route.done;
+
+ expect(route.captured.status).toBe(500);
+ expect(JSON.parse(route.captured.body)).toMatchObject({
+ type: 'urn:ok:error:internal-server-error',
+ });
+ } finally {
+ vi.restoreAllMocks();
+ await index.close();
+ rmSync(projectDir, { recursive: true, force: true });
+ }
+ });
+
+ test('project skill move waits for derived-index projection before responding', async () => {
+ const contentDir = mkdtempSync(join(tmpdir(), 'ok-derived-skill-move-'));
+ const sourceSkillDir = join(contentDir, '.ok', 'skills', 'old-skill');
+ mkdirSync(sourceSkillDir, { recursive: true });
+ writeFileSync(
+ join(sourceSkillDir, 'SKILL.md'),
+ '---\nname: old-skill\ndescription: Use when testing moves.\n---\n\n# Old skill\n',
+ );
+ let releaseProjection!: () => void;
+ const projectionBarrier = new Promise<void>((resolve) => {
+ releaseProjection = resolve;
+ });
+ let projectionStarted!: () => void;
+ const started = new Promise<void>((resolve) => {
+ projectionStarted = resolve;
+ });
+ const recordDirectMutations = vi.fn(async () => {
+ projectionStarted();
+ await projectionBarrier;
+ });
+ const extension = createApiExtension({
+ hocuspocus: {
+ documents: new Map(),
+ closeConnections() {},
+ unloadDocument: async () => {},
+ } as never,
+ sessionManager: {
+ closeSession: async () => {},
+ closeAllForDoc: async () => {},
+ } as never,
+ contentDir,
+ getFileIndex: () => new Map(),
+ derivedDocumentIndex: createDerivedDocumentIndexApiPortStub({
+ recordDirectMutations,
+ }),
+ });
+
+ try {
+ const route = startRoute(extension, '/api/skill', 'POST', {
+ scope: 'project',
+ fromName: 'old-skill',
+ toName: 'new-skill',
+ });
+
+ await started;
+ await Promise.resolve();
+ expect(route.captured.status).toBe(0);
+
+ releaseProjection();
+ await route.done;
+
+ expect(recordDirectMutations).toHaveBeenCalledTimes(1);
+ expect(route.captured.status).toBe(200);
+ expect(JSON.parse(route.captured.body)).toMatchObject({
+ from: '.ok/skills/old-skill',
+ to: '.ok/skills/new-skill',
+ });
+ } finally {
+ releaseProjection();
+ rmSync(contentDir, { recursive: true, force: true });
+ }
+ });
+});
diff --git a/packages/server/src/api-extension.ok-init.test.ts b/packages/server/src/api-extension.ok-init.test.ts
index afc1b4cd2..996a7815d 100644
--- a/packages/server/src/api-extension.ok-init.test.ts
+++ b/packages/server/src/api-extension.ok-init.test.ts
@@ -14,22 +14,9 @@ import { join } from 'node:path';
import { afterEach, describe, expect, test } from 'vitest';
import { listenOnLoopback } from './loopback-rig-test-helpers.ts';
-/**
- * Locate the source body that follows `const <handlerNameConst> = '...'`
- * declaration and return the next ~80 lines as a single string. The
- * 429-emission test uses this to scope a `.toContain` check to the
- * ok-init handler block without false-matching the same URN/header in
- * unrelated handlers.
- */
-function extractHandlerBlock(src: string, handlerNameConst: string): string {
- const anchorRe = new RegExp(`const\\s+${handlerNameConst}\\s*=`);
- const match = anchorRe.exec(src);
- if (!match) {
- throw new Error(`extractHandlerBlock: '${handlerNameConst}' anchor not found`);
- }
- // 80 lines × roughly 80 chars = ~6400 chars. Generous slice to catch
- // both the tryAcquire branch and the finally-release.
- return src.slice(match.index, match.index + 6400);
+interface TestConcurrencyGuard {
+ tryAcquire(key: string): boolean;
+ release(key: string): void;
}
interface TestRig {
@@ -57,7 +44,9 @@ function initRepo(cwd: string): void {
* doesn't read from it (the body's projectPath is the operative target),
* so a minimal git repo there satisfies the boot.
*/
-async function bootRig(): Promise<TestRig> {
+async function bootRig(
+ options: { localOpConcurrencyGuard?: TestConcurrencyGuard } = {},
+): Promise<TestRig> {
// Root tmpRoot under the real home dir so projectPaths constructed beneath
// it pass the handler's `isSafeLocalPath` home-dir containment gate. (A
// tmpdir() root resolves outside $HOME on macOS — `/private/var/...` — and
@@ -85,6 +74,7 @@ async function bootRig(): Promise<TestRig> {
projectDir,
getFileIndex: () => new Map(),
serverInstanceId: 'test-instance',
+ localOpConcurrencyGuard: options.localOpConcurrencyGuard,
});
const { createServer } = await import('node:http');
@@ -116,7 +106,7 @@ async function bootRig(): Promise<TestRig> {
async function postOkInit(
port: number,
body: Record<string, unknown>,
-): Promise<{ status: number; json: Record<string, unknown> }> {
+): Promise<{ status: number; json: Record<string, unknown>; retryAfter: string | null }> {
const res = await fetch(`http://127.0.0.1:${port}/api/local-op/ok-init`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
@@ -124,7 +114,7 @@ async function postOkInit(
});
// biome-ignore lint/suspicious/noExplicitAny: test
const json = (await res.json()) as any;
- return { status: res.status, json };
+ return { status: res.status, json, retryAfter: res.headers.get('retry-after') };
}
let rig: TestRig | null = null;
@@ -234,48 +224,47 @@ describe('POST /api/local-op/ok-init', () => {
expect(res.json.type).toBe('urn:ok:error:invalid-request');
});
- test('429 problem+json contract — handler is wired through localOpGuard with the ok-init key', () => {
- // The localOpGuard-backed 429 path is the user-facing contract when
- // a concurrent ok-init lands while one is in-flight. The race itself
- // is hard to observe deterministically through node:http's serial
- // request dispatch (the synchronous initContent + finally{release}
- // window is shorter than the body-parse + sync-gates window of the
- // next request in the rig), so we validate the contract via source
- // scan instead — three invariants the handler MUST satisfy:
- //
- // 1. A dedicated key constant `LOCAL_OP_OK_INIT_KEY` is declared
- // with the canonical channel string `/api/local-op/ok-init`.
- // 2. The handler acquires the key via `localOpGuard.tryAcquire`
- // and emits a 429 with `urn:ok:error:concurrent-operation`
- // and the `Retry-After: 2` header on contention.
- // 3. The key is released in a `finally` block so a thrown
- // `withParentLock` task can't leak the lock.
- //
- // Source scanning here is in the spirit of the structural-ratchet
- // pattern (`ipc-channel-count-ratchet.test.ts`, `no-loosely-typed-
- // webcontents-ipc.test.ts`): a regression that removes the lock or
- // the 429 envelope shape would silently re-enable the race the
- // guard exists to prevent.
- const apiExtensionSrc = readFileSync(join(__dirname, 'api-extension.ts'), 'utf8');
-
- // (1) Key constant declared with the canonical channel string.
- expect(apiExtensionSrc).toMatch(/LOCAL_OP_OK_INIT_KEY\s*=\s*['"]\/api\/local-op\/ok-init['"]/);
-
- // (2a) Handler calls tryAcquire on the key.
- expect(apiExtensionSrc).toMatch(/localOpGuard\.tryAcquire\(LOCAL_OP_OK_INIT_KEY\)/);
-
- // (2b) 429 + concurrent-operation URN + Retry-After header live in
- // the handler block immediately following the tryAcquire negative
- // branch. Co-located so a regression that breaks the contract
- // surfaces here.
- const okInitBlock = extractHandlerBlock(apiExtensionSrc, 'HANDLE_LOCAL_OP_OK_INIT');
- expect(okInitBlock).toContain('429');
- expect(okInitBlock).toContain("'urn:ok:error:concurrent-operation'");
- expect(okInitBlock).toContain("'Retry-After'");
-
- // (3) Release lives in a finally block (catches both happy-path
- // success and `withParentLock` throws).
- expect(okInitBlock).toMatch(/finally\s*\{[^}]*localOpGuard\.release\(LOCAL_OP_OK_INIT_KEY\)/s);
+ test('returns 429 on contention and releases the guard after init failure', async () => {
+ const key = '/api/local-op/ok-init';
+ const acquired: string[] = [];
+ const released: string[] = [];
+ let rejectAcquisition = true;
+ let held = false;
+ const guard: TestConcurrencyGuard = {
+ tryAcquire(nextKey) {
+ acquired.push(nextKey);
+ if (rejectAcquisition || held) return false;
+ held = true;
+ return true;
+ },
+ release(nextKey) {
+ released.push(nextKey);
+ held = false;
+ },
+ };
+ rig = await bootRig({ localOpConcurrencyGuard: guard });
+ const target = join(rig.tmpRoot, 'guarded-worktree');
+ mkdirSync(target);
+ initRepo(target);
+ writeFileSync(join(target, 'README.md'), '# guarded\n');
+ run(target, 'git add -A');
+ run(target, 'git commit -q -m initial');
+
+ const contended = await postOkInit(rig.port, { projectPath: target });
+ expect(contended.status).toBe(429);
+ expect(contended.json.type).toBe('urn:ok:error:concurrent-operation');
+ expect(contended.retryAfter).toBe('2');
+ expect(acquired).toEqual([key]);
+ expect(released).toEqual([]);
+
+ rejectAcquisition = false;
+ writeFileSync(join(target, '.ok'), 'blocks scaffold directory\n');
+ const failed = await postOkInit(rig.port, { projectPath: target });
+ expect(failed.status).toBe(200);
+ expect(failed.json).toMatchObject({ ok: false, reason: 'init-failed' });
+ expect(acquired).toEqual([key, key]);
+ expect(released).toEqual([key]);
+ expect(held).toBe(false);
});
test('scaffolds inside a linked worktree (FR13 + D12 spirit)', async () => {
diff --git a/packages/server/src/api-extension.test-helper.ts b/packages/server/src/api-extension.test-helper.ts
index 451b4b990..1bb60c1dc 100644
--- a/packages/server/src/api-extension.test-helper.ts
+++ b/packages/server/src/api-extension.test-helper.ts
@@ -1,14 +1,229 @@
import type { ApiExtensionOptions } from './api-extension.ts';
import { createApiExtension as createApiExtensionBase } from './api-extension.ts';
+import type { BacklinkIndex } from './backlink-index.ts';
+import type {
+ DerivedDocumentIndexApiPort,
+ DerivedDocumentIndexMutation,
+} from './derived-document-index.ts';
import { DocumentDurabilityState } from './document-durability-state.ts';
+import type { TagIndex } from './tag-index.ts';
export * from './api-extension.ts';
+interface LegacyIndexOptions {
+ backlinkIndex?: BacklinkIndex | null;
+ tagIndex?: TagIndex | null;
+}
+
+type LegacySignalChannel = (
+ channel: 'files' | 'backlinks' | 'graph' | 'tags' | 'lint-config',
+) => void;
+
+export function createDerivedDocumentIndexApiPortStub(
+ overrides: Partial<DerivedDocumentIndexApiPort> = {},
+): DerivedDocumentIndexApiPort {
+ return {
+ async recordDirectMutations() {},
+ async recordDirectDocument() {},
+ async recordDirectDelete() {},
+ async recordDirectRename() {},
+ async recordLinkRewrite() {},
+ async getBacklinks() {
+ return [];
+ },
+ async getBacklinkCount() {
+ return 0;
+ },
+ async getBacklinkCounts(documentNames) {
+ return Object.fromEntries(documentNames.map((documentName) => [documentName, 0]));
+ },
+ async getForwardLinkEntries() {
+ return [];
+ },
+ async getLinkGraphNeighborhood() {
+ return { nodes: [], links: [] };
+ },
+ async getLinkGraph() {
+ return { nodes: [], links: [] };
+ },
+ async getHubs() {
+ return [];
+ },
+ async getOrphans() {
+ return [];
+ },
+ async getDeadLinks() {
+ return [];
+ },
+ async getIndexedDocNames() {
+ return [];
+ },
+ async getAllTags() {
+ return [];
+ },
+ async getDocsForTagWithMatches() {
+ return [];
+ },
+ ...overrides,
+ };
+}
+
+function createLegacyDerivedIndexPort(
+ backlinkIndex: BacklinkIndex | null | undefined,
+ tagIndex: TagIndex | null | undefined,
+ signalChannel: LegacySignalChannel | undefined,
+): DerivedDocumentIndexApiPort | undefined {
+ if (!backlinkIndex && !tagIndex) return undefined;
+
+ const signalBacklinks = (): void => {
+ signalChannel?.('backlinks');
+ signalChannel?.('graph');
+ };
+ const signalAll = (): void => {
+ signalBacklinks();
+ signalChannel?.('tags');
+ };
+ const saveAll = async (): Promise<void> => {
+ await Promise.allSettled([backlinkIndex?.saveToDisk(), tagIndex?.saveToDisk()]);
+ };
+ const updateAll = (documentName: string, markdown: string): void => {
+ backlinkIndex?.updateDocumentFromMarkdown(documentName, markdown);
+ tagIndex?.updateDocumentFromMarkdown(documentName, markdown);
+ };
+ const deleteAll = (documentName: string): void => {
+ backlinkIndex?.deleteDocument(documentName);
+ tagIndex?.deleteDocument(documentName);
+ };
+ const recordDirectMutations = async (
+ mutations: readonly DerivedDocumentIndexMutation[],
+ ): Promise<void> => {
+ if (mutations.length === 0) return;
+ let tagsChanged = false;
+ for (const mutation of mutations) {
+ switch (mutation.kind) {
+ case 'upsert':
+ updateAll(mutation.documentName, mutation.markdown);
+ tagsChanged = true;
+ break;
+ case 'delete':
+ deleteAll(mutation.documentName);
+ tagsChanged = true;
+ break;
+ case 'rename':
+ backlinkIndex?.renameDocument(
+ mutation.oldDocumentName,
+ mutation.newDocumentName,
+ mutation.markdown,
+ );
+ tagIndex?.renameDocument(
+ mutation.oldDocumentName,
+ mutation.newDocumentName,
+ mutation.markdown,
+ );
+ tagsChanged = true;
+ break;
+ case 'link-rewrite':
+ backlinkIndex?.updateDocumentFromMarkdown(mutation.documentName, mutation.markdown);
+ break;
+ }
+ }
+ if (tagsChanged) {
+ await saveAll();
+ signalAll();
+ } else {
+ await Promise.allSettled([backlinkIndex?.saveToDisk()]);
+ signalBacklinks();
+ }
+ };
+
+ return {
+ recordDirectMutations,
+ testOnly: {
+ async resetDocumentForTest(documentName) {
+ deleteAll(documentName);
+ await saveAll();
+ signalAll();
+ },
+ async rescanBacklinksForTest() {
+ await backlinkIndex?.rebuildFromDisk();
+ await backlinkIndex?.saveToDisk();
+ signalBacklinks();
+ },
+ },
+ async recordDirectDocument(documentName, markdown) {
+ await recordDirectMutations([{ kind: 'upsert', documentName, markdown }]);
+ },
+ async recordDirectDelete(documentName) {
+ await recordDirectMutations([{ kind: 'delete', documentName }]);
+ },
+ async recordDirectRename(oldDocumentName, newDocumentName, markdown) {
+ await recordDirectMutations([{ kind: 'rename', oldDocumentName, newDocumentName, markdown }]);
+ },
+ async recordLinkRewrite(documentName, markdown) {
+ await recordDirectMutations([{ kind: 'link-rewrite', documentName, markdown }]);
+ },
+ async getBacklinks(documentName) {
+ return backlinkIndex?.getBacklinks(documentName) ?? [];
+ },
+ async getBacklinkCount(documentName) {
+ return backlinkIndex?.getBacklinkCount(documentName) ?? 0;
+ },
+ async getBacklinkCounts(documentNames) {
+ return Object.fromEntries(
+ documentNames.map((documentName) => [
+ documentName,
+ backlinkIndex?.getBacklinkCount(documentName) ?? 0,
+ ]),
+ );
+ },
+ async getForwardLinkEntries(documentName) {
+ return backlinkIndex?.getForwardLinkEntries(documentName) ?? [];
+ },
+ async getLinkGraphNeighborhood(documentName, degrees) {
+ return (
+ backlinkIndex?.getLinkGraphNeighborhood(documentName, degrees) ?? {
+ nodes: [],
+ links: [],
+ }
+ );
+ },
+ async getLinkGraph() {
+ return backlinkIndex?.getLinkGraph() ?? { nodes: [], links: [] };
+ },
+ async getHubs(limit) {
+ return backlinkIndex?.getHubs(limit) ?? [];
+ },
+ async getOrphans(documentNames, mode) {
+ return backlinkIndex?.getOrphans(documentNames, mode) ?? [];
+ },
+ async getDeadLinks(admittedDocuments, sourceDocumentNames) {
+ return backlinkIndex?.getDeadLinks(admittedDocuments, sourceDocumentNames) ?? [];
+ },
+ async getIndexedDocNames() {
+ return backlinkIndex?.getIndexedDocNames() ?? [];
+ },
+ async getAllTags() {
+ return tagIndex?.getAllTags() ?? [];
+ },
+ async getDocsForTagWithMatches(tag) {
+ return tagIndex?.getDocsForTagWithMatches(tag) ?? [];
+ },
+ };
+}
+
export function createApiExtension(
- options: Omit<ApiExtensionOptions, 'durabilityState'>,
+ options: Omit<ApiExtensionOptions, 'durabilityState' | 'derivedDocumentIndex' | 'signalChannel'> &
+ LegacyIndexOptions & {
+ derivedDocumentIndex?: DerivedDocumentIndexApiPort;
+ signalChannel?: LegacySignalChannel;
+ },
): ReturnType<typeof createApiExtensionBase> {
+ const { backlinkIndex, tagIndex, derivedDocumentIndex, ...apiOptions } = options;
return createApiExtensionBase({
- ...options,
+ ...apiOptions,
durabilityState: new DocumentDurabilityState(),
+ derivedDocumentIndex:
+ derivedDocumentIndex ??
+ createLegacyDerivedIndexPort(backlinkIndex, tagIndex, options.signalChannel),
});
}
diff --git a/packages/server/src/api-extension.ts b/packages/server/src/api-extension.ts
index dd66dc1a8..600aec7a8 100644
--- a/packages/server/src/api-extension.ts
+++ b/packages/server/src/api-extension.ts
@@ -498,12 +498,7 @@ import {
ManagedRenameSourceTypeMismatchError,
SymlinkEscapeError,
} from './apply-managed-rename.ts';
-import {
- type BacklinkIndex,
- computeBrokenOutboundLinks,
- type GraphNode as IndexedGraphNode,
- isOrphanMode,
-} from './backlink-index.ts';
+import { computeBrokenOutboundLinks } from './backlink-index.ts';
import { getBootTimings } from './boot-timings.ts';
import { composeAndWriteRawBody, type PrecomputedParse, replaceRawBody } from './bridge-intake.ts';
import type { BridgeDeriveLossReporter } from './bridge-loss-detector.ts';
@@ -512,6 +507,13 @@ import { withHiddenWindowsConsole } from './child-process-windows-hide.ts';
import type { ResolveStrategy } from './conflict-storage.ts';
import type { ContentFilter } from './content-filter.ts';
import { isWithinContentDir, safeContentPath } from './content-path.ts';
+import {
+ type DerivedDocumentIndexApiPort,
+ type DerivedDocumentIndexMutation,
+ type DerivedGraphNode,
+ isDerivedDocumentIndexClosedError,
+ isDerivedOrphanMode,
+} from './derived-document-index.ts';
import {
docNameToRelativePath,
extensionlessDocTreePath,
@@ -657,7 +659,6 @@ import { createSingleFlight } from './single-flight.ts';
import { restoreSkillVersion } from './skill-restore.ts';
import { SuggestLinksTargetNotFoundError, suggestLinks } from './suggest-links.ts';
import type { SyncEngine } from './sync-engine.ts';
-import type { TagIndex } from './tag-index.ts';
import { getMeter, getTracer, withSpan, withSpanSync } from './telemetry.ts';
import { getDocumentHistory, getFolderTimeline } from './timeline-query.ts';
import { recordTimelineCoalesced } from './timeline-telemetry.ts';
@@ -2743,6 +2744,7 @@ export interface ApiExtensionOptions {
* Linux CI — see `WatcherHandle.rescanFromDisk` in `file-watcher.ts`.
*/
rescanFiles?: () => void | Promise<void>;
+ localOpConcurrencyGuard?: ReturnType<typeof createConcurrencyGuard>;
/**
* When true, register test-only routes (`/api/test-reset`,
* `/api/test-rescan-backlinks`, `/api/test-rescan-files`). Defaults to
@@ -2776,9 +2778,8 @@ export interface ApiExtensionOptions {
*/
getDiskAckSVs?: () => Record<string, string>;
contentRoot?: string;
- backlinkIndex?: BacklinkIndex;
- tagIndex?: TagIndex;
- signalChannel?: (channel: 'files' | 'backlinks' | 'graph' | 'lint-config') => void;
+ derivedDocumentIndex?: DerivedDocumentIndexApiPort;
+ signalChannel?: (channel: 'files' | 'lint-config') => void;
/**
* Optional. When present, agent write handlers publish per-write attribution
* entries on `__system__` awareness (`agentFocus` map) with writeKind +
@@ -3092,6 +3093,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
getAliasMap,
getFolderAliasIndex,
rescanFiles,
+ localOpConcurrencyGuard,
enableTestRoutes = false,
shadowRef,
flushGitCommit,
@@ -3099,8 +3101,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
getCurrentBranch,
getDiskAckSVs,
contentRoot,
- backlinkIndex,
- tagIndex,
+ derivedDocumentIndex,
signalChannel,
agentFocusBroadcaster,
agentPresenceBroadcaster,
@@ -3134,7 +3135,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
} = options;
// Concurrency guard: at most 1 in-flight request per local-op endpoint
- const localOpGuard = createConcurrencyGuard();
+ const localOpGuard = localOpConcurrencyGuard ?? createConcurrencyGuard();
// Single-flight dedupe for `GET /api/documents?showAll=true`. Keyed per
// server instance (NOT module-global — tests boot several servers in one
@@ -3342,15 +3343,15 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
* Soft orphan-hint: when a written doc has zero backlinks AND a hub
* candidate exists in its folder tree, attach a hint suggesting the hub.
* Returns `undefined` when any prerequisite is unavailable (no
- * backlinkIndex wired, target not in index, has backlinks, or no candidate).
+ * relationship index wired, target not in index, has backlinks, or no candidate).
* Non-throwing — a hint-computation failure must not fail the write.
*/
- function computeOrphanHints(
+ async function computeOrphanHints(
docName: string,
- ): Array<{ type: 'orphan'; parentCandidates: string[]; message: string }> | undefined {
- if (!backlinkIndex) return undefined;
+ ): Promise<Array<{ type: 'orphan'; parentCandidates: string[]; message: string }> | undefined> {
+ if (!derivedDocumentIndex) return undefined;
try {
- const backlinks = backlinkIndex.getBacklinks(docName);
+ const backlinks = await derivedDocumentIndex.getBacklinks(docName);
if (backlinks.length > 0) return undefined;
// This runs on every write — if hub-candidate walking becomes pathological
// on very large file indexes, we want an observable signal. 5ms is well
@@ -3593,7 +3594,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
return contentFilter.isExcluded(relPath);
}
- function collectAdmittedDocNames(): Set<string> {
+ async function collectAdmittedDocNames(): Promise<Set<string>> {
const admitted = new Set<string>();
for (const [docName, entry] of getFileIndex()) {
admitted.add(docName);
@@ -3637,13 +3638,119 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
// its title/frontmatter would leak through the link/title endpoints. Already-
// admitted names (file index, managed artifacts) skip the gate — cheap and
// they're known in-scope.
- for (const docName of backlinkIndex?.getIndexedDocNames() ?? []) {
+ for (const docName of (await derivedDocumentIndex?.getIndexedDocNames()) ?? []) {
if (admitted.has(docName)) continue;
if (!isDocNameContentExcluded(docName)) admitted.add(docName);
}
return admitted;
}
+ async function recordDerivedMutationsBestEffort(
+ mutations: readonly DerivedDocumentIndexMutation[],
+ reason: string,
+ ): Promise<void> {
+ if (!derivedDocumentIndex || mutations.length === 0) return;
+ try {
+ await derivedDocumentIndex.recordDirectMutations(mutations);
+ } catch (err) {
+ logDerivedProjectionFailure(
+ err,
+ { count: mutations.length, reason },
+ '[derived-index] failed to project durable document mutations',
+ );
+ }
+ }
+
+ async function recordDerivedDocumentBestEffort(
+ documentName: string,
+ markdown: string,
+ reason: string,
+ ): Promise<void> {
+ if (!derivedDocumentIndex) return;
+ try {
+ await derivedDocumentIndex.recordDirectDocument(documentName, markdown);
+ } catch (err) {
+ logDerivedProjectionFailure(
+ err,
+ { documentName, reason },
+ '[derived-index] failed to project durable document',
+ );
+ }
+ }
+
+ async function recordDerivedLinkRewriteBestEffort(
+ documentName: string,
+ markdown: string,
+ reason: string,
+ ): Promise<void> {
+ if (!derivedDocumentIndex) return;
+ try {
+ await derivedDocumentIndex.recordLinkRewrite(documentName, markdown);
+ } catch (err) {
+ logDerivedProjectionFailure(
+ err,
+ { documentName, reason },
+ '[derived-index] failed to project link rewrite',
+ );
+ }
+ }
+
+ function logDerivedProjectionFailure(
+ err: unknown,
+ context: Record<string, unknown>,
+ failureMessage: string,
+ ): void {
+ if (isDerivedDocumentIndexClosedError(err)) {
+ log.debug(
+ { err, ...context },
+ '[derived-index] coordinator closed; skipping durable projection',
+ );
+ return;
+ }
+ log.warn({ err, ...context }, failureMessage);
+ }
+
+ function respondToDerivedIndexQueryFailure(
+ res: ServerResponse,
+ err: unknown,
+ options: {
+ handler: string;
+ failureTitle: string;
+ },
+ ): void {
+ if (isDerivedDocumentIndexClosedError(err)) {
+ errorResponse(
+ res,
+ 503,
+ 'urn:ok:error:derived-index-unavailable',
+ 'Derived index is shutting down.',
+ {
+ handler: options.handler,
+ cause: err,
+ logLevel: 'debug',
+ },
+ );
+ return;
+ }
+ errorResponse(res, 500, 'urn:ok:error:internal-server-error', options.failureTitle, {
+ handler: options.handler,
+ cause: err,
+ });
+ }
+
+ async function deleteDerivedDocumentsBestEffort(
+ documentNames: Iterable<string>,
+ reason: string,
+ ): Promise<void> {
+ await recordDerivedMutationsBestEffort(
+ [...documentNames].map((documentName) => ({
+ kind: 'delete',
+ documentName,
+ })),
+ reason,
+ );
+ }
+
// On-disk existence oracle for non-doc outbound link targets (linked assets
// and source files) used by `computeBrokenOutboundLinks`. Doc links resolve
// against the admitted set above; file links have no CRDT presence, so a
@@ -4010,11 +4117,15 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
}
}
- function applyPendingAssetReferenceRewrites(
+ async function applyPendingAssetReferenceRewrites(
pendingRewrites: readonly { docName: string; markdown: string; rewrites: number }[],
renamedAssets: readonly RenamedAssetMapping[],
- ): ManagedRenameRewrittenDoc[] {
+ ): Promise<{
+ rewrittenDocs: ManagedRenameRewrittenDoc[];
+ derivedMutations: DerivedDocumentIndexMutation[];
+ }> {
const rewrittenDocs: ManagedRenameRewrittenDoc[] = [];
+ const derivedMutations: DerivedDocumentIndexMutation[] = [];
for (const pending of pendingRewrites) {
const document = hocuspocus.documents.get(pending.docName);
const rewritten = document
@@ -4022,10 +4133,14 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
: pending;
if (rewritten.rewrites === 0) continue;
writeManagedRenameDocumentToDisk(pending.docName, rewritten.markdown);
- backlinkIndex?.updateDocumentFromMarkdown(pending.docName, rewritten.markdown);
+ derivedMutations.push({
+ kind: 'link-rewrite',
+ documentName: pending.docName,
+ markdown: rewritten.markdown,
+ });
rewrittenDocs.push({ docName: pending.docName, rewrites: rewritten.rewrites });
}
- return rewrittenDocs;
+ return { rewrittenDocs, derivedMutations };
}
function resolveExtensionlessAssetPath(assetPath: string): {
@@ -4169,7 +4284,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
},
},
async (span) => {
- if (!backlinkIndex) {
+ if (!derivedDocumentIndex) {
throw new BacklinkIndexRequiredError();
}
const destinationAssetPath = extname(toPath) ? toPath : `${toPath}${extname(fromPath)}`;
@@ -4226,19 +4341,12 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
renamePathOnDisk(sourcePath, destinationPath);
}
- const rewrittenDocs = applyPendingAssetReferenceRewrites(pendingRewrites, renamedAssets);
-
- void backlinkIndex.saveToDisk().catch((err) => {
- log.warn(
- { fromPath, toPath: destinationAssetPath, err },
- `[backlinks] Failed to persist asset rename cache for ${fromPath} -> ${destinationAssetPath}`,
- );
- });
+ const { rewrittenDocs, derivedMutations } = await applyPendingAssetReferenceRewrites(
+ pendingRewrites,
+ renamedAssets,
+ );
+ await recordDerivedMutationsBestEffort(derivedMutations, 'asset-rename');
signalChannel?.('files');
- if (rewrittenDocs.length > 0) {
- signalChannel?.('backlinks');
- signalChannel?.('graph');
- }
rewrittenDocs.sort((a, b) => a.docName.localeCompare(b.docName));
span.setAttribute('rename.rewrite_count', rewrittenDocs.length);
@@ -4265,7 +4373,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
},
},
async (span) => {
- if (!backlinkIndex) {
+ if (!derivedDocumentIndex) {
throw new BacklinkIndexRequiredError();
}
if (!isSupportedDocFile(fromPath) || isSupportedDocFile(toPath)) {
@@ -4363,7 +4471,6 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
renamePathOnDisk(sourcePath, destinationPath);
}
- backlinkIndex.deleteDocument(sourceDocName);
forgetDocExtension(sourceDocName);
mutateFileIndex?.({ kind: 'delete', path: sourcePath, docName: sourceDocName });
const destinationStat = statSync(destinationPath);
@@ -4376,19 +4483,16 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
inode: destinationStat.ino,
});
- rewrittenDocs = applyPendingAssetReferenceRewrites(pendingRewrites, renamedAssets);
-
- void backlinkIndex.saveToDisk().catch((err) => {
- log.warn(
- { fromPath, toPath, err },
- `[backlinks] Failed to persist document-to-file rename cache for ${fromPath} -> ${toPath}`,
- );
- });
+ const rewriteResult = await applyPendingAssetReferenceRewrites(
+ pendingRewrites,
+ renamedAssets,
+ );
+ rewrittenDocs = rewriteResult.rewrittenDocs;
+ await recordDerivedMutationsBestEffort(
+ [{ kind: 'delete', documentName: sourceDocName }, ...rewriteResult.derivedMutations],
+ 'document-to-file-rename',
+ );
signalChannel?.('files');
- if (rewrittenDocs.length > 0) {
- signalChannel?.('backlinks');
- signalChannel?.('graph');
- }
});
rewrittenDocs.sort((a, b) => a.docName.localeCompare(b.docName));
@@ -4431,7 +4535,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
},
},
async (span) => {
- if (!backlinkIndex) {
+ if (!derivedDocumentIndex) {
throw new BacklinkIndexRequiredError();
}
@@ -4506,7 +4610,6 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
// updated after the folder itself moves.
const pendingAssetRewrites = collectAssetReferenceRewritesForMappings(renamedAssets);
assertRewriteTargetsNotConflicted(pendingAssetRewrites.map((entry) => entry.docName));
- const rewrittenDocs: ManagedRenameRewrittenDoc[] = [];
if (kind === 'folder') {
const renamedWithGit = await renameTrackedPathInGit(
projectDir,
@@ -4519,19 +4622,11 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
renameFolderIndexEntries(fromPath, toPath);
signalChannel?.('files');
}
- rewrittenDocs.push(
- ...applyPendingAssetReferenceRewrites(pendingAssetRewrites, renamedAssets),
+ const { rewrittenDocs, derivedMutations } = await applyPendingAssetReferenceRewrites(
+ pendingAssetRewrites,
+ renamedAssets,
);
- if (rewrittenDocs.length > 0) {
- void backlinkIndex.saveToDisk().catch((err) => {
- log.warn(
- { fromPath, toPath, err },
- `[backlinks] Failed to persist managed rename cache for ${fromPath} -> ${toPath}`,
- );
- });
- signalChannel?.('backlinks');
- signalChannel?.('graph');
- }
+ await recordDerivedMutationsBestEffort(derivedMutations, 'asset-only-folder-rename');
rewrittenDocs.sort((a, b) => a.docName.localeCompare(b.docName));
return { renamed: [], renamedAssets, rewrittenDocs };
}
@@ -4544,7 +4639,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
const backlinkSourceSet = new Set<string>();
for (const { from } of affectedDocs) {
- for (const entry of backlinkIndex.getBacklinks(from)) {
+ for (const entry of await derivedDocumentIndex.getBacklinks(from)) {
if (!renameMap.has(entry.source)) {
backlinkSourceSet.add(entry.source);
}
@@ -4653,10 +4748,11 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
const rewrittenDocs: ManagedRenameRewrittenDoc[] = [];
const rewriteDocNames = [...rewriteDocNameSet].sort((a, b) => a.localeCompare(b));
+ const derivedMutations: DerivedDocumentIndexMutation[] = [];
await withManagedRenameRecovery(projectDir ?? contentDir, recoveryJournal, async () => {
for (const docName of missingBacklinkSources) {
- backlinkIndex.deleteDocument(docName);
+ derivedMutations.push({ kind: 'delete', documentName: docName });
}
for (const docName of rewriteDocNames) {
@@ -4676,7 +4772,11 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
rewrittenDocs.push({ docName, rewrites: rewritten.rewrites });
}
- backlinkIndex.updateDocumentFromMarkdown(docName, rewritten.markdown);
+ derivedMutations.push({
+ kind: 'link-rewrite',
+ documentName: docName,
+ markdown: rewritten.markdown,
+ });
}
// `captureAndCloseDocuments` sends an application-level
@@ -4913,11 +5013,17 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
content: renamedSource.markdown,
});
- backlinkIndex.renameDocument(fromDocName, toDocName, renamedSource.markdown);
+ derivedMutations.push({
+ kind: 'rename',
+ oldDocumentName: fromDocName,
+ newDocumentName: toDocName,
+ markdown: renamedSource.markdown,
+ });
if (renamedSource.rewrites > 0) {
rewrittenDocs.push({ docName: toDocName, rewrites: renamedSource.rewrites });
}
}
+ await recordDerivedMutationsBestEffort(derivedMutations, 'document-rename');
// Second crash-injection seam — fires AFTER the log append +
// AFTER the per-doc sync loop, BEFORE the implicit
@@ -4933,15 +5039,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
}
});
- void backlinkIndex.saveToDisk().catch((err) => {
- log.warn(
- { fromPath, toPath, err },
- `[backlinks] Failed to persist managed rename cache for ${fromPath} -> ${toPath}`,
- );
- });
signalChannel?.('files');
- signalChannel?.('backlinks');
- signalChannel?.('graph');
rewrittenDocs.sort((a, b) => a.docName.localeCompare(b.docName));
span.setAttribute('rename.rewrite_count', rewrittenDocs.length);
@@ -5636,8 +5734,8 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
// Orphan-hint nudge: if this doc now has zero backlinks and a
// plausible hub exists in its folder tree, suggest the hub. Soft —
- // agent can ignore. Silent when no backlinkIndex is wired.
- const hints = computeOrphanHints(resolvedDocName);
+ // agent can ignore. Silent when no relationship index is wired.
+ const hints = await computeOrphanHints(resolvedDocName);
// The converged post-write source (frontmatter region + body), read
// once and reused for both the mermaid render check and the broken-
@@ -5659,7 +5757,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
// the write (authoring a doc before its target exists is legitimate).
// The just-written doc is added to the admitted set so a valid self-link
// isn't falsely flagged before the file-watcher indexes it on disk.
- const admittedForLinks = collectAdmittedDocNames();
+ const admittedForLinks = await collectAdmittedDocNames();
admittedForLinks.add(resolvedDocName);
const brokenLinks = computeBrokenOutboundLinks(
writtenSource,
@@ -6087,7 +6185,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
// the whole batch. Every flushed batch doc joins the admitted set
// first so intra-batch links (doc A -> doc B written in the same
// call) validate regardless of entry order.
- const admittedForLinks = collectAdmittedDocNames();
+ const admittedForLinks = await collectAdmittedDocNames();
for (const p of pending) {
if (flushErrors.get(p.docName) === undefined) admittedForLinks.add(p.docName);
}
@@ -6445,7 +6543,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
// the body unchanged, so this reflects the doc's current body links —
// surfacing the same `brokenLinks` signal on every `edit` path keeps
// the contract uniform rather than returning a misleading empty `[]`.
- const admittedForLinks = collectAdmittedDocNames();
+ const admittedForLinks = await collectAdmittedDocNames();
admittedForLinks.add(resolvedDocName);
const brokenLinks = computeBrokenOutboundLinks(
session.dc.document.getText('source').toString(),
@@ -7193,7 +7291,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
const handleBacklinks = withValidation(
EmptyRequestSchema,
async (req, res) => {
- if (!backlinkIndex) {
+ if (!derivedDocumentIndex) {
errorResponse(
res,
503,
@@ -7218,7 +7316,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
});
return;
}
- const backlinks = backlinkIndex.getBacklinks(docName).map((entry) => ({
+ const backlinks = (await derivedDocumentIndex.getBacklinks(docName)).map((entry) => ({
source: entry.source,
anchor: entry.anchor,
title: readPageTitleForDocName(entry.source),
@@ -7232,9 +7330,9 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
{ handler: 'backlinks' },
);
} catch (e) {
- errorResponse(res, 500, 'urn:ok:error:internal-server-error', 'Failed to read backlinks.', {
+ respondToDerivedIndexQueryFailure(res, e, {
handler: 'backlinks',
- cause: e,
+ failureTitle: 'Failed to read backlinks.',
});
}
},
@@ -7251,7 +7349,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
const handleBacklinkCounts = withValidation(
EmptyRequestSchema,
async (req, res) => {
- if (!backlinkIndex) {
+ if (!derivedDocumentIndex) {
errorResponse(
res,
503,
@@ -7270,12 +7368,11 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
});
return;
}
- const counts: Record<string, number> = {};
- for (const docName of raw.split(',')) {
- const trimmed = docName.trim();
- if (!trimmed || !isSafeDocName(trimmed)) continue;
- counts[trimmed] = backlinkIndex.getBacklinkCount(trimmed);
- }
+ const docNames = raw
+ .split(',')
+ .map((docName) => docName.trim())
+ .filter((docName) => docName && isSafeDocName(docName));
+ const counts = await derivedDocumentIndex.getBacklinkCounts(docNames);
successResponse(
res,
200,
@@ -7284,13 +7381,10 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
{ handler: 'backlink-counts' },
);
} catch (e) {
- errorResponse(
- res,
- 500,
- 'urn:ok:error:internal-server-error',
- 'Failed to read backlink counts.',
- { handler: 'backlink-counts', cause: e },
- );
+ respondToDerivedIndexQueryFailure(res, e, {
+ handler: 'backlink-counts',
+ failureTitle: 'Failed to read backlink counts.',
+ });
}
},
{ handler: 'backlink-counts', method: 'GET', skipBodyParse: true },
@@ -7299,7 +7393,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
const handleForwardLinks = withValidation(
EmptyRequestSchema,
async (req, res) => {
- if (!backlinkIndex) {
+ if (!derivedDocumentIndex) {
errorResponse(
res,
503,
@@ -7324,40 +7418,38 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
});
return;
}
- const admitted = collectAdmittedDocNames();
+ const admitted = await collectAdmittedDocNames();
successResponse(
res,
200,
ForwardLinksSuccessSchema,
{
docName,
- forwardLinks: backlinkIndex.getForwardLinkEntries(docName).map((entry) =>
- entry.kind === 'doc'
- ? {
- kind: 'doc' as const,
- docName: entry.target,
- anchor: entry.anchor,
- title: readPageTitleForLinkedDocName(entry.target, admitted),
- snippet: entry.snippet,
- }
- : {
- kind: 'external' as const,
- url: entry.url,
- title: entry.label ?? entry.url,
- snippet: entry.snippet,
- },
+ forwardLinks: (await derivedDocumentIndex.getForwardLinkEntries(docName)).map(
+ (entry) =>
+ entry.kind === 'doc'
+ ? {
+ kind: 'doc' as const,
+ docName: entry.target,
+ anchor: entry.anchor,
+ title: readPageTitleForLinkedDocName(entry.target, admitted),
+ snippet: entry.snippet,
+ }
+ : {
+ kind: 'external' as const,
+ url: entry.url,
+ title: entry.label ?? entry.url,
+ snippet: entry.snippet,
+ },
),
},
{ handler: 'forward-links' },
);
} catch (e) {
- errorResponse(
- res,
- 500,
- 'urn:ok:error:internal-server-error',
- 'Failed to read forward links.',
- { handler: 'forward-links', cause: e },
- );
+ respondToDerivedIndexQueryFailure(res, e, {
+ handler: 'forward-links',
+ failureTitle: 'Failed to read forward links.',
+ });
}
},
{ handler: 'forward-links', method: 'GET', skipBodyParse: true },
@@ -7366,7 +7458,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
const handleLinkGraph = withValidation(
EmptyRequestSchema,
async (req, res) => {
- if (!backlinkIndex) {
+ if (!derivedDocumentIndex) {
errorResponse(
res,
503,
@@ -7398,7 +7490,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
return;
}
- let nodes: IndexedGraphNode[];
+ let nodes: DerivedGraphNode[];
let links: Array<{ source: string; target: string }>;
if (rawDegrees && docName) {
@@ -7414,12 +7506,15 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
return;
}
- ({ nodes, links } = backlinkIndex.getLinkGraphNeighborhood(docName, degrees));
+ ({ nodes, links } = await derivedDocumentIndex.getLinkGraphNeighborhood(
+ docName,
+ degrees,
+ ));
} else {
- ({ nodes, links } = backlinkIndex.getLinkGraph());
+ ({ nodes, links } = await derivedDocumentIndex.getLinkGraph());
}
- const admitted = collectAdmittedDocNames();
+ const admitted = await collectAdmittedDocNames();
const enrichedNodes = nodes.map((node) => {
if (node.kind === 'doc') {
const meta = readFrontmatterMetadataForLinkedDocName(node.docName, admitted);
@@ -7449,16 +7544,10 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
{ handler: 'link-graph' },
);
} catch (e) {
- errorResponse(
- res,
- 500,
- 'urn:ok:error:internal-server-error',
- 'Failed to read link graph.',
- {
- handler: 'link-graph',
- cause: e,
- },
- );
+ respondToDerivedIndexQueryFailure(res, e, {
+ handler: 'link-graph',
+ failureTitle: 'Failed to read link graph.',
+ });
}
},
{ handler: 'link-graph', method: 'GET', skipBodyParse: true },
@@ -7467,7 +7556,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
const handleOrphans = withValidation(
EmptyRequestSchema,
async (req, res) => {
- if (!backlinkIndex) {
+ if (!derivedDocumentIndex) {
errorResponse(
res,
503,
@@ -7480,7 +7569,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
try {
const url = new URL(req.url ?? '', 'http://localhost');
const mode = url.searchParams.get('mode') ?? 'both';
- if (!isOrphanMode(mode)) {
+ if (!isDerivedOrphanMode(mode)) {
errorResponse(
res,
400,
@@ -7491,21 +7580,18 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
return;
}
- const orphans = backlinkIndex
- .getOrphans([...getFileIndex().keys()], mode)
- .map((docName) => ({
- docName,
- title: readPageTitleForDocName(docName),
- }));
+ const orphans = (
+ await derivedDocumentIndex.getOrphans([...getFileIndex().keys()], mode)
+ ).map((docName) => ({
+ docName,
+ title: readPageTitleForDocName(docName),
+ }));
successResponse(res, 200, OrphansSuccessSchema, { orphans }, { handler: 'orphans' });
} catch (e) {
- errorResponse(
- res,
- 500,
- 'urn:ok:error:internal-server-error',
- 'Failed to read orphan pages.',
- { handler: 'orphans', cause: e },
- );
+ respondToDerivedIndexQueryFailure(res, e, {
+ handler: 'orphans',
+ failureTitle: 'Failed to read orphan pages.',
+ });
}
},
{ handler: 'orphans', method: 'GET', skipBodyParse: true },
@@ -7514,7 +7600,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
const handleHubs = withValidation(
EmptyRequestSchema,
async (req, res) => {
- if (!backlinkIndex) {
+ if (!derivedDocumentIndex) {
errorResponse(
res,
503,
@@ -7529,17 +7615,17 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
const rawLimit = url.searchParams.get('limit');
const parsed = rawLimit ? Number.parseInt(rawLimit, 10) : 20;
const limit = Number.isFinite(parsed) && parsed > 0 ? parsed : 20;
- const admitted = collectAdmittedDocNames();
- const hubs = backlinkIndex.getHubs(limit).map((hub) => ({
+ const admitted = await collectAdmittedDocNames();
+ const hubs = (await derivedDocumentIndex.getHubs(limit)).map((hub) => ({
docName: hub.docName,
title: readPageTitleForLinkedDocName(hub.docName, admitted),
count: hub.count,
}));
successResponse(res, 200, HubsSuccessSchema, { hubs }, { handler: 'hubs' });
} catch (e) {
- errorResponse(res, 500, 'urn:ok:error:internal-server-error', 'Failed to read hub pages.', {
+ respondToDerivedIndexQueryFailure(res, e, {
handler: 'hubs',
- cause: e,
+ failureTitle: 'Failed to read hub pages.',
});
}
},
@@ -7549,7 +7635,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
const handleDeadLinks = withValidation(
EmptyRequestSchema,
async (req, res) => {
- if (!backlinkIndex) {
+ if (!derivedDocumentIndex) {
errorResponse(
res,
503,
@@ -7572,8 +7658,8 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
const sourceDocNameFilter = sourceDocNames.length
? [...new Set(sourceDocNames.map((docName) => resolveAlias(docName)))]
: undefined;
- const deadLinks = backlinkIndex.getDeadLinks(
- collectAdmittedDocNames(),
+ const deadLinks = await derivedDocumentIndex.getDeadLinks(
+ await collectAdmittedDocNames(),
sourceDocNameFilter,
);
@@ -7594,13 +7680,10 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
{ handler: 'dead-links' },
);
} catch (e) {
- errorResponse(
- res,
- 500,
- 'urn:ok:error:internal-server-error',
- 'Failed to read dead links.',
- { handler: 'dead-links', cause: e },
- );
+ respondToDerivedIndexQueryFailure(res, e, {
+ handler: 'dead-links',
+ failureTitle: 'Failed to read dead links.',
+ });
}
},
{ handler: 'dead-links', method: 'GET', skipBodyParse: true },
@@ -7942,7 +8025,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
// Write-time outbound-link validation — synchronous, from the
// just-edited source bytes; see handleAgentWriteMd for the full why.
- const admittedForLinks = collectAdmittedDocNames();
+ const admittedForLinks = await collectAdmittedDocNames();
admittedForLinks.add(docName);
const brokenLinks = computeBrokenOutboundLinks(
patchedSource,
@@ -8436,17 +8519,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
const doc = hocuspocus.documents.get(docName);
if (doc) await (forceUnloadDocument ?? hocuspocus.unloadDocument.bind(hocuspocus))(doc);
writeFileSync(filePath, '', 'utf-8');
- if (backlinkIndex) {
- backlinkIndex.deleteDocument(docName);
- void backlinkIndex.saveToDisk().catch((err) => {
- log.warn(
- { docName, err },
- `[backlinks] Failed to persist cache after test-reset for ${docName}`,
- );
- });
- signalChannel?.('backlinks');
- signalChannel?.('graph');
- }
+ await derivedDocumentIndex?.testOnly?.resetDocumentForTest(docName);
// Also reset the project-root .okignore synthetic doc + on-disk file
// unless the caller explicitly opts out. Without this, patterns added
@@ -8510,7 +8583,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
* backlink-dependent integration tests (e.g. `agent-focus-wiring.test.ts`
* orphan-hint shape) cannot otherwise recover from.
*
- * This endpoint forces `backlinkIndex.rebuildFromDisk()` — authoritative
+ * This endpoint forces an authoritative relationship-index rescan from disk,
* resync from the filesystem that covers dropped events. It is NOT suitable
* for production: rebuild wipes any in-memory backlink state not yet
* debounced to disk (e.g. a live agent-write awaiting persistence). Gated
@@ -8520,7 +8593,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
EmptyRequestSchema,
async (_req, res) => {
try {
- if (!backlinkIndex) {
+ if (!derivedDocumentIndex?.testOnly) {
errorResponse(
res,
503,
@@ -8530,16 +8603,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
);
return;
}
- await backlinkIndex.rebuildFromDisk();
- // A full rebuild replaces branch state, dropping the out-of-contentDir
- // global skill bundle nodes — re-register them (node-only, within-bundle)
- // so a forced rescan keeps the global skill graph intact.
- await backlinkIndex.ingestGlobalSkillBundles([resolve(skillsHome, '.ok', 'skills')]);
- void backlinkIndex.saveToDisk().catch((err) => {
- log.warn({ err }, '[backlinks] Failed to persist cache after test-rescan-backlinks');
- });
- signalChannel?.('backlinks');
- signalChannel?.('graph');
+ await derivedDocumentIndex.testOnly.rescanBacklinksForTest();
successResponse(
res,
200,
@@ -10478,17 +10542,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
}
}
mutateFileIndex?.({ kind: 'create', path: fullPath, docName, content: initialContent });
- if (backlinkIndex) {
- backlinkIndex.updateDocumentFromMarkdown(docName, initialContent);
- void backlinkIndex.saveToDisk().catch((err) => {
- log.warn(
- { docName, err },
- `[backlinks] Failed to persist create-page cache for ${docName}`,
- );
- });
- signalChannel?.('backlinks');
- signalChannel?.('graph');
- }
+ await recordDerivedDocumentBestEffort(docName, initialContent, 'create-page');
signalChannel?.('files');
if (templateScopeForLog !== undefined) {
// Cardinality-bounded structured event — `templateScope` is one of
@@ -10770,7 +10824,6 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
docName: duplicatedPath,
content,
});
- backlinkIndex?.updateDocumentFromMarkdown(duplicatedPath, content);
duplicatedDocNames = [duplicatedPath];
} catch (err) {
try {
@@ -10792,6 +10845,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
});
throw err;
}
+ await recordDerivedDocumentBestEffort(duplicatedPath, content, 'duplicate-path-file');
} else {
const next = nextAvailableDuplicateFolderPath(contentDir, requestedPath);
duplicatedPath = next.folderPath;
@@ -10806,6 +10860,13 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
return;
}
const destinationPath = resolveContentEntryPath(contentDir, 'folder', duplicatedPath);
+ const copiedDocRollbackLedger: Array<{
+ docName: string;
+ fullPath: string;
+ extensionRegistered: boolean;
+ dirCountIncremented: boolean;
+ fileIndexRegistered: boolean;
+ }> = [];
try {
tracedCpSync(sourcePath, destinationPath, {
recursive: true,
@@ -10825,6 +10886,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
}
throw err;
}
+ const derivedMutations: DerivedDocumentIndexMutation[] = [];
try {
for (const folderPath of collectFolderPaths(contentDir, duplicatedPath)) {
upsertFolderIndexPathSegments(folderPath);
@@ -10832,22 +10894,60 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
const copiedDocs = collectMarkdownCopies(contentDir, duplicatedPath);
duplicatedDocNames = copiedDocs.map((doc) => doc.docName);
for (const doc of copiedDocs) {
+ const rollbackEntry = {
+ docName: doc.docName,
+ fullPath: doc.fullPath,
+ extensionRegistered: false,
+ dirCountIncremented: false,
+ fileIndexRegistered: false,
+ };
+ copiedDocRollbackLedger.push(rollbackEntry);
const sourceExtension = extname(doc.fullPath);
registerDocExtension(stripDocExtension(doc.docName), sourceExtension);
+ rollbackEntry.extensionRegistered = true;
recentlyRemovedDocs?.delete(doc.docName);
if (contentFilter) {
contentFilter.incrementMdDir(dirname(doc.docName));
+ rollbackEntry.dirCountIncremented = true;
}
registerWrite(doc.fullPath, contentHash(doc.content));
+ rollbackEntry.fileIndexRegistered = true;
mutateFileIndex?.({
kind: 'create',
path: doc.fullPath,
docName: doc.docName,
content: doc.content,
});
- backlinkIndex?.updateDocumentFromMarkdown(doc.docName, doc.content);
+ derivedMutations.push({
+ kind: 'upsert',
+ documentName: doc.docName,
+ markdown: doc.content,
+ });
}
} catch (err) {
+ for (const rollbackEntry of copiedDocRollbackLedger.reverse()) {
+ if (rollbackEntry.fileIndexRegistered) {
+ try {
+ mutateFileIndex?.({
+ kind: 'delete',
+ path: rollbackEntry.fullPath,
+ docName: rollbackEntry.docName,
+ });
+ } catch (rollbackErr) {
+ log.warn(
+ { docName: rollbackEntry.docName, err: rollbackErr },
+ '[duplicate-path] failed to roll back copied file-index row',
+ );
+ }
+ }
+ if (rollbackEntry.dirCountIncremented) {
+ contentFilter?.decrementMdDir(dirname(rollbackEntry.docName));
+ }
+ if (rollbackEntry.extensionRegistered) {
+ forgetDocExtension(stripDocExtension(rollbackEntry.docName));
+ }
+ }
+ removeFolderIndexEntries(duplicatedPath);
try {
tracedRmSync(destinationPath, { recursive: true, force: true });
} catch (cleanupErr) {
@@ -10858,6 +10958,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
}
throw err;
}
+ await recordDerivedMutationsBestEffort(derivedMutations, 'duplicate-path-folder');
}
switch (actor.kind) {
@@ -10884,13 +10985,6 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
}
}
- if (backlinkIndex && duplicatedDocNames.length > 0) {
- void backlinkIndex.saveToDisk().catch((err) => {
- log.warn({ err }, '[backlinks] Failed to persist duplicate-path cache');
- });
- signalChannel?.('backlinks');
- signalChannel?.('graph');
- }
signalChannel?.('files');
successResponse(
res,
@@ -11499,6 +11593,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
docName,
});
}
+ await deleteDerivedDocumentsBestEffort(deletedDocNames, 'delete-path');
signalChannel?.('files');
successResponse(
@@ -11667,6 +11762,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
if (operationKind === 'folder') {
removeFolderIndexEntries(path);
}
+ await deleteDerivedDocumentsBestEffort(deletedDocNames, 'trash-cleanup');
// Synchronous CC1 emit closes the race where the renderer expects
// the updated tree right after the response. The watcher's later
@@ -14440,7 +14536,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
const handleTagsList = withValidation(
EmptyRequestSchema,
async (_req, res) => {
- if (!tagIndex) {
+ if (!derivedDocumentIndex) {
errorResponse(
res,
503,
@@ -14451,12 +14547,12 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
return;
}
try {
- const tags = tagIndex.getAllTags();
+ const tags = await derivedDocumentIndex.getAllTags();
successResponse(res, 200, TagsListSuccessSchema, { tags }, { handler: 'tags-list' });
} catch (e) {
- errorResponse(res, 500, 'urn:ok:error:internal-server-error', 'Failed to read tags.', {
+ respondToDerivedIndexQueryFailure(res, e, {
handler: 'tags-list',
- cause: e,
+ failureTitle: 'Failed to read tags.',
});
}
},
@@ -14475,7 +14571,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
});
return;
}
- if (!tagIndex) {
+ if (!derivedDocumentIndex) {
errorResponse(
res,
503,
@@ -14501,12 +14597,14 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
return;
}
try {
- const docs = tagIndex.getDocsForTagWithMatches(name).map(({ docName, matchingTags }) => ({
- docName,
- title: readPageTitleForDocName(docName),
- matchingTags,
- snippet: null,
- }));
+ const docs = (await derivedDocumentIndex.getDocsForTagWithMatches(name)).map(
+ ({ docName, matchingTags }) => ({
+ docName,
+ title: readPageTitleForDocName(docName),
+ matchingTags,
+ snippet: null,
+ }),
+ );
successResponse(
res,
200,
@@ -14517,13 +14615,10 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
},
);
} catch (e) {
- errorResponse(
- res,
- 500,
- 'urn:ok:error:internal-server-error',
- 'Failed to read tag membership.',
- { handler: 'tags-for-name', cause: e },
- );
+ respondToDerivedIndexQueryFailure(res, e, {
+ handler: 'tags-for-name',
+ failureTitle: 'Failed to read tag membership.',
+ });
}
}
@@ -16425,12 +16520,8 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
// entries. Global skills live outside the project graph (not content
// docs), so they have nothing to re-index.
if (body.scope === 'project' && !contentEditError) {
- // Best-effort: the git-mv + SKILL.md rewrite already succeeded, so a
- // re-index failure (e.g. a `readFileSync` racing a relocated file)
- // must NOT turn a successful rename into a 500. The next open/rescan
- // re-indexes the moved docs from disk.
try {
- reindexMovedProjectSkillDocs(skillsRoot, body.fromName, body.toName);
+ await reindexMovedProjectSkillDocs(skillsRoot, body.fromName, body.toName);
} catch (err) {
getLogger('skill-move').warn(
{ err, fromName: body.fromName, toName: body.toName },
@@ -16593,46 +16684,47 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
* content docs never desync — disk stays the truth on the next open), and no
* session churn against the just-moved dir.
*/
- function reindexMovedProjectSkillDocs(
+ async function reindexMovedProjectSkillDocs(
skillsRoot: string,
fromName: string,
toName: string,
- ): void {
- if (!backlinkIndex) return;
- const reindexOne = (oldDocName: string, newDocName: string, absFile: string): void => {
+ ): Promise<void> {
+ if (!derivedDocumentIndex) return;
+ const derivedMutations: DerivedDocumentIndexMutation[] = [];
+ const collectReindex = (oldDocName: string, newDocName: string, absFile: string): void => {
let markdown: string;
try {
markdown = readFileSync(absFile, 'utf-8');
} catch {
// Unreadable relocated file: drop the stale old-name entry rather than
// leave it dangling (the next open will index it fresh from disk).
- backlinkIndex.deleteDocument(oldDocName);
- tagIndex?.deleteDocument(oldDocName);
+ derivedMutations.push({ kind: 'delete', documentName: oldDocName });
return;
}
- backlinkIndex.renameDocument(oldDocName, newDocName, markdown);
- tagIndex?.renameDocument(oldDocName, newDocName, markdown);
+ derivedMutations.push({
+ kind: 'rename',
+ oldDocumentName: oldDocName,
+ newDocumentName: newDocName,
+ markdown,
+ });
};
// SKILL.md: its rewrite during the move is fs-direct (applySkillWrite), so
// it never re-enters the index via a CRDT write either. The reference `.md`
// files were git-mv'd verbatim — never rewritten — so they too are stale.
- reindexOne(
+ collectReindex(
projectSkillContentDocName(fromName),
projectSkillContentDocName(toName),
resolve(skillsRoot, toName, 'SKILL.md'),
);
for (const rel of listProjectMdReferences(skillsRoot, toName)) {
- reindexOne(
+ collectReindex(
projectRefContentDocName(fromName, rel),
projectRefContentDocName(toName, rel),
resolve(skillsRoot, toName, rel),
);
}
- // Nudge any client that isn't holding the moved docs open to refresh its
- // graph promptly. (`tags` rides the live-derived-index path, not here.)
- signalChannel?.('backlinks');
- signalChannel?.('graph');
+ await derivedDocumentIndex.recordDirectMutations(derivedMutations);
}
const handleSkillFileGet = withValidation(
@@ -19525,13 +19617,13 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
// (idempotent: the debounced pass re-applies the same bytes).
let linkFindings: ValidationDiagnostic[] = [];
const linksSetting = getLinksValidationSetting?.() ?? DEFAULT_LINKS_VALIDATION;
- if (backlinkIndex && linksSetting !== 'off' && !isLinkIndexExcludedDoc(docName)) {
- backlinkIndex.updateDocumentFromMarkdown(docName, source);
+ if (derivedDocumentIndex && linksSetting !== 'off' && !isLinkIndexExcludedDoc(docName)) {
+ await recordDerivedLinkRewriteBestEffort(docName, source, 'lint-validation');
const linksValidator = createProjectValidators({
projectDir: projectDir ?? contentDir,
contentDir,
baseConfig: base,
- backlinkIndex,
+ derivedDocumentIndex,
linksValidation: linksSetting,
admittedDocNames: collectAdmittedDocNames,
docFilePathFor: (d) => resolveDocFilePath(contentDir, d),
@@ -19955,7 +20047,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
);
// Unified validation audit: every registered project validator (markdownlint
- // walk + backlink-index dead-link read) merged into one source-tagged plane.
+ // walk + derived-index dead-link read) merged into one source-tagged plane.
// Additive alongside /api/lint/audit and /api/dead-links, which keep their
// single-validator contracts.
const handleAudit = withValidation(
@@ -19997,7 +20089,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
contentDir,
baseConfig,
liveSourceFor: liveLintSourceFor,
- backlinkIndex: backlinkIndex ?? null,
+ derivedDocumentIndex: derivedDocumentIndex ?? null,
linksValidation: getLinksValidationSetting?.(),
admittedDocNames: collectAdmittedDocNames,
docFilePathFor: (docName) => resolveDocFilePath(contentDir, docName),
diff --git a/packages/server/src/api-file-ops.test.ts b/packages/server/src/api-file-ops.test.ts
index c658c4d63..f05850a82 100644
--- a/packages/server/src/api-file-ops.test.ts
+++ b/packages/server/src/api-file-ops.test.ts
@@ -15,11 +15,15 @@ import { join, resolve } from 'node:path';
import { Readable } from 'node:stream';
import { Hocuspocus } from '@hocuspocus/server';
import simpleGit from 'simple-git';
-import { afterEach, describe, expect, test } from 'vitest';
+import { afterEach, describe, expect, test, vi } from 'vitest';
import type * as Y from 'yjs';
-import { createApiExtension } from './api-extension.test-helper.ts';
+import {
+ createApiExtension,
+ createDerivedDocumentIndexApiPortStub,
+} from './api-extension.test-helper.ts';
import { BacklinkIndex } from './backlink-index.ts';
import { type ContentFilter, createContentFilter } from './content-filter.ts';
+import type { DerivedDocumentIndexApiPort } from './derived-document-index.ts';
import { _resetDocExtensionsForTests, getDocExtension } from './doc-extensions.ts';
import type { FileIndexEntry, FolderIndexEntry } from './file-watcher.ts';
@@ -125,6 +129,7 @@ type CallApiOptions = {
signalChannel?: Parameters<typeof createApiExtension>[0]['signalChannel'];
projectDir?: string;
contentFilter?: ContentFilter;
+ derivedDocumentIndex?: DerivedDocumentIndexApiPort;
};
async function createTestApiExtension(contentDir: string, options?: CallApiOptions) {
@@ -155,6 +160,19 @@ async function createTestApiExtension(contentDir: string, options?: CallApiOptio
signalChannel: options?.signalChannel,
projectDir: options?.projectDir,
contentFilter: options?.contentFilter,
+ derivedDocumentIndex: options?.derivedDocumentIndex,
+ });
+}
+
+function createClosedDerivedDocumentIndexPort(): DerivedDocumentIndexApiPort {
+ const rejectClosed = async (): Promise<never> => {
+ const error = new Error('Derived document index is closed');
+ error.name = 'DerivedDocumentIndexClosedError';
+ throw error;
+ };
+ return createDerivedDocumentIndexApiPortStub({
+ recordDirectMutations: rejectClosed,
+ recordDirectDocument: rejectClosed,
});
}
@@ -203,6 +221,7 @@ function setupTmpDir(): string {
}
afterEach(() => {
+ vi.restoreAllMocks();
_resetDocExtensionsForTests();
if (tmpDir) {
rmSync(tmpDir, { recursive: true, force: true });
@@ -971,6 +990,27 @@ describe('file operation API routes', () => {
expect(body.renamed).toEqual([{ fromDocName: 'notes', toDocName: 'renamed-notes' }]);
});
+ test('keeps a document rename successful when the derived index closes after disk commit', async () => {
+ const dir = setupTmpDir();
+ writeFileSync(join(dir, 'notes.md'), '# Notes\n', 'utf-8');
+
+ const result = await callApi(
+ dir,
+ '/api/rename-path',
+ 'POST',
+ {
+ kind: 'file',
+ fromPath: 'notes',
+ toPath: 'renamed-notes',
+ },
+ { derivedDocumentIndex: createClosedDerivedDocumentIndexPort() },
+ );
+
+ expect(result.status).toBe(200);
+ expect(existsSync(join(dir, 'notes.md'))).toBe(false);
+ expect(readFileSync(join(dir, 'renamed-notes.md'), 'utf-8')).toBe('# Notes\n');
+ });
+
test('renames a folder and returns descendant mappings', async () => {
const dir = setupTmpDir();
mkdirSync(join(dir, 'docs/nested'), { recursive: true });
@@ -1283,6 +1323,28 @@ describe('file operation API routes', () => {
expect(body.deletedDocNames).toEqual(['trash-me']);
});
+ test('keeps a durable delete successful when relationship projection fails', async () => {
+ const dir = setupTmpDir();
+ writeFileSync(join(dir, 'trash-me.md'), '# Delete me\n', 'utf-8');
+ const failingBacklinkIndex = {
+ deleteDocument: () => {
+ throw new Error('projection failed');
+ },
+ } as unknown as BacklinkIndex;
+
+ const result = await callApi(
+ dir,
+ '/api/delete-path',
+ 'POST',
+ { kind: 'file', path: 'trash-me' },
+ { backlinkIndex: failingBacklinkIndex },
+ );
+
+ expect(result.status).toBe(200);
+ expect(existsSync(join(dir, 'trash-me.md'))).toBe(false);
+ expect(JSON.parse(result.body)).toMatchObject({ deletedDocNames: ['trash-me'] });
+ });
+
test('delete accepts asset-shaped markdown paths as document deletes', async () => {
const dir = setupTmpDir();
writeFileSync(join(dir, 'trash-me.md'), '# Delete me\n', 'utf-8');
@@ -1372,6 +1434,32 @@ describe('file operation API routes', () => {
expect(body.rewrittenDocs).toEqual([{ docName: 'docs/guide', rewrites: 5 }]);
});
+ test('keeps an asset rename successful when the derived index closes after disk commit', async () => {
+ const dir = setupTmpDir();
+ mkdirSync(join(dir, 'docs/media'), { recursive: true });
+ writeFileSync(join(dir, 'docs/media/diagram.png'), 'fake image bytes', 'utf-8');
+ writeFileSync(join(dir, 'docs/guide.md'), '![diagram](./media/diagram.png)\n', 'utf-8');
+
+ const result = await callApi(
+ dir,
+ '/api/rename-path',
+ 'POST',
+ {
+ kind: 'asset',
+ fromPath: 'docs/media/diagram.png',
+ toPath: 'docs/assets/diagram.png',
+ },
+ { derivedDocumentIndex: createClosedDerivedDocumentIndexPort() },
+ );
+
+ expect(result.status).toBe(200);
+ expect(existsSync(join(dir, 'docs/media/diagram.png'))).toBe(false);
+ expect(readFileSync(join(dir, 'docs/assets/diagram.png'), 'utf-8')).toBe('fake image bytes');
+ expect(readFileSync(join(dir, 'docs/guide.md'), 'utf-8')).toBe(
+ '![diagram](./assets/diagram.png)\n',
+ );
+ });
+
test('asset rename rejects destinations excluded by content config', async () => {
const dir = setupTmpDir();
mkdirSync(join(dir, 'docs/media'), { recursive: true });
@@ -1475,6 +1563,9 @@ describe('file operation API routes', () => {
mkdirSync(join(dir, 'docs/media'), { recursive: true });
writeFileSync(join(dir, 'docs/guide.md'), '# Guide\n', 'utf-8');
writeFileSync(join(dir, 'docs/index.md'), '[Guide](./guide.md)\n', 'utf-8');
+ const backlinkIndex = await buildBacklinkIndex(dir);
+ const backlinkSave = vi.spyOn(backlinkIndex, 'saveToDisk');
+ const signals: string[] = [];
const result = await callApi(
dir,
@@ -1485,7 +1576,10 @@ describe('file operation API routes', () => {
fromPath: 'docs/guide.md',
toPath: 'docs/media/guide.custom',
},
- { backlinkIndex: buildBacklinkIndex(dir) },
+ {
+ backlinkIndex,
+ signalChannel: (channel) => signals.push(channel),
+ },
);
expect(result.status).toBe(200);
@@ -1507,6 +1601,12 @@ describe('file operation API routes', () => {
},
]);
expect(body.rewrittenDocs).toEqual([{ docName: 'docs/index', rewrites: 1 }]);
+ expect(backlinkSave).toHaveBeenCalledTimes(1);
+ expect(signals.filter((channel) => channel !== 'files')).toEqual([
+ 'backlinks',
+ 'graph',
+ 'tags',
+ ]);
});
test('document-to-file rename carries live Y.Doc content to the destination file', async () => {
@@ -1545,6 +1645,27 @@ describe('file operation API routes', () => {
}
});
+ test('keeps a document-to-file rename successful when the derived index closes', async () => {
+ const dir = setupTmpDir();
+ writeFileSync(join(dir, 'guide.md'), '# Guide\n', 'utf-8');
+
+ const result = await callApi(
+ dir,
+ '/api/rename-path',
+ 'POST',
+ {
+ kind: 'asset',
+ fromPath: 'guide.md',
+ toPath: 'guide.txt',
+ },
+ { derivedDocumentIndex: createClosedDerivedDocumentIndexPort() },
+ );
+
+ expect(result.status).toBe(200);
+ expect(existsSync(join(dir, 'guide.md'))).toBe(false);
+ expect(readFileSync(join(dir, 'guide.txt'), 'utf-8')).toBe('# Guide\n');
+ });
+
test('document-to-file rename returns 404 when the source document is missing', async () => {
const dir = setupTmpDir();
@@ -1931,6 +2052,30 @@ describe('file operation API routes', () => {
expect(body.rewrittenDocs).toEqual([{ docName: 'index', rewrites: 2 }]);
});
+ test('keeps an asset-only folder rename successful when the derived index closes', async () => {
+ const dir = setupTmpDir();
+ mkdirSync(join(dir, 'media'), { recursive: true });
+ writeFileSync(join(dir, 'media/diagram.png'), 'fake image bytes', 'utf-8');
+ writeFileSync(join(dir, 'index.md'), '![diagram](./media/diagram.png)\n', 'utf-8');
+
+ const result = await callApi(
+ dir,
+ '/api/rename-path',
+ 'POST',
+ {
+ kind: 'folder',
+ fromPath: 'media',
+ toPath: 'assets',
+ },
+ { derivedDocumentIndex: createClosedDerivedDocumentIndexPort() },
+ );
+
+ expect(result.status).toBe(200);
+ expect(existsSync(join(dir, 'media'))).toBe(false);
+ expect(readFileSync(join(dir, 'assets/diagram.png'), 'utf-8')).toBe('fake image bytes');
+ expect(readFileSync(join(dir, 'index.md'), 'utf-8')).toBe('![diagram](./assets/diagram.png)\n');
+ });
+
test('folder rename rewrites references to assets that move with markdown docs', async () => {
const dir = setupTmpDir();
mkdirSync(join(dir, 'docs/media'), { recursive: true });
@@ -2366,15 +2511,13 @@ describe('file operation API routes', () => {
expect(body.duplicatedDocNames).toEqual(['archive copy/index', 'archive copy/old/entry']);
});
- test('removes a copied folder when duplicate-path post-copy registration fails', async () => {
+ test('keeps a copied folder when the derived index closes after registration', async () => {
const dir = setupTmpDir();
mkdirSync(join(dir, 'archive'), { recursive: true });
- writeFileSync(join(dir, 'archive/index.md'), '# Archive\n', 'utf-8');
- const failingBacklinkIndex = {
- updateDocumentFromMarkdown: () => {
- throw new Error('registration failed');
- },
- } as unknown as BacklinkIndex;
+ writeFileSync(join(dir, 'archive/index.mdx'), '# Archive\n', 'utf-8');
+ const fileIndex = new Map(buildFileIndex(dir));
+ const folderIndex = new Map(buildFolderIndex(dir));
+ const contentFilter = createContentFilter({ projectDir: dir, contentDir: dir });
const result = await callApi(
dir,
@@ -2384,55 +2527,42 @@ describe('file operation API routes', () => {
kind: 'folder',
path: 'archive',
},
- { backlinkIndex: failingBacklinkIndex, getFileIndex: () => new Map() },
- );
-
- expect(result.status).toBe(500);
- expect(existsSync(join(dir, 'archive copy'))).toBe(false);
- expect(JSON.parse(result.body)).toMatchObject({
- type: 'urn:ok:error:internal-server-error',
- title: 'Failed to duplicate path.',
- status: 500,
- });
- });
-
- test('removes a copied file when duplicate-path post-copy registration fails', async () => {
- const dir = setupTmpDir();
- writeFileSync(join(dir, 'notes.md'), '# Notes\n', 'utf-8');
- const failingBacklinkIndex = {
- updateDocumentFromMarkdown: () => {
- throw new Error('registration failed');
- },
- } as unknown as BacklinkIndex;
-
- const result = await callApi(
- dir,
- '/api/duplicate-path',
- 'POST',
{
- kind: 'file',
- path: 'notes',
+ derivedDocumentIndex: createClosedDerivedDocumentIndexPort(),
+ contentFilter,
+ getFileIndex: () => fileIndex,
+ getFolderIndex: () => folderIndex,
+ mutateFileIndex: (event) => {
+ if (event.kind === 'create') {
+ fileIndex.set(event.docName, {
+ size: event.content.length,
+ modified: new Date(0).toISOString(),
+ canonicalPath: event.path,
+ inode: 0,
+ aliases: [],
+ });
+ } else if (event.kind === 'delete') {
+ fileIndex.delete(event.docName);
+ }
+ },
},
- { backlinkIndex: failingBacklinkIndex, getFileIndex: () => new Map() },
);
- expect(result.status).toBe(500);
- expect(existsSync(join(dir, 'notes copy.md'))).toBe(false);
+ expect(result.status).toBe(200);
+ expect(readFileSync(join(dir, 'archive copy/index.mdx'), 'utf-8')).toBe('# Archive\n');
+ expect(fileIndex.has('archive copy/index')).toBe(true);
+ expect([...folderIndex.keys()].some((path) => path.startsWith('archive copy'))).toBe(true);
+ expect(getDocExtension('archive copy/index')).toBe('.mdx');
expect(JSON.parse(result.body)).toMatchObject({
- type: 'urn:ok:error:internal-server-error',
- title: 'Failed to duplicate path.',
- status: 500,
+ kind: 'folder',
+ path: 'archive copy',
+ duplicatedDocNames: ['archive copy/index'],
});
});
- test('forgets a copied file extension when duplicate-path post-copy registration fails', async () => {
+ test('keeps a copied file when the derived index closes after registration', async () => {
const dir = setupTmpDir();
writeFileSync(join(dir, 'notes.mdx'), '# Notes\n', 'utf-8');
- const failingBacklinkIndex = {
- updateDocumentFromMarkdown: () => {
- throw new Error('registration failed');
- },
- } as unknown as BacklinkIndex;
const result = await callApi(
dir,
@@ -2442,88 +2572,26 @@ describe('file operation API routes', () => {
kind: 'file',
path: 'notes',
},
- { backlinkIndex: failingBacklinkIndex, getFileIndex: () => new Map() },
- );
-
- expect(result.status).toBe(500);
- expect(existsSync(join(dir, 'notes copy.mdx'))).toBe(false);
- expect(getDocExtension('notes copy')).toBe('.md');
- });
-
- test('returns 507 when duplicate-path registration hits storage exhaustion', async () => {
- const dir = setupTmpDir();
- mkdirSync(join(dir, 'archive'), { recursive: true });
- writeFileSync(join(dir, 'archive/index.md'), '# Archive\n', 'utf-8');
- const fullDiskBacklinkIndex = {
- updateDocumentFromMarkdown: () => {
- const err = new Error('no space left on device') as NodeJS.ErrnoException;
- err.code = 'ENOSPC';
- throw err;
- },
- } as unknown as BacklinkIndex;
-
- const result = await callApi(
- dir,
- '/api/duplicate-path',
- 'POST',
{
- kind: 'folder',
- path: 'archive',
+ derivedDocumentIndex: createClosedDerivedDocumentIndexPort(),
+ getFileIndex: () => new Map(),
},
- { backlinkIndex: fullDiskBacklinkIndex, getFileIndex: () => new Map() },
);
- expect(result.status).toBe(507);
- expect(existsSync(join(dir, 'archive copy'))).toBe(false);
- expect(JSON.parse(result.body)).toMatchObject({
- type: 'urn:ok:error:storage-full',
- title: 'Could not duplicate path because storage is full.',
- status: 507,
- });
- });
-
- test('returns 507 when duplicate-path file registration hits storage exhaustion', async () => {
- const dir = setupTmpDir();
- writeFileSync(join(dir, 'notes.md'), '# Notes\n', 'utf-8');
- const fullDiskBacklinkIndex = {
- updateDocumentFromMarkdown: () => {
- const err = new Error('no space left on device') as NodeJS.ErrnoException;
- err.code = 'ENOSPC';
- throw err;
- },
- } as unknown as BacklinkIndex;
-
- const result = await callApi(
- dir,
- '/api/duplicate-path',
- 'POST',
- {
- kind: 'file',
- path: 'notes',
- },
- { backlinkIndex: fullDiskBacklinkIndex, getFileIndex: () => new Map() },
- );
-
- expect(result.status).toBe(507);
- expect(existsSync(join(dir, 'notes copy.md'))).toBe(false);
+ expect(result.status).toBe(200);
+ expect(readFileSync(join(dir, 'notes copy.mdx'), 'utf-8')).toBe('# Notes\n');
+ expect(getDocExtension('notes copy')).toBe('.mdx');
expect(JSON.parse(result.body)).toMatchObject({
- type: 'urn:ok:error:storage-full',
- title: 'Could not duplicate path because storage is full.',
- status: 507,
+ kind: 'file',
+ path: 'notes copy',
+ duplicatedDocNames: ['notes copy'],
});
});
- test('returns actionable storage-readonly when duplicate-path registration hits permissions', async () => {
+ test('rolls back a copied folder when synchronous registration fails', async () => {
const dir = setupTmpDir();
mkdirSync(join(dir, 'archive'), { recursive: true });
writeFileSync(join(dir, 'archive/index.md'), '# Archive\n', 'utf-8');
- const readOnlyBacklinkIndex = {
- updateDocumentFromMarkdown: () => {
- const err = new Error('permission denied') as NodeJS.ErrnoException;
- err.code = 'EACCES';
- throw err;
- },
- } as unknown as BacklinkIndex;
const result = await callApi(
dir,
@@ -2533,28 +2601,20 @@ describe('file operation API routes', () => {
kind: 'folder',
path: 'archive',
},
- { backlinkIndex: readOnlyBacklinkIndex, getFileIndex: () => new Map() },
+ {
+ mutateFileIndex: (event) => {
+ if (event.kind === 'create') throw new Error('registration failed');
+ },
+ },
);
expect(result.status).toBe(500);
expect(existsSync(join(dir, 'archive copy'))).toBe(false);
- expect(JSON.parse(result.body)).toMatchObject({
- type: 'urn:ok:error:storage-readonly',
- title: 'Could not duplicate path because storage is not writable.',
- status: 500,
- });
});
- test('returns actionable storage-readonly when duplicate-path file registration hits permissions', async () => {
+ test('rolls back a copied file when synchronous registration fails', async () => {
const dir = setupTmpDir();
writeFileSync(join(dir, 'notes.md'), '# Notes\n', 'utf-8');
- const readOnlyBacklinkIndex = {
- updateDocumentFromMarkdown: () => {
- const err = new Error('permission denied') as NodeJS.ErrnoException;
- err.code = 'EACCES';
- throw err;
- },
- } as unknown as BacklinkIndex;
const result = await callApi(
dir,
@@ -2564,16 +2624,15 @@ describe('file operation API routes', () => {
kind: 'file',
path: 'notes',
},
- { backlinkIndex: readOnlyBacklinkIndex, getFileIndex: () => new Map() },
+ {
+ mutateFileIndex: (event) => {
+ if (event.kind === 'create') throw new Error('registration failed');
+ },
+ },
);
expect(result.status).toBe(500);
expect(existsSync(join(dir, 'notes copy.md'))).toBe(false);
- expect(JSON.parse(result.body)).toMatchObject({
- type: 'urn:ok:error:storage-readonly',
- title: 'Could not duplicate path because storage is not writable.',
- status: 500,
- });
});
test('returns 409 when duplicate-path folder destination appears before copy', async () => {
diff --git a/packages/server/src/derived-document-index-boundary.test.ts b/packages/server/src/derived-document-index-boundary.test.ts
new file mode 100644
index 000000000..868d9a03e
--- /dev/null
+++ b/packages/server/src/derived-document-index-boundary.test.ts
@@ -0,0 +1,60 @@
+import { describe, expectTypeOf, test } from 'vitest';
+import type { ApiExtensionOptions } from './api-extension.ts';
+import type {
+ DerivedDocumentIndexApiPort,
+ DerivedDocumentIndexLivePort,
+ DerivedDocumentIndexPersistencePort,
+} from './derived-document-index.ts';
+import type { LiveDerivedIndexOptions } from './live-derived-index.ts';
+import type { PersistenceOptions } from './persistence.ts';
+import type { ServerOptions } from './server-factory.ts';
+
+type RawIndexMember =
+ | 'backlinkIndex'
+ | 'tagIndex'
+ | 'updateDocumentFromMarkdown'
+ | 'deleteDocument'
+ | 'renameDocument'
+ | 'saveToDisk'
+ | 'loadFromDisk'
+ | 'rebuildFromDisk'
+ | 'reconcileWithDisk'
+ | 'ingestGlobalSkillBundles'
+ | 'init'
+ | 'close'
+ | 'switchBranch';
+
+type ConsumerOptionKey =
+ | keyof ApiExtensionOptions
+ | keyof LiveDerivedIndexOptions
+ | keyof PersistenceOptions;
+
+type ConsumerPortKey =
+ | keyof DerivedDocumentIndexApiPort
+ | keyof DerivedDocumentIndexLivePort
+ | keyof DerivedDocumentIndexPersistencePort;
+
+describe('derived document index ownership boundary', () => {
+ test('runtime consumers accept only their coordinator ports', () => {
+ expectTypeOf<ApiExtensionOptions['derivedDocumentIndex']>().toEqualTypeOf<
+ DerivedDocumentIndexApiPort | undefined
+ >();
+ expectTypeOf<
+ LiveDerivedIndexOptions['derivedDocumentIndex']
+ >().toEqualTypeOf<DerivedDocumentIndexLivePort>();
+ expectTypeOf<PersistenceOptions['derivedDocumentIndex']>().toEqualTypeOf<
+ DerivedDocumentIndexPersistencePort | undefined
+ >();
+ });
+
+ test('consumer options and ports do not expose raw relationship indexes', () => {
+ expectTypeOf<Extract<ConsumerOptionKey, RawIndexMember>>().toEqualTypeOf<never>();
+ expectTypeOf<Extract<ConsumerPortKey, RawIndexMember>>().toEqualTypeOf<never>();
+ });
+
+ test('server composition owns coordinator construction', () => {
+ expectTypeOf<
+ Extract<keyof ServerOptions, 'backlinkIndex' | 'tagIndex' | 'derivedDocumentIndex'>
+ >().toEqualTypeOf<never>();
+ });
+});
diff --git a/packages/server/src/derived-document-index.test.ts b/packages/server/src/derived-document-index.test.ts
new file mode 100644
index 000000000..7fd409941
--- /dev/null
+++ b/packages/server/src/derived-document-index.test.ts
@@ -0,0 +1,818 @@
+import { mkdirSync, mkdtempSync, rmSync, unlinkSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join } from 'node:path';
+import { afterEach, describe, expect, test, vi } from 'vitest';
+import { BacklinkIndex } from './backlink-index.ts';
+import { createContentFilter } from './content-filter.ts';
+import {
+ DerivedDocumentIndex,
+ isDerivedDocumentIndexClosedError,
+} from './derived-document-index.ts';
+import { getLogger } from './logger.ts';
+import { TagIndex } from './tag-index.ts';
+
+interface Rig {
+ projectDir: string;
+ contentDir: string;
+ contentFilter: ReturnType<typeof createContentFilter>;
+ signals: string[];
+ index: DerivedDocumentIndex;
+}
+
+interface Deferred {
+ promise: Promise<void>;
+ resolve: () => void;
+}
+
+const cleanups: Array<() => Promise<void>> = [];
+
+function createDeferred(): Deferred {
+ let resolve!: () => void;
+ const promise = new Promise<void>((settle) => {
+ resolve = settle;
+ });
+ return { promise, resolve };
+}
+
+function createRig(): Rig {
+ const projectDir = mkdtempSync(join(tmpdir(), 'ok-derived-index-'));
+ const contentDir = join(projectDir, 'content');
+ mkdirSync(contentDir, { recursive: true });
+ const signals: string[] = [];
+ const contentFilter = createContentFilter({ projectDir, contentDir });
+ const index = new DerivedDocumentIndex({
+ projectDir,
+ contentDir,
+ contentFilter,
+ getGlobalSkillRoots: () => [],
+ signalChannel: (channel) => signals.push(channel),
+ });
+ cleanups.push(async () => {
+ await index.close();
+ rmSync(projectDir, { recursive: true, force: true });
+ });
+ return { projectDir, contentDir, contentFilter, signals, index };
+}
+
+function writeDoc(rig: Rig, relativePath: string, markdown: string): void {
+ const filePath = join(rig.contentDir, relativePath);
+ mkdirSync(dirname(filePath), { recursive: true });
+ writeFileSync(filePath, markdown);
+}
+
+async function settleStartup(rig: Rig): Promise<void> {
+ const startup = rig.index.beginStartup('main');
+ await startup.backlinksReady;
+ await rig.index.settleStartupAfterWatcherSeed();
+ rig.signals.length = 0;
+}
+
+afterEach(async () => {
+ vi.useRealTimers();
+ vi.restoreAllMocks();
+ for (const cleanup of cleanups.splice(0)) await cleanup();
+});
+
+describe('DerivedDocumentIndex', () => {
+ test('direct documents update links and tags before returning and signal B/G/T', async () => {
+ const rig = createRig();
+ await settleStartup(rig);
+
+ await rig.index.recordDirectDocument(
+ 'source',
+ '---\ntags: [project/active]\n---\nSee [[target]].\n',
+ );
+
+ expect(await rig.index.getBacklinks('target')).toEqual([
+ expect.objectContaining({ source: 'source' }),
+ ]);
+ expect(await rig.index.getBacklinkCounts(['target', 'missing'])).toEqual({
+ target: 1,
+ missing: 0,
+ });
+ expect(await rig.index.getDocsForTagWithMatches('project')).toEqual([
+ { docName: 'source', matchingTags: ['project/active'] },
+ ]);
+ expect(rig.signals).toEqual(['backlinks', 'graph', 'tags']);
+ });
+
+ test('direct batches mutate in order with one cache save and one B/G/T signal set', async () => {
+ const backlinkSave = vi.spyOn(BacklinkIndex.prototype, 'saveToDisk');
+ const tagSave = vi.spyOn(TagIndex.prototype, 'saveToDisk');
+ const rig = createRig();
+ await settleStartup(rig);
+ backlinkSave.mockClear();
+ tagSave.mockClear();
+
+ await rig.index.recordDirectMutations([
+ { kind: 'upsert', documentName: 'first', markdown: 'See [[old]]. #old\n' },
+ {
+ kind: 'rename',
+ oldDocumentName: 'first',
+ newDocumentName: 'second',
+ markdown: 'See [[target]]. #moved\n',
+ },
+ {
+ kind: 'link-rewrite',
+ documentName: 'second',
+ markdown: 'See [[rewritten]]. #ignored\n',
+ },
+ { kind: 'delete', documentName: 'missing' },
+ ]);
+
+ expect(await rig.index.getBacklinks('old')).toEqual([]);
+ expect(await rig.index.getBacklinks('target')).toEqual([]);
+ expect((await rig.index.getBacklinks('rewritten')).map((entry) => entry.source)).toEqual([
+ 'second',
+ ]);
+ expect(await rig.index.getDocsForTagWithMatches('moved')).toEqual([
+ { docName: 'second', matchingTags: ['moved'] },
+ ]);
+ expect(await rig.index.getDocsForTagWithMatches('ignored')).toEqual([]);
+ expect(backlinkSave).toHaveBeenCalledTimes(1);
+ expect(tagSave).toHaveBeenCalledTimes(1);
+ expect(rig.signals).toEqual(['backlinks', 'graph', 'tags']);
+ });
+
+ test('empty direct batches are immediate no-ops across readiness, branch, and close gates', async () => {
+ const rig = createRig();
+
+ await expect(rig.index.recordDirectMutations([])).resolves.toBeUndefined();
+ await settleStartup(rig);
+ const transition = await rig.index.beginBranchSwitch('feature');
+ await expect(rig.index.recordDirectMutations([])).resolves.toBeUndefined();
+ rig.index.abortBranchSwitch(transition);
+ await rig.index.close();
+ await expect(rig.index.recordDirectMutations([])).resolves.toBeUndefined();
+ expect(rig.signals).toEqual([]);
+ });
+
+ test('live documents do not persist caches but preserve backlink-first paired updates', async () => {
+ const backlinkSave = vi.spyOn(BacklinkIndex.prototype, 'saveToDisk');
+ const tagSave = vi.spyOn(TagIndex.prototype, 'saveToDisk');
+ const rig = createRig();
+ await settleStartup(rig);
+ backlinkSave.mockClear();
+ tagSave.mockClear();
+
+ const token = rig.index.captureLiveUpdateToken();
+ expect(token).not.toBeNull();
+ await rig.index.recordLiveDocument('source', 'See [[target]]. #live\n', token ?? -1);
+
+ expect(await rig.index.getBacklinks('target')).toHaveLength(1);
+ expect(await rig.index.getDocsForTagWithMatches('live')).toHaveLength(1);
+ expect(backlinkSave).not.toHaveBeenCalled();
+ expect(tagSave).not.toHaveBeenCalled();
+ expect(rig.signals).toEqual(['backlinks', 'graph', 'tags']);
+ });
+
+ test('an ignored incremental document removes existing link and tag memberships', async () => {
+ const rig = createRig();
+ await settleStartup(rig);
+ await rig.index.recordDirectDocument('ignored', 'See [[target]]. #hidden\n');
+ rig.signals.length = 0;
+ writeFileSync(join(rig.projectDir, '.okignore'), 'content/ignored.md\n');
+ const rebuilt = await rig.contentFilter.rebuildIgnorePatterns();
+ expect(rebuilt.ok).toBe(true);
+
+ await rig.index.recordDirectDocument('ignored', 'See [[target]]. #hidden\n');
+
+ expect(await rig.index.getBacklinks('target')).toEqual([]);
+ expect(await rig.index.getDocsForTagWithMatches('hidden')).toEqual([]);
+ expect(rig.signals).toEqual(['backlinks', 'graph', 'tags']);
+ });
+
+ test('system and config docs are excluded while managed artifacts remain admitted', async () => {
+ const rig = createRig();
+ await settleStartup(rig);
+
+ await rig.index.recordDirectDocument('__system__', 'See [[target]]. #internal\n');
+ await rig.index.recordDirectDocument('__config__/project', 'See [[target]]. #internal\n');
+ await rig.index.recordDirectDocument(
+ '__template__/notes/lifecycle',
+ 'See [[target]]. #managed\n',
+ );
+
+ expect((await rig.index.getBacklinks('target')).map((entry) => entry.source)).toEqual([
+ '__template__/notes/lifecycle',
+ ]);
+ expect(await rig.index.getDocsForTagWithMatches('internal')).toEqual([]);
+ expect(await rig.index.getDocsForTagWithMatches('managed')).toEqual([
+ { docName: '__template__/notes/lifecycle', matchingTags: ['managed'] },
+ ]);
+ expect(await rig.index.getIndexedDocNames()).not.toContain('__system__');
+ expect(await rig.index.getIndexedDocNames()).not.toContain('__config__/project');
+ });
+
+ test('durable stores update and persist backlinks without changing tags or signaling', async () => {
+ vi.useFakeTimers();
+ const backlinkSave = vi.spyOn(BacklinkIndex.prototype, 'saveToDisk');
+ const tagSave = vi.spyOn(TagIndex.prototype, 'saveToDisk');
+ const rig = createRig();
+ await settleStartup(rig);
+ backlinkSave.mockClear();
+ tagSave.mockClear();
+
+ await rig.index.recordDurableStore('source', 'See [[target]]. #durable\n');
+
+ expect(await rig.index.getBacklinks('target')).toHaveLength(1);
+ expect(await rig.index.getDocsForTagWithMatches('durable')).toEqual([]);
+ expect(backlinkSave).not.toHaveBeenCalled();
+ expect(tagSave).not.toHaveBeenCalled();
+
+ await vi.advanceTimersByTimeAsync(2000);
+
+ expect(backlinkSave).toHaveBeenCalledTimes(1);
+ expect(tagSave).not.toHaveBeenCalled();
+ expect(rig.signals).toEqual([]);
+ });
+
+ test('disk events debounce paired cache saves while applying and signaling immediately', async () => {
+ vi.useFakeTimers();
+ const backlinkSave = vi.spyOn(BacklinkIndex.prototype, 'saveToDisk');
+ const tagSave = vi.spyOn(TagIndex.prototype, 'saveToDisk');
+ const rig = createRig();
+ await settleStartup(rig);
+ backlinkSave.mockClear();
+ tagSave.mockClear();
+
+ await rig.index.recordDiskUpsert('source', 'See [[first]]. #one\n');
+ await rig.index.recordDiskUpsert('source', 'See [[second]]. #two\n');
+
+ expect(await rig.index.getBacklinks('first')).toEqual([]);
+ expect(await rig.index.getBacklinks('second')).toHaveLength(1);
+ expect(await rig.index.getDocsForTagWithMatches('two')).toHaveLength(1);
+ expect(backlinkSave).not.toHaveBeenCalled();
+ expect(tagSave).not.toHaveBeenCalled();
+
+ await rig.index.recordDiskRename('source', 'renamed', 'See [[third]]. #three\n');
+ await rig.index.recordDiskDelete('renamed');
+
+ expect(await rig.index.getBacklinks('second')).toEqual([]);
+ expect(await rig.index.getBacklinks('third')).toEqual([]);
+ expect(await rig.index.getDocsForTagWithMatches('two')).toEqual([]);
+ expect(await rig.index.getDocsForTagWithMatches('three')).toEqual([]);
+
+ await vi.advanceTimersByTimeAsync(2000);
+ await rig.index.getIndexedDocNames();
+
+ expect(backlinkSave).toHaveBeenCalledTimes(1);
+ expect(tagSave).toHaveBeenCalledTimes(1);
+ expect(rig.signals).toEqual([
+ 'backlinks',
+ 'graph',
+ 'tags',
+ 'backlinks',
+ 'graph',
+ 'tags',
+ 'backlinks',
+ 'graph',
+ 'tags',
+ 'backlinks',
+ 'graph',
+ 'tags',
+ ]);
+ });
+
+ test('link rewrites are backlink-only and retain existing tag membership', async () => {
+ const rig = createRig();
+ await settleStartup(rig);
+ await rig.index.recordDirectDocument('source', 'See [[old]]. #stable\n');
+ rig.signals.length = 0;
+
+ await rig.index.recordLinkRewrite('source', 'See [[new]]. #changed\n');
+
+ expect(await rig.index.getBacklinks('old')).toEqual([]);
+ expect(await rig.index.getBacklinks('new')).toHaveLength(1);
+ expect(await rig.index.getDocsForTagWithMatches('stable')).toHaveLength(1);
+ expect(await rig.index.getDocsForTagWithMatches('changed')).toEqual([]);
+ expect(rig.signals).toEqual(['backlinks', 'graph']);
+ });
+
+ test('direct rename moves link and tag membership atomically', async () => {
+ const rig = createRig();
+ await settleStartup(rig);
+ await rig.index.recordDirectDocument('old', 'See [[target]]. #moved\n');
+ rig.signals.length = 0;
+
+ await rig.index.recordDirectRename('old', 'new', 'See [[target]]. #moved\n');
+
+ expect((await rig.index.getBacklinks('target')).map((entry) => entry.source)).toEqual(['new']);
+ expect(await rig.index.getDocsForTagWithMatches('moved')).toEqual([
+ { docName: 'new', matchingTags: ['moved'] },
+ ]);
+ expect(rig.signals).toEqual(['backlinks', 'graph', 'tags']);
+ });
+
+ test('queries wait for post-watcher startup settlement', async () => {
+ const rig = createRig();
+ writeDoc(rig, 'source.md', 'See [[target]].\n');
+ const startup = rig.index.beginStartup('main');
+ let settled = false;
+ const query = rig.index.getBacklinks('target').then((entries) => {
+ settled = true;
+ return entries;
+ });
+
+ await startup.backlinksReady;
+ await Promise.resolve();
+ expect(settled).toBe(false);
+
+ await rig.index.settleStartupAfterWatcherSeed();
+ await expect(query).resolves.toHaveLength(1);
+ });
+
+ test('branch transitions hold new commands until the replacement indexes settle', async () => {
+ const rig = createRig();
+ await settleStartup(rig);
+ const transition = await rig.index.beginBranchSwitch('feature');
+ let applied = false;
+ let queried = false;
+ const pending = rig.index.recordDirectDocument('after-switch', '# branch\n').then(() => {
+ applied = true;
+ });
+ const pendingQuery = rig.index.getIndexedDocNames().then(() => {
+ queried = true;
+ });
+
+ await Promise.resolve();
+ expect(applied).toBe(false);
+ expect(queried).toBe(false);
+
+ await rig.index.settleBranchFromDisk(transition);
+ await Promise.all([pending, pendingQuery]);
+ expect(applied).toBe(true);
+ expect(queried).toBe(true);
+ });
+
+ test('branch transitions invalidate stale live payloads captured from the old branch', async () => {
+ const rig = createRig();
+ await settleStartup(rig);
+ await rig.index.recordDirectDocument('old-doc', 'See [[target]]. #old-tag\n');
+ const staleToken = rig.index.captureLiveUpdateToken();
+ expect(staleToken).not.toBeNull();
+
+ const transition = await rig.index.beginBranchSwitch('feature');
+ const targetBranchToken = rig.index.captureLiveUpdateToken();
+ expect(targetBranchToken).not.toBeNull();
+ expect(targetBranchToken).not.toBe(staleToken);
+ await rig.index.settleBranchFromDisk(transition);
+ await rig.index.recordLiveDocument('old-doc', 'See [[target]]. #old-tag\n', staleToken ?? -1);
+
+ expect(await rig.index.getBacklinks('target')).toEqual([]);
+ expect(await rig.index.getDocsForTagWithMatches('old-tag')).toEqual([]);
+ });
+
+ test('live mutations captured during a branch transition wait and apply to the target branch', async () => {
+ const rig = createRig();
+ await settleStartup(rig);
+ const transition = await rig.index.beginBranchSwitch('feature');
+ const token = rig.index.captureLiveUpdateToken();
+ expect(token).not.toBeNull();
+ let mutationSettled = false;
+ let querySettled = false;
+ const mutation = rig.index
+ .recordLiveDocument('target-live', 'See [[target]]. #target-live\n', token ?? -1)
+ .then(() => {
+ mutationSettled = true;
+ });
+ const query = rig.index.getBacklinks('target').then((entries) => {
+ querySettled = true;
+ return entries;
+ });
+
+ await Promise.resolve();
+ expect(mutationSettled).toBe(false);
+ expect(querySettled).toBe(false);
+
+ await rig.index.settleBranchFromDisk(transition);
+ await expect(mutation).resolves.toBeUndefined();
+ await expect(query).resolves.toEqual([expect.objectContaining({ source: 'target-live' })]);
+ expect(await rig.index.getDocsForTagWithMatches('target-live')).toHaveLength(1);
+ });
+
+ test('backlink failure short-circuits tags and signals without poisoning the queue', async () => {
+ const rig = createRig();
+ await settleStartup(rig);
+ const warn = vi.spyOn(getLogger('derived-document-index'), 'warn');
+ const update = vi
+ .spyOn(BacklinkIndex.prototype, 'updateDocumentFromMarkdown')
+ .mockImplementationOnce(() => {
+ throw new Error('injected projection failure');
+ });
+ const updateTags = vi.spyOn(TagIndex.prototype, 'updateDocumentFromMarkdown');
+
+ await expect(rig.index.recordDirectDocument('failed', '# failed\n')).rejects.toThrow(
+ 'injected projection failure',
+ );
+ expect(updateTags).not.toHaveBeenCalled();
+ expect(rig.signals).toEqual([]);
+ expect(warn).toHaveBeenCalledWith(
+ expect.objectContaining({ err: expect.any(Error) }),
+ 'Derived document index operation failed; queue remains available',
+ );
+ update.mockRestore();
+
+ await expect(
+ rig.index.recordDirectDocument('recovered', 'See [[target]].\n'),
+ ).resolves.toBeUndefined();
+ expect(updateTags).toHaveBeenCalledTimes(1);
+ expect(await rig.index.getBacklinks('target')).toHaveLength(1);
+ });
+
+ test('refreshContentScope gates later mutations and queries until both replacements settle', async () => {
+ const rig = createRig();
+ await settleStartup(rig);
+ const backlinksReplacement = createDeferred();
+ const tagsReplacement = createDeferred();
+ vi.spyOn(BacklinkIndex.prototype, 'rebuildFromDisk').mockImplementationOnce(
+ async () => backlinksReplacement.promise,
+ );
+ const tagInit = vi
+ .spyOn(TagIndex.prototype, 'init')
+ .mockImplementationOnce(async () => tagsReplacement.promise);
+
+ const refresh = rig.index.refreshContentScope();
+ let mutationSettled = false;
+ let querySettled = false;
+ const mutation = rig.index.recordDurableStore('after-refresh', '# refreshed\n').then(() => {
+ mutationSettled = true;
+ });
+ const query = rig.index.getIndexedDocNames().then(() => {
+ querySettled = true;
+ });
+
+ await Promise.resolve();
+ expect(mutationSettled).toBe(false);
+ expect(querySettled).toBe(false);
+ expect(rig.signals).toEqual([]);
+
+ backlinksReplacement.resolve();
+ await vi.waitFor(() => expect(tagInit).toHaveBeenCalledTimes(1));
+ expect(mutationSettled).toBe(false);
+ expect(querySettled).toBe(false);
+ expect(rig.signals).toEqual([]);
+
+ tagsReplacement.resolve();
+ await Promise.all([refresh, mutation, query]);
+ expect(mutationSettled).toBe(true);
+ expect(querySettled).toBe(true);
+ expect(rig.signals).toEqual(['backlinks', 'graph', 'tags']);
+ });
+
+ test('content-scope refresh surfaces global ingest failure and the queue recovers', async () => {
+ const rig = createRig();
+ await settleStartup(rig);
+ writeDoc(rig, 'hidden.md', 'See [[target]]. #hidden\n');
+ await rig.index.recordDirectDocument('hidden', 'See [[target]]. #hidden\n');
+ writeFileSync(join(rig.projectDir, '.okignore'), 'content/hidden.md\n');
+ const rebuilt = await rig.contentFilter.rebuildIgnorePatterns();
+ expect(rebuilt.ok).toBe(true);
+ rig.signals.length = 0;
+ vi.spyOn(BacklinkIndex.prototype, 'ingestGlobalSkillBundles').mockRejectedValueOnce(
+ new Error('global ingest failed'),
+ );
+
+ await expect(rig.index.refreshContentScope()).rejects.toThrow('global ingest failed');
+ expect(await rig.index.getBacklinks('target')).toEqual([]);
+ expect(await rig.index.getDocsForTagWithMatches('hidden')).toEqual([]);
+ expect(rig.signals).toEqual(['backlinks', 'graph', 'tags']);
+
+ await expect(
+ rig.index.recordDirectDocument('recovered', 'See [[target]]. #recovered\n'),
+ ).resolves.toBeUndefined();
+ expect(await rig.index.getBacklinks('target')).toHaveLength(1);
+ expect(await rig.index.getDocsForTagWithMatches('recovered')).toHaveLength(1);
+ });
+
+ test('test rescan surfaces global ingest failure and leaves the queue available', async () => {
+ const rig = createRig();
+ await settleStartup(rig);
+ vi.spyOn(BacklinkIndex.prototype, 'ingestGlobalSkillBundles').mockRejectedValueOnce(
+ new Error('rescan global ingest failed'),
+ );
+
+ await expect(rig.index.testOnly.rescanBacklinksForTest()).rejects.toThrow(
+ 'rescan global ingest failed',
+ );
+ expect(rig.signals).toEqual([]);
+
+ await expect(
+ rig.index.recordDirectDocument('recovered', 'See [[target]].\n'),
+ ).resolves.toBeUndefined();
+ expect(await rig.index.getBacklinks('target')).toHaveLength(1);
+ });
+
+ test('branch settlement rebuilds, ingests global nodes, then saves and falls back for tags', async () => {
+ const rig = createRig();
+ await settleStartup(rig);
+ const transition = await rig.index.beginBranchSwitch('feature');
+ const load = vi.spyOn(BacklinkIndex.prototype, 'loadFromDisk').mockResolvedValueOnce(false);
+ const rebuild = vi
+ .spyOn(BacklinkIndex.prototype, 'rebuildFromDisk')
+ .mockResolvedValueOnce(undefined);
+ const ingest = vi
+ .spyOn(BacklinkIndex.prototype, 'ingestGlobalSkillBundles')
+ .mockResolvedValueOnce(undefined);
+ const backlinkSave = vi
+ .spyOn(BacklinkIndex.prototype, 'saveToDisk')
+ .mockResolvedValueOnce(undefined);
+ const reconcileTags = vi
+ .spyOn(TagIndex.prototype, 'reconcileWithDisk')
+ .mockRejectedValueOnce(new Error('reconcile failed'));
+ const rebuildTags = vi.spyOn(TagIndex.prototype, 'init').mockResolvedValueOnce(undefined);
+ const tagSave = vi.spyOn(TagIndex.prototype, 'saveToDisk').mockResolvedValueOnce(undefined);
+
+ await rig.index.settleBranchFromDisk(transition);
+
+ expect(load).toHaveBeenCalledWith('feature');
+ expect(rebuild).toHaveBeenCalledWith('feature');
+ expect(ingest).toHaveBeenCalledWith([], 'feature');
+ expect(rebuild.mock.invocationCallOrder[0]).toBeLessThan(
+ ingest.mock.invocationCallOrder[0] ?? 0,
+ );
+ expect(ingest.mock.invocationCallOrder[0]).toBeLessThan(
+ backlinkSave.mock.invocationCallOrder[0] ?? 0,
+ );
+ expect(reconcileTags).toHaveBeenCalledTimes(1);
+ expect(rebuildTags).toHaveBeenCalledTimes(1);
+ expect(tagSave).toHaveBeenCalledTimes(1);
+ });
+
+ test('branch settlement reconciles a warm cache before global ingest and save', async () => {
+ const rig = createRig();
+ await settleStartup(rig);
+ const transition = await rig.index.beginBranchSwitch('warm');
+ vi.spyOn(BacklinkIndex.prototype, 'loadFromDisk').mockResolvedValueOnce(true);
+ const reconcile = vi.spyOn(BacklinkIndex.prototype, 'reconcileWithDisk').mockResolvedValueOnce({
+ added: 0,
+ updated: 0,
+ deleted: 0,
+ deletedDocNames: [],
+ });
+ const ingest = vi
+ .spyOn(BacklinkIndex.prototype, 'ingestGlobalSkillBundles')
+ .mockResolvedValueOnce(undefined);
+ const save = vi.spyOn(BacklinkIndex.prototype, 'saveToDisk').mockResolvedValueOnce(undefined);
+
+ await rig.index.settleBranchFromDisk(transition);
+
+ expect(reconcile.mock.invocationCallOrder[0]).toBeLessThan(
+ ingest.mock.invocationCallOrder[0] ?? 0,
+ );
+ expect(ingest.mock.invocationCallOrder[0]).toBeLessThan(save.mock.invocationCallOrder[0] ?? 0);
+ });
+
+ test('branch global-ingest failure still replaces local indexes and releases callers', async () => {
+ const rig = createRig();
+ writeDoc(rig, 'old.md', '# Old\n\n#old-branch\n');
+ await settleStartup(rig);
+ unlinkSync(join(rig.contentDir, 'old.md'));
+ writeDoc(rig, 'target.md', '# Target\n\n#target-branch\n');
+ const transition = await rig.index.beginBranchSwitch('feature');
+ vi.spyOn(BacklinkIndex.prototype, 'ingestGlobalSkillBundles').mockRejectedValueOnce(
+ new Error('branch global ingest failed'),
+ );
+ const query = rig.index.getDocsForTagWithMatches('target-branch');
+
+ await expect(rig.index.settleBranchFromDisk(transition)).rejects.toThrow(
+ 'branch global ingest failed',
+ );
+ await expect(query).resolves.toEqual([{ docName: 'target', matchingTags: ['target-branch'] }]);
+ expect(await rig.index.getDocsForTagWithMatches('old-branch')).toEqual([]);
+ });
+
+ test('beginBranchSwitch cancels an old-branch save timer before changing scope', async () => {
+ vi.useFakeTimers();
+ const backlinkSave = vi.spyOn(BacklinkIndex.prototype, 'saveToDisk');
+ const tagSave = vi.spyOn(TagIndex.prototype, 'saveToDisk');
+ const rig = createRig();
+ await settleStartup(rig);
+ await rig.index.recordDiskUpsert('old-branch', '# old\n');
+
+ const transition = await rig.index.beginBranchSwitch('feature');
+ await rig.index.settleBranchFromDisk(transition);
+ backlinkSave.mockClear();
+ tagSave.mockClear();
+ await vi.advanceTimersByTimeAsync(2000);
+
+ expect(backlinkSave).not.toHaveBeenCalled();
+ expect(tagSave).not.toHaveBeenCalled();
+ });
+
+ test('abortBranchSwitch releases only the transition owned by its caller', async () => {
+ const rig = createRig();
+ await settleStartup(rig);
+ const transition = await rig.index.beginBranchSwitch('aborted');
+ const mutation = rig.index.recordDirectDocument('released', '# released\n');
+ const query = rig.index.getIndexedDocNames();
+ let mutationSettled = false;
+ void mutation.then(() => {
+ mutationSettled = true;
+ });
+
+ rig.index.abortBranchSwitch({ branch: 'other' });
+ await Promise.resolve();
+ expect(mutationSettled).toBe(false);
+
+ rig.index.abortBranchSwitch(transition);
+ await expect(mutation).resolves.toBeUndefined();
+ await expect(query).resolves.toBeInstanceOf(Array);
+ await expect(rig.index.getIndexedDocNames()).resolves.toContain('released');
+ });
+
+ test('recognizes only coordinator close errors as lifecycle rejections', async () => {
+ const rig = createRig();
+ await rig.index.close();
+
+ const closedError = await rig.index.getIndexedDocNames().catch((err: unknown) => err);
+
+ expect(isDerivedDocumentIndexClosedError(closedError)).toBe(true);
+ const sameName = new Error('lookalike');
+ sameName.name = 'DerivedDocumentIndexClosedError';
+ expect(isDerivedDocumentIndexClosedError(sameName)).toBe(false);
+ });
+
+ test('warm startup reports documents deleted while the server was down', async () => {
+ const projectDir = mkdtempSync(join(tmpdir(), 'ok-derived-warm-'));
+ const contentDir = join(projectDir, 'content');
+ mkdirSync(contentDir, { recursive: true });
+ writeFileSync(join(contentDir, 'stale.md'), '# stale\n');
+ const contentFilter = createContentFilter({ projectDir, contentDir });
+ const rawIndex = new BacklinkIndex({ projectDir, contentDir, contentFilter });
+ await rawIndex.rebuildFromDisk('main');
+ await rawIndex.saveToDisk('main');
+ unlinkSync(join(contentDir, 'stale.md'));
+ const index = new DerivedDocumentIndex({
+ projectDir,
+ contentDir,
+ contentFilter,
+ getGlobalSkillRoots: () => [],
+ signalChannel: () => {},
+ });
+ cleanups.push(async () => {
+ await index.close();
+ rmSync(projectDir, { recursive: true, force: true });
+ });
+
+ const startup = index.beginStartup('main');
+
+ await expect(startup.backlinksReady).resolves.toEqual({
+ deletedDocNames: ['stale'],
+ backlinkIndexDegraded: false,
+ });
+ await index.settleStartupAfterWatcherSeed();
+ });
+
+ test('degraded startup releases query readiness and reports index degradation', async () => {
+ const warn = vi.spyOn(getLogger('derived-document-index'), 'warn');
+ const rig = createRig();
+ vi.spyOn(BacklinkIndex.prototype, 'loadFromDisk').mockRejectedValueOnce(
+ new Error('broken backlink cache'),
+ );
+ vi.spyOn(TagIndex.prototype, 'loadFromDisk').mockRejectedValueOnce(
+ new Error('broken tag cache'),
+ );
+ const startup = rig.index.beginStartup('main');
+ await expect(startup.backlinksReady).resolves.toEqual({
+ deletedDocNames: [],
+ backlinkIndexDegraded: true,
+ });
+ vi.spyOn(TagIndex.prototype, 'reconcileWithDisk').mockRejectedValueOnce(
+ new Error('broken tag scan'),
+ );
+ const query = rig.index.getIndexedDocNames();
+
+ await expect(rig.index.settleStartupAfterWatcherSeed()).resolves.toEqual({
+ tagIndexDegraded: true,
+ });
+ await expect(query).resolves.toEqual([]);
+ expect(warn).toHaveBeenCalledWith(
+ expect.objectContaining({ err: expect.any(Error) }),
+ '[derived-document-index] tag-index warm boot failed; continuing with stale or empty index',
+ );
+ });
+
+ test('close flushes a pending paired save and rejects future commands', async () => {
+ vi.useFakeTimers();
+ const backlinkSave = vi.spyOn(BacklinkIndex.prototype, 'saveToDisk');
+ const tagSave = vi.spyOn(TagIndex.prototype, 'saveToDisk');
+ const rig = createRig();
+ await settleStartup(rig);
+ backlinkSave.mockClear();
+ tagSave.mockClear();
+
+ await rig.index.recordDiskUpsert('source', '# pending\n');
+ const closing = rig.index.close();
+ await expect(rig.index.recordDirectDelete('source')).rejects.toThrow(
+ 'Derived document index is closed',
+ );
+ await closing;
+ await vi.advanceTimersByTimeAsync(2000);
+
+ expect(backlinkSave).toHaveBeenCalledTimes(1);
+ expect(tagSave).toHaveBeenCalledTimes(1);
+ });
+
+ test('close flushes a pending backlink-only durable-store save', async () => {
+ vi.useFakeTimers();
+ const backlinkSave = vi.spyOn(BacklinkIndex.prototype, 'saveToDisk');
+ const tagSave = vi.spyOn(TagIndex.prototype, 'saveToDisk');
+ const rig = createRig();
+ await settleStartup(rig);
+ backlinkSave.mockClear();
+ tagSave.mockClear();
+
+ await rig.index.recordDurableStore('source', 'See [[target]].\n');
+ await rig.index.close();
+ await vi.advanceTimersByTimeAsync(2000);
+
+ expect(backlinkSave).toHaveBeenCalledTimes(1);
+ expect(tagSave).not.toHaveBeenCalled();
+ });
+
+ test('repeated close calls wait for the same dependency drain', async () => {
+ const rig = createRig();
+ await settleStartup(rig);
+ const tagClose = createDeferred();
+ const closeTagIndex = vi
+ .spyOn(TagIndex.prototype, 'close')
+ .mockReturnValueOnce(tagClose.promise);
+
+ const firstClose = rig.index.close();
+ const secondClose = rig.index.close();
+ let secondCloseSettled = false;
+ void secondClose.then(() => {
+ secondCloseSettled = true;
+ });
+ await Promise.resolve();
+
+ expect(closeTagIndex).toHaveBeenCalledTimes(1);
+ expect(secondCloseSettled).toBe(false);
+
+ tagClose.resolve();
+ await Promise.all([firstClose, secondClose]);
+ expect(secondCloseSettled).toBe(true);
+ });
+
+ test('close rejects an already-admitted queued disk mutation before it can arm a timer', async () => {
+ vi.useFakeTimers();
+ const deferredSave = createDeferred();
+ const backlinkSave = vi.spyOn(BacklinkIndex.prototype, 'saveToDisk');
+ const tagSave = vi.spyOn(TagIndex.prototype, 'saveToDisk');
+ const indexLog = getLogger('derived-document-index');
+ const warn = vi.spyOn(indexLog, 'warn');
+ const debug = vi.spyOn(indexLog, 'debug');
+ const rig = createRig();
+ await settleStartup(rig);
+ backlinkSave.mockClear();
+ tagSave.mockClear();
+ backlinkSave.mockImplementationOnce(async () => deferredSave.promise);
+
+ const running = rig.index.recordDirectDocument('running', '# running\n');
+ await vi.waitFor(() => expect(backlinkSave).toHaveBeenCalledTimes(1));
+ const queued = rig.index.recordDiskUpsert('queued', '# queued\n');
+ await Promise.resolve();
+ const closing = rig.index.close();
+ deferredSave.resolve();
+
+ await expect(running).resolves.toBeUndefined();
+ await expect(queued).rejects.toThrow('Derived document index is closed');
+ await closing;
+ const tagSaveCallsAfterClose = tagSave.mock.calls.length;
+ await vi.advanceTimersByTimeAsync(2000);
+
+ expect(backlinkSave).toHaveBeenCalledTimes(1);
+ expect(tagSave).toHaveBeenCalledTimes(tagSaveCallsAfterClose);
+ expect(debug).toHaveBeenCalledWith(
+ expect.objectContaining({ err: expect.any(Error) }),
+ 'Derived document index operation rejected after close',
+ );
+ expect(warn).not.toHaveBeenCalledWith(
+ expect.anything(),
+ 'Derived document index operation failed; queue remains available',
+ );
+ });
+
+ test('close drains a debounced save already enqueued behind in-flight work', async () => {
+ vi.useFakeTimers();
+ const deferredSave = createDeferred();
+ const backlinkSave = vi.spyOn(BacklinkIndex.prototype, 'saveToDisk');
+ const rig = createRig();
+ await settleStartup(rig);
+ backlinkSave.mockClear();
+
+ await rig.index.recordDiskUpsert('disk-event', '# disk\n');
+ backlinkSave.mockImplementationOnce(async () => deferredSave.promise);
+ const running = rig.index.recordDirectDocument('running', '# running\n');
+ await vi.waitFor(() => expect(backlinkSave).toHaveBeenCalledTimes(1));
+ await vi.advanceTimersByTimeAsync(2000);
+
+ const closing = rig.index.close();
+ deferredSave.resolve();
+ await expect(running).resolves.toBeUndefined();
+ await closing;
+
+ expect(backlinkSave).toHaveBeenCalledTimes(2);
+ });
+});
diff --git a/packages/server/src/derived-document-index.ts b/packages/server/src/derived-document-index.ts
new file mode 100644
index 000000000..6955b535c
--- /dev/null
+++ b/packages/server/src/derived-document-index.ts
@@ -0,0 +1,751 @@
+import type { DerivedViewChannel } from '@inkeep/open-knowledge-core';
+import {
+ type BacklinkEntry,
+ BacklinkIndex,
+ type GraphNode,
+ type HubEntry,
+ isOrphanMode,
+ type OrphanMode,
+} from './backlink-index.ts';
+import { isLinkIndexExcludedDoc, isManagedArtifactDoc } from './cc1-broadcast.ts';
+import type { ContentFilter } from './content-filter.ts';
+import { docNameToRelativePath } from './doc-extensions.ts';
+import { getLogger } from './logger.ts';
+import { TagIndex, type TagSummaryEntry } from './tag-index.ts';
+
+const log = getLogger('derived-document-index');
+
+const DERIVED_INDEX_SAVE_DEBOUNCE_MS = 2000;
+
+export type DerivedDocumentRelationChannel = Extract<
+ DerivedViewChannel,
+ 'backlinks' | 'graph' | 'tags'
+>;
+
+type ForwardLinkEntry = ReturnType<BacklinkIndex['getForwardLinkEntries']>[number];
+type LinkGraph = ReturnType<BacklinkIndex['getLinkGraph']>;
+type DeadLinkEntry = ReturnType<BacklinkIndex['getDeadLinks']>[number];
+type TagDocMatch = ReturnType<TagIndex['getDocsForTagWithMatches']>[number];
+
+export type DerivedGraphNode = GraphNode;
+export type DerivedOrphanMode = OrphanMode;
+
+export function isDerivedOrphanMode(value: unknown): value is DerivedOrphanMode {
+ return typeof value === 'string' && isOrphanMode(value);
+}
+
+export interface DerivedIndexStartupBacklinksResult {
+ deletedDocNames: readonly string[];
+ backlinkIndexDegraded: boolean;
+}
+
+export interface DerivedIndexStartupSettlement {
+ tagIndexDegraded: boolean;
+}
+
+export interface DerivedDocumentIndexBranchTransition {
+ readonly branch: string;
+}
+
+export interface DerivedDocumentIndexLivePort {
+ captureLiveUpdateToken(): number | null;
+ recordLiveDocument(documentName: string, markdown: string, token: number): Promise<void>;
+}
+
+export interface DerivedDocumentIndexPersistencePort {
+ recordDurableStore(documentName: string, markdown: string): Promise<void>;
+}
+
+export interface DerivedDocumentIndexTestPort {
+ resetDocumentForTest(documentName: string): Promise<void>;
+ rescanBacklinksForTest(): Promise<void>;
+}
+
+export type DerivedDocumentIndexMutation =
+ | { kind: 'upsert'; documentName: string; markdown: string }
+ | { kind: 'delete'; documentName: string }
+ | {
+ kind: 'rename';
+ oldDocumentName: string;
+ newDocumentName: string;
+ markdown: string;
+ }
+ | { kind: 'link-rewrite'; documentName: string; markdown: string };
+
+export interface DerivedDocumentIndexApiPort {
+ readonly testOnly?: DerivedDocumentIndexTestPort;
+ recordDirectMutations(mutations: readonly DerivedDocumentIndexMutation[]): Promise<void>;
+ recordDirectDocument(documentName: string, markdown: string): Promise<void>;
+ recordDirectDelete(documentName: string): Promise<void>;
+ recordDirectRename(
+ oldDocumentName: string,
+ newDocumentName: string,
+ markdown: string,
+ ): Promise<void>;
+ recordLinkRewrite(documentName: string, markdown: string): Promise<void>;
+ getBacklinks(documentName: string): Promise<BacklinkEntry[]>;
+ getBacklinkCount(documentName: string): Promise<number>;
+ getBacklinkCounts(documentNames: readonly string[]): Promise<Record<string, number>>;
+ getForwardLinkEntries(documentName: string): Promise<ForwardLinkEntry[]>;
+ getLinkGraphNeighborhood(documentName: string, degrees: number): Promise<LinkGraph>;
+ getLinkGraph(): Promise<LinkGraph>;
+ getHubs(limit?: number): Promise<HubEntry[]>;
+ getOrphans(documentNames: string[], mode: OrphanMode): Promise<string[]>;
+ getDeadLinks(
+ admittedDocuments: Iterable<string>,
+ sourceDocumentNames?: readonly string[],
+ ): Promise<DeadLinkEntry[]>;
+ getIndexedDocNames(): Promise<string[]>;
+ getAllTags(): Promise<TagSummaryEntry[]>;
+ getDocsForTagWithMatches(tag: string): Promise<TagDocMatch[]>;
+}
+
+export interface DerivedDocumentIndexOptions {
+ projectDir: string;
+ contentDir: string;
+ contentFilter: ContentFilter;
+ getGlobalSkillRoots: () => string[];
+ signalChannel: (channel: DerivedDocumentRelationChannel) => void;
+}
+
+interface Deferred {
+ promise: Promise<void>;
+ resolve: () => void;
+}
+
+interface BranchTransition extends DerivedDocumentIndexBranchTransition {
+ barrier: Deferred;
+}
+
+function createDeferred(): Deferred {
+ let resolve!: () => void;
+ const promise = new Promise<void>((settle) => {
+ resolve = settle;
+ });
+ return { promise, resolve };
+}
+
+class DerivedDocumentIndexClosedError extends Error {
+ constructor() {
+ super('Derived document index is closed');
+ this.name = 'DerivedDocumentIndexClosedError';
+ }
+}
+
+export function isDerivedDocumentIndexClosedError(err: unknown): boolean {
+ return err instanceof DerivedDocumentIndexClosedError;
+}
+
+export class DerivedDocumentIndex
+ implements
+ DerivedDocumentIndexApiPort,
+ DerivedDocumentIndexLivePort,
+ DerivedDocumentIndexPersistencePort
+{
+ readonly testOnly: DerivedDocumentIndexTestPort;
+
+ private readonly backlinkIndex: BacklinkIndex;
+ private readonly tagIndex: TagIndex;
+ private readonly contentFilter: ContentFilter;
+ private readonly getGlobalSkillRoots: () => string[];
+ private readonly signalChannel: (channel: DerivedDocumentRelationChannel) => void;
+ private readonly startupAdmission = createDeferred();
+ private readonly readyBarrier = createDeferred();
+ private tail: Promise<void> = Promise.resolve();
+ private saveTimer: ReturnType<typeof setTimeout> | null = null;
+ private saveTagsOnNextDebounce = false;
+ private branchTransition: BranchTransition | null = null;
+ private startupBegun = false;
+ private startupSettled = false;
+ private liveUpdateToken = 0;
+ private closed = false;
+ private closePromise: Promise<void> | null = null;
+
+ constructor(options: DerivedDocumentIndexOptions) {
+ this.backlinkIndex = new BacklinkIndex({
+ projectDir: options.projectDir,
+ contentDir: options.contentDir,
+ contentFilter: options.contentFilter,
+ });
+ this.tagIndex = new TagIndex({
+ projectDir: options.projectDir,
+ contentDir: options.contentDir,
+ contentFilter: options.contentFilter,
+ });
+ this.contentFilter = options.contentFilter;
+ this.getGlobalSkillRoots = options.getGlobalSkillRoots;
+ this.signalChannel = options.signalChannel;
+ this.testOnly = {
+ resetDocumentForTest: (documentName) => this.resetDocumentForTest(documentName),
+ rescanBacklinksForTest: () => this.rescanBacklinksForTest(),
+ };
+ }
+
+ beginStartup(branch: string): { backlinksReady: Promise<DerivedIndexStartupBacklinksResult> } {
+ this.assertOpen();
+ if (this.startupBegun) {
+ throw new Error('Derived document index startup has already begun');
+ }
+ this.startupBegun = true;
+ this.backlinkIndex.switchBranch(branch);
+
+ const backlinksReady = this.initializeBacklinks(branch);
+ const tagsWarm = this.warmTags();
+ this.tail = Promise.all([backlinksReady.then(() => undefined), tagsWarm]).then(() => undefined);
+ this.startupAdmission.resolve();
+ return { backlinksReady };
+ }
+
+ async settleStartupAfterWatcherSeed(): Promise<DerivedIndexStartupSettlement> {
+ this.assertOpen();
+ if (!this.startupBegun) {
+ throw new Error('Derived document index startup has not begun');
+ }
+ if (this.startupSettled) {
+ throw new Error('Derived document index startup has already settled');
+ }
+ this.startupSettled = true;
+
+ try {
+ return await this.enqueueInternal(async () => {
+ try {
+ const diff = await this.tagIndex.reconcileWithDisk();
+ if (diff.added > 0 || diff.updated > 0 || diff.deleted > 0) {
+ log.info(diff, '[tag-index] startup reconcile: offline changes applied');
+ }
+ await this.saveTagsLogOnly('startup');
+ return { tagIndexDegraded: false };
+ } catch (err) {
+ log.error(
+ { err },
+ '[tag-index] startup reconcile failed; tag index updates incrementally via watcher events',
+ );
+ return { tagIndexDegraded: true };
+ }
+ });
+ } finally {
+ this.readyBarrier.resolve();
+ }
+ }
+
+ async beginBranchSwitch(branch: string): Promise<DerivedDocumentIndexBranchTransition> {
+ this.assertOpen();
+ if (this.branchTransition) {
+ throw new Error(`Derived document index branch switch already active`);
+ }
+ const transition: BranchTransition = { branch, barrier: createDeferred() };
+ this.liveUpdateToken++;
+ this.branchTransition = transition;
+ this.cancelScheduledSave();
+
+ try {
+ await this.startupAdmission.promise;
+ await this.readyBarrier.promise;
+ this.assertOpen();
+ await this.enqueueInternal(async () => {
+ this.backlinkIndex.switchBranch(branch);
+ });
+ return transition;
+ } catch (err) {
+ this.releaseBranchTransition(transition);
+ throw err;
+ }
+ }
+
+ async settleBranchFromDisk(transition: DerivedDocumentIndexBranchTransition): Promise<void> {
+ this.assertOpen();
+ if (this.branchTransition !== transition) {
+ throw new Error('No derived document index branch switch is active');
+ }
+
+ try {
+ await this.enqueueInternal(async () => {
+ const branch = transition.branch;
+ const loaded = await this.backlinkIndex.loadFromDisk(branch);
+ if (loaded) {
+ const diff = await this.backlinkIndex.reconcileWithDisk(branch);
+ if (diff.added > 0 || diff.updated > 0 || diff.deleted > 0) {
+ log.info(diff, `[backlinks] branch-switch reconcile for ${branch}`);
+ }
+ } else {
+ await this.backlinkIndex.rebuildFromDisk(branch);
+ }
+ let globalIngestFailed = false;
+ let globalIngestFailure: unknown;
+ try {
+ await this.backlinkIndex.ingestGlobalSkillBundles(this.getGlobalSkillRoots(), branch);
+ } catch (err) {
+ globalIngestFailed = true;
+ globalIngestFailure = err;
+ }
+ await this.saveBacklinksLogOnly('branch-switch', branch);
+
+ try {
+ const tagDiff = await this.tagIndex.reconcileWithDisk();
+ if (tagDiff.added > 0 || tagDiff.updated > 0 || tagDiff.deleted > 0) {
+ log.info(tagDiff, `[tag-index] branch-switch reconcile for ${branch}`);
+ }
+ } catch (err) {
+ log.warn(
+ { err, branch },
+ '[tag-index] branch-switch reconcile failed; falling back to full rebuild',
+ );
+ await this.tagIndex.init();
+ }
+ await this.saveTagsLogOnly('branch-switch');
+ if (globalIngestFailed) throw globalIngestFailure;
+ });
+ } finally {
+ this.releaseBranchTransition(transition);
+ }
+ }
+
+ abortBranchSwitch(transition: DerivedDocumentIndexBranchTransition | undefined): void {
+ if (transition) this.releaseBranchTransition(transition);
+ }
+
+ captureLiveUpdateToken(): number | null {
+ if (this.closed) return null;
+ return this.liveUpdateToken;
+ }
+
+ recordLiveDocument(documentName: string, markdown: string, token: number): Promise<void> {
+ if (this.closed || token !== this.liveUpdateToken) {
+ return Promise.resolve();
+ }
+ return this.runCommand(async () => {
+ if (token !== this.liveUpdateToken) return;
+ this.updateBoth(documentName, markdown);
+ this.signalAllRelations();
+ });
+ }
+
+ recordDurableStore(documentName: string, markdown: string): Promise<void> {
+ return this.runCommand(async () => {
+ this.updateBacklinks(documentName, markdown);
+ this.scheduleSave(false);
+ });
+ }
+
+ recordDiskUpsert(documentName: string, markdown: string): Promise<void> {
+ return this.runCommand(async () => {
+ this.updateBoth(documentName, markdown);
+ this.scheduleSave();
+ this.signalAllRelations();
+ });
+ }
+
+ recordDiskDelete(documentName: string): Promise<void> {
+ return this.runCommand(async () => {
+ this.deleteBoth(documentName);
+ this.scheduleSave();
+ this.signalAllRelations();
+ });
+ }
+
+ recordDiskRename(
+ oldDocumentName: string,
+ newDocumentName: string,
+ markdown: string,
+ ): Promise<void> {
+ return this.runCommand(async () => {
+ this.renameBoth(oldDocumentName, newDocumentName, markdown);
+ this.scheduleSave();
+ this.signalAllRelations();
+ });
+ }
+
+ recordDirectDocument(documentName: string, markdown: string): Promise<void> {
+ return this.recordDirectMutations([{ kind: 'upsert', documentName, markdown }]);
+ }
+
+ recordDirectDelete(documentName: string): Promise<void> {
+ return this.recordDirectMutations([{ kind: 'delete', documentName }]);
+ }
+
+ recordDirectRename(
+ oldDocumentName: string,
+ newDocumentName: string,
+ markdown: string,
+ ): Promise<void> {
+ return this.recordDirectMutations([
+ { kind: 'rename', oldDocumentName, newDocumentName, markdown },
+ ]);
+ }
+
+ recordLinkRewrite(documentName: string, markdown: string): Promise<void> {
+ return this.recordDirectMutations([{ kind: 'link-rewrite', documentName, markdown }]);
+ }
+
+ recordDirectMutations(mutations: readonly DerivedDocumentIndexMutation[]): Promise<void> {
+ if (mutations.length === 0) return Promise.resolve();
+ return this.runCommand(async () => {
+ let tagsChanged = false;
+ for (const mutation of mutations) {
+ switch (mutation.kind) {
+ case 'upsert':
+ this.updateBoth(mutation.documentName, mutation.markdown);
+ tagsChanged = true;
+ break;
+ case 'delete':
+ this.deleteBoth(mutation.documentName);
+ tagsChanged = true;
+ break;
+ case 'rename':
+ this.renameBoth(mutation.oldDocumentName, mutation.newDocumentName, mutation.markdown);
+ tagsChanged = true;
+ break;
+ case 'link-rewrite':
+ this.updateBacklinks(mutation.documentName, mutation.markdown);
+ break;
+ }
+ }
+
+ if (tagsChanged) {
+ await this.saveBothLogOnly('direct-batch');
+ this.signalAllRelations();
+ } else {
+ await this.saveBacklinksLogOnly('direct-batch');
+ this.signalBacklinks();
+ }
+ });
+ }
+
+ refreshGlobalSkillNodes(): Promise<void> {
+ return this.runCommand(async () => {
+ await this.backlinkIndex.ingestGlobalSkillBundles(this.getGlobalSkillRoots());
+ this.signalBacklinks();
+ });
+ }
+
+ refreshContentScope(): Promise<void> {
+ return this.runCommand(async () => {
+ const branch = this.backlinkIndex.getActiveBranch();
+ await this.backlinkIndex.rebuildFromDisk(branch);
+ let globalIngestFailed = false;
+ let globalIngestFailure: unknown;
+ try {
+ await this.backlinkIndex.ingestGlobalSkillBundles(this.getGlobalSkillRoots(), branch);
+ } catch (err) {
+ globalIngestFailed = true;
+ globalIngestFailure = err;
+ }
+ await this.tagIndex.init();
+ await this.saveBothLogOnly('content-filter', branch);
+ this.signalAllRelations();
+ if (globalIngestFailed) throw globalIngestFailure;
+ });
+ }
+
+ announceReadyViews(): void {
+ // The server factory calls this only after startup settlement has resolved.
+ this.assertOpen();
+ this.signalAllRelations();
+ }
+
+ getBacklinks(documentName: string): Promise<BacklinkEntry[]> {
+ return this.runQuery(() => this.backlinkIndex.getBacklinks(documentName));
+ }
+
+ getBacklinkCount(documentName: string): Promise<number> {
+ return this.runQuery(() => this.backlinkIndex.getBacklinkCount(documentName));
+ }
+
+ getBacklinkCounts(documentNames: readonly string[]): Promise<Record<string, number>> {
+ return this.runQuery(() =>
+ Object.fromEntries(
+ documentNames.map((documentName) => [
+ documentName,
+ this.backlinkIndex.getBacklinkCount(documentName),
+ ]),
+ ),
+ );
+ }
+
+ getForwardLinkEntries(documentName: string): Promise<ForwardLinkEntry[]> {
+ return this.runQuery(() => this.backlinkIndex.getForwardLinkEntries(documentName));
+ }
+
+ getLinkGraphNeighborhood(documentName: string, degrees: number): Promise<LinkGraph> {
+ return this.runQuery(() => this.backlinkIndex.getLinkGraphNeighborhood(documentName, degrees));
+ }
+
+ getLinkGraph(): Promise<LinkGraph> {
+ return this.runQuery(() => this.backlinkIndex.getLinkGraph());
+ }
+
+ getHubs(limit?: number): Promise<HubEntry[]> {
+ return this.runQuery(() => this.backlinkIndex.getHubs(limit));
+ }
+
+ getOrphans(documentNames: string[], mode: OrphanMode): Promise<string[]> {
+ return this.runQuery(() => this.backlinkIndex.getOrphans(documentNames, mode));
+ }
+
+ getDeadLinks(
+ admittedDocuments: Iterable<string>,
+ sourceDocumentNames?: readonly string[],
+ ): Promise<DeadLinkEntry[]> {
+ return this.runQuery(() =>
+ this.backlinkIndex.getDeadLinks(admittedDocuments, sourceDocumentNames),
+ );
+ }
+
+ getIndexedDocNames(): Promise<string[]> {
+ return this.runQuery(() => this.backlinkIndex.getIndexedDocNames());
+ }
+
+ getAllTags(): Promise<TagSummaryEntry[]> {
+ return this.runQuery(() => this.tagIndex.getAllTags());
+ }
+
+ getDocsForTagWithMatches(tag: string): Promise<TagDocMatch[]> {
+ return this.runQuery(() => this.tagIndex.getDocsForTagWithMatches(tag));
+ }
+
+ close(): Promise<void> {
+ if (this.closePromise) return this.closePromise;
+ const pendingSave = this.cancelScheduledSave();
+ if (pendingSave !== null) {
+ void this.enqueueInternal(() => this.saveScheduledCache(pendingSave, 'close-flush'));
+ }
+ this.closed = true;
+ this.startupAdmission.resolve();
+ this.readyBarrier.resolve();
+ this.releaseBranchTransition(this.branchTransition);
+ this.closePromise = this.tail
+ .then(() => this.tagIndex.close())
+ .then(
+ () => undefined,
+ () => undefined,
+ );
+ return this.closePromise;
+ }
+
+ private resetDocumentForTest(documentName: string): Promise<void> {
+ return this.runCommand(async () => {
+ this.deleteBoth(documentName);
+ await this.saveBothLogOnly('test-reset');
+ this.signalAllRelations();
+ });
+ }
+
+ private rescanBacklinksForTest(): Promise<void> {
+ return this.runCommand(async () => {
+ const branch = this.backlinkIndex.getActiveBranch();
+ await this.backlinkIndex.rebuildFromDisk(branch);
+ await this.backlinkIndex.ingestGlobalSkillBundles(this.getGlobalSkillRoots(), branch);
+ await this.saveBacklinksLogOnly('test-rescan', branch);
+ this.signalBacklinks();
+ });
+ }
+
+ private async initializeBacklinks(branch: string): Promise<DerivedIndexStartupBacklinksResult> {
+ try {
+ let deletedDocNames: readonly string[] = [];
+ if (await this.backlinkIndex.loadFromDisk(branch)) {
+ const diff = await this.backlinkIndex.reconcileWithDisk(branch);
+ deletedDocNames = diff.deletedDocNames;
+ if (diff.added > 0 || diff.updated > 0 || diff.deleted > 0) {
+ log.info(
+ { added: diff.added, updated: diff.updated, deleted: diff.deleted },
+ '[backlinks] startup reconcile: offline changes applied',
+ );
+ }
+ } else {
+ await this.backlinkIndex.rebuildFromDisk(branch);
+ }
+ await this.ingestGlobalSkillNodesLogOnly(branch, 'startup');
+ await this.saveBacklinksLogOnly('startup', branch);
+ return { deletedDocNames, backlinkIndexDegraded: false };
+ } catch (err) {
+ log.error(
+ { err, branch },
+ '[backlinks] startup init failed; index will populate incrementally via watcher',
+ );
+ return { deletedDocNames: [], backlinkIndexDegraded: true };
+ }
+ }
+
+ private async warmTags(): Promise<void> {
+ try {
+ if (await this.tagIndex.loadFromDisk()) {
+ await this.tagIndex.reconcileWithDisk();
+ } else {
+ await this.tagIndex.init();
+ }
+ } catch (err) {
+ log.warn(
+ { err },
+ '[derived-document-index] tag-index warm boot failed; continuing with stale or empty index',
+ );
+ }
+ }
+
+ private updateBoth(documentName: string, markdown: string): void {
+ this.updateBacklinks(documentName, markdown);
+ if (this.isAdmitted(documentName)) {
+ this.tagIndex.updateDocumentFromMarkdown(documentName, markdown);
+ } else {
+ this.tagIndex.deleteDocument(documentName);
+ }
+ }
+
+ private updateBacklinks(documentName: string, markdown: string): void {
+ if (this.isAdmitted(documentName)) {
+ this.backlinkIndex.updateDocumentFromMarkdown(documentName, markdown);
+ } else {
+ this.backlinkIndex.deleteDocument(documentName);
+ }
+ }
+
+ private deleteBoth(documentName: string): void {
+ this.backlinkIndex.deleteDocument(documentName);
+ this.tagIndex.deleteDocument(documentName);
+ }
+
+ private renameBoth(oldDocumentName: string, newDocumentName: string, markdown: string): void {
+ if (this.isAdmitted(newDocumentName)) {
+ this.backlinkIndex.renameDocument(oldDocumentName, newDocumentName, markdown);
+ this.tagIndex.renameDocument(oldDocumentName, newDocumentName, markdown);
+ } else {
+ this.backlinkIndex.deleteDocument(oldDocumentName);
+ this.backlinkIndex.deleteDocument(newDocumentName);
+ this.tagIndex.deleteDocument(oldDocumentName);
+ this.tagIndex.deleteDocument(newDocumentName);
+ }
+ }
+
+ private isAdmitted(documentName: string): boolean {
+ if (isLinkIndexExcludedDoc(documentName)) return false;
+ if (isManagedArtifactDoc(documentName)) return true;
+ return !this.contentFilter.isExcluded(docNameToRelativePath(documentName));
+ }
+
+ private signalBacklinks(): void {
+ this.signalChannel('backlinks');
+ this.signalChannel('graph');
+ }
+
+ private signalAllRelations(): void {
+ this.signalBacklinks();
+ this.signalChannel('tags');
+ }
+
+ private scheduleSave(includeTags = true): void {
+ const saveTags = this.saveTagsOnNextDebounce || includeTags;
+ this.cancelScheduledSave();
+ this.saveTagsOnNextDebounce = saveTags;
+ this.saveTimer = setTimeout(() => {
+ this.saveTimer = null;
+ const saveTags = this.saveTagsOnNextDebounce;
+ this.saveTagsOnNextDebounce = false;
+ void this.enqueueInternal(() => this.saveScheduledCache(saveTags, 'debounced')).catch(
+ (err) => {
+ log.warn({ err }, 'Failed to enqueue debounced derived-index cache save');
+ },
+ );
+ }, DERIVED_INDEX_SAVE_DEBOUNCE_MS);
+ }
+
+ private cancelScheduledSave(): boolean | null {
+ const pendingSave = this.saveTimer === null ? null : this.saveTagsOnNextDebounce;
+ if (this.saveTimer !== null) clearTimeout(this.saveTimer);
+ this.saveTimer = null;
+ this.saveTagsOnNextDebounce = false;
+ return pendingSave;
+ }
+
+ private saveScheduledCache(includeTags: boolean, reason: string): Promise<void> {
+ return includeTags
+ ? this.saveBothLogOnly(reason)
+ : this.saveBacklinksLogOnly(`${reason}-backlinks`);
+ }
+
+ private releaseBranchTransition(transition: DerivedDocumentIndexBranchTransition | null): void {
+ const active = this.branchTransition;
+ if (!transition || active !== transition) return;
+ this.branchTransition = null;
+ active.barrier.resolve();
+ }
+
+ private async saveBothLogOnly(reason: string, branch?: string): Promise<void> {
+ await this.saveBacklinksLogOnly(reason, branch);
+ await this.saveTagsLogOnly(reason);
+ }
+
+ private async saveBacklinksLogOnly(reason: string, branch?: string): Promise<void> {
+ try {
+ await this.backlinkIndex.saveToDisk(branch);
+ } catch (err) {
+ log.warn({ err, reason, branch }, 'Failed to persist backlink cache');
+ }
+ }
+
+ private async saveTagsLogOnly(reason: string): Promise<void> {
+ try {
+ await this.tagIndex.saveToDisk();
+ } catch (err) {
+ log.warn({ err, reason }, 'Failed to persist tag snapshot');
+ }
+ }
+
+ private async ingestGlobalSkillNodesLogOnly(branch: string, reason: string): Promise<void> {
+ try {
+ await this.backlinkIndex.ingestGlobalSkillBundles(this.getGlobalSkillRoots(), branch);
+ } catch (err) {
+ log.warn({ err, branch, reason }, 'Global skill bundle ingest failed');
+ }
+ }
+
+ private async runCommand<T>(operation: () => Promise<T> | T): Promise<T> {
+ this.assertOpen();
+ await this.startupAdmission.promise;
+ this.assertOpen();
+ return this.enqueueAfterBranchAdmission(operation);
+ }
+
+ private async runQuery<T>(query: () => Promise<T> | T): Promise<T> {
+ this.assertOpen();
+ await this.startupAdmission.promise;
+ await this.readyBarrier.promise;
+ this.assertOpen();
+ return this.enqueueAfterBranchAdmission(query);
+ }
+
+ private async enqueueAfterBranchAdmission<T>(operation: () => Promise<T> | T): Promise<T> {
+ while (true) {
+ const transition = this.branchTransition;
+ if (!transition) {
+ this.assertOpen();
+ return this.enqueueInternal(() => {
+ this.assertOpen();
+ return operation();
+ });
+ }
+ await transition.barrier.promise;
+ this.assertOpen();
+ }
+ }
+
+ private enqueueInternal<T>(operation: () => Promise<T> | T): Promise<T> {
+ const run = this.tail.then(operation);
+ this.tail = run.then(
+ () => undefined,
+ (err) => {
+ if (isDerivedDocumentIndexClosedError(err)) {
+ log.debug({ err }, 'Derived document index operation rejected after close');
+ } else {
+ log.warn({ err }, 'Derived document index operation failed; queue remains available');
+ }
+ },
+ );
+ return run;
+ }
+
+ private assertOpen(): void {
+ if (this.closed) {
+ throw new DerivedDocumentIndexClosedError();
+ }
+ }
+}
diff --git a/packages/server/src/head-watcher.test.ts b/packages/server/src/head-watcher.test.ts
index 8900aa628..0e245784d 100644
--- a/packages/server/src/head-watcher.test.ts
+++ b/packages/server/src/head-watcher.test.ts
@@ -3,7 +3,7 @@ import { mkdirSync, writeFileSync } from 'node:fs';
import { mkdtemp, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { resolve } from 'node:path';
-import { afterEach, beforeEach, describe, expect, test } from 'vitest';
+import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';
import { readProjectHeadState, startHeadWatcher, watchedGitFile } from './head-watcher';
let tmpDir: string;
@@ -140,3 +140,68 @@ describe('startHeadWatcher chokidar fallback', () => {
}
});
});
+
+describe('startHeadWatcher batch serialization', () => {
+ test('queues a second HEAD batch until the first branch-switch callback settles', async () => {
+ const projectRoot = resolve(tmpDir, 'repo');
+ mkdirSync(projectRoot, { recursive: true });
+ const git = (args: string) => execSync(`git ${args}`, { cwd: projectRoot, stdio: 'ignore' });
+ git('init -q');
+ git('config user.email [email protected]');
+ git('config user.name t');
+ writeFileSync(resolve(projectRoot, 'a.md'), 'hello\n');
+ git('add -A');
+ git('commit -qm init');
+ git('branch -M main');
+ git('branch feature');
+ git('branch second');
+
+ let dispatch!: (rawPath: string) => void;
+ let releaseFirst!: () => void;
+ const firstCanSettle = new Promise<void>((resolve) => {
+ releaseFirst = resolve;
+ });
+ const branches: Array<string | null> = [];
+ let activeCallbacks = 0;
+ let maxActiveCallbacks = 0;
+
+ const handle = await startHeadWatcher(
+ projectRoot,
+ () => {},
+ async (info) => {
+ activeCallbacks++;
+ maxActiveCallbacks = Math.max(maxActiveCallbacks, activeCallbacks);
+ branches.push(info.newBranch);
+ try {
+ if (branches.length === 1) await firstCanSettle;
+ } finally {
+ activeCallbacks--;
+ }
+ },
+ {
+ subscribeForTest: async (_gitDir, nextDispatch) => {
+ dispatch = nextDispatch;
+ return async () => {};
+ },
+ },
+ );
+
+ try {
+ git('checkout -q feature');
+ dispatch(resolve(projectRoot, '.git', 'HEAD'));
+ await vi.waitFor(() => expect(branches).toEqual(['feature']));
+
+ git('checkout -q second');
+ dispatch(resolve(projectRoot, '.git', 'HEAD'));
+ await new Promise((resolveWait) => setTimeout(resolveWait, 150));
+ expect(branches).toEqual(['feature']);
+
+ releaseFirst();
+ await vi.waitFor(() => expect(branches).toEqual(['feature', 'second']));
+ expect(maxActiveCallbacks).toBe(1);
+ } finally {
+ releaseFirst();
+ await handle.unsubscribe();
+ }
+ });
+});
diff --git a/packages/server/src/head-watcher.ts b/packages/server/src/head-watcher.ts
index ed6e3d33f..8726b3726 100644
--- a/packages/server/src/head-watcher.ts
+++ b/packages/server/src/head-watcher.ts
@@ -179,7 +179,13 @@ export async function startHeadWatcher(
projectRoot: string,
onBatchBegin: OnBatchBegin,
onBatchEnd: OnBatchEnd,
- opts: { forceBackend?: 'parcel' | 'chokidar' } = {},
+ opts: {
+ forceBackend?: 'parcel' | 'chokidar';
+ subscribeForTest?: (
+ gitDir: string,
+ dispatch: HeadEventDispatch,
+ ) => Promise<() => Promise<void>>;
+ } = {},
): Promise<HeadWatcherHandle> {
const inspected = discoverGitRepository(projectRoot);
if (inspected.kind !== 'repository') {
@@ -194,10 +200,15 @@ export async function startHeadWatcher(
let timeoutTimer: ReturnType<typeof setTimeout> | null = null;
let oldHead: string | null = null;
let lastKnownBranch: string | null = null;
+ let batchEndInFlight: Promise<void> | null = null;
async function emitBatchEnd(timeout: boolean): Promise<void> {
// Wait for onBatchBegin to finish before proceeding
if (beginInFlight) await beginInFlight;
+ if (batchEndInFlight) {
+ await batchEndInFlight;
+ return;
+ }
if (!inBatch) return;
if (quietTimer) {
@@ -226,24 +237,32 @@ export async function startHeadWatcher(
const oldBranch = lastKnownBranch;
+ const end = Promise.resolve().then(async () => {
+ try {
+ await onBatchEnd({
+ headMoved,
+ oldHead,
+ newHead,
+ timeout,
+ batchKind,
+ oldBranch,
+ newBranch,
+ });
+ } catch (e) {
+ log.error({ err: e }, 'onBatchEnd callback failed');
+ } finally {
+ // Set inBatch = false AFTER the async callback completes
+ // so new file events stay buffered during branch-switch orchestration
+ inBatch = false;
+ oldHead = newHead;
+ lastKnownBranch = newBranch;
+ }
+ });
+ batchEndInFlight = end;
try {
- await onBatchEnd({
- headMoved,
- oldHead,
- newHead,
- timeout,
- batchKind,
- oldBranch,
- newBranch,
- });
- } catch (e) {
- log.error({ err: e }, 'onBatchEnd callback failed');
+ await end;
} finally {
- // Set inBatch = false AFTER the async callback completes
- // so new file events stay buffered during branch-switch orchestration
- inBatch = false;
- oldHead = newHead;
- lastKnownBranch = newBranch;
+ if (batchEndInFlight === end) batchEndInFlight = null;
}
}
@@ -258,6 +277,9 @@ export async function startHeadWatcher(
let beginInFlight: Promise<void> | null = null;
async function handleGitEvent(trigger: string): Promise<void> {
+ // A second HEAD event belongs to a new batch, but it must not re-enter the
+ // still-running branch-switch callback for the preceding batch.
+ if (batchEndInFlight) await batchEndInFlight;
if (!inBatch) {
inBatch = true;
// Do NOT re-read HEAD here. `oldHead` already holds the last-settled HEAD
@@ -297,8 +319,11 @@ export async function startHeadWatcher(
// Prefer @parcel/watcher; fall back to chokidar when it can't start so HEAD
// watching stays functional in packaged builds that omit the native addon.
let resolvedUnsub: (() => Promise<void>) | null = null;
- let backend: 'parcel' | 'chokidar' = 'chokidar';
- if (opts.forceBackend !== 'chokidar') {
+ let backend: 'parcel' | 'chokidar' | 'test' = 'chokidar';
+ if (opts.subscribeForTest) {
+ resolvedUnsub = await opts.subscribeForTest(gitDir, dispatch);
+ backend = 'test';
+ } else if (opts.forceBackend !== 'chokidar') {
resolvedUnsub = await tryStartParcelHeadWatcher(gitDir, dispatch);
if (resolvedUnsub) backend = 'parcel';
}
diff --git a/packages/server/src/http/error-response.ts b/packages/server/src/http/error-response.ts
index 9ceb6a9cd..c8ca42ba2 100644
--- a/packages/server/src/http/error-response.ts
+++ b/packages/server/src/http/error-response.ts
@@ -126,6 +126,12 @@ interface ErrorResponseOptions {
* line. Surfaces underlying errno / syscall on storage failures.
*/
cause?: unknown;
+ /**
+ * Override the status-derived log level for expected lifecycle responses.
+ * For example, a 503 emitted while a subsystem is shutting down is useful
+ * to the caller but is not an operational fault.
+ */
+ logLevel?: 'debug' | 'warn' | 'error';
}
/**
@@ -299,7 +305,7 @@ export function errorResponse(
// The Pino line is for grep correlation via `instance`. Separating
// levels avoids drowning monitoring in routine 4xx noise (mirrors
// pino-http's `customLogLevel` defaults).
- const logLevel = status >= 500 ? 'error' : 'warn';
+ const logLevel = options.logLevel ?? (status >= 500 ? 'error' : 'warn');
log()[logLevel](
{
event: 'api.error',
diff --git a/packages/server/src/lint/validation-audit.test.ts b/packages/server/src/lint/validation-audit.test.ts
index 2e0d44f78..e0176c164 100644
--- a/packages/server/src/lint/validation-audit.test.ts
+++ b/packages/server/src/lint/validation-audit.test.ts
@@ -67,7 +67,7 @@ function deps(overrides: Partial<ValidationAuditDeps> = {}): ValidationAuditDeps
projectDir: root,
contentDir: root,
baseConfig: lintOn,
- backlinkIndex: index,
+ derivedDocumentIndex: index,
admittedDocNames: () => admitted,
docFilePathFor,
...overrides,
@@ -214,7 +214,9 @@ describe('runValidationAudit', () => {
seedDoc('dirty', DOC_WITH_TAB);
seedDoc('linker', '# Linker\n\nSee [[ghost]].\n');
- const result = await runValidationAudit(createProjectValidators(deps({ backlinkIndex: null })));
+ const result = await runValidationAudit(
+ createProjectValidators(deps({ derivedDocumentIndex: null })),
+ );
expect(result.files.map((f) => f.file)).toEqual(['dirty.md']);
expect(result.errorCount).toBe(0);
@@ -302,7 +304,7 @@ describe('runValidationAudit', () => {
const result = await runValidationAudit(
createProjectValidators(
deps({
- backlinkIndex: fakeIndex,
+ derivedDocumentIndex: fakeIndex,
admittedDocNames: () => ['unsaved'],
docFilePathFor: () => null,
}),
@@ -328,7 +330,7 @@ describe('runValidationAudit', () => {
const result = await runValidationAudit(
createProjectValidators(
deps({
- backlinkIndex: fakeIndex,
+ derivedDocumentIndex: fakeIndex,
admittedDocNames: () => ['legacy'],
docFilePathFor: () => 'legacy.md',
}),
diff --git a/packages/server/src/lint/validation-audit.ts b/packages/server/src/lint/validation-audit.ts
index 763f1ee79..c78aae652 100644
--- a/packages/server/src/lint/validation-audit.ts
+++ b/packages/server/src/lint/validation-audit.ts
@@ -4,7 +4,7 @@
* "given the project (or a sub-path), what per-file diagnostics hold?" behind
* one interface while owning its own execution model — the lint validator
* walks the content tree via `auditProject`; the links validator reads the
- * in-memory `BacklinkIndex`. The engine fans out and merges everything into
+ * derived document index. The engine fans out and merges everything into
* one source-tagged diagnostic plane. Backs `GET /api/audit` + MCP `audit`.
*/
@@ -15,10 +15,19 @@ import {
SUPPORTED_DOC_EXTENSIONS,
type ValidationDiagnostic,
} from '@inkeep/open-knowledge-core';
-import type { BacklinkIndex } from '../backlink-index.ts';
+import type { DerivedDocumentIndexApiPort } from '../derived-document-index.ts';
import { getLogger } from '../logger.ts';
import { auditProject } from './audit.ts';
+type DeadLinksResult = Awaited<ReturnType<DerivedDocumentIndexApiPort['getDeadLinks']>>;
+
+interface ValidationDerivedIndexReader {
+ getDeadLinks(
+ admittedDocuments: Iterable<string>,
+ sourceDocumentNames?: readonly string[],
+ ): DeadLinksResult | Promise<DeadLinksResult>;
+}
+
interface FileValidationResult {
/** Path relative to `contentDir`. */
file: string;
@@ -60,15 +69,15 @@ export interface ValidationAuditDeps {
baseConfig: LinterConfig;
/** Live CRDT source overlay for loaded docs (see `AuditOptions.liveSourceFor`). */
liveSourceFor?: (docRelPath: string) => string | null;
- /** Null when the server booted without a backlink index; links findings degrade to a warning. */
- backlinkIndex: Pick<BacklinkIndex, 'getDeadLinks'> | null;
+ /** Null when the server booted without a derived index; links findings degrade to a warning. */
+ derivedDocumentIndex: ValidationDerivedIndexReader | null;
/**
* Project posture for broken links (`validation.links`): 'off' silences the
* links validator entirely, 'warning' (default) / 'error' set the severity.
*/
linksValidation?: LinksValidationSetting;
/** Every docName that currently exists — the dead-link existence oracle. */
- admittedDocNames: () => Iterable<string>;
+ admittedDocNames: () => Iterable<string> | Promise<Iterable<string>>;
/** docName → on-disk contentDir-relative path, null when no file exists yet. */
docFilePathFor: (docName: string) => string | null;
}
@@ -177,14 +186,14 @@ function createLinksValidator(deps: ValidationAuditDeps): ProjectValidator {
return { files: [], fileCount: 0, warnings: [] };
}
const severity = setting === 'error' ? 'error' : 'warning';
- if (!deps.backlinkIndex) {
+ if (!deps.derivedDocumentIndex) {
return {
files: [],
fileCount: 0,
warnings: ['links validation unavailable: backlink index is not configured'],
};
}
- const admitted = [...deps.admittedDocNames()];
+ const admitted = [...(await deps.admittedDocNames())];
const sourceFilter = scopedSourceDocNames(admitted, scope.targetPath);
// `getDeadLinks` reads an empty source filter as "no filter" — a scope
// matching zero docs must short-circuit here or it would silently widen
@@ -192,7 +201,7 @@ function createLinksValidator(deps: ValidationAuditDeps): ProjectValidator {
if (sourceFilter !== undefined && sourceFilter.length === 0) {
return { files: [], fileCount: 0, warnings: [] };
}
- const deadLinks = deps.backlinkIndex.getDeadLinks(admitted, sourceFilter);
+ const deadLinks = await deps.derivedDocumentIndex.getDeadLinks(admitted, sourceFilter);
const byFile = new Map<string, ValidationDiagnostic[]>();
for (const { target, sources } of deadLinks) {
diff --git a/packages/server/src/live-derived-index.test.ts b/packages/server/src/live-derived-index.test.ts
index 07891fc26..10aeaf3b9 100644
--- a/packages/server/src/live-derived-index.test.ts
+++ b/packages/server/src/live-derived-index.test.ts
@@ -2,6 +2,7 @@ import { setTimeout as wait } from 'node:timers/promises';
import { Hocuspocus } from '@hocuspocus/server';
import { beforeEach, describe, expect, test, vi } from 'vitest';
import type * as Y from 'yjs';
+import type { DerivedDocumentIndexLivePort } from './derived-document-index.ts';
import { DocumentDurabilityState } from './document-durability-state.ts';
import { applyExternalChange } from './external-change.ts';
import { createLiveDerivedIndexExtension } from './live-derived-index.ts';
@@ -36,6 +37,13 @@ function makeOnChangePayload(
};
}
+function createDerivedIndexPort(
+ recordLiveDocument = vi.fn(async (_documentName: string, _markdown: string) => {}),
+ captureLiveUpdateToken = vi.fn((): number | null => 0),
+): DerivedDocumentIndexLivePort {
+ return { captureLiveUpdateToken, recordLiveDocument };
+}
+
describe('createLiveDerivedIndexExtension', () => {
let hp: Hocuspocus;
let durabilityState: DocumentDurabilityState;
@@ -46,12 +54,10 @@ describe('createLiveDerivedIndexExtension', () => {
});
test('skips file-watcher origin transactions', async () => {
- const updateDocumentFromMarkdown = vi.fn(() => {});
- const signalChannel = vi.fn(() => {});
+ const recordLiveDocument = vi.fn(async () => {});
const extension = createLiveDerivedIndexExtension({
- backlinkIndex: { updateDocumentFromMarkdown } as unknown as never,
+ derivedDocumentIndex: createDerivedIndexPort(recordLiveDocument),
debounceMs: 5,
- signalChannel,
});
const conn = await hp.openDirectConnection('skip-file-watcher');
const doc = getDoc(conn);
@@ -65,18 +71,40 @@ describe('createLiveDerivedIndexExtension', () => {
);
await wait(20);
- expect(updateDocumentFromMarkdown).not.toHaveBeenCalled();
- expect(signalChannel).not.toHaveBeenCalled();
+ expect(recordLiveDocument).not.toHaveBeenCalled();
+ await conn.disconnect();
+ });
+
+ test('drops live changes while the coordinator withholds a branch token', async () => {
+ const recordLiveDocument = vi.fn(async () => {});
+ const extension = createLiveDerivedIndexExtension({
+ derivedDocumentIndex: createDerivedIndexPort(
+ recordLiveDocument,
+ vi.fn(() => null),
+ ),
+ debounceMs: 5,
+ });
+ const conn = await hp.openDirectConnection('branch-transition');
+ const doc = getDoc(conn);
+
+ applyExternalChange(durabilityState, hp, 'branch-transition', '# Old branch\n');
+ await extension.onChange?.(
+ makeOnChangePayload(hp, doc, 'branch-transition', {
+ source: 'local',
+ context: { origin: 'branch-switch' },
+ }),
+ );
+ await wait(20);
+
+ expect(recordLiveDocument).not.toHaveBeenCalled();
await conn.disconnect();
});
test('debounces rapid changes to a single update and preserves frontmatter', async () => {
- const updateDocumentFromMarkdown = vi.fn(() => {});
- const signalChannel = vi.fn(() => {});
+ const recordLiveDocument = vi.fn(async () => {});
const extension = createLiveDerivedIndexExtension({
- backlinkIndex: { updateDocumentFromMarkdown } as unknown as never,
+ derivedDocumentIndex: createDerivedIndexPort(recordLiveDocument),
debounceMs: 5,
- signalChannel,
});
const conn = await hp.openDirectConnection('debounced-doc');
const doc = getDoc(conn);
@@ -97,25 +125,20 @@ describe('createLiveDerivedIndexExtension', () => {
await extension.onChange?.(payload);
await wait(20);
- expect(updateDocumentFromMarkdown).toHaveBeenCalledTimes(1);
- expect(updateDocumentFromMarkdown).toHaveBeenCalledWith(
+ expect(recordLiveDocument).toHaveBeenCalledTimes(1);
+ expect(recordLiveDocument).toHaveBeenCalledWith(
'debounced-doc',
'---\ntitle: Debounced\n---\n# Hello\n\n[[beta]]\n',
+ 0,
);
- expect(signalChannel).toHaveBeenCalledTimes(2);
- expect(signalChannel.mock.calls).toEqual([['backlinks'], ['graph']]);
await conn.disconnect();
});
- test('feeds tagIndex with the same markdown and signals the tags channel', async () => {
- const updateBacklink = vi.fn(() => {});
- const updateTag = vi.fn(() => {});
- const signalChannel = vi.fn(() => {});
+ test('hands one settled raw markdown string to the derived-index boundary', async () => {
+ const recordLiveDocument = vi.fn(async () => {});
const extension = createLiveDerivedIndexExtension({
- backlinkIndex: { updateDocumentFromMarkdown: updateBacklink } as unknown as never,
- tagIndex: { updateDocumentFromMarkdown: updateTag } as unknown as never,
+ derivedDocumentIndex: createDerivedIndexPort(recordLiveDocument),
debounceMs: 5,
- signalChannel,
});
const conn = await hp.openDirectConnection('tag-derived-doc');
const doc = getDoc(conn);
@@ -129,17 +152,18 @@ describe('createLiveDerivedIndexExtension', () => {
await extension.onChange?.(payload);
await wait(20);
- expect(updateBacklink).toHaveBeenCalledTimes(1);
- expect(updateTag).toHaveBeenCalledTimes(1);
- expect(updateTag).toHaveBeenCalledWith('tag-derived-doc', '# Hello\n\nA #typescript note.\n');
- expect(signalChannel.mock.calls).toEqual([['backlinks'], ['graph'], ['tags']]);
+ expect(recordLiveDocument).toHaveBeenCalledExactlyOnceWith(
+ 'tag-derived-doc',
+ '# Hello\n\nA #typescript note.\n',
+ 0,
+ );
await conn.disconnect();
});
test('beforeUnloadDocument cancels pending timers', async () => {
- const updateDocumentFromMarkdown = vi.fn(() => {});
+ const recordLiveDocument = vi.fn(async () => {});
const extension = createLiveDerivedIndexExtension({
- backlinkIndex: { updateDocumentFromMarkdown } as unknown as never,
+ derivedDocumentIndex: createDerivedIndexPort(recordLiveDocument),
debounceMs: 20,
});
const conn = await hp.openDirectConnection('unload-doc');
@@ -159,14 +183,14 @@ describe('createLiveDerivedIndexExtension', () => {
});
await wait(40);
- expect(updateDocumentFromMarkdown).not.toHaveBeenCalled();
+ expect(recordLiveDocument).not.toHaveBeenCalled();
await conn.disconnect();
});
test('onDestroy clears pending timers across documents', async () => {
- const updateDocumentFromMarkdown = vi.fn(() => {});
+ const recordLiveDocument = vi.fn(async () => {});
const extension = createLiveDerivedIndexExtension({
- backlinkIndex: { updateDocumentFromMarkdown } as unknown as never,
+ derivedDocumentIndex: createDerivedIndexPort(recordLiveDocument),
debounceMs: 20,
});
const first = await hp.openDirectConnection('destroy-a');
@@ -195,7 +219,7 @@ describe('createLiveDerivedIndexExtension', () => {
});
await wait(40);
- expect(updateDocumentFromMarkdown).not.toHaveBeenCalled();
+ expect(recordLiveDocument).not.toHaveBeenCalled();
await first.disconnect();
await second.disconnect();
});
@@ -206,9 +230,9 @@ describe('createLiveDerivedIndexExtension', () => {
// serializer never emits them back). Under contract, body source is
// `Y.Text('source').toString()` — CRLF survives byte-equal. Discriminating
// because CRLF is not preserved through parse → serialize (parser strips).
- const updateDocumentFromMarkdown = vi.fn(() => {});
+ const recordLiveDocument = vi.fn(async () => {});
const extension = createLiveDerivedIndexExtension({
- backlinkIndex: { updateDocumentFromMarkdown } as unknown as never,
+ derivedDocumentIndex: createDerivedIndexPort(recordLiveDocument),
debounceMs: 5,
});
const conn = await hp.openDirectConnection('crlf-doc');
@@ -227,8 +251,8 @@ describe('createLiveDerivedIndexExtension', () => {
);
await wait(20);
- expect(updateDocumentFromMarkdown).toHaveBeenCalledTimes(1);
- const [, bodyArg] = updateDocumentFromMarkdown.mock.calls[0] as [string, string];
+ expect(recordLiveDocument).toHaveBeenCalledTimes(1);
+ const [, bodyArg] = recordLiveDocument.mock.calls[0] as [string, string];
// CRLF survives — ytext is the source-of-truth.
expect(bodyArg).toContain('\r\n');
// would have produced LF-only canonical bytes from
@@ -242,9 +266,9 @@ describe('createLiveDerivedIndexExtension', () => {
// doc starting with `---\n# H\n` would round-trip through
// serialize(fragment) as `***\n\n# H\n`. Under contract, ytext keeps
// the user's typed `---\n` byte-equal — discriminating.
- const updateDocumentFromMarkdown = vi.fn(() => {});
+ const recordLiveDocument = vi.fn(async () => {});
const extension = createLiveDerivedIndexExtension({
- backlinkIndex: { updateDocumentFromMarkdown } as unknown as never,
+ derivedDocumentIndex: createDerivedIndexPort(recordLiveDocument),
debounceMs: 5,
});
const conn = await hp.openDirectConnection('thematic-doc');
@@ -259,8 +283,8 @@ describe('createLiveDerivedIndexExtension', () => {
);
await wait(20);
- expect(updateDocumentFromMarkdown).toHaveBeenCalledTimes(1);
- const [, bodyArg] = updateDocumentFromMarkdown.mock.calls[0] as [string, string];
+ expect(recordLiveDocument).toHaveBeenCalledTimes(1);
+ const [, bodyArg] = recordLiveDocument.mock.calls[0] as [string, string];
expect(bodyArg).toBe('---\n# Title\n');
expect(bodyArg).not.toContain('***');
await conn.disconnect();
@@ -272,9 +296,9 @@ describe('createLiveDerivedIndexExtension', () => {
// paths in steady state. It still validates the spec acceptance: the
// SNIPPET CONTENT (what consumers see) reflects what the user typed.
// '<https://x>' in snippet text, not '[https://x](https://x)'.
- const updateDocumentFromMarkdown = vi.fn(() => {});
+ const recordLiveDocument = vi.fn(async () => {});
const extension = createLiveDerivedIndexExtension({
- backlinkIndex: { updateDocumentFromMarkdown } as unknown as never,
+ derivedDocumentIndex: createDerivedIndexPort(recordLiveDocument),
debounceMs: 5,
});
const conn = await hp.openDirectConnection('autolink-doc');
@@ -294,22 +318,20 @@ describe('createLiveDerivedIndexExtension', () => {
);
await wait(20);
- expect(updateDocumentFromMarkdown).toHaveBeenCalledTimes(1);
- const [, bodyArg] = updateDocumentFromMarkdown.mock.calls[0] as [string, string];
+ expect(recordLiveDocument).toHaveBeenCalledTimes(1);
+ const [, bodyArg] = recordLiveDocument.mock.calls[0] as [string, string];
expect(bodyArg).toContain('<https://example.com>');
expect(bodyArg).not.toContain('[https://example.com](https://example.com)');
await conn.disconnect();
});
test('logs and swallows callback errors', async () => {
- const updateDocumentFromMarkdown = vi.fn(() => {
+ const recordLiveDocument = vi.fn(async () => {
throw new Error('boom');
});
- const signalChannel = vi.fn(() => {});
const extension = createLiveDerivedIndexExtension({
- backlinkIndex: { updateDocumentFromMarkdown } as unknown as never,
+ derivedDocumentIndex: createDerivedIndexPort(recordLiveDocument),
debounceMs: 5,
- signalChannel,
});
const conn = await hp.openDirectConnection('error-doc');
const doc = getDoc(conn);
@@ -325,8 +347,7 @@ describe('createLiveDerivedIndexExtension', () => {
);
await wait(20);
- expect(updateDocumentFromMarkdown).toHaveBeenCalledTimes(1);
- expect(signalChannel).not.toHaveBeenCalled();
+ expect(recordLiveDocument).toHaveBeenCalledTimes(1);
expect(errorSpy).toHaveBeenCalled();
expect(String(errorSpy.mock.calls[0]?.[1])).toContain(
'Failed to update derived views for error-doc',
diff --git a/packages/server/src/live-derived-index.ts b/packages/server/src/live-derived-index.ts
index 8014f4c64..501c2c6ba 100644
--- a/packages/server/src/live-derived-index.ts
+++ b/packages/server/src/live-derived-index.ts
@@ -1,20 +1,12 @@
import type { Document, Extension } from '@hocuspocus/server';
-import type { BacklinkIndex } from './backlink-index.ts';
import { isLinkIndexExcludedDoc } from './cc1-broadcast.ts';
+import type { DerivedDocumentIndexLivePort } from './derived-document-index.ts';
import { getLogger } from './logger.ts';
-import type { TagIndex } from './tag-index.ts';
export const LIVE_DERIVED_INDEX_DEBOUNCE_MS = 100;
export interface LiveDerivedIndexOptions {
- backlinkIndex: BacklinkIndex;
- /**
- * Optional. When wired, every backlink-update tick also re-extracts tags
- * for the changed doc and broadcasts the `'tags'` derived-view channel so
- * tag-aware UIs can invalidate their caches alongside backlinks/graph.
- */
- tagIndex?: TagIndex;
- signalChannel?: (channel: 'files' | 'backlinks' | 'graph' | 'tags') => void;
+ derivedDocumentIndex: DerivedDocumentIndexLivePort;
debounceMs?: number;
}
@@ -40,12 +32,7 @@ function serializeLiveDocument(document: Document): string {
}
export function createLiveDerivedIndexExtension(options: LiveDerivedIndexOptions): Extension {
- const {
- backlinkIndex,
- tagIndex,
- signalChannel,
- debounceMs = LIVE_DERIVED_INDEX_DEBOUNCE_MS,
- } = options;
+ const { derivedDocumentIndex, debounceMs = LIVE_DERIVED_INDEX_DEBOUNCE_MS } = options;
const pendingByDoc = new Map<string, ReturnType<typeof setTimeout>>();
function clearPending(docName: string): void {
@@ -56,7 +43,7 @@ export function createLiveDerivedIndexExtension(options: LiveDerivedIndexOptions
}
}
- function schedule(docName: string, document: Document): void {
+ function schedule(docName: string, document: Document, token: number): void {
clearPending(docName);
pendingByDoc.set(
docName,
@@ -64,13 +51,12 @@ export function createLiveDerivedIndexExtension(options: LiveDerivedIndexOptions
pendingByDoc.delete(docName);
try {
const markdown = serializeLiveDocument(document);
- backlinkIndex.updateDocumentFromMarkdown(docName, markdown);
- signalChannel?.('backlinks');
- signalChannel?.('graph');
- if (tagIndex) {
- tagIndex.updateDocumentFromMarkdown(docName, markdown);
- signalChannel?.('tags');
- }
+ void derivedDocumentIndex.recordLiveDocument(docName, markdown, token).catch((err) => {
+ getLogger('live-derived-index').error(
+ { docName, err },
+ `Failed to update derived views for ${docName}`,
+ );
+ });
} catch (err) {
getLogger('live-derived-index').error(
{ docName, err },
@@ -93,9 +79,15 @@ export function createLiveDerivedIndexExtension(options: LiveDerivedIndexOptions
return;
}
+ const token = derivedDocumentIndex.captureLiveUpdateToken();
+ if (token === null) {
+ clearPending(documentName);
+ return;
+ }
+
// Give the source/tree bridge a short trailing window to converge so we
// derive links from settled live document state instead of the 2s store debounce.
- schedule(documentName, document);
+ schedule(documentName, document, token);
},
async beforeUnloadDocument({ documentName }) {
diff --git a/packages/server/src/persistence.test.ts b/packages/server/src/persistence.test.ts
index 8ed1fcd73..c60f87f31 100644
--- a/packages/server/src/persistence.test.ts
+++ b/packages/server/src/persistence.test.ts
@@ -12,11 +12,22 @@ import {
import { mkdir, realpath, rename, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { dirname, join, resolve, sep } from 'node:path';
-import { afterEach, beforeEach, describe, expect, test } from 'vitest';
+import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';
import * as Y from 'yjs';
+import { composeAndWriteRawBody } from './bridge-intake.ts';
+import { createContentFilter } from './content-filter.ts';
import { isWithinContentDir, safeContentPath } from './content-path.ts';
+import {
+ DerivedDocumentIndex,
+ type DerivedDocumentIndexPersistencePort,
+} from './derived-document-index.ts';
+import { DocumentDurabilityState } from './document-durability-state.ts';
import { contentHash, isSelfWrite, registerWrite } from './file-watcher';
-import { captureDocSnapshotForPersistence, resolveWriterFromOrigin } from './persistence';
+import {
+ captureDocSnapshotForPersistence,
+ createPersistenceExtension,
+ resolveWriterFromOrigin,
+} from './persistence';
import { FILE_SYSTEM_WRITER, GIT_UPSTREAM_WRITER, SERVICE_WRITER } from './shadow-repo';
describe('safeContentPath', () => {
@@ -221,6 +232,112 @@ describe('symlink-safe atomic write', () => {
});
});
+describe('durable derived-index projection', () => {
+ let contentDir: string;
+
+ beforeEach(() => {
+ contentDir = realpathSync(mkdtempSync(join(tmpdir(), 'ok-derived-persistence-')));
+ });
+
+ afterEach(() => {
+ rmSync(contentDir, { recursive: true, force: true });
+ });
+
+ async function storeDocument(
+ persistence: ReturnType<typeof createPersistenceExtension>,
+ document: Y.Doc,
+ documentName: string,
+ ): Promise<void> {
+ await persistence.extension.onStoreDocument?.({
+ document,
+ documentName,
+ lastTransactionOrigin: {
+ source: 'connection',
+ connection: { context: { principalId: 'principal-test' } },
+ },
+ lastContext: {},
+ } as never);
+ }
+
+ test('advances reconciled durability state before projecting the successful store', async () => {
+ const documentName = 'projected';
+ const markdown = '# Projected\n';
+ const durabilityState = new DocumentDurabilityState();
+ const recordDurableStore = vi.fn(async (receivedName: string, receivedMarkdown: string) => {
+ expect(durabilityState.getReconciledBase(receivedName)).toBe(receivedMarkdown);
+ });
+ const derivedDocumentIndex: DerivedDocumentIndexPersistencePort = {
+ recordDurableStore,
+ };
+ const persistence = createPersistenceExtension({
+ contentDir,
+ projectDir: contentDir,
+ gitEnabled: false,
+ durabilityState,
+ derivedDocumentIndex,
+ });
+ const document = new Y.Doc();
+ composeAndWriteRawBody(document, markdown, 'test');
+
+ await storeDocument(persistence, document, documentName);
+
+ expect(readFileSync(join(contentDir, `${documentName}.md`), 'utf-8')).toBe(markdown);
+ expect(recordDurableStore).toHaveBeenCalledWith(documentName, markdown);
+ document.destroy();
+ });
+
+ test('does not project a store whose durable disk write failed', async () => {
+ const documentName = 'blocked';
+ mkdirSync(join(contentDir, `${documentName}.md`));
+ const durabilityState = new DocumentDurabilityState();
+ const recordDurableStore = vi.fn();
+ const persistence = createPersistenceExtension({
+ contentDir,
+ projectDir: contentDir,
+ gitEnabled: false,
+ durabilityState,
+ derivedDocumentIndex: { recordDurableStore },
+ });
+ const document = new Y.Doc();
+ composeAndWriteRawBody(document, '# Blocked\n', 'test');
+
+ await expect(storeDocument(persistence, document, documentName)).rejects.toThrow();
+
+ expect(recordDurableStore).not.toHaveBeenCalled();
+ expect(durabilityState.getReconciledBase(documentName)).toBeUndefined();
+ document.destroy();
+ });
+
+ test('keeps a successful store durable when the derived index has already closed', async () => {
+ const documentName = 'closed-projection';
+ const markdown = '# Closed projection\n';
+ const durabilityState = new DocumentDurabilityState();
+ const derivedDocumentIndex = new DerivedDocumentIndex({
+ projectDir: contentDir,
+ contentDir,
+ contentFilter: createContentFilter({ projectDir: contentDir, contentDir }),
+ getGlobalSkillRoots: () => [],
+ signalChannel: () => {},
+ });
+ await derivedDocumentIndex.close();
+ const persistence = createPersistenceExtension({
+ contentDir,
+ projectDir: contentDir,
+ gitEnabled: false,
+ durabilityState,
+ derivedDocumentIndex,
+ });
+ const document = new Y.Doc();
+ composeAndWriteRawBody(document, markdown, 'test');
+
+ await expect(storeDocument(persistence, document, documentName)).resolves.toBeUndefined();
+
+ expect(readFileSync(join(contentDir, `${documentName}.md`), 'utf-8')).toBe(markdown);
+ expect(durabilityState.getReconciledBase(documentName)).toBe(markdown);
+ document.destroy();
+ });
+});
+
// ─── resolveWriterFromOrigin ───────────────────────
describe('resolveWriterFromOrigin', () => {
diff --git a/packages/server/src/persistence.ts b/packages/server/src/persistence.ts
index e12b0278e..c2aad1ab1 100644
--- a/packages/server/src/persistence.ts
+++ b/packages/server/src/persistence.ts
@@ -45,7 +45,6 @@ import type { JSONContent } from '@tiptap/core';
import { updateYFragment, yXmlFragmentToProseMirrorRootNode } from '@tiptap/y-tiptap';
import * as Y from 'yjs';
import { LINEAGE_EPOCH_KEY } from './auth-token-schema.ts';
-import type { BacklinkIndex } from './backlink-index.ts';
import { type DeriveLossDetectOptions, detectPairedIntakeLoss } from './bridge-loss-detector.ts';
import { getMsSinceLastUserTx, isDocQuiescent } from './bridge-quiescence.ts';
import { assertBridgeInvariant, createDocCanonicalizer } from './bridge-watchdog.ts';
@@ -68,6 +67,7 @@ import {
restoreContributors,
swapContributors,
} from './contributor-tracker.ts';
+import type { DerivedDocumentIndexPersistencePort } from './derived-document-index.ts';
import { applyDiskContentToDoc, FILE_WATCHER_ORIGIN } from './disk-content-intake.ts';
import { DocumentDurabilityState, type StoreFailure } from './document-durability-state.ts';
import { contentHash, registerWrite } from './file-watcher.ts';
@@ -315,7 +315,7 @@ export interface PersistenceOptions {
shadowRef?: ShadowRef;
/** Content root relative to project dir (e.g., 'content/docs'). Used for shadow repo staging. */
contentRoot?: string;
- backlinkIndex?: BacklinkIndex;
+ derivedDocumentIndex?: DerivedDocumentIndexPersistencePort;
/** Accessor for the current branch from the HEAD watcher. Used to scope WIP refs per branch. */
getCurrentBranch?: () => string | null;
/**
@@ -610,7 +610,7 @@ export function createPersistenceExtension(options?: PersistenceOptions): Persis
// project root must fall back to '.' — using the literal pathspec would
// make `git add <fallback>` look for a non-existent subfolder.
const contentRoot = options?.contentRoot ?? (toPosix(relative(projectDir, contentDir)) || '.');
- const backlinkIndex = options?.backlinkIndex;
+ const derivedDocumentIndex = options?.derivedDocumentIndex;
const getPrincipal = options?.getPrincipal;
const onAgentCommit = options?.onAgentCommit;
const onFlushCommit = options?.onFlushCommit;
@@ -2462,14 +2462,13 @@ export function createPersistenceExtension(options?: PersistenceOptions): Persis
tripwireResetFailedDocs.delete(documentName);
persistenceDeferCounts.delete(documentName);
- if (backlinkIndex) {
- backlinkIndex.updateDocumentFromMarkdown(documentName, markdown);
- void backlinkIndex.saveToDisk().catch((err) => {
- log.warn(
- { err, documentName },
- `[backlinks] Failed to persist cache for ${documentName}`,
- );
- });
+ try {
+ await derivedDocumentIndex?.recordDurableStore(documentName, markdown);
+ } catch (err) {
+ log.warn(
+ { err, documentName },
+ '[derived-index] durable-store projection failed; disk write remains authoritative',
+ );
}
setActiveSpanAttributes({ 'persistence.bytes': markdown.length });
diff --git a/packages/server/src/server-factory.test.ts b/packages/server/src/server-factory.test.ts
index 17cffdacd..a34edb63b 100644
--- a/packages/server/src/server-factory.test.ts
+++ b/packages/server/src/server-factory.test.ts
@@ -1,12 +1,25 @@
-import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import {
+ existsSync,
+ mkdirSync,
+ mkdtempSync,
+ readFileSync,
+ rmSync,
+ unlinkSync,
+ writeFileSync,
+} from 'node:fs';
import { mkdtemp, readFile, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join, resolve } from 'node:path';
import { LOCAL_DIR } from '@inkeep/open-knowledge-core';
+import { readConfigSafely, resolveConfigPath } from '@inkeep/open-knowledge-core/server';
import shellQuote from 'shell-quote';
import simpleGit from 'simple-git';
-import { afterEach, beforeEach, describe, expect, test } from 'vitest';
+import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';
import * as Y from 'yjs';
+import { BacklinkIndex } from './backlink-index.ts';
+import { getBootTimings, resetBootTimingsForTest, startBootTimings } from './boot-timings.ts';
+import { DerivedDocumentIndex } from './derived-document-index.ts';
+import { classifyGitError } from './error-classification.ts';
import { applyExternalChange } from './external-change.ts';
import type {
CheckPushPermissionOptions,
@@ -21,8 +34,39 @@ import {
writeManagedRenameJournal,
} from './managed-rename-journal.ts';
import { ensureProjectGit } from './project-git.ts';
+import { saveRemovedDocsJournal } from './removed-docs-journal.ts';
import { buildSyncCredentialArgs, createServer, type ServerInstance } from './server-factory.ts';
+import { releaseServerLock } from './server-lock.ts';
import { initShadowRepo, shadowGit } from './shadow-repo.ts';
+import { TagIndex } from './tag-index.ts';
+
+const watcherStartupFailures = vi.hoisted(() => ({ file: false, head: false }));
+
+vi.mock('./file-watcher.ts', async (importOriginal) => {
+ const actual = await importOriginal<typeof import('./file-watcher.ts')>();
+ return {
+ ...actual,
+ startWatcher: async (...args: Parameters<typeof actual.startWatcher>) => {
+ if (watcherStartupFailures.file) {
+ throw new Error('injected file-watcher startup failure');
+ }
+ return actual.startWatcher(...args);
+ },
+ };
+});
+
+vi.mock('./head-watcher.ts', async (importOriginal) => {
+ const actual = await importOriginal<typeof import('./head-watcher.ts')>();
+ return {
+ ...actual,
+ startHeadWatcher: async (...args: Parameters<typeof actual.startHeadWatcher>) => {
+ if (watcherStartupFailures.head) {
+ throw new Error('injected head-watcher startup failure');
+ }
+ return actual.startHeadWatcher(...args);
+ },
+ };
+});
// ─── CaptureLogger infrastructure ───────────────────────────────────────────
// Uses loggerFactory.configure() pattern from logger.test.ts.
@@ -119,6 +163,7 @@ describe('createServer() — document durability state isolation', () => {
const diskB = '# Disk B\n';
let serverA: ServerInstance | null = null;
let serverB: ServerInstance | null = null;
+ const beginStartup = vi.spyOn(DerivedDocumentIndex.prototype, 'beginStartup');
try {
writeFileSync(join(projectA, `${docName}.md`), diskA, 'utf-8');
@@ -137,6 +182,8 @@ describe('createServer() — document durability state isolation', () => {
});
await Promise.all([serverA.ready, serverB.ready]);
+ expect(beginStartup).toHaveBeenCalledTimes(2);
+ expect(beginStartup.mock.instances[0]).not.toBe(beginStartup.mock.instances[1]);
const [connectionA, connectionB] = await Promise.all([
serverA.hocuspocus.openDirectConnection(docName),
serverB.hocuspocus.openDirectConnection(docName),
@@ -178,10 +225,141 @@ describe('createServer() — document durability state isolation', () => {
rm(projectA, { recursive: true, force: true }),
rm(projectB, { recursive: true, force: true }),
]);
+ beginStartup.mockRestore();
}
});
});
+describe('createServer() — derived-index branch lifecycle', () => {
+ let projectDir: string;
+ let git: ReturnType<typeof simpleGit>;
+ let server: ServerInstance | null;
+
+ beforeEach(async () => {
+ projectDir = await mkdtemp(join(tmpdir(), 'ok-derived-branch-'));
+ git = simpleGit(projectDir);
+ await git.init(['--initial-branch=main']);
+ await git.raw('config', 'user.name', 'Test');
+ await git.raw('config', 'user.email', '[email protected]');
+ writeFileSync(join(projectDir, 'main.md'), '# Main\n\n#main-branch\n', 'utf-8');
+ await git.add('.');
+ await git.commit('main content');
+ await git.checkoutLocalBranch('feature');
+ unlinkSync(join(projectDir, 'main.md'));
+ writeFileSync(join(projectDir, 'feature.md'), '# Feature\n\n#feature-branch\n', 'utf-8');
+ await git.add(['-A']);
+ await git.commit('feature content');
+ await git.checkout('main');
+ server = null;
+ });
+
+ afterEach(async () => {
+ await server?.destroy();
+ vi.restoreAllMocks();
+ await rm(projectDir, { recursive: true, force: true });
+ });
+
+ test('settles target indexes before broadcasting branch-switched', async () => {
+ const beginStartup = vi.spyOn(DerivedDocumentIndex.prototype, 'beginStartup');
+ server = createServer({
+ contentDir: projectDir,
+ projectDir,
+ quiet: true,
+ gitEnabled: false,
+ skipStateManifestCheck: true,
+ });
+ await server.ready;
+ const coordinator = beginStartup.mock.instances[0] as DerivedDocumentIndex;
+ beginStartup.mockRestore();
+ const settle = vi.spyOn(DerivedDocumentIndex.prototype, 'settleBranchFromDisk');
+ const emit = vi.spyOn(server.cc1Broadcaster, 'emitBranchSwitched');
+
+ await git.checkout('feature');
+ await vi.waitFor(() => expect(emit).toHaveBeenCalledWith('feature'), {
+ timeout: 10_000,
+ interval: 25,
+ });
+
+ expect(settle).toHaveBeenCalledTimes(1);
+ expect(settle.mock.invocationCallOrder[0]).toBeLessThan(emit.mock.invocationCallOrder[0] ?? 0);
+ expect(server.durabilityState.isBatchInProgress()).toBe(false);
+ expect(await coordinator.getDocsForTagWithMatches('feature-branch')).toEqual([
+ { docName: 'feature', matchingTags: ['feature-branch'] },
+ ]);
+ expect(await coordinator.getDocsForTagWithMatches('main-branch')).toEqual([]);
+ }, 20_000);
+
+ test('aborts a degraded branch settlement and releases coordinator queries', async () => {
+ const beginStartup = vi.spyOn(DerivedDocumentIndex.prototype, 'beginStartup');
+ server = createServer({
+ contentDir: projectDir,
+ projectDir,
+ quiet: true,
+ gitEnabled: false,
+ skipStateManifestCheck: true,
+ });
+ await server.ready;
+ const coordinator = beginStartup.mock.instances[0] as DerivedDocumentIndex;
+ beginStartup.mockRestore();
+ vi.spyOn(DerivedDocumentIndex.prototype, 'settleBranchFromDisk').mockRejectedValueOnce(
+ new Error('injected branch settlement failure'),
+ );
+ const abort = vi.spyOn(DerivedDocumentIndex.prototype, 'abortBranchSwitch');
+ const emit = vi.spyOn(server.cc1Broadcaster, 'emitBranchSwitched');
+
+ await git.checkout('feature');
+ await vi.waitFor(() => expect(emit).toHaveBeenCalledWith('feature'), {
+ timeout: 10_000,
+ interval: 25,
+ });
+
+ expect(abort).toHaveBeenCalled();
+ expect(server.durabilityState.isBatchInProgress()).toBe(false);
+ await expect(coordinator.getIndexedDocNames()).resolves.toBeInstanceOf(Array);
+ }, 20_000);
+
+ test('begin-branch failure always restores durability admission and releases queries', async () => {
+ const beginStartup = vi.spyOn(DerivedDocumentIndex.prototype, 'beginStartup');
+ server = createServer({
+ contentDir: projectDir,
+ projectDir,
+ quiet: true,
+ gitEnabled: false,
+ skipStateManifestCheck: true,
+ });
+ await server.ready;
+ const coordinator = beginStartup.mock.instances[0] as DerivedDocumentIndex;
+ beginStartup.mockRestore();
+ let rejectBegin!: (error: Error) => void;
+ const beginBranch = vi
+ .spyOn(DerivedDocumentIndex.prototype, 'beginBranchSwitch')
+ .mockImplementationOnce(
+ () =>
+ new Promise<never>((_resolve, reject) => {
+ rejectBegin = reject;
+ }),
+ );
+ const abort = vi.spyOn(DerivedDocumentIndex.prototype, 'abortBranchSwitch');
+
+ await git.checkout('feature');
+ await vi.waitFor(() => expect(beginBranch).toHaveBeenCalledWith('feature'), {
+ timeout: 10_000,
+ interval: 25,
+ });
+ expect(server.durabilityState.isBatchInProgress()).toBe(true);
+
+ rejectBegin(new Error('injected branch begin failure'));
+ await vi.waitFor(
+ () => {
+ expect(abort).toHaveBeenCalled();
+ expect(server?.durabilityState.isBatchInProgress()).toBe(false);
+ },
+ { timeout: 10_000, interval: 25 },
+ );
+ await expect(coordinator.getIndexedDocNames()).resolves.toBeInstanceOf(Array);
+ }, 20_000);
+});
+
describe('createServer().destroy() — graceful shutdown flush', () => {
let tmpDir: string;
let logCapture: ReturnType<typeof captureAllLoggers>;
@@ -520,6 +698,39 @@ describe('createServer().destroy() — graceful shutdown flush', () => {
expect(shutdownLogs[0].payload.documentCount).toBe(5);
});
+ test('destroy does not await the derived-index cache drain', async () => {
+ const beginStartup = vi.spyOn(DerivedDocumentIndex.prototype, 'beginStartup');
+ const server = createServer({
+ contentDir: tmpDir,
+ projectDir: tmpDir,
+ quiet: true,
+ destroyTimeoutMs: 500,
+ });
+ await server.ready;
+ const coordinator = beginStartup.mock.instances[0] as DerivedDocumentIndex;
+ beginStartup.mockRestore();
+ const close = vi
+ .spyOn(DerivedDocumentIndex.prototype, 'close')
+ .mockImplementation(() => new Promise<void>(() => {}));
+ let timeout: ReturnType<typeof setTimeout> | undefined;
+
+ try {
+ const result = await Promise.race([
+ server.destroy().then(() => 'destroyed' as const),
+ new Promise<'timed-out'>((resolveTimeout) => {
+ timeout = setTimeout(() => resolveTimeout('timed-out'), 2_000);
+ }),
+ ]);
+
+ expect(result).toBe('destroyed');
+ expect(close).toHaveBeenCalledTimes(1);
+ } finally {
+ if (timeout) clearTimeout(timeout);
+ close.mockRestore();
+ await coordinator.close();
+ }
+ });
+
test('destroy() flushes multiple documents before resolving (multi-doc drain)', async () => {
const server = createServer({
contentDir: tmpDir,
@@ -589,13 +800,8 @@ describe('createServer().destroy() — graceful shutdown flush', () => {
* Failure injection:
* - shadow-repo: forced via invalid path (file-as-dir). This subsystem's
* init throws on invalid paths, so preferred technique works.
- * - file-watcher + head-watcher: cannot be forced via invalid paths because
- * startWatcher falls back from @parcel/watcher to chokidar (tolerates
- * invalid paths) and startHeadWatcher returns a no-op handle on missing
- * .git. The degraded.push wiring for these subsystems is verified by
- * the shadow-repo test (same push pattern) + code-level assertions.
- * vi.doMock was attempted but leaks across all test files in the same
- * `bun test` process, breaking file-watcher.test.ts.
+ * - file-watcher + head-watcher: hoisted, call-through module mocks throw
+ * only when the corresponding per-test flag is enabled.
*/
describe('createServer() degraded signal', () => {
@@ -606,6 +812,9 @@ describe('createServer() degraded signal', () => {
});
afterEach(() => {
+ watcherStartupFailures.file = false;
+ watcherStartupFailures.head = false;
+ resetBootTimingsForTest();
rmSync(testProjectDir, { recursive: true, force: true });
});
@@ -625,6 +834,54 @@ describe('createServer() degraded signal', () => {
await srv.destroy();
});
+ test('backlink startup failure labels the server as backlink-index degraded', async () => {
+ const contentDir = mkdtempSync(resolve(testProjectDir, 'content-'));
+ const load = vi
+ .spyOn(BacklinkIndex.prototype, 'loadFromDisk')
+ .mockRejectedValueOnce(new Error('injected backlink startup failure'));
+ const srv = createServer({
+ contentDir,
+ projectDir: testProjectDir,
+ quiet: true,
+ });
+
+ try {
+ await srv.ready;
+ expect(load).toHaveBeenCalled();
+ expect(srv.degraded).toContain('backlink-index');
+ expect(srv.degraded.filter((name) => name === 'backlink-index')).toHaveLength(1);
+ } finally {
+ load.mockRestore();
+ await srv.destroy();
+ }
+ });
+
+ test('tag reconciliation failure labels the server as tag-index degraded', async () => {
+ const contentDir = mkdtempSync(resolve(testProjectDir, 'content-'));
+ const beginStartup = vi.spyOn(DerivedDocumentIndex.prototype, 'beginStartup');
+ const reconcile = vi
+ .spyOn(TagIndex.prototype, 'reconcileWithDisk')
+ .mockRejectedValueOnce(new Error('injected tag reconciliation failure'));
+ const srv = createServer({
+ contentDir,
+ projectDir: testProjectDir,
+ quiet: true,
+ });
+
+ try {
+ await srv.ready;
+ const coordinator = beginStartup.mock.instances[0] as DerivedDocumentIndex;
+ expect(reconcile).toHaveBeenCalled();
+ expect(srv.degraded).toContain('tag-index');
+ expect(srv.degraded.filter((name) => name === 'tag-index')).toHaveLength(1);
+ await expect(coordinator.getAllTags()).resolves.toBeInstanceOf(Array);
+ } finally {
+ await srv.destroy();
+ beginStartup.mockRestore();
+ reconcile.mockRestore();
+ }
+ });
+
test('shadow-repo init failure — degraded includes "shadow-repo"', async () => {
// Force shadow-repo init to fail by making `.git/ok` a file (not a dir).
// `resolveShadowDir` returns `<projectDir>/.git/ok` for the directory case;
@@ -650,28 +907,28 @@ describe('createServer() degraded signal', () => {
await srv.destroy();
});
- test('degraded push wiring exists for all three subsystems', () => {
- // Verify at the source level that the degraded.push calls exist in
- // initAsync for file-watcher and head-watcher. This is a code-level
- // assertion — not as strong as a runtime test, but vi.doMock leaks
- // make runtime testing impractical without process isolation.
- const dir = import.meta.dirname ?? new URL('.', import.meta.url).pathname;
- const src = readFileSync(resolve(dir, 'server-factory.ts'), 'utf-8');
-
- // Each subsystem's catch block should push to the degraded array
- expect(src).toContain("degraded.push('shadow-repo')");
- expect(src).toContain("degraded.push('file-watcher')");
- expect(src).toContain("degraded.push('head-watcher')");
-
- // The factory return should include degraded
- expect(src).toMatch(/return\s*\{[^}]*degraded[^}]*\}/s);
+ test.each([
+ { failure: 'file' as const, label: 'file-watcher' },
+ { failure: 'head' as const, label: 'head-watcher' },
+ ])('$label startup failure is reported at runtime', async ({ failure, label }) => {
+ watcherStartupFailures[failure] = true;
+ if (failure === 'file') startBootTimings('2026-07-23T00:00:00.000Z');
+ const contentDir = mkdtempSync(resolve(testProjectDir, 'content-'));
+ const srv = createServer({
+ contentDir,
+ projectDir: testProjectDir,
+ quiet: true,
+ });
- // The index-phase boot timing is recorded in a `finally`, not the `try`, so
- // a partial/failed watcher start still yields a timing for the desktop
- // startup waterfall. Same code-level-assertion rationale as above: pin the
- // placement so a refactor that moves it into the try (dropping it on the
- // degraded path) is caught here.
- expect(src).toMatch(/finally\s*\{[\s\S]*?recordBootPhase\('indexesMs'/);
+ try {
+ await srv.ready;
+ expect(srv.degraded).toEqual([label]);
+ if (failure === 'file') {
+ expect(getBootTimings()?.indexesMs).toEqual(expect.any(Number));
+ }
+ } finally {
+ await srv.destroy();
+ }
});
test('degraded is readonly — push and reassignment are compile-time errors', async () => {
@@ -2305,23 +2562,75 @@ describe('createServer() — readProjectAutoSyncMode precedence', () => {
});
});
-// onAutoDisable invocation requires a protected-branch git remote, which is
-// expensive to set up in a unit test. The callback body is small and its
-// effect (writeConfigPatch with scope: 'project-local') is fully covered by
-// writeConfigPatch's own round-trip test. Pinning the call-site at the source
-// level catches scope drift without provisioning git fixtures.
-describe('createServer() — onAutoDisable scope pinning', () => {
- test('onAutoDisable callback writes via scope: project-local', () => {
- const dir = import.meta.dirname ?? new URL('.', import.meta.url).pathname;
- const src = readFileSync(resolve(dir, 'server-factory.ts'), 'utf-8');
- const onAutoDisableMatch = src.match(
- /onAutoDisable:\s*async\s*\([^)]*\)\s*=>\s*\{[\s\S]*?\n\s{8}\},/,
- );
- expect(onAutoDisableMatch).not.toBeNull();
- const body = onAutoDisableMatch?.[0] ?? '';
- expect(body).toContain("scope: 'project-local'");
- expect(body).toContain('autoSync: { enabled: false }');
- expect(body).not.toMatch(/scope:\s*'project'(?!-local)/);
+describe('createServer() — protected-branch auto-disable persistence', () => {
+ test('persists autoSync.enabled=false to project-local config only', async () => {
+ const projectDir = mkdtempSync(resolve(tmpdir(), 'ok-auto-disable-test-'));
+ const homedir = mkdtempSync(resolve(tmpdir(), 'ok-auto-disable-home-'));
+ const contentDir = mkdtempSync(resolve(projectDir, 'content-'));
+ const projectConfigPath = resolveConfigPath('project', projectDir);
+ const localConfigPath = resolveConfigPath('project-local', projectDir);
+ mkdirSync(join(projectDir, '.ok', LOCAL_DIR), { recursive: true });
+ writeFileSync(projectConfigPath, 'autoSync:\n default: true\n', 'utf-8');
+ writeFileSync(localConfigPath, 'autoSync:\n enabled: true\n', 'utf-8');
+
+ const server = createServer({
+ contentDir,
+ projectDir,
+ quiet: true,
+ gitEnabled: false,
+ configHomedirOverride: homedir,
+ destroyTimeoutMs: 1_000,
+ });
+
+ try {
+ await server.ready;
+ const engine = server.syncEngine;
+ expect(engine).not.toBeNull();
+ expect(engine?.getStatus().syncEnabled).toBe(true);
+
+ const testEngine = engine as unknown as {
+ handleError: (
+ classified: ReturnType<typeof classifyGitError>,
+ operation: 'push' | 'pull',
+ ) => void;
+ };
+ testEngine.handleError(
+ classifyGitError(new Error('remote: error: protected branch')),
+ 'push',
+ );
+
+ expect(engine?.getStatus()).toMatchObject({
+ state: 'disabled',
+ syncEnabled: false,
+ pausedReason: 'protected-branch',
+ pushErrorCode: 'semantic-protected-branch',
+ });
+ await vi.waitFor(
+ () => {
+ const local = readConfigSafely({
+ absPath: localConfigPath,
+ sideline: false,
+ warn: () => {},
+ });
+ expect(local.valid).toBe(true);
+ expect(local.value.autoSync.enabled).toBe(false);
+ },
+ { timeout: 2_000, interval: 20 },
+ );
+
+ const project = readConfigSafely({
+ absPath: projectConfigPath,
+ sideline: false,
+ warn: () => {},
+ });
+ expect(project.valid).toBe(true);
+ expect(project.value.autoSync.default).toBe(true);
+ expect(project.value.autoSync.enabled).toBeNull();
+ } finally {
+ await server.destroy();
+ rmSync(projectDir, { recursive: true, force: true });
+ rmSync(homedir, { recursive: true, force: true });
+ }
});
});
describe('createServer() — phantom-doc unload', () => {
@@ -2647,6 +2956,93 @@ describe('createServer() — removalRedirectGuard registration', () => {
await server.destroy();
}
});
+
+ test('warm backlink reconciliation arms deletion rejection for a file removed offline', async () => {
+ writeFileSync(join(tmpDir, 'removed-offline.md'), '# Removed offline\n', 'utf-8');
+ const first = createServer({
+ contentDir: tmpDir,
+ projectDir: tmpDir,
+ quiet: true,
+ gitEnabled: false,
+ });
+ await first.ready;
+ await first.destroy();
+ releaseServerLock(first.lockDir);
+ unlinkSync(join(tmpDir, 'removed-offline.md'));
+
+ const restarted = createServer({
+ contentDir: tmpDir,
+ projectDir: tmpDir,
+ quiet: true,
+ gitEnabled: false,
+ });
+ try {
+ await restarted.ready;
+ const ext = restarted.hocuspocus.configuration.extensions.find(
+ (entry) => (entry as { __kind?: string }).__kind === 'removal-redirect-guard',
+ ) as { onAuthenticate: (payload: unknown) => Promise<void> } | undefined;
+ if (!ext) throw new Error('removal-redirect-guard not registered');
+
+ await expect(
+ ext.onAuthenticate({
+ token: undefined,
+ context: {},
+ documentName: 'removed-offline',
+ }),
+ ).rejects.toMatchObject({ reason: 'doc-deleted' });
+ } finally {
+ await restarted.destroy();
+ }
+ });
+
+ test('warm removal journal preserves rename redirects', async () => {
+ writeFileSync(join(tmpDir, 'renamed-source.md'), '# Renamed source\n', 'utf-8');
+ const first = createServer({
+ contentDir: tmpDir,
+ projectDir: tmpDir,
+ quiet: true,
+ gitEnabled: false,
+ });
+ await first.ready;
+ await first.destroy();
+ releaseServerLock(first.lockDir);
+ unlinkSync(join(tmpDir, 'renamed-source.md'));
+ writeFileSync(join(tmpDir, 'renamed-target.md'), '# Renamed target\n', 'utf-8');
+ saveRemovedDocsJournal(tmpDir, [
+ [
+ 'renamed-source',
+ {
+ kind: 'renamed',
+ newDocName: 'renamed-target',
+ addedAt: Date.now(),
+ },
+ ],
+ ]);
+ const server = createServer({
+ contentDir: tmpDir,
+ projectDir: tmpDir,
+ quiet: true,
+ gitEnabled: false,
+ });
+
+ try {
+ await server.ready;
+ const ext = server.hocuspocus.configuration.extensions.find(
+ (entry) => (entry as { __kind?: string }).__kind === 'removal-redirect-guard',
+ ) as { onAuthenticate: (payload: unknown) => Promise<void> } | undefined;
+ if (!ext) throw new Error('removal-redirect-guard not registered');
+
+ await expect(
+ ext.onAuthenticate({
+ token: undefined,
+ context: {},
+ documentName: 'renamed-source',
+ }),
+ ).rejects.toMatchObject({ reason: 'rename-redirect:renamed-target' });
+ } finally {
+ await server.destroy();
+ }
+ });
});
// ─── Production wiring: push-permission auth ────────────────────────────────
diff --git a/packages/server/src/server-factory.ts b/packages/server/src/server-factory.ts
index a7e18d759..90a386601 100644
--- a/packages/server/src/server-factory.ts
+++ b/packages/server/src/server-factory.ts
@@ -59,7 +59,6 @@ import { createApiExtension, isSafeDocName } from './api-extension.ts';
import { assetReferencesChanged } from './asset-references.ts';
import { seedBasenameIndex, seedSingleDirBasenameIndex } from './asset-walk.ts';
import { HocuspocusAuthRejection, parseHocuspocusAuthToken } from './auth-token-schema.ts';
-import { BacklinkIndex } from './backlink-index.ts';
import { shellEscape } from './bash/shell-escape.ts';
import { bootElapsedMs, recordBootPhase, setBootField } from './boot-timings.ts';
import {
@@ -90,6 +89,10 @@ import { resolveProjectTemplates } from './content/templates-resolver.ts';
import { type ContentFilter, createContentFilter } from './content-filter.ts';
import { isWithinContentDir, safeContentPath } from './content-path.ts';
import { dropPendingDocs, recordContributor } from './contributor-tracker.ts';
+import {
+ DerivedDocumentIndex,
+ type DerivedDocumentIndexBranchTransition,
+} from './derived-document-index.ts';
import { applyDiskContentToDoc } from './disk-content-intake.ts';
import { docNameToRelativePath, getDocExtension, stripDocExtension } from './doc-extensions.ts';
import { runDocLineageGuard } from './doc-lineage-guard.ts';
@@ -201,7 +204,6 @@ import {
import { assertCompatibleStateManifest } from './state-manifest.ts';
import { SyncEngine } from './sync-engine.ts';
import { createSyncHandshakeSpanExtension } from './sync-handshake-span-extension.ts';
-import { TagIndex } from './tag-index.ts';
import { initTelemetry, shutdownTelemetry, withSpan } from './telemetry.ts';
import { trustSystemCertificates } from './trust-system-ca.ts';
import { cleanupOrphanUploadTempfiles } from './upload-streaming.ts';
@@ -416,7 +418,7 @@ export interface ServerInstance {
* Read AFTER `await ready` for a stable list; reads before may return a partial result.
* Empty array means all subsystems initialized successfully.
* Possible values: `'shadow-repo'`, `'managed-rename-recovery'`, `'file-watcher'`,
- * `'head-watcher'`.
+ * `'head-watcher'`, `'backlink-index'`, `'tag-index'`.
*/
readonly degraded: readonly string[];
/**
@@ -974,8 +976,7 @@ export function createServer(options: ServerOptions): ServerInstance {
// Synchronous init — if any constructor throws, release the lock before propagating.
let contentFilter: ReturnType<typeof createContentFilter>;
- let backlinkIndex: BacklinkIndex;
- let tagIndex: TagIndex;
+ let derivedDocumentIndex: DerivedDocumentIndex;
let shadowRef: ShadowRef;
let maintenanceCoordinator: MaintenanceCoordinator | undefined;
let persistence: ReturnType<typeof createPersistenceExtension>;
@@ -1045,27 +1046,6 @@ export function createServer(options: ServerOptions): ServerInstance {
cc1Broadcaster?.signal(channel);
}
- // Debounced saveToDisk for watcher-event paths. Collapses bursts (e.g. a git
- // clone landing many files) into a single write. Startup and branch-switch
- // paths call backlinkIndex.saveToDisk() directly — those are deliberate
- // full-state transitions that should not be deferred. The tag snapshot
- // rides the same debounce: every scheduleSaveToDisk call site pairs with a
- // tagIndex mutation on the same disk event.
- const BACKLINK_SAVE_DEBOUNCE_MS = 2000;
- let backlinkSaveTimer: ReturnType<typeof setTimeout> | null = null;
- function scheduleSaveToDisk(): void {
- if (backlinkSaveTimer !== null) clearTimeout(backlinkSaveTimer);
- backlinkSaveTimer = setTimeout(() => {
- backlinkSaveTimer = null;
- void backlinkIndex.saveToDisk().catch((err) => {
- getLogger('backlinks').warn({ err }, 'Failed to persist debounced cache');
- });
- void tagIndex.saveToDisk().catch((err) => {
- getLogger('tag-index').warn({ err }, 'Failed to persist debounced tag snapshot');
- });
- }, BACKLINK_SAVE_DEBOUNCE_MS);
- }
-
// LRU cache of docNames renamed away or deleted, durable across restarts
// via the removal journal at `.ok/local/removed-docs.json`. Read by
// `removalRedirectGuard` (registered below) to reject WebSocket
@@ -1176,28 +1156,12 @@ export function createServer(options: ServerOptions): ServerInstance {
contentDir,
singleDocRelPath,
onAfterRebuild: () => {
- // Re-derive backlink + tag indexes against the new visible-set.
- // Both indexes hold a live reference to ContentFilter, so they
- // read the freshly-rebuilt state on their next call. Failures
- // are logged but never roll back the rebuild — the in-memory
- // ignore filter is correct; derived views may go stale until
- // the next external trigger.
- //
- // All three re-derivations below are fire-and-forget and run
- // concurrently. Rapid successive ignore-file edits can overlap
- // them; that's fine — each walk reads the live ContentFilter, so
- // the last rebuild's visible-set wins and the end state converges
- // (TagIndex additionally serializes its init calls internally).
- void backlinkIndex.rebuildFromDisk(getActiveBranch()).catch((err) => {
+ // Re-derive relationship views against the freshly rebuilt admission
+ // boundary. The coordinator serializes rapid successive rebuilds.
+ void derivedDocumentIndex.refreshContentScope().catch((err) => {
getLogger('server-factory').warn(
{ err },
- '[content-filter] backlink-index rebuild failed after onAfterRebuild',
- );
- });
- void tagIndex.init().catch((err) => {
- getLogger('server-factory').warn(
- { err },
- '[content-filter] tag-index rebuild failed after onAfterRebuild',
+ '[content-filter] derived-index rebuild failed after onAfterRebuild',
);
});
// Reconcile the watcher's in-memory file/folder indexes with the
@@ -1235,25 +1199,12 @@ export function createServer(options: ServerOptions): ServerInstance {
});
},
});
- backlinkIndex = new BacklinkIndex({ projectDir, contentDir, contentFilter });
- tagIndex = new TagIndex({ projectDir, contentDir, contentFilter });
- // Boot-time scan, fire-and-forget: the factory itself is synchronous.
- // Warm start restores the persisted snapshot and mtime-reconciles offline
- // changes; cold start (no/corrupt snapshot) falls back to the full scan.
- // `initAsync` awaits a reconcile after the watcher starts; TagIndex
- // serializes the passes internally, so by the time `ready` resolves the
- // index reflects disk.
- void (async () => {
- if (await tagIndex.loadFromDisk()) {
- await tagIndex.reconcileWithDisk();
- } else {
- await tagIndex.init();
- }
- })().catch((err) => {
- getLogger('server-factory').warn(
- { err },
- '[server-factory] tag-index warm boot failed; continuing with empty index',
- );
+ derivedDocumentIndex = new DerivedDocumentIndex({
+ projectDir,
+ contentDir,
+ contentFilter,
+ getGlobalSkillRoots: () => managedArtifactSkillsRoots(persistence.managedArtifactCtx),
+ signalChannel,
});
shadowRef = { current: shadowRepo };
@@ -1305,7 +1256,7 @@ export function createServer(options: ServerOptions): ServerInstance {
shadowRef,
ephemeral,
contentRoot,
- backlinkIndex,
+ derivedDocumentIndex,
configHomedirOverride,
getCurrentBranch: () => headWatcher?.getLastKnownBranch() ?? null,
resolveEmbed,
@@ -1545,9 +1496,7 @@ export function createServer(options: ServerOptions): ServerInstance {
})),
);
const liveDerivedIndexExtension = createLiveDerivedIndexExtension({
- backlinkIndex,
- tagIndex,
- signalChannel,
+ derivedDocumentIndex,
});
hocuspocus.configuration.extensions.push(liveDerivedIndexExtension);
@@ -1862,8 +1811,7 @@ export function createServer(options: ServerOptions): ServerInstance {
// empty-object case.
getDiskAckSVs: () => cc1Broadcaster?.getLatestDiskAckSVsAsBase64() ?? {},
contentRoot,
- backlinkIndex,
- tagIndex,
+ derivedDocumentIndex,
signalChannel,
agentFocusBroadcaster,
agentPresenceBroadcaster,
@@ -2220,13 +2168,8 @@ export function createServer(options: ServerOptions): ServerInstance {
switch (event.kind) {
case 'create': {
log.info({ docName: event.docName }, `[reconcile] create: ${event.docName}`);
- backlinkIndex.updateDocumentFromMarkdown(event.docName, event.content);
- scheduleSaveToDisk();
- tagIndex.updateDocumentFromMarkdown(event.docName, event.content);
+ await derivedDocumentIndex.recordDiskUpsert(event.docName, event.content);
signalChannel('files');
- signalChannel('backlinks');
- signalChannel('graph');
- signalChannel('tags');
onUpstreamAdd(event.docName);
break;
}
@@ -2235,12 +2178,7 @@ export function createServer(options: ServerOptions): ServerInstance {
const { docName, content: theirs } = event;
const document = hocuspocus.documents.get(docName);
if (!document) {
- backlinkIndex.updateDocumentFromMarkdown(docName, theirs);
- scheduleSaveToDisk();
- tagIndex.updateDocumentFromMarkdown(docName, theirs);
- signalChannel('backlinks');
- signalChannel('graph');
- signalChannel('tags');
+ await derivedDocumentIndex.recordDiskUpsert(docName, theirs);
return;
}
@@ -2261,83 +2199,76 @@ export function createServer(options: ServerOptions): ServerInstance {
switch (result.kind) {
case 'noop':
clearLifecycleConflict(document);
- backlinkIndex.updateDocumentFromMarkdown(docName, theirs);
- scheduleSaveToDisk();
- tagIndex.updateDocumentFromMarkdown(docName, theirs);
- signalChannel('backlinks');
- signalChannel('graph');
- signalChannel('tags');
+ await derivedDocumentIndex.recordDiskUpsert(docName, theirs);
break;
case 'clean':
- try {
- applyToDoc(docName, result.newContent);
- setReconciledBase(docName, result.newContent);
- incrementReconcile();
- clearLifecycleConflict(document);
- backlinkIndex.updateDocumentFromMarkdown(docName, theirs);
- scheduleSaveToDisk();
- tagIndex.updateDocumentFromMarkdown(docName, theirs);
- signalChannel('backlinks');
- signalChannel('graph');
- signalChannel('tags');
- } catch (e) {
- log.error(
- { err: e, docName },
- `[reconcile] failed to apply clean content to Y.Doc for ${docName}`,
- );
- // Disk is source of truth — keep base in sync even if Y.Doc update failed
- setReconciledBase(docName, theirs);
- clearLifecycleConflict(document);
+ {
+ let applied = false;
+ try {
+ applyToDoc(docName, result.newContent);
+ setReconciledBase(docName, result.newContent);
+ incrementReconcile();
+ clearLifecycleConflict(document);
+ applied = true;
+ } catch (e) {
+ log.error(
+ { err: e, docName },
+ `[reconcile] failed to apply clean content to Y.Doc for ${docName}`,
+ );
+ // Disk is source of truth — keep base in sync even if Y.Doc update failed
+ setReconciledBase(docName, theirs);
+ clearLifecycleConflict(document);
+ }
+ if (applied) {
+ await derivedDocumentIndex.recordDiskUpsert(docName, theirs);
+ }
}
break;
case 'merged':
- try {
- applyToDoc(docName, result.newContent);
- // Base tracks the DISK bytes (theirs), not the merged content
- // — the merge exists only in memory until a later store
- // flushes it. A base pointing past disk makes every
- // disk-vs-base comparator misread the world: the
- // L1 before-agent-write reconcile would see a phantom
- // divergence and clean-ingest disk (reverting this merge),
- // the L3 store backstop would abort the next agent flush the
- // same way, and the no-op store skip (ytext === base) would
- // keep the merged content off disk indefinitely. With
- // base = theirs, the next store writes the merge through and
- // re-events reconcile to noop (theirs === base).
- setReconciledBase(docName, theirs);
- incrementReconcile();
- clearLifecycleConflict(document);
- backlinkIndex.updateDocumentFromMarkdown(docName, theirs);
- scheduleSaveToDisk();
- tagIndex.updateDocumentFromMarkdown(docName, theirs);
- signalChannel('backlinks');
- signalChannel('graph');
- signalChannel('tags');
- } catch (e) {
- log.error(
- { err: e, docName },
- `[reconcile] failed to apply merged content to Y.Doc for ${docName}`,
- );
- // Disk is source of truth — keep base in sync even if Y.Doc update failed
- setReconciledBase(docName, theirs);
- clearLifecycleConflict(document);
+ {
+ let applied = false;
+ try {
+ applyToDoc(docName, result.newContent);
+ // Base tracks the DISK bytes (theirs), not the merged content
+ // — the merge exists only in memory until a later store
+ // flushes it. A base pointing past disk makes every
+ // disk-vs-base comparator misread the world: the
+ // L1 before-agent-write reconcile would see a phantom
+ // divergence and clean-ingest disk (reverting this merge),
+ // the L3 store backstop would abort the next agent flush the
+ // same way, and the no-op store skip (ytext === base) would
+ // keep the merged content off disk indefinitely. With
+ // base = theirs, the next store writes the merge through and
+ // re-events reconcile to noop (theirs === base).
+ setReconciledBase(docName, theirs);
+ incrementReconcile();
+ clearLifecycleConflict(document);
+ applied = true;
+ } catch (e) {
+ log.error(
+ { err: e, docName },
+ `[reconcile] failed to apply merged content to Y.Doc for ${docName}`,
+ );
+ // Disk is source of truth — keep base in sync even if Y.Doc update failed
+ setReconciledBase(docName, theirs);
+ clearLifecycleConflict(document);
+ }
+ if (applied) {
+ await derivedDocumentIndex.recordDiskUpsert(docName, theirs);
+ }
}
break;
case 'conflicts': {
+ let applied = false;
try {
applyToDoc(docName, result.newContent);
setReconciledBase(docName, result.newContent);
incrementReconcile();
incrementConflict();
- backlinkIndex.updateDocumentFromMarkdown(docName, theirs);
- scheduleSaveToDisk();
- tagIndex.updateDocumentFromMarkdown(docName, theirs);
- signalChannel('backlinks');
- signalChannel('graph');
- signalChannel('tags');
+ applied = true;
} catch (e) {
log.error(
{ err: e, docName },
@@ -2357,6 +2288,9 @@ export function createServer(options: ServerOptions): ServerInstance {
lifecycleMap.set('status', 'conflict');
lifecycleMap.set('reason', 'merged-with-markers');
}
+ if (applied) {
+ await derivedDocumentIndex.recordDiskUpsert(docName, theirs);
+ }
break;
}
@@ -2375,13 +2309,8 @@ export function createServer(options: ServerOptions): ServerInstance {
const { docName } = event;
const document = hocuspocus.documents.get(docName);
if (!document) {
- backlinkIndex.deleteDocument(docName);
- scheduleSaveToDisk();
- tagIndex.deleteDocument(docName);
+ await derivedDocumentIndex.recordDiskDelete(docName);
signalChannel('files');
- signalChannel('backlinks');
- signalChannel('graph');
- signalChannel('tags');
onUpstreamDelete(docName);
console.info(
JSON.stringify({
@@ -2435,18 +2364,13 @@ export function createServer(options: ServerOptions): ServerInstance {
lifecycleMap.set('status', 'deleted-upstream');
deleteReconciledBase(docName);
- backlinkIndex.deleteDocument(docName);
- scheduleSaveToDisk();
- tagIndex.deleteDocument(docName);
+ await derivedDocumentIndex.recordDiskDelete(docName);
log.info({ docName, isDirty }, `[reconcile] delete: ${docName} (dirty=${isDirty})`);
// Unload document to prevent re-creation on next persistence cycle
hocuspocus.closeConnections(docName);
await forceUnloadDocument(document);
signalChannel('files');
- signalChannel('backlinks');
- signalChannel('graph');
- signalChannel('tags');
onUpstreamDelete(docName);
console.info(
JSON.stringify({
@@ -2465,9 +2389,7 @@ export function createServer(options: ServerOptions): ServerInstance {
deleteReconciledBase(oldDocName);
setReconciledBase(newDocName, content);
- backlinkIndex.renameDocument(oldDocName, newDocName, content);
- scheduleSaveToDisk();
- tagIndex.renameDocument(oldDocName, newDocName, content);
+ await derivedDocumentIndex.recordDiskRename(oldDocName, newDocName, content);
if (document) {
const lifecycleMap = document.getMap('lifecycle');
@@ -2477,9 +2399,6 @@ export function createServer(options: ServerOptions): ServerInstance {
log.info({ oldDocName, newDocName }, `[reconcile] rename: ${oldDocName} → ${newDocName}`);
signalChannel('files');
- signalChannel('backlinks');
- signalChannel('graph');
- signalChannel('tags');
onUpstreamRename(oldDocName, newDocName);
console.info(
JSON.stringify({
@@ -2523,7 +2442,7 @@ export function createServer(options: ServerOptions): ServerInstance {
}
// Asset events update the basename index and fire CC1 'files' only.
- // They do NOT touch backlinkIndex (markdown-only). They DO trigger
+ // They do NOT touch relationship views (markdown-only). They DO trigger
// a fallback re-render of any open doc that references the changed
// basename via `[[name.ext]]` (see
// `rerenderDocsReferencingAssetBasename`) — without this, a doc
@@ -2550,8 +2469,8 @@ export function createServer(options: ServerOptions): ServerInstance {
}
// file-* events maintain the in-memory fileIndex as `kind:'file'`. Like
// asset events they signal `files` (cache-invalidate /api/documents and
- // the workspace search corpus) but do NOT touch backlinkIndex or
- // tagIndex (relationship surfaces stay markdown-scoped). updateFileIndex
+ // the workspace search corpus) but do NOT touch relationship views
+ // (those surfaces stay markdown-scoped). updateFileIndex
// in handleRawEvents already mutated the index by the time we arrive
// here.
case 'file-create':
@@ -2778,24 +2697,6 @@ export function createServer(options: ServerOptions): ServerInstance {
log.warn({ err }, '[server] failed to mark server.lock draining');
}
- // Cancel any pending debounced backlink cache write so the timer doesn't
- // fire after resources are torn down.
- if (backlinkSaveTimer !== null) {
- clearTimeout(backlinkSaveTimer);
- backlinkSaveTimer = null;
- }
-
- // Close tag-index persistence: queued scans/saves become no-ops from
- // here, so a fire-and-forget snapshot save from the boot path can't
- // recreate files under `.ok/` while a caller tears the state dir down.
- // Deliberately NOT awaited: this whole destroy runs under boot.ts's 5s
- // per-step budget (destroyHocuspocus), and draining a boot-busy chain
- // here stacks on top of the ready-wait below and blows that budget on
- // slow machines. The synchronous flag flip is the load-bearing part;
- // an already-in-flight write finishes long before callers reach their
- // post-destroy cleanup.
- void tagIndex.close();
-
// Flush the removal journal synchronously — a tombstone recorded just
// before shutdown must survive the restart or a reconnecting stale
// client resurrects the doc. Failure feeds `phaseErrors`: this flush is
@@ -2954,6 +2855,14 @@ export function createServer(options: ServerOptions): ServerInstance {
log.error({ err }, '[server] shutdown phase-3 flush failed');
}
+ // Close relationship-index persistence only after the final L1
+ // stores have projected their durable markdown into backlinks.
+ // close() moves any pending cache save onto the coordinator queue
+ // before rejecting later work. The async drain is deliberately not
+ // awaited here: this destroy runs under boot.ts's bounded per-step
+ // shutdown budget.
+ void derivedDocumentIndex.close();
+
// Phase 4: drain L2 (disk → git) — only meaningful AFTER L1 has run
// Bounded to destroyTimeoutMs so a stuck git process doesn't hang shutdown.
let l2TimeoutId: ReturnType<typeof setTimeout> | undefined;
@@ -3432,18 +3341,9 @@ export function createServer(options: ServerOptions): ServerInstance {
// so reference-only edits ride the next SKILL.md touch / restart; a SKILL.md
// unlink does NOT fire onChange, so a deleted skill is pruned on restart.
if (parseGlobalSkillBundleDoc(docName)) {
- void backlinkIndex
- .ingestGlobalSkillBundles(
- managedArtifactSkillsRoots(persistence.managedArtifactCtx),
- getActiveBranch(),
- )
- .then(() => {
- signalChannel('backlinks');
- signalChannel('graph');
- })
- .catch((err) => {
- log.warn({ err, docName }, '[backlinks] global skill bundle re-ingest failed');
- });
+ void derivedDocumentIndex.refreshGlobalSkillNodes().catch((err) => {
+ log.warn({ err, docName }, '[backlinks] global skill bundle re-ingest failed');
+ });
}
};
@@ -3643,11 +3543,12 @@ export function createServer(options: ServerOptions): ServerInstance {
degraded.push('ignore-files-watcher');
}
- // Align this server's branch-scoped durability state and backlink index
- // with the project's current HEAD before either is read or written.
+ // Align branch-scoped durability and relationship views with the project's
+ // current HEAD before either is read or written.
const startupBranch = readProjectHeadState(projectDir).branch ?? 'main';
switchReconciledBaseScope(startupBranch);
- backlinkIndex.switchBranch(startupBranch);
+ const derivedIndexStartup = derivedDocumentIndex.beginStartup(startupBranch);
+ let derivedIndexStartupSettled = false;
// Boot-timing scope for the index phases (backlink load/rebuild, tag
// re-init, basename seed) plus the watcher's startup seed walk. The whole
@@ -3660,85 +3561,21 @@ export function createServer(options: ServerOptions): ServerInstance {
// `ok.boot.indexes` spans the whole index-building phase; the nested
// `ok.boot.seed-walk` span (around `startWatcher`) is its child.
await withSpan('ok.boot.indexes', undefined, async () => {
- // Warm start: load the on-disk cache and reconcile (stat-only for
- // unchanged files, async bounded reads for changed ones).
- // Cold start (no cache): full async rebuild with bounded concurrency.
- // Runs BEFORE startWatcher to avoid a race where watcher events mutate
- // the backlink index while an async rebuild is in progress.
- // Isolated in its own try/catch so a corrupt cache never prevents the
- // watcher from starting — watcher events will populate the index
- // incrementally from that point.
- const branch = getActiveBranch();
- try {
- const cacheLoaded = await backlinkIndex.loadFromDisk(branch);
- if (cacheLoaded) {
- const diff = await backlinkIndex.reconcileWithDisk(branch);
- if (diff.added > 0 || diff.updated > 0 || diff.deleted > 0) {
- log.info(
- { added: diff.added, updated: diff.updated, deleted: diff.deleted },
- '[backlinks] startup reconcile: offline changes applied',
- );
- }
- // Files deleted while the server was down leave no tombstone —
- // the delete was never observed, so nothing stops a reconnecting
- // client that still holds the doc's Yjs state from re-creating
- // the file as a "legitimate first write". The reconcile's
- // deleted set is exactly the last-known content docs whose file
- // vanished during downtime; arm the removal guard for each (the
- // journal hook makes the inference durable, and the guard's
- // file-existence-first self-heal admits any legitimate
- // re-creation).
- let tombstonedOffline = 0;
- for (const deletedDocName of diff.deletedDocNames) {
- // Same doc-kind gate as every other populate site:
- // `isReservedForUserTree` excludes synthetic, config, and
- // managed-artifact docs — none may enter the removal cache.
- // (Managed artifacts live under `.ok/` outside the content
- // walk and have their own reconcile ownership.)
- if (isReservedForUserTree(deletedDocName)) continue;
- // A journaled rename-redirect is strictly more informative
- // than an inferred flat delete — a rename inside the
- // debounced backlink-save window leaves the persisted cache
- // stale with the old name, so the inference sees it as
- // deleted-while-down. Refuse the downgrade (mirrors the
- // watcher unpaired-delete peek-guard above).
- if (recentlyRemovedDocs.peek(deletedDocName)?.kind === 'renamed') continue;
- recentlyRemovedDocs.setDeleted(deletedDocName);
- tombstonedOffline++;
- }
- if (tombstonedOffline > 0) {
- log.info(
- { count: tombstonedOffline },
- '[removal-guard] tombstoned docs deleted while the server was down',
- );
- }
- } else {
- await backlinkIndex.rebuildFromDisk(branch);
- }
- // Register GLOBAL skill bundle docs (SKILL + references/**) as graph
- // nodes. They live at `<home>/.ok/skills`, OUTSIDE contentDir, so the
- // content rebuild/reconcile above never touches them — this runs AFTER
- // those (both replace branch state) so the nodes aren't dropped. Their
- // bodies are deliberately not parsed (within-bundle-only). The cache is
- // persisted after, but a warm restart re-ingests here regardless.
- try {
- await backlinkIndex.ingestGlobalSkillBundles(
- managedArtifactSkillsRoots(persistence.managedArtifactCtx),
- branch,
- );
- } catch (err) {
- log.warn({ err, branch }, '[backlinks] global skill bundle ingest failed');
- }
- void backlinkIndex.saveToDisk().catch((err) => {
- getLogger('backlinks').warn(
- { branch, err },
- `Failed to persist startup cache for ${branch}`,
- );
- });
- } catch (err) {
- log.error(
- { err, branch },
- '[backlinks] startup init failed; index will populate incrementally via watcher',
+ const { deletedDocNames, backlinkIndexDegraded } = await derivedIndexStartup.backlinksReady;
+ if (backlinkIndexDegraded) degraded.push('backlink-index');
+ // Files deleted while the server was down leave no watcher tombstone.
+ // Arm the removal guard from the coordinator's startup reconciliation.
+ let tombstonedOffline = 0;
+ for (const deletedDocName of deletedDocNames) {
+ if (isReservedForUserTree(deletedDocName)) continue;
+ if (recentlyRemovedDocs.peek(deletedDocName)?.kind === 'renamed') continue;
+ recentlyRemovedDocs.setDeleted(deletedDocName);
+ tombstonedOffline++;
+ }
+ if (tombstonedOffline > 0) {
+ log.info(
+ { count: tombstonedOffline },
+ '[removal-guard] tombstoned docs deleted while the server was down',
);
}
// `startWatcher` performs the file-watcher's startup seed walk — the
@@ -3781,30 +3618,9 @@ export function createServer(options: ServerOptions): ServerInstance {
);
}
}
- // Reconcile tagIndex once the watcher has settled. The warm-boot pass
- // in the synchronous boot path covers the snapshot restore / cold
- // scan; this second pass is a cheap stat-only sweep that picks up any
- // disk content that landed between constructor time and watcher
- // startup (rare, but cheap to cover), then persists the snapshot for
- // the next boot. Isolated in its own try/catch so a failed tag scan
- // degrades tag search only — letting it reach the outer catch would
- // skip the basename-index seed below and misreport a healthy watcher
- // as `file-watcher` in `degraded[]`.
- try {
- const tagDiff = await tagIndex.reconcileWithDisk();
- if (tagDiff.added > 0 || tagDiff.updated > 0 || tagDiff.deleted > 0) {
- log.info(tagDiff, '[tag-index] startup reconcile: offline changes applied');
- }
- void tagIndex.saveToDisk().catch((err) => {
- getLogger('tag-index').warn({ err }, 'Failed to persist startup tag snapshot');
- });
- } catch (err) {
- log.error(
- { err },
- '[tag-index] startup reconcile failed; tag index updates incrementally via watcher events',
- );
- degraded.push('tag-index');
- }
+ const derivedIndexSettlement = await derivedDocumentIndex.settleStartupAfterWatcherSeed();
+ derivedIndexStartupSettled = true;
+ if (derivedIndexSettlement.tagIndexDegraded) degraded.push('tag-index');
// Seed the basename index from disk once the watcher's startup walk
// has finished. The watcher's fileIndex is markdown-only, so we walk
// the contentDir directly for assets.
@@ -3869,6 +3685,11 @@ export function createServer(options: ServerOptions): ServerInstance {
} catch (err) {
log.error({ err }, '[server] disk bridge watcher failed to start');
degraded.push('file-watcher');
+ if (!derivedIndexStartupSettled) {
+ const settlement = await derivedDocumentIndex.settleStartupAfterWatcherSeed();
+ derivedIndexStartupSettled = true;
+ if (settlement.tagIndexDegraded) degraded.push('tag-index');
+ }
} finally {
// Record the index-phase duration + final markdown file count even on a
// partial/failed watcher start, so the waterfall still has a value. The
@@ -4012,312 +3833,271 @@ export function createServer(options: ServerOptions): ServerInstance {
// Cross-branch or detached-head — discard buffered events (wrong branch state)
incrementBranchSwitch();
eventBuffer.splice(0, eventBuffer.length);
-
- // Switch reconciledBase scope to target branch
- switchReconciledBaseScope(newBranch);
- // Cancel any pending debounced save before switching branches.
- // Without this, the timer fires after activeBranch is updated and
- // saves the old branch's graph state into the new branch's cache.
- if (backlinkSaveTimer !== null) {
- clearTimeout(backlinkSaveTimer);
- backlinkSaveTimer = null;
- }
- backlinkIndex.switchBranch(newBranch);
-
- // Rebuild `ContentFilter`'s sibling-asset refcount BEFORE the
- // basenameIndex reseed. ContentFilter's `dirCount` is normally
- // maintained incrementally via `incrementMdDir` /
- // `decrementMdDir` calls fired by the file watcher's create /
- // delete events, but the cross-branch path discarded those
- // events above (`eventBuffer.splice`). Without a rebuild, the
- // refcount holds the previous branch's directory shape and
- // legitimate sibling-asset pairs on the new branch
- // (`assets/cover.md` next to `assets/photo.png`) are rejected
- // by `seedBasenameIndex`'s admission check, leaving the asset
- // unresolved.
- contentFilter.rebuildDirCount();
-
- // Reseed `basenameIndex` BEFORE the doc-reset loop. The reset
- // calls `applyToDoc` → `applyExternalChange` → mdast→PM with
- // `resolveEmbed`, which resolves `![[photo.png]]` against the
- // basename index. With the previous (stale) branch's paths
- // still in the index, the PM image `src` carries the
- // pre-switch resolution until the next user edit — disk
- // markdown round-trips fine, but the rendered preview is
- // wrong.
- //
- // Asset DiskEvents from the switch itself are discarded
- // (`eventBuffer.splice` above) and `basenameIndex` is a flat
- // Map without branch scope, so the explicit walk is the only
- // mechanism by which post-switch paths enter the index.
- // Mirror backlinkIndex's branch-scoped reset: drop the index,
- // walk the new branch's disk, re-seed.
- //
- // `onSkip` wiring is symmetric with the boot path — a mid-
- // session permission flip (EACCES), fd exhaustion (EMFILE),
- // or root-scope read failure during the reseed walk surfaces
- // the same `basename-index-partial` degraded indicator the
- // boot path uses.
+ let deferredStoresFlushed = false;
+ let branchTransition: DerivedDocumentIndexBranchTransition | undefined;
try {
- let reseedSkipCount = 0;
- basenameIndex.clear();
- await seedBasenameIndex({
- contentDir,
- contentFilter,
- basenameIndex,
- onSkip: (reason, code, path) => {
- reseedSkipCount++;
+ // Switch reconciledBase scope to target branch
+ switchReconciledBaseScope(newBranch);
+ branchTransition = await derivedDocumentIndex.beginBranchSwitch(newBranch);
+
+ // Rebuild `ContentFilter`'s sibling-asset refcount BEFORE the
+ // basenameIndex reseed. ContentFilter's `dirCount` is normally
+ // maintained incrementally via `incrementMdDir` /
+ // `decrementMdDir` calls fired by the file watcher's create /
+ // delete events, but the cross-branch path discarded those
+ // events above (`eventBuffer.splice`). Without a rebuild, the
+ // refcount holds the previous branch's directory shape and
+ // legitimate sibling-asset pairs on the new branch
+ // (`assets/cover.md` next to `assets/photo.png`) are rejected
+ // by `seedBasenameIndex`'s admission check, leaving the asset
+ // unresolved.
+ contentFilter.rebuildDirCount();
+
+ // Reseed `basenameIndex` BEFORE the doc-reset loop. The reset
+ // calls `applyToDoc` → `applyExternalChange` → mdast→PM with
+ // `resolveEmbed`, which resolves `![[photo.png]]` against the
+ // basename index. With the previous (stale) branch's paths
+ // still in the index, the PM image `src` carries the
+ // pre-switch resolution until the next user edit — disk
+ // markdown round-trips fine, but the rendered preview is
+ // wrong.
+ //
+ // Asset DiskEvents from the switch itself are discarded
+ // (`eventBuffer.splice` above) and `basenameIndex` is a flat
+ // Map without branch scope, so the explicit walk is the only
+ // mechanism by which post-switch paths enter the index.
+ // Mirror the relationship coordinator's branch-scoped reset: drop
+ // the basename index, walk the new branch's disk, and re-seed.
+ //
+ // `onSkip` wiring is symmetric with the boot path — a mid-
+ // session permission flip (EACCES), fd exhaustion (EMFILE),
+ // or root-scope read failure during the reseed walk surfaces
+ // the same `basename-index-partial` degraded indicator the
+ // boot path uses.
+ try {
+ let reseedSkipCount = 0;
+ basenameIndex.clear();
+ await seedBasenameIndex({
+ contentDir,
+ contentFilter,
+ basenameIndex,
+ onSkip: (reason, code, path) => {
+ reseedSkipCount++;
+ log.warn(
+ { reason, code, path, branch: newBranch },
+ `[basename-index] skipped entry during branch-switch reseed (${reason}${code ? ` ${code}` : ''})`,
+ );
+ },
+ });
+ if (reseedSkipCount > 0) {
log.warn(
- { reason, code, path, branch: newBranch },
- `[basename-index] skipped entry during branch-switch reseed (${reason}${code ? ` ${code}` : ''})`,
+ { count: reseedSkipCount, branch: newBranch },
+ `[basename-index] branch-switch reseed completed with ${reseedSkipCount} skipped entries — embeds under inaccessible subtrees will not resolve on this branch`,
);
- },
- });
- if (reseedSkipCount > 0) {
- log.warn(
- { count: reseedSkipCount, branch: newBranch },
- `[basename-index] branch-switch reseed completed with ${reseedSkipCount} skipped entries — embeds under inaccessible subtrees will not resolve on this branch`,
- );
- if (!degraded.includes('basename-index-partial')) {
- degraded.push('basename-index-partial');
+ if (!degraded.includes('basename-index-partial')) {
+ degraded.push('basename-index-partial');
+ }
}
+ } catch (err) {
+ log.error(
+ { err, branch: newBranch },
+ '[basename-index] branch-switch reseed failed',
+ );
}
- } catch (err) {
- log.error({ err, branch: newBranch }, '[basename-index] branch-switch reseed failed');
- }
- // Reset all open Y.Docs from the target branch's disk content
- for (const [docName, document] of hocuspocus.documents) {
- if (isReservedForUserTree(docName)) continue;
- try {
- const filePath = safeContentPath(docName, contentDir);
- if (!existsSync(filePath)) {
- // File doesn't exist on target branch — tombstone
- const base = getReconciledBase(docName) ?? '';
- const ours = serializeDoc(docName) ?? '';
- const isDirty = ours !== base;
-
- if (isDirty && shadowRef.current) {
- // Silent rescue checkpoint on branch-switch tombstone
- // Same pattern as reconcile-delete above.
- const shadowForCheckpoint = shadowRef.current;
- queueMicrotask(() => {
- saveInMemoryCheckpoint(shadowForCheckpoint, contentRoot ?? '', {
- kind: 'external-change-rescue',
- docName,
- contents: ours,
- label: `External change recovered @ ${new Date().toISOString()}`,
- branch: newBranch,
- metadata: { incomingDiskSha: '' },
- })
- .then(() => {
- incrementRescueBuffer();
- log.info(
- { docName },
- `[reconcile] rescue checkpoint saved on branch switch: ${docName}`,
- );
+ // Reset all open Y.Docs from the target branch's disk content
+ for (const [docName, document] of hocuspocus.documents) {
+ if (isReservedForUserTree(docName)) continue;
+ try {
+ const filePath = safeContentPath(docName, contentDir);
+ if (!existsSync(filePath)) {
+ // File doesn't exist on target branch — tombstone
+ const base = getReconciledBase(docName) ?? '';
+ const ours = serializeDoc(docName) ?? '';
+ const isDirty = ours !== base;
+
+ if (isDirty && shadowRef.current) {
+ // Silent rescue checkpoint on branch-switch tombstone
+ // Same pattern as reconcile-delete above.
+ const shadowForCheckpoint = shadowRef.current;
+ queueMicrotask(() => {
+ saveInMemoryCheckpoint(shadowForCheckpoint, contentRoot ?? '', {
+ kind: 'external-change-rescue',
+ docName,
+ contents: ours,
+ label: `External change recovered @ ${new Date().toISOString()}`,
+ branch: newBranch,
+ metadata: { incomingDiskSha: '' },
})
- .catch((e: unknown) => {
- log.error(
- { docName, err: e },
- `[reconcile] rescue checkpoint write failed: ${docName}`,
- );
- });
- });
+ .then(() => {
+ incrementRescueBuffer();
+ log.info(
+ { docName },
+ `[reconcile] rescue checkpoint saved on branch switch: ${docName}`,
+ );
+ })
+ .catch((e: unknown) => {
+ log.error(
+ { docName, err: e },
+ `[reconcile] rescue checkpoint write failed: ${docName}`,
+ );
+ });
+ });
+ }
+
+ const lifecycleMap = document.getMap('lifecycle');
+ lifecycleMap.set('status', 'deleted-upstream');
+ log.info(
+ { docName, branch: newBranch },
+ `[branch-switch] tombstone: ${docName} (not on ${newBranch})`,
+ );
+ continue;
}
- const lifecycleMap = document.getMap('lifecycle');
- lifecycleMap.set('status', 'deleted-upstream');
- log.info(
- { docName, branch: newBranch },
- `[branch-switch] tombstone: ${docName} (not on ${newBranch})`,
- );
- continue;
+ // Reset Y.Doc from disk
+ const diskContent = readFileSync(filePath, 'utf-8');
+ applyToDoc(docName, diskContent);
+ setReconciledBase(docName, diskContent);
+ log.info({ docName }, `[branch-switch] reset: ${docName}`);
+ } catch (e) {
+ log.error({ err: e, docName }, `[branch-switch] failed to reset ${docName}`);
}
-
- // Reset Y.Doc from disk
- const diskContent = readFileSync(filePath, 'utf-8');
- applyToDoc(docName, diskContent);
- setReconciledBase(docName, diskContent);
- log.info({ docName }, `[branch-switch] reset: ${docName}`);
- } catch (e) {
- log.error({ err: e, docName }, `[branch-switch] failed to reset ${docName}`);
}
- }
- log.info(
- { branch: newBranch, docCount: hocuspocus.documents.size },
- `[branch-switch] loaded branch ${newBranch} (${hocuspocus.documents.size} docs)`,
- );
- try {
- const branchCacheLoaded = await backlinkIndex.loadFromDisk(newBranch);
- if (branchCacheLoaded) {
- const diff = await backlinkIndex.reconcileWithDisk(newBranch);
- if (diff.added > 0 || diff.updated > 0 || diff.deleted > 0) {
- log.info(diff, `[backlinks] branch-switch reconcile for ${newBranch}`);
- }
- } else {
- await backlinkIndex.rebuildFromDisk(newBranch);
- }
- // Global skill bundle nodes are user-global, not branch-scoped — the
- // freshly-built branch state lacks them, so re-ingest for this branch
- // (same node-only, within-bundle-only path as boot).
+ log.info(
+ { branch: newBranch, docCount: hocuspocus.documents.size },
+ `[branch-switch] loaded branch ${newBranch} (${hocuspocus.documents.size} docs)`,
+ );
try {
- await backlinkIndex.ingestGlobalSkillBundles(
- managedArtifactSkillsRoots(persistence.managedArtifactCtx),
- newBranch,
- );
+ await derivedDocumentIndex.settleBranchFromDisk(branchTransition);
} catch (err) {
- log.warn(
+ derivedDocumentIndex.abortBranchSwitch(branchTransition);
+ log.error(
{ err, branch: newBranch },
- '[backlinks] branch-switch global skill bundle ingest failed',
+ '[derived-index] branch-switch rebuild failed; relationship views may be stale',
);
}
- void backlinkIndex.saveToDisk(newBranch).catch((err) => {
- getLogger('backlinks').warn(
- { branch: newBranch, err },
- `Failed to persist branch cache for ${newBranch}`,
- );
- });
- } catch (err) {
- log.error(
- { err, branch: newBranch },
- '[backlinks] branch-switch rebuild failed; backlinks may be stale',
- );
- }
- // TagIndex is branch-agnostic but its source-of-truth is the
- // contentDir that just changed underneath it. The checkout
- // rewrote changed files' mtimes, so a reconcile re-parses exactly
- // the docs that differ between branches instead of the whole
- // corpus; a failed reconcile falls back to the full scan so the
- // index never silently keeps the old branch's tags.
- try {
- const tagDiff = await tagIndex.reconcileWithDisk();
- if (tagDiff.added > 0 || tagDiff.updated > 0 || tagDiff.deleted > 0) {
- log.info(tagDiff, `[tag-index] branch-switch reconcile for ${newBranch}`);
- }
- } catch (err) {
- log.warn(
- { err, branch: newBranch },
- '[tag-index] branch-switch reconcile failed; falling back to full rebuild',
- );
- await tagIndex.init();
- }
- void tagIndex.saveToDisk().catch((err) => {
- getLogger('tag-index').warn(
- { branch: newBranch, err },
- 'Failed to persist tag snapshot after branch switch',
- );
- });
- // Restore parked WIP if exists (three-way merge parked state against current disk)
- if (shadowRef.current && info.batchKind === 'cross-branch') {
- let restoredCount = 0;
- for (const [docName] of hocuspocus.documents) {
- if (isReservedForUserTree(docName)) continue;
- try {
- const parked = await readParkedState(
- shadowRef.current,
- newBranch,
- SERVICE_WRITER.id,
- docName,
- );
- if (!parked) continue;
- // Skip if no in-flight edits were parked
- if (parked.markdown === parked.diskSnapshot) continue;
-
- const currentDisk = getReconciledBase(docName);
- if (!currentDisk) continue;
+ // Restore parked WIP if exists (three-way merge parked state against current disk)
+ if (shadowRef.current && info.batchKind === 'cross-branch') {
+ let restoredCount = 0;
+ for (const [docName] of hocuspocus.documents) {
+ if (isReservedForUserTree(docName)) continue;
+ try {
+ const parked = await readParkedState(
+ shadowRef.current,
+ newBranch,
+ SERVICE_WRITER.id,
+ docName,
+ );
+ if (!parked) continue;
+ // Skip if no in-flight edits were parked
+ if (parked.markdown === parked.diskSnapshot) continue;
+
+ const currentDisk = getReconciledBase(docName);
+ if (!currentDisk) continue;
+
+ const outcome = reconcile({
+ docName,
+ base: parked.diskSnapshot,
+ ours: parked.markdown,
+ theirs: currentDisk,
+ });
- const outcome = reconcile({
- docName,
- base: parked.diskSnapshot,
- ours: parked.markdown,
- theirs: currentDisk,
- });
-
- switch (outcome.kind) {
- case 'merged':
- case 'clean':
- applyToDoc(docName, outcome.newContent);
- setReconciledBase(docName, outcome.newContent);
- restoredCount++;
- break;
- case 'conflicts': {
- applyToDoc(docName, outcome.newContent);
- setReconciledBase(docName, outcome.newContent);
- incrementConflict();
- restoredCount++;
- // Mirror the file-watcher `case 'conflicts'` lifecycle set
- // so block-level reconcile failures during branch-switch
- // WIP restore also fire the UI swap + mutating-handler
- // refusal gate. Raw Y.Map.set, no transact — matches the
- // sibling convention.
- {
- const restoredDoc = hocuspocus.documents.get(docName);
- if (restoredDoc) {
- const lifecycleMap = restoredDoc.getMap('lifecycle');
- lifecycleMap.set('status', 'conflict');
- lifecycleMap.set('reason', 'merged-with-markers');
+ switch (outcome.kind) {
+ case 'merged':
+ case 'clean':
+ applyToDoc(docName, outcome.newContent);
+ setReconciledBase(docName, outcome.newContent);
+ restoredCount++;
+ break;
+ case 'conflicts': {
+ applyToDoc(docName, outcome.newContent);
+ setReconciledBase(docName, outcome.newContent);
+ incrementConflict();
+ restoredCount++;
+ // Mirror the file-watcher `case 'conflicts'` lifecycle set
+ // so block-level reconcile failures during branch-switch
+ // WIP restore also fire the UI swap + mutating-handler
+ // refusal gate. Raw Y.Map.set, no transact — matches the
+ // sibling convention.
+ {
+ const restoredDoc = hocuspocus.documents.get(docName);
+ if (restoredDoc) {
+ const lifecycleMap = restoredDoc.getMap('lifecycle');
+ lifecycleMap.set('status', 'conflict');
+ lifecycleMap.set('reason', 'merged-with-markers');
+ }
}
+ break;
}
- break;
+ case 'noop':
+ case 'refused':
+ break;
}
- case 'noop':
- case 'refused':
- break;
+ } catch (e) {
+ log.error(
+ { err: e, docName },
+ `[branch-switch] restore WIP failed for ${docName}`,
+ );
}
- } catch (e) {
- log.error(
- { err: e, docName },
- `[branch-switch] restore WIP failed for ${docName}`,
+ }
+ if (restoredCount > 0) {
+ log.info(
+ { count: restoredCount, branch: newBranch },
+ `[branch-switch] restored ${restoredCount} parked docs on ${newBranch}`,
);
}
}
- if (restoredCount > 0) {
- log.info(
- { count: restoredCount, branch: newBranch },
- `[branch-switch] restored ${restoredCount} parked docs on ${newBranch}`,
- );
- }
- }
- // Clean up detached HEAD context if switching FROM detached TO named branch
- if (info.oldBranch?.startsWith('detached-') && shadowRef.current) {
- try {
- const sg = shadowGit(shadowRef.current);
- // List refs under the detached context
- const refs = (
- await sg.raw('for-each-ref', `refs/wip/${info.oldBranch}/`, '--format=%(refname)')
- ).trim();
- if (refs) {
- // Ref deletion is a shadow mutation — take the op gate so it
- // cannot interleave with a maintenance gc run.
- await shadowOpGateFor(shadowRef.current).withMutator(async () => {
- for (const ref of refs.split('\n')) {
- if (ref) {
- await sg.raw('update-ref', '-d', ref);
+ // Clean up detached HEAD context if switching FROM detached TO named branch
+ if (info.oldBranch?.startsWith('detached-') && shadowRef.current) {
+ try {
+ const sg = shadowGit(shadowRef.current);
+ // List refs under the detached context
+ const refs = (
+ await sg.raw(
+ 'for-each-ref',
+ `refs/wip/${info.oldBranch}/`,
+ '--format=%(refname)',
+ )
+ ).trim();
+ if (refs) {
+ // Ref deletion is a shadow mutation — take the op gate so it
+ // cannot interleave with a maintenance gc run.
+ await shadowOpGateFor(shadowRef.current).withMutator(async () => {
+ for (const ref of refs.split('\n')) {
+ if (ref) {
+ await sg.raw('update-ref', '-d', ref);
+ }
}
- }
- });
- log.info(
- { context: info.oldBranch },
- `[branch-switch] cleaned up detached context ${info.oldBranch}`,
- );
+ });
+ log.info(
+ { context: info.oldBranch },
+ `[branch-switch] cleaned up detached context ${info.oldBranch}`,
+ );
+ }
+ } catch (e) {
+ log.error({ err: e }, '[branch-switch] detached cleanup failed');
}
- } catch (e) {
- log.error({ err: e }, '[branch-switch] detached cleanup failed');
}
- }
- // Notify connected clients that the branch scope changed so they can
- // invalidate their IDB persistence caches. Emit AFTER all server-side
- // state transitions (Y.Doc reset, backlink rebuild, WIP restore,
- // detached-ref cleanup) so a client's recycle-triggered reconnect
- // synchronizes against the new branch's fully-settled state.
- setBatchInProgress(false);
- await persistence.flushDeferredStores('discard-stale');
- cc1Broadcaster?.emitBranchSwitched(newBranch);
+ // Notify connected clients that the branch scope changed so they can
+ // invalidate their IDB persistence caches. Emit AFTER all server-side
+ // state transitions (Y.Doc reset, backlink rebuild, WIP restore,
+ // detached-ref cleanup) so a client's recycle-triggered reconnect
+ // synchronizes against the new branch's fully-settled state.
+ setBatchInProgress(false);
+ await persistence.flushDeferredStores('discard-stale');
+ deferredStoresFlushed = true;
+ cc1Broadcaster?.emitBranchSwitched(newBranch);
+ } finally {
+ derivedDocumentIndex.abortBranchSwitch(branchTransition);
+ if (!deferredStoresFlushed) {
+ setBatchInProgress(false);
+ await persistence.flushDeferredStores('discard-stale');
+ }
+ }
}
// Record upstream import if HEAD moved AND content files were affected.
@@ -4522,9 +4302,7 @@ export function createServer(options: ServerOptions): ServerInstance {
// nothing else would trigger a refresh until the next focus /
// visibilitychange.
signalChannel('files');
- signalChannel('backlinks');
- signalChannel('graph');
- signalChannel('tags');
+ derivedDocumentIndex.announceReadyViews();
// initAsync has reached the point where `resolveReady` will fire — record
// the elapsed-from-boot-start so the waterfall has a server-ready mark.
@@ -4539,7 +4317,13 @@ export function createServer(options: ServerOptions): ServerInstance {
// (so it could be passed into createApiExtension before initAsync ran).
// Settle it now from initAsync's completion. Errors propagate through the
// same channel callers awaited before.
- initAsync().then(resolveReady, rejectReady);
+ initAsync().then(resolveReady, (err) => {
+ // A pre-index startup failure would otherwise strand coordinator callers
+ // behind unresolved admission/readiness barriers. Failed server startup
+ // has no recovery path, so close releases every waiter with a rejection.
+ void derivedDocumentIndex.close();
+ rejectReady(err);
+ });
return {
hocuspocus,