Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
Centralize the derived-document index lifecycle (#2881)
Dimitri POSTOLOV59dd42269f0b
diff --git a/.changeset/deepen-derived-index-lifecycle.md b/.changeset/deepen-derived-index-lifecycle.mdnew file mode 100644index 000000000..83280a174--- /dev/null+++ b/.changeset/deepen-derived-index-lifecycle.md@@ -0,0 +1,10 @@+---+"@inkeep/open-knowledge": patch+"@inkeep/open-knowledge-server": patch+---++Keep backlinks, graph data, and tags consistent across every document lifecycle path. Direct creates and duplicates, disk reconciliation, deletes, folder renames, project-skill moves, ignore-file changes, startup, and branch switches now pass through one ordered derived-index coordinator with shared admission, readiness, persistence, and invalidation rules.++Derived-view queries wait for startup and branch replacement to settle instead of observing a partially rebuilt index. Rapid consecutive branch switches are serialized so one transition cannot release another transition's barrier.++Durable creates, duplicates, renames, deletes, and persistence stores now return according to authoritative disk state. Derived projection and cache persistence remain best-effort, so coordinator shutdown or a delayed cache save cannot turn an already successful write into a failure.diff --git a/packages/app/tests/integration/derived-document-index-lifecycle.test.ts b/packages/app/tests/integration/derived-document-index-lifecycle.test.tsnew file mode 100644index 000000000..49741ecc8--- /dev/null+++ b/packages/app/tests/integration/derived-document-index-lifecycle.test.ts@@ -0,0 +1,254 @@+import { mkdirSync, writeFileSync } from 'node:fs';+import { join } from 'node:path';+import { setTimeout as wait } from 'node:timers/promises';+import { HocuspocusProvider } from '@hocuspocus/provider';+import {+ type CC1DerivedViewPayload,+ CC1DerivedViewPayloadSchema,+ SYSTEM_DOC_NAME,+} from '@inkeep/open-knowledge-core';+import { expect, test } from 'vitest';+import * as Y from 'yjs';+import { createTestServer, pollUntil, waitForSync } from './test-harness.ts';++type RelationChannel = 'backlinks' | 'graph' | 'tags';+const RELATION_CHANNELS: readonly RelationChannel[] = ['backlinks', 'graph', 'tags'];++function connectRelationSignals(port: number): {+ provider: HocuspocusProvider;+ signals: CC1DerivedViewPayload[];+ destroy: () => void;+} {+ const document = new Y.Doc();+ const signals: CC1DerivedViewPayload[] = [];+ const provider = new HocuspocusProvider({+ url: `ws://127.0.0.1:${port}/collab`,+ name: SYSTEM_DOC_NAME,+ document,+ connect: true,+ onStateless: ({ payload }) => {+ let raw: unknown;+ try {+ raw = JSON.parse(payload);+ } catch {+ return;+ }+ const parsed = CC1DerivedViewPayloadSchema.safeParse(raw);+ if (parsed.success && RELATION_CHANNELS.includes(parsed.data.ch as RelationChannel)) {+ signals.push(parsed.data);+ }+ },+ });+ return {+ provider,+ signals,+ destroy: () => {+ provider.destroy();+ document.destroy();+ },+ };+}++async function expectRelationSignalsAfter<T>(+ signals: readonly CC1DerivedViewPayload[],+ action: () => Promise<T>,+): Promise<T> {+ const baseline = new Map(+ RELATION_CHANNELS.map((channel) => [+ channel,+ signals.filter((signal) => signal.ch === channel).length,+ ]),+ );+ const result = await action();+ await pollUntil(+ () =>+ RELATION_CHANNELS.every(+ (channel) =>+ signals.filter((signal) => signal.ch === channel).length > (baseline.get(channel) ?? 0),+ ),+ 5000,+ 20,+ );+ return result;+}++async function postJson(+ baseUrl: string,+ path: string,+ body: Record<string, unknown>,+ method = 'POST',+): Promise<{ status: number; body: Record<string, unknown> }> {+ const response = await fetch(`${baseUrl}${path}`, {+ method,+ headers: { 'Content-Type': 'application/json' },+ body: JSON.stringify(body),+ });+ return {+ status: response.status,+ body: (await response.json()) as Record<string, unknown>,+ };+}++async function tagDocuments(baseUrl: string, tag: string): Promise<string[]> {+ const response = await fetch(`${baseUrl}/api/tags/${encodeURIComponent(tag)}`);+ expect(response.status).toBe(200);+ const body = (await response.json()) as { docs: Array<{ docName: string }> };+ return body.docs.map((entry) => entry.docName).sort();+}++async function backlinkSources(baseUrl: string, docName: string): Promise<string[]> {+ const response = await fetch(`${baseUrl}/api/backlinks?docName=${encodeURIComponent(docName)}`);+ expect(response.status).toBe(200);+ const body = (await response.json()) as { backlinks: Array<{ source: string }> };+ return body.backlinks.map((entry) => entry.source).sort();+}++test('direct create, duplicate, rename, and delete keep links and tags in lockstep', async () => {+ const server = await createTestServer();+ const relationSignals = connectRelationSignals(server.port);+ const templateDir = join(server.contentDir, '.ok', 'templates');+ mkdirSync(templateDir, { recursive: true });+ writeFileSync(+ join(templateDir, 'lifecycle.md'),+ '---\ntemplate:\n title: Lifecycle\ntags: [lifecycle]\n---\n\nSee [[target]].\n',+ 'utf-8',+ );++ try {+ await waitForSync(relationSignals.provider);+ await wait(300);++ const created = await expectRelationSignalsAfter(relationSignals.signals, () =>+ postJson(server.baseUrl, '/api/create-page', {+ path: 'folder/item.md',+ template: 'lifecycle',+ }),+ );+ expect(created.status).toBe(200);+ expect(await tagDocuments(server.baseUrl, 'lifecycle')).toEqual(['folder/item']);+ expect(await backlinkSources(server.baseUrl, 'target')).toEqual(['folder/item']);++ const fileDuplicate = await expectRelationSignalsAfter(relationSignals.signals, () =>+ postJson(server.baseUrl, '/api/duplicate-path', {+ kind: 'file',+ path: 'folder/item',+ }),+ );+ expect(fileDuplicate.status).toBe(200);+ expect(await tagDocuments(server.baseUrl, 'lifecycle')).toEqual([+ 'folder/item',+ 'folder/item copy',+ ]);++ const folderDuplicate = await expectRelationSignalsAfter(relationSignals.signals, () =>+ postJson(server.baseUrl, '/api/duplicate-path', {+ kind: 'folder',+ path: 'folder',+ }),+ );+ expect(folderDuplicate.status).toBe(200);+ expect(await tagDocuments(server.baseUrl, 'lifecycle')).toEqual([+ 'folder copy/item',+ 'folder copy/item copy',+ 'folder/item',+ 'folder/item copy',+ ]);++ const renamed = await expectRelationSignalsAfter(relationSignals.signals, () =>+ postJson(server.baseUrl, '/api/rename-path', {+ kind: 'folder',+ fromPath: 'folder copy',+ toPath: 'archive',+ }),+ );+ expect(renamed.status).toBe(200);+ expect(await tagDocuments(server.baseUrl, 'lifecycle')).toEqual([+ 'archive/item',+ 'archive/item copy',+ 'folder/item',+ 'folder/item copy',+ ]);+ expect(await backlinkSources(server.baseUrl, 'target')).toEqual([+ 'archive/item',+ 'archive/item copy',+ 'folder/item',+ 'folder/item copy',+ ]);++ const deleted = await expectRelationSignalsAfter(relationSignals.signals, () =>+ postJson(server.baseUrl, '/api/delete-path', {+ kind: 'folder',+ path: 'archive',+ }),+ );+ expect(deleted.status).toBe(200);+ expect(await tagDocuments(server.baseUrl, 'lifecycle')).toEqual([+ 'folder/item',+ 'folder/item copy',+ ]);+ expect(await backlinkSources(server.baseUrl, 'target')).toEqual([+ 'folder/item',+ 'folder/item copy',+ ]);+ } finally {+ relationSignals.destroy();+ await server.cleanup();+ }+});++test('project skill moves refresh backlinks and tags at the relocated content doc name', async () => {+ const server = await createTestServer();+ const relationSignals = connectRelationSignals(server.port);++ try {+ await waitForSync(relationSignals.provider);+ await wait(300);++ const created = await expectRelationSignalsAfter(relationSignals.signals, () =>+ postJson(+ server.baseUrl,+ '/api/skill',+ {+ scope: 'project',+ name: 'lifecycle-skill',+ frontmatter: {+ name: 'lifecycle-skill',+ description: 'Exercises derived relationship views.',+ },+ body: 'See [[skill-target]]. #skill-lifecycle\n',+ },+ 'PUT',+ ),+ );+ expect(created.status).toBe(200);++ await pollUntil(async () => {+ const tags = await tagDocuments(server.baseUrl, 'skill-lifecycle');+ const backlinks = await backlinkSources(server.baseUrl, 'skill-target');+ return (+ tags.includes('.ok/skills/lifecycle-skill/SKILL') &&+ backlinks.includes('.ok/skills/lifecycle-skill/SKILL')+ );+ });++ const moved = await expectRelationSignalsAfter(relationSignals.signals, () =>+ postJson(server.baseUrl, '/api/skill', {+ scope: 'project',+ fromName: 'lifecycle-skill',+ toName: 'relocated-skill',+ }),+ );+ expect(moved.status).toBe(200);++ await pollUntil(async () => {+ const expected = '.ok/skills/relocated-skill/SKILL';+ const old = '.ok/skills/lifecycle-skill/SKILL';+ const tags = await tagDocuments(server.baseUrl, 'skill-lifecycle');+ const backlinks = await backlinkSources(server.baseUrl, 'skill-target');+ return tags.includes(expected) && !tags.includes(old) && backlinks.includes(expected);+ });+ } finally {+ relationSignals.destroy();+ await server.cleanup();+ }+});diff --git a/packages/app/tests/integration/managed-artifact-reconcile-anchor.test.ts b/packages/app/tests/integration/managed-artifact-reconcile-anchor.test.tsindex d674072c1..e7a8b6ae4 100644--- a/packages/app/tests/integration/managed-artifact-reconcile-anchor.test.ts+++ b/packages/app/tests/integration/managed-artifact-reconcile-anchor.test.ts@@ -22,9 +22,9 @@** Fidelity: real boot, real WS client, real `onStoreDocument` managed-artifact* branch, real file lock, real shadow repo. The divergent write is a plain- * `writeFileSync` — a second writer's edit needs no seam, because `.ok/templates`- * is not on the content watcher, so the divergence survives until the next store- * exactly as it does in production.+ * `writeFileSync`. The test invokes the real store hook immediately afterward+ * so the managed-artifact watcher cannot consume the same change first and turn+ * this into coverage of its separate disk-intake path.*/import { execFileSync } from 'node:child_process';@@ -108,6 +108,20 @@ async function pollUntil(throw new Error(`pollUntil timed out after ${timeoutMs}ms`);}+async function storeImmediately(rig: TestServer, docName: string): Promise<void> {+ const document = rig.instance.hocuspocus.documents.get(docName);+ if (!document) throw new Error(`document not loaded: ${docName}`);++ await rig.instance.hocuspocus.hooks('onStoreDocument', {+ clientsCount: document.getConnectionsCount(),+ document,+ lastContext: {},+ lastTransactionOrigin: undefined,+ documentName: docName,+ instance: rig.instance.hocuspocus,+ });+}+describe('managed-artifact reconcile-on-divergence — content-loss conventions', () => {test('the reconcile checkpoints the live artifact and reports the loss before discarding it',@@ -124,14 +138,7 @@ describe('managed-artifact reconcile-on-divergence — content-loss conventions'client.doc.transact(() => client.ytext.insert(0, v1));await pollUntil(() => existsSync(tplFile) && readFileSync(tplFile, 'utf-8') === v1);- // A second writer changes the file underneath us. `.ok/templates` is not on- // the content watcher, so this stays invisible until the next store — the- // production shape of a concurrent OK window or a hand edit.- mkdirSync(resolve(tplFile, '..'), { recursive: true });- const foreign = `---\ntitle: T\ndescription: d\n---\n\n# Template\n\n${FOREIGN_LINE}\n`;- writeFileSync(tplFile, foreign, 'utf-8');-- // Meanwhile the author keeps typing. These bytes are live and unstored.+ // The author keeps typing. These bytes are live and unstored.const live = client.ytext.toString();client.doc.transact(() => client.ytext.insert(live.length, `\n${AUTHOR_PENDING_LINE}\n`));await pollUntil(() =>@@ -144,6 +151,15 @@ describe('managed-artifact reconcile-on-divergence — content-loss conventions'),);+ // A second writer changes the file underneath us. Invoke the real store+ // hook in the same turn so this test deterministically reaches the+ // concurrent-writer reconcile before the separate template watcher can+ // import the file.+ mkdirSync(resolve(tplFile, '..'), { recursive: true });+ const foreign = `---\ntitle: T\ndescription: d\n---\n\n# Template\n\n${FOREIGN_LINE}\n`;+ writeFileSync(tplFile, foreign, 'utf-8');+ await storeImmediately(server, docName);+// The next store reads disk, sees `disk !== lkg && disk !== content`, and// reconciles: the foreign bytes win and the author's sentence is discarded.// This premise assertion PASSES on the unfixed tree — it documents the harm.diff --git a/packages/cli/src/commands/init.test.ts b/packages/cli/src/commands/init.test.tsindex fa637fc26..7115921c9 100644--- a/packages/cli/src/commands/init.test.ts+++ b/packages/cli/src/commands/init.test.ts@@ -8,6 +8,7 @@ import {realpathSync,rmSync,symlinkSync,+ unlinkSync,writeFileSync,} from 'node:fs';import { tmpdir } from 'node:os';@@ -2094,7 +2095,7 @@ describe('resolveRequestedContentDir', () => {expect(resolveRequestedContentDir('.', root, linkParent)).toBe('.');expect(resolveRequestedContentDir('sub', root, linkParent)).toBe('sub');} finally {- rmSync(linkParent, { force: true });+ unlinkSync(linkParent);}});});diff --git a/packages/core/src/handoff/urn-ipc-registry.ts b/packages/core/src/handoff/urn-ipc-registry.tsindex e2dab6a79..c71c8aeb6 100644--- a/packages/core/src/handoff/urn-ipc-registry.ts+++ b/packages/core/src/handoff/urn-ipc-registry.ts@@ -206,6 +206,7 @@ export const URN_HTTP_ONLY: ReadonlySet<ProblemType> = new Set<ProblemType>(['urn:ok:error:payload-too-large','urn:ok:error:request-timeout','urn:ok:error:internal-server-error',+ 'urn:ok:error:derived-index-unavailable',// Local-op security gates (fire before IPC; bypassed by IPC channels by design)'urn:ok:error:loopback-required','urn:ok:error:invalid-origin',diff --git a/packages/core/src/schemas/api/_envelope.test.ts b/packages/core/src/schemas/api/_envelope.test.tsindex 5458f3d61..405198c4f 100644--- a/packages/core/src/schemas/api/_envelope.test.ts+++ b/packages/core/src/schemas/api/_envelope.test.ts@@ -141,6 +141,7 @@ describe('ProblemTypeSchema', () => {'urn:ok:error:method-not-allowed','urn:ok:error:invalid-request','urn:ok:error:internal-server-error',+ 'urn:ok:error:derived-index-unavailable',];for (const t of tokens) {const result = ProblemTypeSchema.safeParse(t);diff --git a/packages/core/src/schemas/api/_envelope.ts b/packages/core/src/schemas/api/_envelope.tsindex 72ff7128a..06ff6a588 100644--- a/packages/core/src/schemas/api/_envelope.ts+++ b/packages/core/src/schemas/api/_envelope.ts@@ -204,6 +204,10 @@ export const ProblemTypeSchema = z.enum([// size-class (413 → reduce payload).'urn:ok:error:request-timeout','urn:ok:error:internal-server-error',+ // Derived-index queries reject with this retryable lifecycle state once+ // coordinator shutdown begins. Configuration absence retains the+ // index-specific `*-not-configured` tokens below.+ 'urn:ok:error:derived-index-unavailable',// /api/local-op/* security gate (shared by all local-op endpoints)'urn:ok:error:loopback-required','urn:ok:error:invalid-origin',diff --git a/packages/server/src/api-backlinks.test.ts b/packages/server/src/api-backlinks.test.tsindex 6041270b4..5c237dff3 100644--- a/packages/server/src/api-backlinks.test.ts+++ b/packages/server/src/api-backlinks.test.ts@@ -3,11 +3,19 @@ import type { IncomingMessage, ServerResponse } from 'node:http';import { tmpdir } from 'node:os';import { join } from 'node:path';import { Readable } from 'node:stream';-import { describe, expect, test } from 'vitest';-import { createApiExtension } from './api-extension.test-helper.ts';+import { describe, expect, test, vi } from 'vitest';+import {+ createApiExtension,+ createDerivedDocumentIndexApiPortStub,+} from './api-extension.test-helper.ts';import { BacklinkIndex } from './backlink-index.ts';import { type ContentFilter, createContentFilter } from './content-filter.ts';+import {+ DerivedDocumentIndex,+ type DerivedDocumentIndexApiPort,+} from './derived-document-index.ts';import type { FileIndexEntry } from './file-watcher.ts';+import { getLogger } from './logger.ts';interface CapturedResponse {status: number;@@ -42,7 +50,12 @@ async function callRoute(url: string,fileIndex: ReadonlyMap<string, FileIndexEntry>,backlinkIndex?: BacklinkIndex,- options?: { method?: string; enableTestRoutes?: boolean; contentFilter?: ContentFilter },+ options?: {+ method?: string;+ enableTestRoutes?: boolean;+ contentFilter?: ContentFilter;+ derivedDocumentIndex?: DerivedDocumentIndexApiPort;+ },): Promise<CapturedResponse> {const ext = createApiExtension({hocuspocus: {} as never,@@ -50,6 +63,7 @@ async function callRoute(contentDir,getFileIndex: () => fileIndex,backlinkIndex,+ derivedDocumentIndex: options?.derivedDocumentIndex,enableTestRoutes: options?.enableTestRoutes,...(options?.contentFilter ? { contentFilter: options.contentFilter } : {}),});@@ -464,6 +478,91 @@ describe('graph endpoints', () => {}});+ test('returns a quiet 503 when derived-index query routes race shutdown', async () => {+ const projectDir = mkdtempSync(join(tmpdir(), 'ok-derived-query-shutdown-'));+ const contentDir = join(projectDir, 'content');+ mkdirSync(contentDir, { recursive: true });+ const contentFilter = createContentFilter({ projectDir, contentDir });+ const derivedDocumentIndex = new DerivedDocumentIndex({+ projectDir,+ contentDir,+ contentFilter,+ getGlobalSkillRoots: () => [],+ signalChannel: () => {},+ });+ await derivedDocumentIndex.close();+ const httpLogger = getLogger('http');+ const errorLog = vi.spyOn(httpLogger, 'error');+ const debugLog = vi.spyOn(httpLogger, 'debug');++ const routes = [+ '/api/backlinks?docName=doc',+ '/api/backlink-counts?docNames=doc',+ '/api/forward-links?docName=doc',+ '/api/link-graph',+ '/api/orphans',+ '/api/hubs',+ '/api/dead-links',+ '/api/tags',+ '/api/tags/example',+ ] as const;++ try {+ for (const route of routes) {+ const response = await callRoute(contentDir, route, new Map(), undefined, {+ derivedDocumentIndex,+ });+ expect(response.status).toBe(503);+ expect(JSON.parse(response.body)).toMatchObject({+ type: 'urn:ok:error:derived-index-unavailable',+ title: 'Derived index is shutting down.',+ status: 503,+ });+ }+ expect(errorLog).not.toHaveBeenCalled();+ expect(debugLog).toHaveBeenCalledTimes(routes.length);+ } finally {+ errorLog.mockRestore();+ debugLog.mockRestore();+ rmSync(projectDir, { recursive: true, force: true });+ }+ });++ test('keeps unexpected derived-index query failures as 500 faults', async () => {+ const projectDir = mkdtempSync(join(tmpdir(), 'ok-derived-query-failure-'));+ const contentDir = join(projectDir, 'content');+ mkdirSync(contentDir, { recursive: true });+ const failure = new Error('unexpected query failure');+ const derivedDocumentIndex = createDerivedDocumentIndexApiPortStub({+ async getBacklinks() {+ throw failure;+ },+ });+ const errorLog = vi.spyOn(getLogger('http'), 'error');++ try {+ const response = await callRoute(+ contentDir,+ '/api/backlinks?docName=doc',+ new Map(),+ undefined,+ { derivedDocumentIndex },+ );+ expect(response.status).toBe(500);+ expect(JSON.parse(response.body)).toMatchObject({+ type: 'urn:ok:error:internal-server-error',+ status: 500,+ });+ expect(errorLog).toHaveBeenCalledWith(+ expect.objectContaining({ event: 'api.error', err: failure }),+ 'Failed to read backlinks.',+ );+ } finally {+ errorLog.mockRestore();+ rmSync(projectDir, { recursive: true, force: true });+ }+ });+test('forward-links / hubs / link-graph fall back to docName for excluded targets', async () => {// Wiki-link targets parsed from indexed docs may name a doc that is// itself excluded from the content scope (e.g. by `.gitignore` /diff --git a/packages/server/src/api-create-page.test.ts b/packages/server/src/api-create-page.test.tsindex 7999222f6..7131fa643 100644--- a/packages/server/src/api-create-page.test.ts+++ b/packages/server/src/api-create-page.test.ts@@ -13,9 +13,13 @@ import { join } from 'node:path';import { Readable } from 'node:stream';import type { Principal } from '@inkeep/open-knowledge-core';import { afterEach, beforeEach, describe, expect, test } from 'vitest';-import { createApiExtension } from './api-extension.test-helper.ts';+import {+ createApiExtension,+ createDerivedDocumentIndexApiPortStub,+} from './api-extension.test-helper.ts';import { BacklinkIndex } from './backlink-index.ts';import { contributorCount, hasContributor, swapContributors } from './contributor-tracker.ts';+import type { DerivedDocumentIndexApiPort } from './derived-document-index.ts';import type { FileIndexEntry } from './file-watcher.ts';// ---------------------------------------------------------------------------@@ -58,6 +62,7 @@ async function callCreatePage(options?: {fileIndex?: Map<string, FileIndexEntry>;backlinkIndex?: BacklinkIndex;+ derivedDocumentIndex?: DerivedDocumentIndexApiPort;getPrincipal?: () => Principal | null;},): Promise<CapturedResponse> {@@ -67,6 +72,7 @@ async function callCreatePage(contentDir,getFileIndex: () => options?.fileIndex ?? new Map<string, FileIndexEntry>(),backlinkIndex: options?.backlinkIndex,+ derivedDocumentIndex: options?.derivedDocumentIndex,getPrincipal: options?.getPrincipal,});const req = makeReq(method, body);@@ -123,6 +129,28 @@ describe('POST /api/create-page', () => {expect(existsSync(join(dir, 'my-page.md'))).toBe(true);});+ test('keeps a created file successful when the derived index is closed', async () => {+ const dir = setupTmpDir();+ const derivedDocumentIndex = createDerivedDocumentIndexApiPortStub({+ async recordDirectDocument() {+ const error = new Error('Derived document index is closed');+ error.name = 'DerivedDocumentIndexClosedError';+ throw error;+ },+ });++ const result = await callCreatePage(+ dir,+ 'POST',+ { path: 'survives-shutdown.md' },+ { derivedDocumentIndex },+ );++ expect(result.status).toBe(200);+ expect(existsSync(join(dir, 'survives-shutdown.md'))).toBe(true);+ expect(JSON.parse(result.body)).toMatchObject({ docName: 'survives-shutdown' });+ });+test('creates a .mdx file and returns the extension-less docName', async () => {const dir = setupTmpDir();const result = await callCreatePage(dir, 'POST', { path: 'component.mdx' });diff --git a/packages/server/src/api-derived-index-ready-gate.test.ts b/packages/server/src/api-derived-index-ready-gate.test.tsnew file mode 100644index 000000000..421664b1f--- /dev/null+++ b/packages/server/src/api-derived-index-ready-gate.test.ts@@ -0,0 +1,204 @@+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';+import type { IncomingMessage, ServerResponse } from 'node:http';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { Readable } from 'node:stream';+import { describe, expect, test, vi } from 'vitest';+import {+ createApiExtension,+ createDerivedDocumentIndexApiPortStub,+} from './api-extension.test-helper.ts';+import { BacklinkIndex } from './backlink-index.ts';+import { createContentFilter } from './content-filter.ts';+import { DerivedDocumentIndex } from './derived-document-index.ts';++interface PendingRoute {+ captured: { status: number; body: string };+ done: Promise<void>;+}++function startRoute(+ extension: ReturnType<typeof createApiExtension>,+ url: string,+ method = 'GET',+ body?: unknown,+): PendingRoute {+ const request = Readable.from(+ Buffer.from(body === undefined ? '' : JSON.stringify(body)),+ ) as unknown as IncomingMessage;+ request.method = method;+ request.url = url;+ request.headers = { host: 'localhost' };+ const captured = { status: 0, body: '' };+ const response = {+ writeHead(status: number) {+ captured.status = status;+ },+ setHeader() {},+ end(body?: string) {+ captured.body = body ?? '';+ },+ } as unknown as ServerResponse;+ const done = (+ extension as {+ onRequest: (ctx: { request: IncomingMessage; response: ServerResponse }) => Promise<void>;+ }+ ).onRequest({ request, response });+ return { captured, done };+}++describe('derived-index API readiness gate', () => {+ test('backlink and tag queries wait for post-watcher startup settlement', async () => {+ const projectDir = mkdtempSync(join(tmpdir(), 'ok-derived-ready-'));+ const contentDir = join(projectDir, 'content');+ mkdirSync(contentDir, { recursive: true });+ writeFileSync(join(contentDir, 'source.md'), 'See [[target]]. #startup\n');+ const index = new DerivedDocumentIndex({+ projectDir,+ contentDir,+ contentFilter: createContentFilter({ projectDir, contentDir }),+ getGlobalSkillRoots: () => [],+ signalChannel: () => {},+ });++ try {+ const startup = index.beginStartup('main');+ const extension = createApiExtension({+ hocuspocus: {} as never,+ sessionManager: {} as never,+ contentDir,+ getFileIndex: () => new Map(),+ derivedDocumentIndex: index,+ });+ const backlinksRoute = startRoute(extension, '/api/backlinks?docName=target');+ const tagsRoute = startRoute(extension, '/api/tags/startup');++ await startup.backlinksReady;+ await Promise.resolve();+ expect(backlinksRoute.captured.status).toBe(0);+ expect(tagsRoute.captured.status).toBe(0);++ await index.settleStartupAfterWatcherSeed();+ await Promise.all([backlinksRoute.done, tagsRoute.done]);++ expect(backlinksRoute.captured.status).toBe(200);+ expect(JSON.parse(backlinksRoute.captured.body)).toMatchObject({+ docName: 'target',+ backlinks: [{ source: 'source' }],+ });+ expect(tagsRoute.captured.status).toBe(200);+ expect(JSON.parse(tagsRoute.captured.body)).toMatchObject({+ name: 'startup',+ docs: [{ docName: 'source', matchingTags: ['startup'] }],+ });+ } finally {+ await index.close();+ rmSync(projectDir, { recursive: true, force: true });+ }+ });++ test('test rescan retains the route-level 500 boundary for coordinator failures', async () => {+ const projectDir = mkdtempSync(join(tmpdir(), 'ok-derived-rescan-error-'));+ const contentDir = join(projectDir, 'content');+ mkdirSync(contentDir, { recursive: true });+ const index = new DerivedDocumentIndex({+ projectDir,+ contentDir,+ contentFilter: createContentFilter({ projectDir, contentDir }),+ getGlobalSkillRoots: () => [],+ signalChannel: () => {},+ });++ try {+ const startup = index.beginStartup('main');+ await startup.backlinksReady;+ await index.settleStartupAfterWatcherSeed();+ vi.spyOn(BacklinkIndex.prototype, 'ingestGlobalSkillBundles').mockRejectedValueOnce(+ new Error('global rescan failure'),+ );+ const extension = createApiExtension({+ hocuspocus: {} as never,+ sessionManager: {} as never,+ contentDir,+ getFileIndex: () => new Map(),+ derivedDocumentIndex: index,+ enableTestRoutes: true,+ });++ const route = startRoute(extension, '/api/test-rescan-backlinks', 'POST');+ await route.done;++ expect(route.captured.status).toBe(500);+ expect(JSON.parse(route.captured.body)).toMatchObject({+ type: 'urn:ok:error:internal-server-error',+ });+ } finally {+ vi.restoreAllMocks();+ await index.close();+ rmSync(projectDir, { recursive: true, force: true });+ }+ });++ test('project skill move waits for derived-index projection before responding', async () => {+ const contentDir = mkdtempSync(join(tmpdir(), 'ok-derived-skill-move-'));+ const sourceSkillDir = join(contentDir, '.ok', 'skills', 'old-skill');+ mkdirSync(sourceSkillDir, { recursive: true });+ writeFileSync(+ join(sourceSkillDir, 'SKILL.md'),+ '---\nname: old-skill\ndescription: Use when testing moves.\n---\n\n# Old skill\n',+ );+ let releaseProjection!: () => void;+ const projectionBarrier = new Promise<void>((resolve) => {+ releaseProjection = resolve;+ });+ let projectionStarted!: () => void;+ const started = new Promise<void>((resolve) => {+ projectionStarted = resolve;+ });+ const recordDirectMutations = vi.fn(async () => {+ projectionStarted();+ await projectionBarrier;+ });+ const extension = createApiExtension({+ hocuspocus: {+ documents: new Map(),+ closeConnections() {},+ unloadDocument: async () => {},+ } as never,+ sessionManager: {+ closeSession: async () => {},+ closeAllForDoc: async () => {},+ } as never,+ contentDir,+ getFileIndex: () => new Map(),+ derivedDocumentIndex: createDerivedDocumentIndexApiPortStub({+ recordDirectMutations,+ }),+ });++ try {+ const route = startRoute(extension, '/api/skill', 'POST', {+ scope: 'project',+ fromName: 'old-skill',+ toName: 'new-skill',+ });++ await started;+ await Promise.resolve();+ expect(route.captured.status).toBe(0);++ releaseProjection();+ await route.done;++ expect(recordDirectMutations).toHaveBeenCalledTimes(1);+ expect(route.captured.status).toBe(200);+ expect(JSON.parse(route.captured.body)).toMatchObject({+ from: '.ok/skills/old-skill',+ to: '.ok/skills/new-skill',+ });+ } finally {+ releaseProjection();+ rmSync(contentDir, { recursive: true, force: true });+ }+ });+});diff --git a/packages/server/src/api-extension.ok-init.test.ts b/packages/server/src/api-extension.ok-init.test.tsindex afc1b4cd2..996a7815d 100644--- a/packages/server/src/api-extension.ok-init.test.ts+++ b/packages/server/src/api-extension.ok-init.test.ts@@ -14,22 +14,9 @@ import { join } from 'node:path';import { afterEach, describe, expect, test } from 'vitest';import { listenOnLoopback } from './loopback-rig-test-helpers.ts';-/**- * Locate the source body that follows `const <handlerNameConst> = '...'`- * declaration and return the next ~80 lines as a single string. The- * 429-emission test uses this to scope a `.toContain` check to the- * ok-init handler block without false-matching the same URN/header in- * unrelated handlers.- */-function extractHandlerBlock(src: string, handlerNameConst: string): string {- const anchorRe = new RegExp(`const\\s+${handlerNameConst}\\s*=`);- const match = anchorRe.exec(src);- if (!match) {- throw new Error(`extractHandlerBlock: '${handlerNameConst}' anchor not found`);- }- // 80 lines × roughly 80 chars = ~6400 chars. Generous slice to catch- // both the tryAcquire branch and the finally-release.- return src.slice(match.index, match.index + 6400);+interface TestConcurrencyGuard {+ tryAcquire(key: string): boolean;+ release(key: string): void;}interface TestRig {@@ -57,7 +44,9 @@ function initRepo(cwd: string): void {* doesn't read from it (the body's projectPath is the operative target),* so a minimal git repo there satisfies the boot.*/-async function bootRig(): Promise<TestRig> {+async function bootRig(+ options: { localOpConcurrencyGuard?: TestConcurrencyGuard } = {},+): Promise<TestRig> {// Root tmpRoot under the real home dir so projectPaths constructed beneath// it pass the handler's `isSafeLocalPath` home-dir containment gate. (A// tmpdir() root resolves outside $HOME on macOS — `/private/var/...` — and@@ -85,6 +74,7 @@ async function bootRig(): Promise<TestRig> {projectDir,getFileIndex: () => new Map(),serverInstanceId: 'test-instance',+ localOpConcurrencyGuard: options.localOpConcurrencyGuard,});const { createServer } = await import('node:http');@@ -116,7 +106,7 @@ async function bootRig(): Promise<TestRig> {async function postOkInit(port: number,body: Record<string, unknown>,-): Promise<{ status: number; json: Record<string, unknown> }> {+): Promise<{ status: number; json: Record<string, unknown>; retryAfter: string | null }> {const res = await fetch(`http://127.0.0.1:${port}/api/local-op/ok-init`, {method: 'POST',headers: { 'Content-Type': 'application/json' },@@ -124,7 +114,7 @@ async function postOkInit(});// biome-ignore lint/suspicious/noExplicitAny: testconst json = (await res.json()) as any;- return { status: res.status, json };+ return { status: res.status, json, retryAfter: res.headers.get('retry-after') };}let rig: TestRig | null = null;@@ -234,48 +224,47 @@ describe('POST /api/local-op/ok-init', () => {expect(res.json.type).toBe('urn:ok:error:invalid-request');});- test('429 problem+json contract — handler is wired through localOpGuard with the ok-init key', () => {- // The localOpGuard-backed 429 path is the user-facing contract when- // a concurrent ok-init lands while one is in-flight. The race itself- // is hard to observe deterministically through node:http's serial- // request dispatch (the synchronous initContent + finally{release}- // window is shorter than the body-parse + sync-gates window of the- // next request in the rig), so we validate the contract via source- // scan instead — three invariants the handler MUST satisfy:- //- // 1. A dedicated key constant `LOCAL_OP_OK_INIT_KEY` is declared- // with the canonical channel string `/api/local-op/ok-init`.- // 2. The handler acquires the key via `localOpGuard.tryAcquire`- // and emits a 429 with `urn:ok:error:concurrent-operation`- // and the `Retry-After: 2` header on contention.- // 3. The key is released in a `finally` block so a thrown- // `withParentLock` task can't leak the lock.- //- // Source scanning here is in the spirit of the structural-ratchet- // pattern (`ipc-channel-count-ratchet.test.ts`, `no-loosely-typed-- // webcontents-ipc.test.ts`): a regression that removes the lock or- // the 429 envelope shape would silently re-enable the race the- // guard exists to prevent.- const apiExtensionSrc = readFileSync(join(__dirname, 'api-extension.ts'), 'utf8');-- // (1) Key constant declared with the canonical channel string.- expect(apiExtensionSrc).toMatch(/LOCAL_OP_OK_INIT_KEY\s*=\s*['"]\/api\/local-op\/ok-init['"]/);-- // (2a) Handler calls tryAcquire on the key.- expect(apiExtensionSrc).toMatch(/localOpGuard\.tryAcquire\(LOCAL_OP_OK_INIT_KEY\)/);-- // (2b) 429 + concurrent-operation URN + Retry-After header live in- // the handler block immediately following the tryAcquire negative- // branch. Co-located so a regression that breaks the contract- // surfaces here.- const okInitBlock = extractHandlerBlock(apiExtensionSrc, 'HANDLE_LOCAL_OP_OK_INIT');- expect(okInitBlock).toContain('429');- expect(okInitBlock).toContain("'urn:ok:error:concurrent-operation'");- expect(okInitBlock).toContain("'Retry-After'");-- // (3) Release lives in a finally block (catches both happy-path- // success and `withParentLock` throws).- expect(okInitBlock).toMatch(/finally\s*\{[^}]*localOpGuard\.release\(LOCAL_OP_OK_INIT_KEY\)/s);+ test('returns 429 on contention and releases the guard after init failure', async () => {+ const key = '/api/local-op/ok-init';+ const acquired: string[] = [];+ const released: string[] = [];+ let rejectAcquisition = true;+ let held = false;+ const guard: TestConcurrencyGuard = {+ tryAcquire(nextKey) {+ acquired.push(nextKey);+ if (rejectAcquisition || held) return false;+ held = true;+ return true;+ },+ release(nextKey) {+ released.push(nextKey);+ held = false;+ },+ };+ rig = await bootRig({ localOpConcurrencyGuard: guard });+ const target = join(rig.tmpRoot, 'guarded-worktree');+ mkdirSync(target);+ initRepo(target);+ writeFileSync(join(target, 'README.md'), '# guarded\n');+ run(target, 'git add -A');+ run(target, 'git commit -q -m initial');++ const contended = await postOkInit(rig.port, { projectPath: target });+ expect(contended.status).toBe(429);+ expect(contended.json.type).toBe('urn:ok:error:concurrent-operation');+ expect(contended.retryAfter).toBe('2');+ expect(acquired).toEqual([key]);+ expect(released).toEqual([]);++ rejectAcquisition = false;+ writeFileSync(join(target, '.ok'), 'blocks scaffold directory\n');+ const failed = await postOkInit(rig.port, { projectPath: target });+ expect(failed.status).toBe(200);+ expect(failed.json).toMatchObject({ ok: false, reason: 'init-failed' });+ expect(acquired).toEqual([key, key]);+ expect(released).toEqual([key]);+ expect(held).toBe(false);});test('scaffolds inside a linked worktree (FR13 + D12 spirit)', async () => {diff --git a/packages/server/src/api-extension.test-helper.ts b/packages/server/src/api-extension.test-helper.tsindex 451b4b990..1bb60c1dc 100644--- a/packages/server/src/api-extension.test-helper.ts+++ b/packages/server/src/api-extension.test-helper.ts@@ -1,14 +1,229 @@import type { ApiExtensionOptions } from './api-extension.ts';import { createApiExtension as createApiExtensionBase } from './api-extension.ts';+import type { BacklinkIndex } from './backlink-index.ts';+import type {+ DerivedDocumentIndexApiPort,+ DerivedDocumentIndexMutation,+} from './derived-document-index.ts';import { DocumentDurabilityState } from './document-durability-state.ts';+import type { TagIndex } from './tag-index.ts';export * from './api-extension.ts';+interface LegacyIndexOptions {+ backlinkIndex?: BacklinkIndex | null;+ tagIndex?: TagIndex | null;+}++type LegacySignalChannel = (+ channel: 'files' | 'backlinks' | 'graph' | 'tags' | 'lint-config',+) => void;++export function createDerivedDocumentIndexApiPortStub(+ overrides: Partial<DerivedDocumentIndexApiPort> = {},+): DerivedDocumentIndexApiPort {+ return {+ async recordDirectMutations() {},+ async recordDirectDocument() {},+ async recordDirectDelete() {},+ async recordDirectRename() {},+ async recordLinkRewrite() {},+ async getBacklinks() {+ return [];+ },+ async getBacklinkCount() {+ return 0;+ },+ async getBacklinkCounts(documentNames) {+ return Object.fromEntries(documentNames.map((documentName) => [documentName, 0]));+ },+ async getForwardLinkEntries() {+ return [];+ },+ async getLinkGraphNeighborhood() {+ return { nodes: [], links: [] };+ },+ async getLinkGraph() {+ return { nodes: [], links: [] };+ },+ async getHubs() {+ return [];+ },+ async getOrphans() {+ return [];+ },+ async getDeadLinks() {+ return [];+ },+ async getIndexedDocNames() {+ return [];+ },+ async getAllTags() {+ return [];+ },+ async getDocsForTagWithMatches() {+ return [];+ },+ ...overrides,+ };+}++function createLegacyDerivedIndexPort(+ backlinkIndex: BacklinkIndex | null | undefined,+ tagIndex: TagIndex | null | undefined,+ signalChannel: LegacySignalChannel | undefined,+): DerivedDocumentIndexApiPort | undefined {+ if (!backlinkIndex && !tagIndex) return undefined;++ const signalBacklinks = (): void => {+ signalChannel?.('backlinks');+ signalChannel?.('graph');+ };+ const signalAll = (): void => {+ signalBacklinks();+ signalChannel?.('tags');+ };+ const saveAll = async (): Promise<void> => {+ await Promise.allSettled([backlinkIndex?.saveToDisk(), tagIndex?.saveToDisk()]);+ };+ const updateAll = (documentName: string, markdown: string): void => {+ backlinkIndex?.updateDocumentFromMarkdown(documentName, markdown);+ tagIndex?.updateDocumentFromMarkdown(documentName, markdown);+ };+ const deleteAll = (documentName: string): void => {+ backlinkIndex?.deleteDocument(documentName);+ tagIndex?.deleteDocument(documentName);+ };+ const recordDirectMutations = async (+ mutations: readonly DerivedDocumentIndexMutation[],+ ): Promise<void> => {+ if (mutations.length === 0) return;+ let tagsChanged = false;+ for (const mutation of mutations) {+ switch (mutation.kind) {+ case 'upsert':+ updateAll(mutation.documentName, mutation.markdown);+ tagsChanged = true;+ break;+ case 'delete':+ deleteAll(mutation.documentName);+ tagsChanged = true;+ break;+ case 'rename':+ backlinkIndex?.renameDocument(+ mutation.oldDocumentName,+ mutation.newDocumentName,+ mutation.markdown,+ );+ tagIndex?.renameDocument(+ mutation.oldDocumentName,+ mutation.newDocumentName,+ mutation.markdown,+ );+ tagsChanged = true;+ break;+ case 'link-rewrite':+ backlinkIndex?.updateDocumentFromMarkdown(mutation.documentName, mutation.markdown);+ break;+ }+ }+ if (tagsChanged) {+ await saveAll();+ signalAll();+ } else {+ await Promise.allSettled([backlinkIndex?.saveToDisk()]);+ signalBacklinks();+ }+ };++ return {+ recordDirectMutations,+ testOnly: {+ async resetDocumentForTest(documentName) {+ deleteAll(documentName);+ await saveAll();+ signalAll();+ },+ async rescanBacklinksForTest() {+ await backlinkIndex?.rebuildFromDisk();+ await backlinkIndex?.saveToDisk();+ signalBacklinks();+ },+ },+ async recordDirectDocument(documentName, markdown) {+ await recordDirectMutations([{ kind: 'upsert', documentName, markdown }]);+ },+ async recordDirectDelete(documentName) {+ await recordDirectMutations([{ kind: 'delete', documentName }]);+ },+ async recordDirectRename(oldDocumentName, newDocumentName, markdown) {+ await recordDirectMutations([{ kind: 'rename', oldDocumentName, newDocumentName, markdown }]);+ },+ async recordLinkRewrite(documentName, markdown) {+ await recordDirectMutations([{ kind: 'link-rewrite', documentName, markdown }]);+ },+ async getBacklinks(documentName) {+ return backlinkIndex?.getBacklinks(documentName) ?? [];+ },+ async getBacklinkCount(documentName) {+ return backlinkIndex?.getBacklinkCount(documentName) ?? 0;+ },+ async getBacklinkCounts(documentNames) {+ return Object.fromEntries(+ documentNames.map((documentName) => [+ documentName,+ backlinkIndex?.getBacklinkCount(documentName) ?? 0,+ ]),+ );+ },+ async getForwardLinkEntries(documentName) {+ return backlinkIndex?.getForwardLinkEntries(documentName) ?? [];+ },+ async getLinkGraphNeighborhood(documentName, degrees) {+ return (+ backlinkIndex?.getLinkGraphNeighborhood(documentName, degrees) ?? {+ nodes: [],+ links: [],+ }+ );+ },+ async getLinkGraph() {+ return backlinkIndex?.getLinkGraph() ?? { nodes: [], links: [] };+ },+ async getHubs(limit) {+ return backlinkIndex?.getHubs(limit) ?? [];+ },+ async getOrphans(documentNames, mode) {+ return backlinkIndex?.getOrphans(documentNames, mode) ?? [];+ },+ async getDeadLinks(admittedDocuments, sourceDocumentNames) {+ return backlinkIndex?.getDeadLinks(admittedDocuments, sourceDocumentNames) ?? [];+ },+ async getIndexedDocNames() {+ return backlinkIndex?.getIndexedDocNames() ?? [];+ },+ async getAllTags() {+ return tagIndex?.getAllTags() ?? [];+ },+ async getDocsForTagWithMatches(tag) {+ return tagIndex?.getDocsForTagWithMatches(tag) ?? [];+ },+ };+}+export function createApiExtension(- options: Omit<ApiExtensionOptions, 'durabilityState'>,+ options: Omit<ApiExtensionOptions, 'durabilityState' | 'derivedDocumentIndex' | 'signalChannel'> &+ LegacyIndexOptions & {+ derivedDocumentIndex?: DerivedDocumentIndexApiPort;+ signalChannel?: LegacySignalChannel;+ },): ReturnType<typeof createApiExtensionBase> {+ const { backlinkIndex, tagIndex, derivedDocumentIndex, ...apiOptions } = options;return createApiExtensionBase({- ...options,+ ...apiOptions,durabilityState: new DocumentDurabilityState(),+ derivedDocumentIndex:+ derivedDocumentIndex ??+ createLegacyDerivedIndexPort(backlinkIndex, tagIndex, options.signalChannel),});}diff --git a/packages/server/src/api-extension.ts b/packages/server/src/api-extension.tsindex dd66dc1a8..600aec7a8 100644--- a/packages/server/src/api-extension.ts+++ b/packages/server/src/api-extension.ts@@ -498,12 +498,7 @@ import {ManagedRenameSourceTypeMismatchError,SymlinkEscapeError,} from './apply-managed-rename.ts';-import {- type BacklinkIndex,- computeBrokenOutboundLinks,- type GraphNode as IndexedGraphNode,- isOrphanMode,-} from './backlink-index.ts';+import { computeBrokenOutboundLinks } from './backlink-index.ts';import { getBootTimings } from './boot-timings.ts';import { composeAndWriteRawBody, type PrecomputedParse, replaceRawBody } from './bridge-intake.ts';import type { BridgeDeriveLossReporter } from './bridge-loss-detector.ts';@@ -512,6 +507,13 @@ import { withHiddenWindowsConsole } from './child-process-windows-hide.ts';import type { ResolveStrategy } from './conflict-storage.ts';import type { ContentFilter } from './content-filter.ts';import { isWithinContentDir, safeContentPath } from './content-path.ts';+import {+ type DerivedDocumentIndexApiPort,+ type DerivedDocumentIndexMutation,+ type DerivedGraphNode,+ isDerivedDocumentIndexClosedError,+ isDerivedOrphanMode,+} from './derived-document-index.ts';import {docNameToRelativePath,extensionlessDocTreePath,@@ -657,7 +659,6 @@ import { createSingleFlight } from './single-flight.ts';import { restoreSkillVersion } from './skill-restore.ts';import { SuggestLinksTargetNotFoundError, suggestLinks } from './suggest-links.ts';import type { SyncEngine } from './sync-engine.ts';-import type { TagIndex } from './tag-index.ts';import { getMeter, getTracer, withSpan, withSpanSync } from './telemetry.ts';import { getDocumentHistory, getFolderTimeline } from './timeline-query.ts';import { recordTimelineCoalesced } from './timeline-telemetry.ts';@@ -2743,6 +2744,7 @@ export interface ApiExtensionOptions {* Linux CI — see `WatcherHandle.rescanFromDisk` in `file-watcher.ts`.*/rescanFiles?: () => void | Promise<void>;+ localOpConcurrencyGuard?: ReturnType<typeof createConcurrencyGuard>;/*** When true, register test-only routes (`/api/test-reset`,* `/api/test-rescan-backlinks`, `/api/test-rescan-files`). Defaults to@@ -2776,9 +2778,8 @@ export interface ApiExtensionOptions {*/getDiskAckSVs?: () => Record<string, string>;contentRoot?: string;- backlinkIndex?: BacklinkIndex;- tagIndex?: TagIndex;- signalChannel?: (channel: 'files' | 'backlinks' | 'graph' | 'lint-config') => void;+ derivedDocumentIndex?: DerivedDocumentIndexApiPort;+ signalChannel?: (channel: 'files' | 'lint-config') => void;/*** Optional. When present, agent write handlers publish per-write attribution* entries on `__system__` awareness (`agentFocus` map) with writeKind +@@ -3092,6 +3093,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {getAliasMap,getFolderAliasIndex,rescanFiles,+ localOpConcurrencyGuard,enableTestRoutes = false,shadowRef,flushGitCommit,@@ -3099,8 +3101,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {getCurrentBranch,getDiskAckSVs,contentRoot,- backlinkIndex,- tagIndex,+ derivedDocumentIndex,signalChannel,agentFocusBroadcaster,agentPresenceBroadcaster,@@ -3134,7 +3135,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {} = options;// Concurrency guard: at most 1 in-flight request per local-op endpoint- const localOpGuard = createConcurrencyGuard();+ const localOpGuard = localOpConcurrencyGuard ?? createConcurrencyGuard();// Single-flight dedupe for `GET /api/documents?showAll=true`. Keyed per// server instance (NOT module-global — tests boot several servers in one@@ -3342,15 +3343,15 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {* Soft orphan-hint: when a written doc has zero backlinks AND a hub* candidate exists in its folder tree, attach a hint suggesting the hub.* Returns `undefined` when any prerequisite is unavailable (no- * backlinkIndex wired, target not in index, has backlinks, or no candidate).+ * relationship index wired, target not in index, has backlinks, or no candidate).* Non-throwing — a hint-computation failure must not fail the write.*/- function computeOrphanHints(+ async function computeOrphanHints(docName: string,- ): Array<{ type: 'orphan'; parentCandidates: string[]; message: string }> | undefined {- if (!backlinkIndex) return undefined;+ ): Promise<Array<{ type: 'orphan'; parentCandidates: string[]; message: string }> | undefined> {+ if (!derivedDocumentIndex) return undefined;try {- const backlinks = backlinkIndex.getBacklinks(docName);+ const backlinks = await derivedDocumentIndex.getBacklinks(docName);if (backlinks.length > 0) return undefined;// This runs on every write — if hub-candidate walking becomes pathological// on very large file indexes, we want an observable signal. 5ms is well@@ -3593,7 +3594,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {return contentFilter.isExcluded(relPath);}- function collectAdmittedDocNames(): Set<string> {+ async function collectAdmittedDocNames(): Promise<Set<string>> {const admitted = new Set<string>();for (const [docName, entry] of getFileIndex()) {admitted.add(docName);@@ -3637,13 +3638,119 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {// its title/frontmatter would leak through the link/title endpoints. Already-// admitted names (file index, managed artifacts) skip the gate — cheap and// they're known in-scope.- for (const docName of backlinkIndex?.getIndexedDocNames() ?? []) {+ for (const docName of (await derivedDocumentIndex?.getIndexedDocNames()) ?? []) {if (admitted.has(docName)) continue;if (!isDocNameContentExcluded(docName)) admitted.add(docName);}return admitted;}+ async function recordDerivedMutationsBestEffort(+ mutations: readonly DerivedDocumentIndexMutation[],+ reason: string,+ ): Promise<void> {+ if (!derivedDocumentIndex || mutations.length === 0) return;+ try {+ await derivedDocumentIndex.recordDirectMutations(mutations);+ } catch (err) {+ logDerivedProjectionFailure(+ err,+ { count: mutations.length, reason },+ '[derived-index] failed to project durable document mutations',+ );+ }+ }++ async function recordDerivedDocumentBestEffort(+ documentName: string,+ markdown: string,+ reason: string,+ ): Promise<void> {+ if (!derivedDocumentIndex) return;+ try {+ await derivedDocumentIndex.recordDirectDocument(documentName, markdown);+ } catch (err) {+ logDerivedProjectionFailure(+ err,+ { documentName, reason },+ '[derived-index] failed to project durable document',+ );+ }+ }++ async function recordDerivedLinkRewriteBestEffort(+ documentName: string,+ markdown: string,+ reason: string,+ ): Promise<void> {+ if (!derivedDocumentIndex) return;+ try {+ await derivedDocumentIndex.recordLinkRewrite(documentName, markdown);+ } catch (err) {+ logDerivedProjectionFailure(+ err,+ { documentName, reason },+ '[derived-index] failed to project link rewrite',+ );+ }+ }++ function logDerivedProjectionFailure(+ err: unknown,+ context: Record<string, unknown>,+ failureMessage: string,+ ): void {+ if (isDerivedDocumentIndexClosedError(err)) {+ log.debug(+ { err, ...context },+ '[derived-index] coordinator closed; skipping durable projection',+ );+ return;+ }+ log.warn({ err, ...context }, failureMessage);+ }++ function respondToDerivedIndexQueryFailure(+ res: ServerResponse,+ err: unknown,+ options: {+ handler: string;+ failureTitle: string;+ },+ ): void {+ if (isDerivedDocumentIndexClosedError(err)) {+ errorResponse(+ res,+ 503,+ 'urn:ok:error:derived-index-unavailable',+ 'Derived index is shutting down.',+ {+ handler: options.handler,+ cause: err,+ logLevel: 'debug',+ },+ );+ return;+ }+ errorResponse(res, 500, 'urn:ok:error:internal-server-error', options.failureTitle, {+ handler: options.handler,+ cause: err,+ });+ }++ async function deleteDerivedDocumentsBestEffort(+ documentNames: Iterable<string>,+ reason: string,+ ): Promise<void> {+ await recordDerivedMutationsBestEffort(+ [...documentNames].map((documentName) => ({+ kind: 'delete',+ documentName,+ })),+ reason,+ );+ }+// On-disk existence oracle for non-doc outbound link targets (linked assets// and source files) used by `computeBrokenOutboundLinks`. Doc links resolve// against the admitted set above; file links have no CRDT presence, so a@@ -4010,11 +4117,15 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {}}- function applyPendingAssetReferenceRewrites(+ async function applyPendingAssetReferenceRewrites(pendingRewrites: readonly { docName: string; markdown: string; rewrites: number }[],renamedAssets: readonly RenamedAssetMapping[],- ): ManagedRenameRewrittenDoc[] {+ ): Promise<{+ rewrittenDocs: ManagedRenameRewrittenDoc[];+ derivedMutations: DerivedDocumentIndexMutation[];+ }> {const rewrittenDocs: ManagedRenameRewrittenDoc[] = [];+ const derivedMutations: DerivedDocumentIndexMutation[] = [];for (const pending of pendingRewrites) {const document = hocuspocus.documents.get(pending.docName);const rewritten = document@@ -4022,10 +4133,14 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {: pending;if (rewritten.rewrites === 0) continue;writeManagedRenameDocumentToDisk(pending.docName, rewritten.markdown);- backlinkIndex?.updateDocumentFromMarkdown(pending.docName, rewritten.markdown);+ derivedMutations.push({+ kind: 'link-rewrite',+ documentName: pending.docName,+ markdown: rewritten.markdown,+ });rewrittenDocs.push({ docName: pending.docName, rewrites: rewritten.rewrites });}- return rewrittenDocs;+ return { rewrittenDocs, derivedMutations };}function resolveExtensionlessAssetPath(assetPath: string): {@@ -4169,7 +4284,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {},},async (span) => {- if (!backlinkIndex) {+ if (!derivedDocumentIndex) {throw new BacklinkIndexRequiredError();}const destinationAssetPath = extname(toPath) ? toPath : `${toPath}${extname(fromPath)}`;@@ -4226,19 +4341,12 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {renamePathOnDisk(sourcePath, destinationPath);}- const rewrittenDocs = applyPendingAssetReferenceRewrites(pendingRewrites, renamedAssets);-- void backlinkIndex.saveToDisk().catch((err) => {- log.warn(- { fromPath, toPath: destinationAssetPath, err },- `[backlinks] Failed to persist asset rename cache for ${fromPath} -> ${destinationAssetPath}`,- );- });+ const { rewrittenDocs, derivedMutations } = await applyPendingAssetReferenceRewrites(+ pendingRewrites,+ renamedAssets,+ );+ await recordDerivedMutationsBestEffort(derivedMutations, 'asset-rename');signalChannel?.('files');- if (rewrittenDocs.length > 0) {- signalChannel?.('backlinks');- signalChannel?.('graph');- }rewrittenDocs.sort((a, b) => a.docName.localeCompare(b.docName));span.setAttribute('rename.rewrite_count', rewrittenDocs.length);@@ -4265,7 +4373,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {},},async (span) => {- if (!backlinkIndex) {+ if (!derivedDocumentIndex) {throw new BacklinkIndexRequiredError();}if (!isSupportedDocFile(fromPath) || isSupportedDocFile(toPath)) {@@ -4363,7 +4471,6 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {renamePathOnDisk(sourcePath, destinationPath);}- backlinkIndex.deleteDocument(sourceDocName);forgetDocExtension(sourceDocName);mutateFileIndex?.({ kind: 'delete', path: sourcePath, docName: sourceDocName });const destinationStat = statSync(destinationPath);@@ -4376,19 +4483,16 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {inode: destinationStat.ino,});- rewrittenDocs = applyPendingAssetReferenceRewrites(pendingRewrites, renamedAssets);-- void backlinkIndex.saveToDisk().catch((err) => {- log.warn(- { fromPath, toPath, err },- `[backlinks] Failed to persist document-to-file rename cache for ${fromPath} -> ${toPath}`,- );- });+ const rewriteResult = await applyPendingAssetReferenceRewrites(+ pendingRewrites,+ renamedAssets,+ );+ rewrittenDocs = rewriteResult.rewrittenDocs;+ await recordDerivedMutationsBestEffort(+ [{ kind: 'delete', documentName: sourceDocName }, ...rewriteResult.derivedMutations],+ 'document-to-file-rename',+ );signalChannel?.('files');- if (rewrittenDocs.length > 0) {- signalChannel?.('backlinks');- signalChannel?.('graph');- }});rewrittenDocs.sort((a, b) => a.docName.localeCompare(b.docName));@@ -4431,7 +4535,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {},},async (span) => {- if (!backlinkIndex) {+ if (!derivedDocumentIndex) {throw new BacklinkIndexRequiredError();}@@ -4506,7 +4610,6 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {// updated after the folder itself moves.const pendingAssetRewrites = collectAssetReferenceRewritesForMappings(renamedAssets);assertRewriteTargetsNotConflicted(pendingAssetRewrites.map((entry) => entry.docName));- const rewrittenDocs: ManagedRenameRewrittenDoc[] = [];if (kind === 'folder') {const renamedWithGit = await renameTrackedPathInGit(projectDir,@@ -4519,19 +4622,11 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {renameFolderIndexEntries(fromPath, toPath);signalChannel?.('files');}- rewrittenDocs.push(- ...applyPendingAssetReferenceRewrites(pendingAssetRewrites, renamedAssets),+ const { rewrittenDocs, derivedMutations } = await applyPendingAssetReferenceRewrites(+ pendingAssetRewrites,+ renamedAssets,);- if (rewrittenDocs.length > 0) {- void backlinkIndex.saveToDisk().catch((err) => {- log.warn(- { fromPath, toPath, err },- `[backlinks] Failed to persist managed rename cache for ${fromPath} -> ${toPath}`,- );- });- signalChannel?.('backlinks');- signalChannel?.('graph');- }+ await recordDerivedMutationsBestEffort(derivedMutations, 'asset-only-folder-rename');rewrittenDocs.sort((a, b) => a.docName.localeCompare(b.docName));return { renamed: [], renamedAssets, rewrittenDocs };}@@ -4544,7 +4639,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {const backlinkSourceSet = new Set<string>();for (const { from } of affectedDocs) {- for (const entry of backlinkIndex.getBacklinks(from)) {+ for (const entry of await derivedDocumentIndex.getBacklinks(from)) {if (!renameMap.has(entry.source)) {backlinkSourceSet.add(entry.source);}@@ -4653,10 +4748,11 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {const rewrittenDocs: ManagedRenameRewrittenDoc[] = [];const rewriteDocNames = [...rewriteDocNameSet].sort((a, b) => a.localeCompare(b));+ const derivedMutations: DerivedDocumentIndexMutation[] = [];await withManagedRenameRecovery(projectDir ?? contentDir, recoveryJournal, async () => {for (const docName of missingBacklinkSources) {- backlinkIndex.deleteDocument(docName);+ derivedMutations.push({ kind: 'delete', documentName: docName });}for (const docName of rewriteDocNames) {@@ -4676,7 +4772,11 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {rewrittenDocs.push({ docName, rewrites: rewritten.rewrites });}- backlinkIndex.updateDocumentFromMarkdown(docName, rewritten.markdown);+ derivedMutations.push({+ kind: 'link-rewrite',+ documentName: docName,+ markdown: rewritten.markdown,+ });}// `captureAndCloseDocuments` sends an application-level@@ -4913,11 +5013,17 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {content: renamedSource.markdown,});- backlinkIndex.renameDocument(fromDocName, toDocName, renamedSource.markdown);+ derivedMutations.push({+ kind: 'rename',+ oldDocumentName: fromDocName,+ newDocumentName: toDocName,+ markdown: renamedSource.markdown,+ });if (renamedSource.rewrites > 0) {rewrittenDocs.push({ docName: toDocName, rewrites: renamedSource.rewrites });}}+ await recordDerivedMutationsBestEffort(derivedMutations, 'document-rename');// Second crash-injection seam — fires AFTER the log append +// AFTER the per-doc sync loop, BEFORE the implicit@@ -4933,15 +5039,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {}});- void backlinkIndex.saveToDisk().catch((err) => {- log.warn(- { fromPath, toPath, err },- `[backlinks] Failed to persist managed rename cache for ${fromPath} -> ${toPath}`,- );- });signalChannel?.('files');- signalChannel?.('backlinks');- signalChannel?.('graph');rewrittenDocs.sort((a, b) => a.docName.localeCompare(b.docName));span.setAttribute('rename.rewrite_count', rewrittenDocs.length);@@ -5636,8 +5734,8 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {// Orphan-hint nudge: if this doc now has zero backlinks and a// plausible hub exists in its folder tree, suggest the hub. Soft —- // agent can ignore. Silent when no backlinkIndex is wired.- const hints = computeOrphanHints(resolvedDocName);+ // agent can ignore. Silent when no relationship index is wired.+ const hints = await computeOrphanHints(resolvedDocName);// The converged post-write source (frontmatter region + body), read// once and reused for both the mermaid render check and the broken-@@ -5659,7 +5757,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {// the write (authoring a doc before its target exists is legitimate).// The just-written doc is added to the admitted set so a valid self-link// isn't falsely flagged before the file-watcher indexes it on disk.- const admittedForLinks = collectAdmittedDocNames();+ const admittedForLinks = await collectAdmittedDocNames();admittedForLinks.add(resolvedDocName);const brokenLinks = computeBrokenOutboundLinks(writtenSource,@@ -6087,7 +6185,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {// the whole batch. Every flushed batch doc joins the admitted set// first so intra-batch links (doc A -> doc B written in the same// call) validate regardless of entry order.- const admittedForLinks = collectAdmittedDocNames();+ const admittedForLinks = await collectAdmittedDocNames();for (const p of pending) {if (flushErrors.get(p.docName) === undefined) admittedForLinks.add(p.docName);}@@ -6445,7 +6543,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {// the body unchanged, so this reflects the doc's current body links —// surfacing the same `brokenLinks` signal on every `edit` path keeps// the contract uniform rather than returning a misleading empty `[]`.- const admittedForLinks = collectAdmittedDocNames();+ const admittedForLinks = await collectAdmittedDocNames();admittedForLinks.add(resolvedDocName);const brokenLinks = computeBrokenOutboundLinks(session.dc.document.getText('source').toString(),@@ -7193,7 +7291,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {const handleBacklinks = withValidation(EmptyRequestSchema,async (req, res) => {- if (!backlinkIndex) {+ if (!derivedDocumentIndex) {errorResponse(res,503,@@ -7218,7 +7316,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {});return;}- const backlinks = backlinkIndex.getBacklinks(docName).map((entry) => ({+ const backlinks = (await derivedDocumentIndex.getBacklinks(docName)).map((entry) => ({source: entry.source,anchor: entry.anchor,title: readPageTitleForDocName(entry.source),@@ -7232,9 +7330,9 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {{ handler: 'backlinks' },);} catch (e) {- errorResponse(res, 500, 'urn:ok:error:internal-server-error', 'Failed to read backlinks.', {+ respondToDerivedIndexQueryFailure(res, e, {handler: 'backlinks',- cause: e,+ failureTitle: 'Failed to read backlinks.',});}},@@ -7251,7 +7349,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {const handleBacklinkCounts = withValidation(EmptyRequestSchema,async (req, res) => {- if (!backlinkIndex) {+ if (!derivedDocumentIndex) {errorResponse(res,503,@@ -7270,12 +7368,11 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {});return;}- const counts: Record<string, number> = {};- for (const docName of raw.split(',')) {- const trimmed = docName.trim();- if (!trimmed || !isSafeDocName(trimmed)) continue;- counts[trimmed] = backlinkIndex.getBacklinkCount(trimmed);- }+ const docNames = raw+ .split(',')+ .map((docName) => docName.trim())+ .filter((docName) => docName && isSafeDocName(docName));+ const counts = await derivedDocumentIndex.getBacklinkCounts(docNames);successResponse(res,200,@@ -7284,13 +7381,10 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {{ handler: 'backlink-counts' },);} catch (e) {- errorResponse(- res,- 500,- 'urn:ok:error:internal-server-error',- 'Failed to read backlink counts.',- { handler: 'backlink-counts', cause: e },- );+ respondToDerivedIndexQueryFailure(res, e, {+ handler: 'backlink-counts',+ failureTitle: 'Failed to read backlink counts.',+ });}},{ handler: 'backlink-counts', method: 'GET', skipBodyParse: true },@@ -7299,7 +7393,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {const handleForwardLinks = withValidation(EmptyRequestSchema,async (req, res) => {- if (!backlinkIndex) {+ if (!derivedDocumentIndex) {errorResponse(res,503,@@ -7324,40 +7418,38 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {});return;}- const admitted = collectAdmittedDocNames();+ const admitted = await collectAdmittedDocNames();successResponse(res,200,ForwardLinksSuccessSchema,{docName,- forwardLinks: backlinkIndex.getForwardLinkEntries(docName).map((entry) =>- entry.kind === 'doc'- ? {- kind: 'doc' as const,- docName: entry.target,- anchor: entry.anchor,- title: readPageTitleForLinkedDocName(entry.target, admitted),- snippet: entry.snippet,- }- : {- kind: 'external' as const,- url: entry.url,- title: entry.label ?? entry.url,- snippet: entry.snippet,- },+ forwardLinks: (await derivedDocumentIndex.getForwardLinkEntries(docName)).map(+ (entry) =>+ entry.kind === 'doc'+ ? {+ kind: 'doc' as const,+ docName: entry.target,+ anchor: entry.anchor,+ title: readPageTitleForLinkedDocName(entry.target, admitted),+ snippet: entry.snippet,+ }+ : {+ kind: 'external' as const,+ url: entry.url,+ title: entry.label ?? entry.url,+ snippet: entry.snippet,+ },),},{ handler: 'forward-links' },);} catch (e) {- errorResponse(- res,- 500,- 'urn:ok:error:internal-server-error',- 'Failed to read forward links.',- { handler: 'forward-links', cause: e },- );+ respondToDerivedIndexQueryFailure(res, e, {+ handler: 'forward-links',+ failureTitle: 'Failed to read forward links.',+ });}},{ handler: 'forward-links', method: 'GET', skipBodyParse: true },@@ -7366,7 +7458,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {const handleLinkGraph = withValidation(EmptyRequestSchema,async (req, res) => {- if (!backlinkIndex) {+ if (!derivedDocumentIndex) {errorResponse(res,503,@@ -7398,7 +7490,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {return;}- let nodes: IndexedGraphNode[];+ let nodes: DerivedGraphNode[];let links: Array<{ source: string; target: string }>;if (rawDegrees && docName) {@@ -7414,12 +7506,15 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {return;}- ({ nodes, links } = backlinkIndex.getLinkGraphNeighborhood(docName, degrees));+ ({ nodes, links } = await derivedDocumentIndex.getLinkGraphNeighborhood(+ docName,+ degrees,+ ));} else {- ({ nodes, links } = backlinkIndex.getLinkGraph());+ ({ nodes, links } = await derivedDocumentIndex.getLinkGraph());}- const admitted = collectAdmittedDocNames();+ const admitted = await collectAdmittedDocNames();const enrichedNodes = nodes.map((node) => {if (node.kind === 'doc') {const meta = readFrontmatterMetadataForLinkedDocName(node.docName, admitted);@@ -7449,16 +7544,10 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {{ handler: 'link-graph' },);} catch (e) {- errorResponse(- res,- 500,- 'urn:ok:error:internal-server-error',- 'Failed to read link graph.',- {- handler: 'link-graph',- cause: e,- },- );+ respondToDerivedIndexQueryFailure(res, e, {+ handler: 'link-graph',+ failureTitle: 'Failed to read link graph.',+ });}},{ handler: 'link-graph', method: 'GET', skipBodyParse: true },@@ -7467,7 +7556,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {const handleOrphans = withValidation(EmptyRequestSchema,async (req, res) => {- if (!backlinkIndex) {+ if (!derivedDocumentIndex) {errorResponse(res,503,@@ -7480,7 +7569,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {try {const url = new URL(req.url ?? '', 'http://localhost');const mode = url.searchParams.get('mode') ?? 'both';- if (!isOrphanMode(mode)) {+ if (!isDerivedOrphanMode(mode)) {errorResponse(res,400,@@ -7491,21 +7580,18 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {return;}- const orphans = backlinkIndex- .getOrphans([...getFileIndex().keys()], mode)- .map((docName) => ({- docName,- title: readPageTitleForDocName(docName),- }));+ const orphans = (+ await derivedDocumentIndex.getOrphans([...getFileIndex().keys()], mode)+ ).map((docName) => ({+ docName,+ title: readPageTitleForDocName(docName),+ }));successResponse(res, 200, OrphansSuccessSchema, { orphans }, { handler: 'orphans' });} catch (e) {- errorResponse(- res,- 500,- 'urn:ok:error:internal-server-error',- 'Failed to read orphan pages.',- { handler: 'orphans', cause: e },- );+ respondToDerivedIndexQueryFailure(res, e, {+ handler: 'orphans',+ failureTitle: 'Failed to read orphan pages.',+ });}},{ handler: 'orphans', method: 'GET', skipBodyParse: true },@@ -7514,7 +7600,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {const handleHubs = withValidation(EmptyRequestSchema,async (req, res) => {- if (!backlinkIndex) {+ if (!derivedDocumentIndex) {errorResponse(res,503,@@ -7529,17 +7615,17 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {const rawLimit = url.searchParams.get('limit');const parsed = rawLimit ? Number.parseInt(rawLimit, 10) : 20;const limit = Number.isFinite(parsed) && parsed > 0 ? parsed : 20;- const admitted = collectAdmittedDocNames();- const hubs = backlinkIndex.getHubs(limit).map((hub) => ({+ const admitted = await collectAdmittedDocNames();+ const hubs = (await derivedDocumentIndex.getHubs(limit)).map((hub) => ({docName: hub.docName,title: readPageTitleForLinkedDocName(hub.docName, admitted),count: hub.count,}));successResponse(res, 200, HubsSuccessSchema, { hubs }, { handler: 'hubs' });} catch (e) {- errorResponse(res, 500, 'urn:ok:error:internal-server-error', 'Failed to read hub pages.', {+ respondToDerivedIndexQueryFailure(res, e, {handler: 'hubs',- cause: e,+ failureTitle: 'Failed to read hub pages.',});}},@@ -7549,7 +7635,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {const handleDeadLinks = withValidation(EmptyRequestSchema,async (req, res) => {- if (!backlinkIndex) {+ if (!derivedDocumentIndex) {errorResponse(res,503,@@ -7572,8 +7658,8 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {const sourceDocNameFilter = sourceDocNames.length? [...new Set(sourceDocNames.map((docName) => resolveAlias(docName)))]: undefined;- const deadLinks = backlinkIndex.getDeadLinks(- collectAdmittedDocNames(),+ const deadLinks = await derivedDocumentIndex.getDeadLinks(+ await collectAdmittedDocNames(),sourceDocNameFilter,);@@ -7594,13 +7680,10 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {{ handler: 'dead-links' },);} catch (e) {- errorResponse(- res,- 500,- 'urn:ok:error:internal-server-error',- 'Failed to read dead links.',- { handler: 'dead-links', cause: e },- );+ respondToDerivedIndexQueryFailure(res, e, {+ handler: 'dead-links',+ failureTitle: 'Failed to read dead links.',+ });}},{ handler: 'dead-links', method: 'GET', skipBodyParse: true },@@ -7942,7 +8025,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {// Write-time outbound-link validation — synchronous, from the// just-edited source bytes; see handleAgentWriteMd for the full why.- const admittedForLinks = collectAdmittedDocNames();+ const admittedForLinks = await collectAdmittedDocNames();admittedForLinks.add(docName);const brokenLinks = computeBrokenOutboundLinks(patchedSource,@@ -8436,17 +8519,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {const doc = hocuspocus.documents.get(docName);if (doc) await (forceUnloadDocument ?? hocuspocus.unloadDocument.bind(hocuspocus))(doc);writeFileSync(filePath, '', 'utf-8');- if (backlinkIndex) {- backlinkIndex.deleteDocument(docName);- void backlinkIndex.saveToDisk().catch((err) => {- log.warn(- { docName, err },- `[backlinks] Failed to persist cache after test-reset for ${docName}`,- );- });- signalChannel?.('backlinks');- signalChannel?.('graph');- }+ await derivedDocumentIndex?.testOnly?.resetDocumentForTest(docName);// Also reset the project-root .okignore synthetic doc + on-disk file// unless the caller explicitly opts out. Without this, patterns added@@ -8510,7 +8583,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {* backlink-dependent integration tests (e.g. `agent-focus-wiring.test.ts`* orphan-hint shape) cannot otherwise recover from.*- * This endpoint forces `backlinkIndex.rebuildFromDisk()` — authoritative+ * This endpoint forces an authoritative relationship-index rescan from disk,* resync from the filesystem that covers dropped events. It is NOT suitable* for production: rebuild wipes any in-memory backlink state not yet* debounced to disk (e.g. a live agent-write awaiting persistence). Gated@@ -8520,7 +8593,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {EmptyRequestSchema,async (_req, res) => {try {- if (!backlinkIndex) {+ if (!derivedDocumentIndex?.testOnly) {errorResponse(res,503,@@ -8530,16 +8603,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {);return;}- await backlinkIndex.rebuildFromDisk();- // A full rebuild replaces branch state, dropping the out-of-contentDir- // global skill bundle nodes — re-register them (node-only, within-bundle)- // so a forced rescan keeps the global skill graph intact.- await backlinkIndex.ingestGlobalSkillBundles([resolve(skillsHome, '.ok', 'skills')]);- void backlinkIndex.saveToDisk().catch((err) => {- log.warn({ err }, '[backlinks] Failed to persist cache after test-rescan-backlinks');- });- signalChannel?.('backlinks');- signalChannel?.('graph');+ await derivedDocumentIndex.testOnly.rescanBacklinksForTest();successResponse(res,200,@@ -10478,17 +10542,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {}}mutateFileIndex?.({ kind: 'create', path: fullPath, docName, content: initialContent });- if (backlinkIndex) {- backlinkIndex.updateDocumentFromMarkdown(docName, initialContent);- void backlinkIndex.saveToDisk().catch((err) => {- log.warn(- { docName, err },- `[backlinks] Failed to persist create-page cache for ${docName}`,- );- });- signalChannel?.('backlinks');- signalChannel?.('graph');- }+ await recordDerivedDocumentBestEffort(docName, initialContent, 'create-page');signalChannel?.('files');if (templateScopeForLog !== undefined) {// Cardinality-bounded structured event — `templateScope` is one of@@ -10770,7 +10824,6 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {docName: duplicatedPath,content,});- backlinkIndex?.updateDocumentFromMarkdown(duplicatedPath, content);duplicatedDocNames = [duplicatedPath];} catch (err) {try {@@ -10792,6 +10845,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {});throw err;}+ await recordDerivedDocumentBestEffort(duplicatedPath, content, 'duplicate-path-file');} else {const next = nextAvailableDuplicateFolderPath(contentDir, requestedPath);duplicatedPath = next.folderPath;@@ -10806,6 +10860,13 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {return;}const destinationPath = resolveContentEntryPath(contentDir, 'folder', duplicatedPath);+ const copiedDocRollbackLedger: Array<{+ docName: string;+ fullPath: string;+ extensionRegistered: boolean;+ dirCountIncremented: boolean;+ fileIndexRegistered: boolean;+ }> = [];try {tracedCpSync(sourcePath, destinationPath, {recursive: true,@@ -10825,6 +10886,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {}throw err;}+ const derivedMutations: DerivedDocumentIndexMutation[] = [];try {for (const folderPath of collectFolderPaths(contentDir, duplicatedPath)) {upsertFolderIndexPathSegments(folderPath);@@ -10832,22 +10894,60 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {const copiedDocs = collectMarkdownCopies(contentDir, duplicatedPath);duplicatedDocNames = copiedDocs.map((doc) => doc.docName);for (const doc of copiedDocs) {+ const rollbackEntry = {+ docName: doc.docName,+ fullPath: doc.fullPath,+ extensionRegistered: false,+ dirCountIncremented: false,+ fileIndexRegistered: false,+ };+ copiedDocRollbackLedger.push(rollbackEntry);const sourceExtension = extname(doc.fullPath);registerDocExtension(stripDocExtension(doc.docName), sourceExtension);+ rollbackEntry.extensionRegistered = true;recentlyRemovedDocs?.delete(doc.docName);if (contentFilter) {contentFilter.incrementMdDir(dirname(doc.docName));+ rollbackEntry.dirCountIncremented = true;}registerWrite(doc.fullPath, contentHash(doc.content));+ rollbackEntry.fileIndexRegistered = true;mutateFileIndex?.({kind: 'create',path: doc.fullPath,docName: doc.docName,content: doc.content,});- backlinkIndex?.updateDocumentFromMarkdown(doc.docName, doc.content);+ derivedMutations.push({+ kind: 'upsert',+ documentName: doc.docName,+ markdown: doc.content,+ });}} catch (err) {+ for (const rollbackEntry of copiedDocRollbackLedger.reverse()) {+ if (rollbackEntry.fileIndexRegistered) {+ try {+ mutateFileIndex?.({+ kind: 'delete',+ path: rollbackEntry.fullPath,+ docName: rollbackEntry.docName,+ });+ } catch (rollbackErr) {+ log.warn(+ { docName: rollbackEntry.docName, err: rollbackErr },+ '[duplicate-path] failed to roll back copied file-index row',+ );+ }+ }+ if (rollbackEntry.dirCountIncremented) {+ contentFilter?.decrementMdDir(dirname(rollbackEntry.docName));+ }+ if (rollbackEntry.extensionRegistered) {+ forgetDocExtension(stripDocExtension(rollbackEntry.docName));+ }+ }+ removeFolderIndexEntries(duplicatedPath);try {tracedRmSync(destinationPath, { recursive: true, force: true });} catch (cleanupErr) {@@ -10858,6 +10958,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {}throw err;}+ await recordDerivedMutationsBestEffort(derivedMutations, 'duplicate-path-folder');}switch (actor.kind) {@@ -10884,13 +10985,6 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {}}- if (backlinkIndex && duplicatedDocNames.length > 0) {- void backlinkIndex.saveToDisk().catch((err) => {- log.warn({ err }, '[backlinks] Failed to persist duplicate-path cache');- });- signalChannel?.('backlinks');- signalChannel?.('graph');- }signalChannel?.('files');successResponse(res,@@ -11499,6 +11593,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {docName,});}+ await deleteDerivedDocumentsBestEffort(deletedDocNames, 'delete-path');signalChannel?.('files');successResponse(@@ -11667,6 +11762,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {if (operationKind === 'folder') {removeFolderIndexEntries(path);}+ await deleteDerivedDocumentsBestEffort(deletedDocNames, 'trash-cleanup');// Synchronous CC1 emit closes the race where the renderer expects// the updated tree right after the response. The watcher's later@@ -14440,7 +14536,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {const handleTagsList = withValidation(EmptyRequestSchema,async (_req, res) => {- if (!tagIndex) {+ if (!derivedDocumentIndex) {errorResponse(res,503,@@ -14451,12 +14547,12 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {return;}try {- const tags = tagIndex.getAllTags();+ const tags = await derivedDocumentIndex.getAllTags();successResponse(res, 200, TagsListSuccessSchema, { tags }, { handler: 'tags-list' });} catch (e) {- errorResponse(res, 500, 'urn:ok:error:internal-server-error', 'Failed to read tags.', {+ respondToDerivedIndexQueryFailure(res, e, {handler: 'tags-list',- cause: e,+ failureTitle: 'Failed to read tags.',});}},@@ -14475,7 +14571,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {});return;}- if (!tagIndex) {+ if (!derivedDocumentIndex) {errorResponse(res,503,@@ -14501,12 +14597,14 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {return;}try {- const docs = tagIndex.getDocsForTagWithMatches(name).map(({ docName, matchingTags }) => ({- docName,- title: readPageTitleForDocName(docName),- matchingTags,- snippet: null,- }));+ const docs = (await derivedDocumentIndex.getDocsForTagWithMatches(name)).map(+ ({ docName, matchingTags }) => ({+ docName,+ title: readPageTitleForDocName(docName),+ matchingTags,+ snippet: null,+ }),+ );successResponse(res,200,@@ -14517,13 +14615,10 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {},);} catch (e) {- errorResponse(- res,- 500,- 'urn:ok:error:internal-server-error',- 'Failed to read tag membership.',- { handler: 'tags-for-name', cause: e },- );+ respondToDerivedIndexQueryFailure(res, e, {+ handler: 'tags-for-name',+ failureTitle: 'Failed to read tag membership.',+ });}}@@ -16425,12 +16520,8 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {// entries. Global skills live outside the project graph (not content// docs), so they have nothing to re-index.if (body.scope === 'project' && !contentEditError) {- // Best-effort: the git-mv + SKILL.md rewrite already succeeded, so a- // re-index failure (e.g. a `readFileSync` racing a relocated file)- // must NOT turn a successful rename into a 500. The next open/rescan- // re-indexes the moved docs from disk.try {- reindexMovedProjectSkillDocs(skillsRoot, body.fromName, body.toName);+ await reindexMovedProjectSkillDocs(skillsRoot, body.fromName, body.toName);} catch (err) {getLogger('skill-move').warn({ err, fromName: body.fromName, toName: body.toName },@@ -16593,46 +16684,47 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {* content docs never desync — disk stays the truth on the next open), and no* session churn against the just-moved dir.*/- function reindexMovedProjectSkillDocs(+ async function reindexMovedProjectSkillDocs(skillsRoot: string,fromName: string,toName: string,- ): void {- if (!backlinkIndex) return;- const reindexOne = (oldDocName: string, newDocName: string, absFile: string): void => {+ ): Promise<void> {+ if (!derivedDocumentIndex) return;+ const derivedMutations: DerivedDocumentIndexMutation[] = [];+ const collectReindex = (oldDocName: string, newDocName: string, absFile: string): void => {let markdown: string;try {markdown = readFileSync(absFile, 'utf-8');} catch {// Unreadable relocated file: drop the stale old-name entry rather than// leave it dangling (the next open will index it fresh from disk).- backlinkIndex.deleteDocument(oldDocName);- tagIndex?.deleteDocument(oldDocName);+ derivedMutations.push({ kind: 'delete', documentName: oldDocName });return;}- backlinkIndex.renameDocument(oldDocName, newDocName, markdown);- tagIndex?.renameDocument(oldDocName, newDocName, markdown);+ derivedMutations.push({+ kind: 'rename',+ oldDocumentName: oldDocName,+ newDocumentName: newDocName,+ markdown,+ });};// SKILL.md: its rewrite during the move is fs-direct (applySkillWrite), so// it never re-enters the index via a CRDT write either. The reference `.md`// files were git-mv'd verbatim — never rewritten — so they too are stale.- reindexOne(+ collectReindex(projectSkillContentDocName(fromName),projectSkillContentDocName(toName),resolve(skillsRoot, toName, 'SKILL.md'),);for (const rel of listProjectMdReferences(skillsRoot, toName)) {- reindexOne(+ collectReindex(projectRefContentDocName(fromName, rel),projectRefContentDocName(toName, rel),resolve(skillsRoot, toName, rel),);}- // Nudge any client that isn't holding the moved docs open to refresh its- // graph promptly. (`tags` rides the live-derived-index path, not here.)- signalChannel?.('backlinks');- signalChannel?.('graph');+ await derivedDocumentIndex.recordDirectMutations(derivedMutations);}const handleSkillFileGet = withValidation(@@ -19525,13 +19617,13 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {// (idempotent: the debounced pass re-applies the same bytes).let linkFindings: ValidationDiagnostic[] = [];const linksSetting = getLinksValidationSetting?.() ?? DEFAULT_LINKS_VALIDATION;- if (backlinkIndex && linksSetting !== 'off' && !isLinkIndexExcludedDoc(docName)) {- backlinkIndex.updateDocumentFromMarkdown(docName, source);+ if (derivedDocumentIndex && linksSetting !== 'off' && !isLinkIndexExcludedDoc(docName)) {+ await recordDerivedLinkRewriteBestEffort(docName, source, 'lint-validation');const linksValidator = createProjectValidators({projectDir: projectDir ?? contentDir,contentDir,baseConfig: base,- backlinkIndex,+ derivedDocumentIndex,linksValidation: linksSetting,admittedDocNames: collectAdmittedDocNames,docFilePathFor: (d) => resolveDocFilePath(contentDir, d),@@ -19955,7 +20047,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {);// Unified validation audit: every registered project validator (markdownlint- // walk + backlink-index dead-link read) merged into one source-tagged plane.+ // walk + derived-index dead-link read) merged into one source-tagged plane.// Additive alongside /api/lint/audit and /api/dead-links, which keep their// single-validator contracts.const handleAudit = withValidation(@@ -19997,7 +20089,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {contentDir,baseConfig,liveSourceFor: liveLintSourceFor,- backlinkIndex: backlinkIndex ?? null,+ derivedDocumentIndex: derivedDocumentIndex ?? null,linksValidation: getLinksValidationSetting?.(),admittedDocNames: collectAdmittedDocNames,docFilePathFor: (docName) => resolveDocFilePath(contentDir, docName),diff --git a/packages/server/src/api-file-ops.test.ts b/packages/server/src/api-file-ops.test.tsindex c658c4d63..f05850a82 100644--- a/packages/server/src/api-file-ops.test.ts+++ b/packages/server/src/api-file-ops.test.ts@@ -15,11 +15,15 @@ import { join, resolve } from 'node:path';import { Readable } from 'node:stream';import { Hocuspocus } from '@hocuspocus/server';import simpleGit from 'simple-git';-import { afterEach, describe, expect, test } from 'vitest';+import { afterEach, describe, expect, test, vi } from 'vitest';import type * as Y from 'yjs';-import { createApiExtension } from './api-extension.test-helper.ts';+import {+ createApiExtension,+ createDerivedDocumentIndexApiPortStub,+} from './api-extension.test-helper.ts';import { BacklinkIndex } from './backlink-index.ts';import { type ContentFilter, createContentFilter } from './content-filter.ts';+import type { DerivedDocumentIndexApiPort } from './derived-document-index.ts';import { _resetDocExtensionsForTests, getDocExtension } from './doc-extensions.ts';import type { FileIndexEntry, FolderIndexEntry } from './file-watcher.ts';@@ -125,6 +129,7 @@ type CallApiOptions = {signalChannel?: Parameters<typeof createApiExtension>[0]['signalChannel'];projectDir?: string;contentFilter?: ContentFilter;+ derivedDocumentIndex?: DerivedDocumentIndexApiPort;};async function createTestApiExtension(contentDir: string, options?: CallApiOptions) {@@ -155,6 +160,19 @@ async function createTestApiExtension(contentDir: string, options?: CallApiOptiosignalChannel: options?.signalChannel,projectDir: options?.projectDir,contentFilter: options?.contentFilter,+ derivedDocumentIndex: options?.derivedDocumentIndex,+ });+}++function createClosedDerivedDocumentIndexPort(): DerivedDocumentIndexApiPort {+ const rejectClosed = async (): Promise<never> => {+ const error = new Error('Derived document index is closed');+ error.name = 'DerivedDocumentIndexClosedError';+ throw error;+ };+ return createDerivedDocumentIndexApiPortStub({+ recordDirectMutations: rejectClosed,+ recordDirectDocument: rejectClosed,});}@@ -203,6 +221,7 @@ function setupTmpDir(): string {}afterEach(() => {+ vi.restoreAllMocks();_resetDocExtensionsForTests();if (tmpDir) {rmSync(tmpDir, { recursive: true, force: true });@@ -971,6 +990,27 @@ describe('file operation API routes', () => {expect(body.renamed).toEqual([{ fromDocName: 'notes', toDocName: 'renamed-notes' }]);});+ test('keeps a document rename successful when the derived index closes after disk commit', async () => {+ const dir = setupTmpDir();+ writeFileSync(join(dir, 'notes.md'), '# Notes\n', 'utf-8');++ const result = await callApi(+ dir,+ '/api/rename-path',+ 'POST',+ {+ kind: 'file',+ fromPath: 'notes',+ toPath: 'renamed-notes',+ },+ { derivedDocumentIndex: createClosedDerivedDocumentIndexPort() },+ );++ expect(result.status).toBe(200);+ expect(existsSync(join(dir, 'notes.md'))).toBe(false);+ expect(readFileSync(join(dir, 'renamed-notes.md'), 'utf-8')).toBe('# Notes\n');+ });+test('renames a folder and returns descendant mappings', async () => {const dir = setupTmpDir();mkdirSync(join(dir, 'docs/nested'), { recursive: true });@@ -1283,6 +1323,28 @@ describe('file operation API routes', () => {expect(body.deletedDocNames).toEqual(['trash-me']);});+ test('keeps a durable delete successful when relationship projection fails', async () => {+ const dir = setupTmpDir();+ writeFileSync(join(dir, 'trash-me.md'), '# Delete me\n', 'utf-8');+ const failingBacklinkIndex = {+ deleteDocument: () => {+ throw new Error('projection failed');+ },+ } as unknown as BacklinkIndex;++ const result = await callApi(+ dir,+ '/api/delete-path',+ 'POST',+ { kind: 'file', path: 'trash-me' },+ { backlinkIndex: failingBacklinkIndex },+ );++ expect(result.status).toBe(200);+ expect(existsSync(join(dir, 'trash-me.md'))).toBe(false);+ expect(JSON.parse(result.body)).toMatchObject({ deletedDocNames: ['trash-me'] });+ });+test('delete accepts asset-shaped markdown paths as document deletes', async () => {const dir = setupTmpDir();writeFileSync(join(dir, 'trash-me.md'), '# Delete me\n', 'utf-8');@@ -1372,6 +1434,32 @@ describe('file operation API routes', () => {expect(body.rewrittenDocs).toEqual([{ docName: 'docs/guide', rewrites: 5 }]);});+ test('keeps an asset rename successful when the derived index closes after disk commit', async () => {+ const dir = setupTmpDir();+ mkdirSync(join(dir, 'docs/media'), { recursive: true });+ writeFileSync(join(dir, 'docs/media/diagram.png'), 'fake image bytes', 'utf-8');+ writeFileSync(join(dir, 'docs/guide.md'), '\n', 'utf-8');++ const result = await callApi(+ dir,+ '/api/rename-path',+ 'POST',+ {+ kind: 'asset',+ fromPath: 'docs/media/diagram.png',+ toPath: 'docs/assets/diagram.png',+ },+ { derivedDocumentIndex: createClosedDerivedDocumentIndexPort() },+ );++ expect(result.status).toBe(200);+ expect(existsSync(join(dir, 'docs/media/diagram.png'))).toBe(false);+ expect(readFileSync(join(dir, 'docs/assets/diagram.png'), 'utf-8')).toBe('fake image bytes');+ expect(readFileSync(join(dir, 'docs/guide.md'), 'utf-8')).toBe(+ '\n',+ );+ });+test('asset rename rejects destinations excluded by content config', async () => {const dir = setupTmpDir();mkdirSync(join(dir, 'docs/media'), { recursive: true });@@ -1475,6 +1563,9 @@ describe('file operation API routes', () => {mkdirSync(join(dir, 'docs/media'), { recursive: true });writeFileSync(join(dir, 'docs/guide.md'), '# Guide\n', 'utf-8');writeFileSync(join(dir, 'docs/index.md'), '[Guide](./guide.md)\n', 'utf-8');+ const backlinkIndex = await buildBacklinkIndex(dir);+ const backlinkSave = vi.spyOn(backlinkIndex, 'saveToDisk');+ const signals: string[] = [];const result = await callApi(dir,@@ -1485,7 +1576,10 @@ describe('file operation API routes', () => {fromPath: 'docs/guide.md',toPath: 'docs/media/guide.custom',},- { backlinkIndex: buildBacklinkIndex(dir) },+ {+ backlinkIndex,+ signalChannel: (channel) => signals.push(channel),+ },);expect(result.status).toBe(200);@@ -1507,6 +1601,12 @@ describe('file operation API routes', () => {},]);expect(body.rewrittenDocs).toEqual([{ docName: 'docs/index', rewrites: 1 }]);+ expect(backlinkSave).toHaveBeenCalledTimes(1);+ expect(signals.filter((channel) => channel !== 'files')).toEqual([+ 'backlinks',+ 'graph',+ 'tags',+ ]);});test('document-to-file rename carries live Y.Doc content to the destination file', async () => {@@ -1545,6 +1645,27 @@ describe('file operation API routes', () => {}});+ test('keeps a document-to-file rename successful when the derived index closes', async () => {+ const dir = setupTmpDir();+ writeFileSync(join(dir, 'guide.md'), '# Guide\n', 'utf-8');++ const result = await callApi(+ dir,+ '/api/rename-path',+ 'POST',+ {+ kind: 'asset',+ fromPath: 'guide.md',+ toPath: 'guide.txt',+ },+ { derivedDocumentIndex: createClosedDerivedDocumentIndexPort() },+ );++ expect(result.status).toBe(200);+ expect(existsSync(join(dir, 'guide.md'))).toBe(false);+ expect(readFileSync(join(dir, 'guide.txt'), 'utf-8')).toBe('# Guide\n');+ });+test('document-to-file rename returns 404 when the source document is missing', async () => {const dir = setupTmpDir();@@ -1931,6 +2052,30 @@ describe('file operation API routes', () => {expect(body.rewrittenDocs).toEqual([{ docName: 'index', rewrites: 2 }]);});+ test('keeps an asset-only folder rename successful when the derived index closes', async () => {+ const dir = setupTmpDir();+ mkdirSync(join(dir, 'media'), { recursive: true });+ writeFileSync(join(dir, 'media/diagram.png'), 'fake image bytes', 'utf-8');+ writeFileSync(join(dir, 'index.md'), '\n', 'utf-8');++ const result = await callApi(+ dir,+ '/api/rename-path',+ 'POST',+ {+ kind: 'folder',+ fromPath: 'media',+ toPath: 'assets',+ },+ { derivedDocumentIndex: createClosedDerivedDocumentIndexPort() },+ );++ expect(result.status).toBe(200);+ expect(existsSync(join(dir, 'media'))).toBe(false);+ expect(readFileSync(join(dir, 'assets/diagram.png'), 'utf-8')).toBe('fake image bytes');+ expect(readFileSync(join(dir, 'index.md'), 'utf-8')).toBe('\n');+ });+test('folder rename rewrites references to assets that move with markdown docs', async () => {const dir = setupTmpDir();mkdirSync(join(dir, 'docs/media'), { recursive: true });@@ -2366,15 +2511,13 @@ describe('file operation API routes', () => {expect(body.duplicatedDocNames).toEqual(['archive copy/index', 'archive copy/old/entry']);});- test('removes a copied folder when duplicate-path post-copy registration fails', async () => {+ test('keeps a copied folder when the derived index closes after registration', async () => {const dir = setupTmpDir();mkdirSync(join(dir, 'archive'), { recursive: true });- writeFileSync(join(dir, 'archive/index.md'), '# Archive\n', 'utf-8');- const failingBacklinkIndex = {- updateDocumentFromMarkdown: () => {- throw new Error('registration failed');- },- } as unknown as BacklinkIndex;+ writeFileSync(join(dir, 'archive/index.mdx'), '# Archive\n', 'utf-8');+ const fileIndex = new Map(buildFileIndex(dir));+ const folderIndex = new Map(buildFolderIndex(dir));+ const contentFilter = createContentFilter({ projectDir: dir, contentDir: dir });const result = await callApi(dir,@@ -2384,55 +2527,42 @@ describe('file operation API routes', () => {kind: 'folder',path: 'archive',},- { backlinkIndex: failingBacklinkIndex, getFileIndex: () => new Map() },- );-- expect(result.status).toBe(500);- expect(existsSync(join(dir, 'archive copy'))).toBe(false);- expect(JSON.parse(result.body)).toMatchObject({- type: 'urn:ok:error:internal-server-error',- title: 'Failed to duplicate path.',- status: 500,- });- });-- test('removes a copied file when duplicate-path post-copy registration fails', async () => {- const dir = setupTmpDir();- writeFileSync(join(dir, 'notes.md'), '# Notes\n', 'utf-8');- const failingBacklinkIndex = {- updateDocumentFromMarkdown: () => {- throw new Error('registration failed');- },- } as unknown as BacklinkIndex;-- const result = await callApi(- dir,- '/api/duplicate-path',- 'POST',{- kind: 'file',- path: 'notes',+ derivedDocumentIndex: createClosedDerivedDocumentIndexPort(),+ contentFilter,+ getFileIndex: () => fileIndex,+ getFolderIndex: () => folderIndex,+ mutateFileIndex: (event) => {+ if (event.kind === 'create') {+ fileIndex.set(event.docName, {+ size: event.content.length,+ modified: new Date(0).toISOString(),+ canonicalPath: event.path,+ inode: 0,+ aliases: [],+ });+ } else if (event.kind === 'delete') {+ fileIndex.delete(event.docName);+ }+ },},- { backlinkIndex: failingBacklinkIndex, getFileIndex: () => new Map() },);- expect(result.status).toBe(500);- expect(existsSync(join(dir, 'notes copy.md'))).toBe(false);+ expect(result.status).toBe(200);+ expect(readFileSync(join(dir, 'archive copy/index.mdx'), 'utf-8')).toBe('# Archive\n');+ expect(fileIndex.has('archive copy/index')).toBe(true);+ expect([...folderIndex.keys()].some((path) => path.startsWith('archive copy'))).toBe(true);+ expect(getDocExtension('archive copy/index')).toBe('.mdx');expect(JSON.parse(result.body)).toMatchObject({- type: 'urn:ok:error:internal-server-error',- title: 'Failed to duplicate path.',- status: 500,+ kind: 'folder',+ path: 'archive copy',+ duplicatedDocNames: ['archive copy/index'],});});- test('forgets a copied file extension when duplicate-path post-copy registration fails', async () => {+ test('keeps a copied file when the derived index closes after registration', async () => {const dir = setupTmpDir();writeFileSync(join(dir, 'notes.mdx'), '# Notes\n', 'utf-8');- const failingBacklinkIndex = {- updateDocumentFromMarkdown: () => {- throw new Error('registration failed');- },- } as unknown as BacklinkIndex;const result = await callApi(dir,@@ -2442,88 +2572,26 @@ describe('file operation API routes', () => {kind: 'file',path: 'notes',},- { backlinkIndex: failingBacklinkIndex, getFileIndex: () => new Map() },- );-- expect(result.status).toBe(500);- expect(existsSync(join(dir, 'notes copy.mdx'))).toBe(false);- expect(getDocExtension('notes copy')).toBe('.md');- });-- test('returns 507 when duplicate-path registration hits storage exhaustion', async () => {- const dir = setupTmpDir();- mkdirSync(join(dir, 'archive'), { recursive: true });- writeFileSync(join(dir, 'archive/index.md'), '# Archive\n', 'utf-8');- const fullDiskBacklinkIndex = {- updateDocumentFromMarkdown: () => {- const err = new Error('no space left on device') as NodeJS.ErrnoException;- err.code = 'ENOSPC';- throw err;- },- } as unknown as BacklinkIndex;-- const result = await callApi(- dir,- '/api/duplicate-path',- 'POST',{- kind: 'folder',- path: 'archive',+ derivedDocumentIndex: createClosedDerivedDocumentIndexPort(),+ getFileIndex: () => new Map(),},- { backlinkIndex: fullDiskBacklinkIndex, getFileIndex: () => new Map() },);- expect(result.status).toBe(507);- expect(existsSync(join(dir, 'archive copy'))).toBe(false);- expect(JSON.parse(result.body)).toMatchObject({- type: 'urn:ok:error:storage-full',- title: 'Could not duplicate path because storage is full.',- status: 507,- });- });-- test('returns 507 when duplicate-path file registration hits storage exhaustion', async () => {- const dir = setupTmpDir();- writeFileSync(join(dir, 'notes.md'), '# Notes\n', 'utf-8');- const fullDiskBacklinkIndex = {- updateDocumentFromMarkdown: () => {- const err = new Error('no space left on device') as NodeJS.ErrnoException;- err.code = 'ENOSPC';- throw err;- },- } as unknown as BacklinkIndex;-- const result = await callApi(- dir,- '/api/duplicate-path',- 'POST',- {- kind: 'file',- path: 'notes',- },- { backlinkIndex: fullDiskBacklinkIndex, getFileIndex: () => new Map() },- );-- expect(result.status).toBe(507);- expect(existsSync(join(dir, 'notes copy.md'))).toBe(false);+ expect(result.status).toBe(200);+ expect(readFileSync(join(dir, 'notes copy.mdx'), 'utf-8')).toBe('# Notes\n');+ expect(getDocExtension('notes copy')).toBe('.mdx');expect(JSON.parse(result.body)).toMatchObject({- type: 'urn:ok:error:storage-full',- title: 'Could not duplicate path because storage is full.',- status: 507,+ kind: 'file',+ path: 'notes copy',+ duplicatedDocNames: ['notes copy'],});});- test('returns actionable storage-readonly when duplicate-path registration hits permissions', async () => {+ test('rolls back a copied folder when synchronous registration fails', async () => {const dir = setupTmpDir();mkdirSync(join(dir, 'archive'), { recursive: true });writeFileSync(join(dir, 'archive/index.md'), '# Archive\n', 'utf-8');- const readOnlyBacklinkIndex = {- updateDocumentFromMarkdown: () => {- const err = new Error('permission denied') as NodeJS.ErrnoException;- err.code = 'EACCES';- throw err;- },- } as unknown as BacklinkIndex;const result = await callApi(dir,@@ -2533,28 +2601,20 @@ describe('file operation API routes', () => {kind: 'folder',path: 'archive',},- { backlinkIndex: readOnlyBacklinkIndex, getFileIndex: () => new Map() },+ {+ mutateFileIndex: (event) => {+ if (event.kind === 'create') throw new Error('registration failed');+ },+ },);expect(result.status).toBe(500);expect(existsSync(join(dir, 'archive copy'))).toBe(false);- expect(JSON.parse(result.body)).toMatchObject({- type: 'urn:ok:error:storage-readonly',- title: 'Could not duplicate path because storage is not writable.',- status: 500,- });});- test('returns actionable storage-readonly when duplicate-path file registration hits permissions', async () => {+ test('rolls back a copied file when synchronous registration fails', async () => {const dir = setupTmpDir();writeFileSync(join(dir, 'notes.md'), '# Notes\n', 'utf-8');- const readOnlyBacklinkIndex = {- updateDocumentFromMarkdown: () => {- const err = new Error('permission denied') as NodeJS.ErrnoException;- err.code = 'EACCES';- throw err;- },- } as unknown as BacklinkIndex;const result = await callApi(dir,@@ -2564,16 +2624,15 @@ describe('file operation API routes', () => {kind: 'file',path: 'notes',},- { backlinkIndex: readOnlyBacklinkIndex, getFileIndex: () => new Map() },+ {+ mutateFileIndex: (event) => {+ if (event.kind === 'create') throw new Error('registration failed');+ },+ },);expect(result.status).toBe(500);expect(existsSync(join(dir, 'notes copy.md'))).toBe(false);- expect(JSON.parse(result.body)).toMatchObject({- type: 'urn:ok:error:storage-readonly',- title: 'Could not duplicate path because storage is not writable.',- status: 500,- });});test('returns 409 when duplicate-path folder destination appears before copy', async () => {diff --git a/packages/server/src/derived-document-index-boundary.test.ts b/packages/server/src/derived-document-index-boundary.test.tsnew file mode 100644index 000000000..868d9a03e--- /dev/null+++ b/packages/server/src/derived-document-index-boundary.test.ts@@ -0,0 +1,60 @@+import { describe, expectTypeOf, test } from 'vitest';+import type { ApiExtensionOptions } from './api-extension.ts';+import type {+ DerivedDocumentIndexApiPort,+ DerivedDocumentIndexLivePort,+ DerivedDocumentIndexPersistencePort,+} from './derived-document-index.ts';+import type { LiveDerivedIndexOptions } from './live-derived-index.ts';+import type { PersistenceOptions } from './persistence.ts';+import type { ServerOptions } from './server-factory.ts';++type RawIndexMember =+ | 'backlinkIndex'+ | 'tagIndex'+ | 'updateDocumentFromMarkdown'+ | 'deleteDocument'+ | 'renameDocument'+ | 'saveToDisk'+ | 'loadFromDisk'+ | 'rebuildFromDisk'+ | 'reconcileWithDisk'+ | 'ingestGlobalSkillBundles'+ | 'init'+ | 'close'+ | 'switchBranch';++type ConsumerOptionKey =+ | keyof ApiExtensionOptions+ | keyof LiveDerivedIndexOptions+ | keyof PersistenceOptions;++type ConsumerPortKey =+ | keyof DerivedDocumentIndexApiPort+ | keyof DerivedDocumentIndexLivePort+ | keyof DerivedDocumentIndexPersistencePort;++describe('derived document index ownership boundary', () => {+ test('runtime consumers accept only their coordinator ports', () => {+ expectTypeOf<ApiExtensionOptions['derivedDocumentIndex']>().toEqualTypeOf<+ DerivedDocumentIndexApiPort | undefined+ >();+ expectTypeOf<+ LiveDerivedIndexOptions['derivedDocumentIndex']+ >().toEqualTypeOf<DerivedDocumentIndexLivePort>();+ expectTypeOf<PersistenceOptions['derivedDocumentIndex']>().toEqualTypeOf<+ DerivedDocumentIndexPersistencePort | undefined+ >();+ });++ test('consumer options and ports do not expose raw relationship indexes', () => {+ expectTypeOf<Extract<ConsumerOptionKey, RawIndexMember>>().toEqualTypeOf<never>();+ expectTypeOf<Extract<ConsumerPortKey, RawIndexMember>>().toEqualTypeOf<never>();+ });++ test('server composition owns coordinator construction', () => {+ expectTypeOf<+ Extract<keyof ServerOptions, 'backlinkIndex' | 'tagIndex' | 'derivedDocumentIndex'>+ >().toEqualTypeOf<never>();+ });+});diff --git a/packages/server/src/derived-document-index.test.ts b/packages/server/src/derived-document-index.test.tsnew file mode 100644index 000000000..7fd409941--- /dev/null+++ b/packages/server/src/derived-document-index.test.ts@@ -0,0 +1,818 @@+import { mkdirSync, mkdtempSync, rmSync, unlinkSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { dirname, join } from 'node:path';+import { afterEach, describe, expect, test, vi } from 'vitest';+import { BacklinkIndex } from './backlink-index.ts';+import { createContentFilter } from './content-filter.ts';+import {+ DerivedDocumentIndex,+ isDerivedDocumentIndexClosedError,+} from './derived-document-index.ts';+import { getLogger } from './logger.ts';+import { TagIndex } from './tag-index.ts';++interface Rig {+ projectDir: string;+ contentDir: string;+ contentFilter: ReturnType<typeof createContentFilter>;+ signals: string[];+ index: DerivedDocumentIndex;+}++interface Deferred {+ promise: Promise<void>;+ resolve: () => void;+}++const cleanups: Array<() => Promise<void>> = [];++function createDeferred(): Deferred {+ let resolve!: () => void;+ const promise = new Promise<void>((settle) => {+ resolve = settle;+ });+ return { promise, resolve };+}++function createRig(): Rig {+ const projectDir = mkdtempSync(join(tmpdir(), 'ok-derived-index-'));+ const contentDir = join(projectDir, 'content');+ mkdirSync(contentDir, { recursive: true });+ const signals: string[] = [];+ const contentFilter = createContentFilter({ projectDir, contentDir });+ const index = new DerivedDocumentIndex({+ projectDir,+ contentDir,+ contentFilter,+ getGlobalSkillRoots: () => [],+ signalChannel: (channel) => signals.push(channel),+ });+ cleanups.push(async () => {+ await index.close();+ rmSync(projectDir, { recursive: true, force: true });+ });+ return { projectDir, contentDir, contentFilter, signals, index };+}++function writeDoc(rig: Rig, relativePath: string, markdown: string): void {+ const filePath = join(rig.contentDir, relativePath);+ mkdirSync(dirname(filePath), { recursive: true });+ writeFileSync(filePath, markdown);+}++async function settleStartup(rig: Rig): Promise<void> {+ const startup = rig.index.beginStartup('main');+ await startup.backlinksReady;+ await rig.index.settleStartupAfterWatcherSeed();+ rig.signals.length = 0;+}++afterEach(async () => {+ vi.useRealTimers();+ vi.restoreAllMocks();+ for (const cleanup of cleanups.splice(0)) await cleanup();+});++describe('DerivedDocumentIndex', () => {+ test('direct documents update links and tags before returning and signal B/G/T', async () => {+ const rig = createRig();+ await settleStartup(rig);++ await rig.index.recordDirectDocument(+ 'source',+ '---\ntags: [project/active]\n---\nSee [[target]].\n',+ );++ expect(await rig.index.getBacklinks('target')).toEqual([+ expect.objectContaining({ source: 'source' }),+ ]);+ expect(await rig.index.getBacklinkCounts(['target', 'missing'])).toEqual({+ target: 1,+ missing: 0,+ });+ expect(await rig.index.getDocsForTagWithMatches('project')).toEqual([+ { docName: 'source', matchingTags: ['project/active'] },+ ]);+ expect(rig.signals).toEqual(['backlinks', 'graph', 'tags']);+ });++ test('direct batches mutate in order with one cache save and one B/G/T signal set', async () => {+ const backlinkSave = vi.spyOn(BacklinkIndex.prototype, 'saveToDisk');+ const tagSave = vi.spyOn(TagIndex.prototype, 'saveToDisk');+ const rig = createRig();+ await settleStartup(rig);+ backlinkSave.mockClear();+ tagSave.mockClear();++ await rig.index.recordDirectMutations([+ { kind: 'upsert', documentName: 'first', markdown: 'See [[old]]. #old\n' },+ {+ kind: 'rename',+ oldDocumentName: 'first',+ newDocumentName: 'second',+ markdown: 'See [[target]]. #moved\n',+ },+ {+ kind: 'link-rewrite',+ documentName: 'second',+ markdown: 'See [[rewritten]]. #ignored\n',+ },+ { kind: 'delete', documentName: 'missing' },+ ]);++ expect(await rig.index.getBacklinks('old')).toEqual([]);+ expect(await rig.index.getBacklinks('target')).toEqual([]);+ expect((await rig.index.getBacklinks('rewritten')).map((entry) => entry.source)).toEqual([+ 'second',+ ]);+ expect(await rig.index.getDocsForTagWithMatches('moved')).toEqual([+ { docName: 'second', matchingTags: ['moved'] },+ ]);+ expect(await rig.index.getDocsForTagWithMatches('ignored')).toEqual([]);+ expect(backlinkSave).toHaveBeenCalledTimes(1);+ expect(tagSave).toHaveBeenCalledTimes(1);+ expect(rig.signals).toEqual(['backlinks', 'graph', 'tags']);+ });++ test('empty direct batches are immediate no-ops across readiness, branch, and close gates', async () => {+ const rig = createRig();++ await expect(rig.index.recordDirectMutations([])).resolves.toBeUndefined();+ await settleStartup(rig);+ const transition = await rig.index.beginBranchSwitch('feature');+ await expect(rig.index.recordDirectMutations([])).resolves.toBeUndefined();+ rig.index.abortBranchSwitch(transition);+ await rig.index.close();+ await expect(rig.index.recordDirectMutations([])).resolves.toBeUndefined();+ expect(rig.signals).toEqual([]);+ });++ test('live documents do not persist caches but preserve backlink-first paired updates', async () => {+ const backlinkSave = vi.spyOn(BacklinkIndex.prototype, 'saveToDisk');+ const tagSave = vi.spyOn(TagIndex.prototype, 'saveToDisk');+ const rig = createRig();+ await settleStartup(rig);+ backlinkSave.mockClear();+ tagSave.mockClear();++ const token = rig.index.captureLiveUpdateToken();+ expect(token).not.toBeNull();+ await rig.index.recordLiveDocument('source', 'See [[target]]. #live\n', token ?? -1);++ expect(await rig.index.getBacklinks('target')).toHaveLength(1);+ expect(await rig.index.getDocsForTagWithMatches('live')).toHaveLength(1);+ expect(backlinkSave).not.toHaveBeenCalled();+ expect(tagSave).not.toHaveBeenCalled();+ expect(rig.signals).toEqual(['backlinks', 'graph', 'tags']);+ });++ test('an ignored incremental document removes existing link and tag memberships', async () => {+ const rig = createRig();+ await settleStartup(rig);+ await rig.index.recordDirectDocument('ignored', 'See [[target]]. #hidden\n');+ rig.signals.length = 0;+ writeFileSync(join(rig.projectDir, '.okignore'), 'content/ignored.md\n');+ const rebuilt = await rig.contentFilter.rebuildIgnorePatterns();+ expect(rebuilt.ok).toBe(true);++ await rig.index.recordDirectDocument('ignored', 'See [[target]]. #hidden\n');++ expect(await rig.index.getBacklinks('target')).toEqual([]);+ expect(await rig.index.getDocsForTagWithMatches('hidden')).toEqual([]);+ expect(rig.signals).toEqual(['backlinks', 'graph', 'tags']);+ });++ test('system and config docs are excluded while managed artifacts remain admitted', async () => {+ const rig = createRig();+ await settleStartup(rig);++ await rig.index.recordDirectDocument('__system__', 'See [[target]]. #internal\n');+ await rig.index.recordDirectDocument('__config__/project', 'See [[target]]. #internal\n');+ await rig.index.recordDirectDocument(+ '__template__/notes/lifecycle',+ 'See [[target]]. #managed\n',+ );++ expect((await rig.index.getBacklinks('target')).map((entry) => entry.source)).toEqual([+ '__template__/notes/lifecycle',+ ]);+ expect(await rig.index.getDocsForTagWithMatches('internal')).toEqual([]);+ expect(await rig.index.getDocsForTagWithMatches('managed')).toEqual([+ { docName: '__template__/notes/lifecycle', matchingTags: ['managed'] },+ ]);+ expect(await rig.index.getIndexedDocNames()).not.toContain('__system__');+ expect(await rig.index.getIndexedDocNames()).not.toContain('__config__/project');+ });++ test('durable stores update and persist backlinks without changing tags or signaling', async () => {+ vi.useFakeTimers();+ const backlinkSave = vi.spyOn(BacklinkIndex.prototype, 'saveToDisk');+ const tagSave = vi.spyOn(TagIndex.prototype, 'saveToDisk');+ const rig = createRig();+ await settleStartup(rig);+ backlinkSave.mockClear();+ tagSave.mockClear();++ await rig.index.recordDurableStore('source', 'See [[target]]. #durable\n');++ expect(await rig.index.getBacklinks('target')).toHaveLength(1);+ expect(await rig.index.getDocsForTagWithMatches('durable')).toEqual([]);+ expect(backlinkSave).not.toHaveBeenCalled();+ expect(tagSave).not.toHaveBeenCalled();++ await vi.advanceTimersByTimeAsync(2000);++ expect(backlinkSave).toHaveBeenCalledTimes(1);+ expect(tagSave).not.toHaveBeenCalled();+ expect(rig.signals).toEqual([]);+ });++ test('disk events debounce paired cache saves while applying and signaling immediately', async () => {+ vi.useFakeTimers();+ const backlinkSave = vi.spyOn(BacklinkIndex.prototype, 'saveToDisk');+ const tagSave = vi.spyOn(TagIndex.prototype, 'saveToDisk');+ const rig = createRig();+ await settleStartup(rig);+ backlinkSave.mockClear();+ tagSave.mockClear();++ await rig.index.recordDiskUpsert('source', 'See [[first]]. #one\n');+ await rig.index.recordDiskUpsert('source', 'See [[second]]. #two\n');++ expect(await rig.index.getBacklinks('first')).toEqual([]);+ expect(await rig.index.getBacklinks('second')).toHaveLength(1);+ expect(await rig.index.getDocsForTagWithMatches('two')).toHaveLength(1);+ expect(backlinkSave).not.toHaveBeenCalled();+ expect(tagSave).not.toHaveBeenCalled();++ await rig.index.recordDiskRename('source', 'renamed', 'See [[third]]. #three\n');+ await rig.index.recordDiskDelete('renamed');++ expect(await rig.index.getBacklinks('second')).toEqual([]);+ expect(await rig.index.getBacklinks('third')).toEqual([]);+ expect(await rig.index.getDocsForTagWithMatches('two')).toEqual([]);+ expect(await rig.index.getDocsForTagWithMatches('three')).toEqual([]);++ await vi.advanceTimersByTimeAsync(2000);+ await rig.index.getIndexedDocNames();++ expect(backlinkSave).toHaveBeenCalledTimes(1);+ expect(tagSave).toHaveBeenCalledTimes(1);+ expect(rig.signals).toEqual([+ 'backlinks',+ 'graph',+ 'tags',+ 'backlinks',+ 'graph',+ 'tags',+ 'backlinks',+ 'graph',+ 'tags',+ 'backlinks',+ 'graph',+ 'tags',+ ]);+ });++ test('link rewrites are backlink-only and retain existing tag membership', async () => {+ const rig = createRig();+ await settleStartup(rig);+ await rig.index.recordDirectDocument('source', 'See [[old]]. #stable\n');+ rig.signals.length = 0;++ await rig.index.recordLinkRewrite('source', 'See [[new]]. #changed\n');++ expect(await rig.index.getBacklinks('old')).toEqual([]);+ expect(await rig.index.getBacklinks('new')).toHaveLength(1);+ expect(await rig.index.getDocsForTagWithMatches('stable')).toHaveLength(1);+ expect(await rig.index.getDocsForTagWithMatches('changed')).toEqual([]);+ expect(rig.signals).toEqual(['backlinks', 'graph']);+ });++ test('direct rename moves link and tag membership atomically', async () => {+ const rig = createRig();+ await settleStartup(rig);+ await rig.index.recordDirectDocument('old', 'See [[target]]. #moved\n');+ rig.signals.length = 0;++ await rig.index.recordDirectRename('old', 'new', 'See [[target]]. #moved\n');++ expect((await rig.index.getBacklinks('target')).map((entry) => entry.source)).toEqual(['new']);+ expect(await rig.index.getDocsForTagWithMatches('moved')).toEqual([+ { docName: 'new', matchingTags: ['moved'] },+ ]);+ expect(rig.signals).toEqual(['backlinks', 'graph', 'tags']);+ });++ test('queries wait for post-watcher startup settlement', async () => {+ const rig = createRig();+ writeDoc(rig, 'source.md', 'See [[target]].\n');+ const startup = rig.index.beginStartup('main');+ let settled = false;+ const query = rig.index.getBacklinks('target').then((entries) => {+ settled = true;+ return entries;+ });++ await startup.backlinksReady;+ await Promise.resolve();+ expect(settled).toBe(false);++ await rig.index.settleStartupAfterWatcherSeed();+ await expect(query).resolves.toHaveLength(1);+ });++ test('branch transitions hold new commands until the replacement indexes settle', async () => {+ const rig = createRig();+ await settleStartup(rig);+ const transition = await rig.index.beginBranchSwitch('feature');+ let applied = false;+ let queried = false;+ const pending = rig.index.recordDirectDocument('after-switch', '# branch\n').then(() => {+ applied = true;+ });+ const pendingQuery = rig.index.getIndexedDocNames().then(() => {+ queried = true;+ });++ await Promise.resolve();+ expect(applied).toBe(false);+ expect(queried).toBe(false);++ await rig.index.settleBranchFromDisk(transition);+ await Promise.all([pending, pendingQuery]);+ expect(applied).toBe(true);+ expect(queried).toBe(true);+ });++ test('branch transitions invalidate stale live payloads captured from the old branch', async () => {+ const rig = createRig();+ await settleStartup(rig);+ await rig.index.recordDirectDocument('old-doc', 'See [[target]]. #old-tag\n');+ const staleToken = rig.index.captureLiveUpdateToken();+ expect(staleToken).not.toBeNull();++ const transition = await rig.index.beginBranchSwitch('feature');+ const targetBranchToken = rig.index.captureLiveUpdateToken();+ expect(targetBranchToken).not.toBeNull();+ expect(targetBranchToken).not.toBe(staleToken);+ await rig.index.settleBranchFromDisk(transition);+ await rig.index.recordLiveDocument('old-doc', 'See [[target]]. #old-tag\n', staleToken ?? -1);++ expect(await rig.index.getBacklinks('target')).toEqual([]);+ expect(await rig.index.getDocsForTagWithMatches('old-tag')).toEqual([]);+ });++ test('live mutations captured during a branch transition wait and apply to the target branch', async () => {+ const rig = createRig();+ await settleStartup(rig);+ const transition = await rig.index.beginBranchSwitch('feature');+ const token = rig.index.captureLiveUpdateToken();+ expect(token).not.toBeNull();+ let mutationSettled = false;+ let querySettled = false;+ const mutation = rig.index+ .recordLiveDocument('target-live', 'See [[target]]. #target-live\n', token ?? -1)+ .then(() => {+ mutationSettled = true;+ });+ const query = rig.index.getBacklinks('target').then((entries) => {+ querySettled = true;+ return entries;+ });++ await Promise.resolve();+ expect(mutationSettled).toBe(false);+ expect(querySettled).toBe(false);++ await rig.index.settleBranchFromDisk(transition);+ await expect(mutation).resolves.toBeUndefined();+ await expect(query).resolves.toEqual([expect.objectContaining({ source: 'target-live' })]);+ expect(await rig.index.getDocsForTagWithMatches('target-live')).toHaveLength(1);+ });++ test('backlink failure short-circuits tags and signals without poisoning the queue', async () => {+ const rig = createRig();+ await settleStartup(rig);+ const warn = vi.spyOn(getLogger('derived-document-index'), 'warn');+ const update = vi+ .spyOn(BacklinkIndex.prototype, 'updateDocumentFromMarkdown')+ .mockImplementationOnce(() => {+ throw new Error('injected projection failure');+ });+ const updateTags = vi.spyOn(TagIndex.prototype, 'updateDocumentFromMarkdown');++ await expect(rig.index.recordDirectDocument('failed', '# failed\n')).rejects.toThrow(+ 'injected projection failure',+ );+ expect(updateTags).not.toHaveBeenCalled();+ expect(rig.signals).toEqual([]);+ expect(warn).toHaveBeenCalledWith(+ expect.objectContaining({ err: expect.any(Error) }),+ 'Derived document index operation failed; queue remains available',+ );+ update.mockRestore();++ await expect(+ rig.index.recordDirectDocument('recovered', 'See [[target]].\n'),+ ).resolves.toBeUndefined();+ expect(updateTags).toHaveBeenCalledTimes(1);+ expect(await rig.index.getBacklinks('target')).toHaveLength(1);+ });++ test('refreshContentScope gates later mutations and queries until both replacements settle', async () => {+ const rig = createRig();+ await settleStartup(rig);+ const backlinksReplacement = createDeferred();+ const tagsReplacement = createDeferred();+ vi.spyOn(BacklinkIndex.prototype, 'rebuildFromDisk').mockImplementationOnce(+ async () => backlinksReplacement.promise,+ );+ const tagInit = vi+ .spyOn(TagIndex.prototype, 'init')+ .mockImplementationOnce(async () => tagsReplacement.promise);++ const refresh = rig.index.refreshContentScope();+ let mutationSettled = false;+ let querySettled = false;+ const mutation = rig.index.recordDurableStore('after-refresh', '# refreshed\n').then(() => {+ mutationSettled = true;+ });+ const query = rig.index.getIndexedDocNames().then(() => {+ querySettled = true;+ });++ await Promise.resolve();+ expect(mutationSettled).toBe(false);+ expect(querySettled).toBe(false);+ expect(rig.signals).toEqual([]);++ backlinksReplacement.resolve();+ await vi.waitFor(() => expect(tagInit).toHaveBeenCalledTimes(1));+ expect(mutationSettled).toBe(false);+ expect(querySettled).toBe(false);+ expect(rig.signals).toEqual([]);++ tagsReplacement.resolve();+ await Promise.all([refresh, mutation, query]);+ expect(mutationSettled).toBe(true);+ expect(querySettled).toBe(true);+ expect(rig.signals).toEqual(['backlinks', 'graph', 'tags']);+ });++ test('content-scope refresh surfaces global ingest failure and the queue recovers', async () => {+ const rig = createRig();+ await settleStartup(rig);+ writeDoc(rig, 'hidden.md', 'See [[target]]. #hidden\n');+ await rig.index.recordDirectDocument('hidden', 'See [[target]]. #hidden\n');+ writeFileSync(join(rig.projectDir, '.okignore'), 'content/hidden.md\n');+ const rebuilt = await rig.contentFilter.rebuildIgnorePatterns();+ expect(rebuilt.ok).toBe(true);+ rig.signals.length = 0;+ vi.spyOn(BacklinkIndex.prototype, 'ingestGlobalSkillBundles').mockRejectedValueOnce(+ new Error('global ingest failed'),+ );++ await expect(rig.index.refreshContentScope()).rejects.toThrow('global ingest failed');+ expect(await rig.index.getBacklinks('target')).toEqual([]);+ expect(await rig.index.getDocsForTagWithMatches('hidden')).toEqual([]);+ expect(rig.signals).toEqual(['backlinks', 'graph', 'tags']);++ await expect(+ rig.index.recordDirectDocument('recovered', 'See [[target]]. #recovered\n'),+ ).resolves.toBeUndefined();+ expect(await rig.index.getBacklinks('target')).toHaveLength(1);+ expect(await rig.index.getDocsForTagWithMatches('recovered')).toHaveLength(1);+ });++ test('test rescan surfaces global ingest failure and leaves the queue available', async () => {+ const rig = createRig();+ await settleStartup(rig);+ vi.spyOn(BacklinkIndex.prototype, 'ingestGlobalSkillBundles').mockRejectedValueOnce(+ new Error('rescan global ingest failed'),+ );++ await expect(rig.index.testOnly.rescanBacklinksForTest()).rejects.toThrow(+ 'rescan global ingest failed',+ );+ expect(rig.signals).toEqual([]);++ await expect(+ rig.index.recordDirectDocument('recovered', 'See [[target]].\n'),+ ).resolves.toBeUndefined();+ expect(await rig.index.getBacklinks('target')).toHaveLength(1);+ });++ test('branch settlement rebuilds, ingests global nodes, then saves and falls back for tags', async () => {+ const rig = createRig();+ await settleStartup(rig);+ const transition = await rig.index.beginBranchSwitch('feature');+ const load = vi.spyOn(BacklinkIndex.prototype, 'loadFromDisk').mockResolvedValueOnce(false);+ const rebuild = vi+ .spyOn(BacklinkIndex.prototype, 'rebuildFromDisk')+ .mockResolvedValueOnce(undefined);+ const ingest = vi+ .spyOn(BacklinkIndex.prototype, 'ingestGlobalSkillBundles')+ .mockResolvedValueOnce(undefined);+ const backlinkSave = vi+ .spyOn(BacklinkIndex.prototype, 'saveToDisk')+ .mockResolvedValueOnce(undefined);+ const reconcileTags = vi+ .spyOn(TagIndex.prototype, 'reconcileWithDisk')+ .mockRejectedValueOnce(new Error('reconcile failed'));+ const rebuildTags = vi.spyOn(TagIndex.prototype, 'init').mockResolvedValueOnce(undefined);+ const tagSave = vi.spyOn(TagIndex.prototype, 'saveToDisk').mockResolvedValueOnce(undefined);++ await rig.index.settleBranchFromDisk(transition);++ expect(load).toHaveBeenCalledWith('feature');+ expect(rebuild).toHaveBeenCalledWith('feature');+ expect(ingest).toHaveBeenCalledWith([], 'feature');+ expect(rebuild.mock.invocationCallOrder[0]).toBeLessThan(+ ingest.mock.invocationCallOrder[0] ?? 0,+ );+ expect(ingest.mock.invocationCallOrder[0]).toBeLessThan(+ backlinkSave.mock.invocationCallOrder[0] ?? 0,+ );+ expect(reconcileTags).toHaveBeenCalledTimes(1);+ expect(rebuildTags).toHaveBeenCalledTimes(1);+ expect(tagSave).toHaveBeenCalledTimes(1);+ });++ test('branch settlement reconciles a warm cache before global ingest and save', async () => {+ const rig = createRig();+ await settleStartup(rig);+ const transition = await rig.index.beginBranchSwitch('warm');+ vi.spyOn(BacklinkIndex.prototype, 'loadFromDisk').mockResolvedValueOnce(true);+ const reconcile = vi.spyOn(BacklinkIndex.prototype, 'reconcileWithDisk').mockResolvedValueOnce({+ added: 0,+ updated: 0,+ deleted: 0,+ deletedDocNames: [],+ });+ const ingest = vi+ .spyOn(BacklinkIndex.prototype, 'ingestGlobalSkillBundles')+ .mockResolvedValueOnce(undefined);+ const save = vi.spyOn(BacklinkIndex.prototype, 'saveToDisk').mockResolvedValueOnce(undefined);++ await rig.index.settleBranchFromDisk(transition);++ expect(reconcile.mock.invocationCallOrder[0]).toBeLessThan(+ ingest.mock.invocationCallOrder[0] ?? 0,+ );+ expect(ingest.mock.invocationCallOrder[0]).toBeLessThan(save.mock.invocationCallOrder[0] ?? 0);+ });++ test('branch global-ingest failure still replaces local indexes and releases callers', async () => {+ const rig = createRig();+ writeDoc(rig, 'old.md', '# Old\n\n#old-branch\n');+ await settleStartup(rig);+ unlinkSync(join(rig.contentDir, 'old.md'));+ writeDoc(rig, 'target.md', '# Target\n\n#target-branch\n');+ const transition = await rig.index.beginBranchSwitch('feature');+ vi.spyOn(BacklinkIndex.prototype, 'ingestGlobalSkillBundles').mockRejectedValueOnce(+ new Error('branch global ingest failed'),+ );+ const query = rig.index.getDocsForTagWithMatches('target-branch');++ await expect(rig.index.settleBranchFromDisk(transition)).rejects.toThrow(+ 'branch global ingest failed',+ );+ await expect(query).resolves.toEqual([{ docName: 'target', matchingTags: ['target-branch'] }]);+ expect(await rig.index.getDocsForTagWithMatches('old-branch')).toEqual([]);+ });++ test('beginBranchSwitch cancels an old-branch save timer before changing scope', async () => {+ vi.useFakeTimers();+ const backlinkSave = vi.spyOn(BacklinkIndex.prototype, 'saveToDisk');+ const tagSave = vi.spyOn(TagIndex.prototype, 'saveToDisk');+ const rig = createRig();+ await settleStartup(rig);+ await rig.index.recordDiskUpsert('old-branch', '# old\n');++ const transition = await rig.index.beginBranchSwitch('feature');+ await rig.index.settleBranchFromDisk(transition);+ backlinkSave.mockClear();+ tagSave.mockClear();+ await vi.advanceTimersByTimeAsync(2000);++ expect(backlinkSave).not.toHaveBeenCalled();+ expect(tagSave).not.toHaveBeenCalled();+ });++ test('abortBranchSwitch releases only the transition owned by its caller', async () => {+ const rig = createRig();+ await settleStartup(rig);+ const transition = await rig.index.beginBranchSwitch('aborted');+ const mutation = rig.index.recordDirectDocument('released', '# released\n');+ const query = rig.index.getIndexedDocNames();+ let mutationSettled = false;+ void mutation.then(() => {+ mutationSettled = true;+ });++ rig.index.abortBranchSwitch({ branch: 'other' });+ await Promise.resolve();+ expect(mutationSettled).toBe(false);++ rig.index.abortBranchSwitch(transition);+ await expect(mutation).resolves.toBeUndefined();+ await expect(query).resolves.toBeInstanceOf(Array);+ await expect(rig.index.getIndexedDocNames()).resolves.toContain('released');+ });++ test('recognizes only coordinator close errors as lifecycle rejections', async () => {+ const rig = createRig();+ await rig.index.close();++ const closedError = await rig.index.getIndexedDocNames().catch((err: unknown) => err);++ expect(isDerivedDocumentIndexClosedError(closedError)).toBe(true);+ const sameName = new Error('lookalike');+ sameName.name = 'DerivedDocumentIndexClosedError';+ expect(isDerivedDocumentIndexClosedError(sameName)).toBe(false);+ });++ test('warm startup reports documents deleted while the server was down', async () => {+ const projectDir = mkdtempSync(join(tmpdir(), 'ok-derived-warm-'));+ const contentDir = join(projectDir, 'content');+ mkdirSync(contentDir, { recursive: true });+ writeFileSync(join(contentDir, 'stale.md'), '# stale\n');+ const contentFilter = createContentFilter({ projectDir, contentDir });+ const rawIndex = new BacklinkIndex({ projectDir, contentDir, contentFilter });+ await rawIndex.rebuildFromDisk('main');+ await rawIndex.saveToDisk('main');+ unlinkSync(join(contentDir, 'stale.md'));+ const index = new DerivedDocumentIndex({+ projectDir,+ contentDir,+ contentFilter,+ getGlobalSkillRoots: () => [],+ signalChannel: () => {},+ });+ cleanups.push(async () => {+ await index.close();+ rmSync(projectDir, { recursive: true, force: true });+ });++ const startup = index.beginStartup('main');++ await expect(startup.backlinksReady).resolves.toEqual({+ deletedDocNames: ['stale'],+ backlinkIndexDegraded: false,+ });+ await index.settleStartupAfterWatcherSeed();+ });++ test('degraded startup releases query readiness and reports index degradation', async () => {+ const warn = vi.spyOn(getLogger('derived-document-index'), 'warn');+ const rig = createRig();+ vi.spyOn(BacklinkIndex.prototype, 'loadFromDisk').mockRejectedValueOnce(+ new Error('broken backlink cache'),+ );+ vi.spyOn(TagIndex.prototype, 'loadFromDisk').mockRejectedValueOnce(+ new Error('broken tag cache'),+ );+ const startup = rig.index.beginStartup('main');+ await expect(startup.backlinksReady).resolves.toEqual({+ deletedDocNames: [],+ backlinkIndexDegraded: true,+ });+ vi.spyOn(TagIndex.prototype, 'reconcileWithDisk').mockRejectedValueOnce(+ new Error('broken tag scan'),+ );+ const query = rig.index.getIndexedDocNames();++ await expect(rig.index.settleStartupAfterWatcherSeed()).resolves.toEqual({+ tagIndexDegraded: true,+ });+ await expect(query).resolves.toEqual([]);+ expect(warn).toHaveBeenCalledWith(+ expect.objectContaining({ err: expect.any(Error) }),+ '[derived-document-index] tag-index warm boot failed; continuing with stale or empty index',+ );+ });++ test('close flushes a pending paired save and rejects future commands', async () => {+ vi.useFakeTimers();+ const backlinkSave = vi.spyOn(BacklinkIndex.prototype, 'saveToDisk');+ const tagSave = vi.spyOn(TagIndex.prototype, 'saveToDisk');+ const rig = createRig();+ await settleStartup(rig);+ backlinkSave.mockClear();+ tagSave.mockClear();++ await rig.index.recordDiskUpsert('source', '# pending\n');+ const closing = rig.index.close();+ await expect(rig.index.recordDirectDelete('source')).rejects.toThrow(+ 'Derived document index is closed',+ );+ await closing;+ await vi.advanceTimersByTimeAsync(2000);++ expect(backlinkSave).toHaveBeenCalledTimes(1);+ expect(tagSave).toHaveBeenCalledTimes(1);+ });++ test('close flushes a pending backlink-only durable-store save', async () => {+ vi.useFakeTimers();+ const backlinkSave = vi.spyOn(BacklinkIndex.prototype, 'saveToDisk');+ const tagSave = vi.spyOn(TagIndex.prototype, 'saveToDisk');+ const rig = createRig();+ await settleStartup(rig);+ backlinkSave.mockClear();+ tagSave.mockClear();++ await rig.index.recordDurableStore('source', 'See [[target]].\n');+ await rig.index.close();+ await vi.advanceTimersByTimeAsync(2000);++ expect(backlinkSave).toHaveBeenCalledTimes(1);+ expect(tagSave).not.toHaveBeenCalled();+ });++ test('repeated close calls wait for the same dependency drain', async () => {+ const rig = createRig();+ await settleStartup(rig);+ const tagClose = createDeferred();+ const closeTagIndex = vi+ .spyOn(TagIndex.prototype, 'close')+ .mockReturnValueOnce(tagClose.promise);++ const firstClose = rig.index.close();+ const secondClose = rig.index.close();+ let secondCloseSettled = false;+ void secondClose.then(() => {+ secondCloseSettled = true;+ });+ await Promise.resolve();++ expect(closeTagIndex).toHaveBeenCalledTimes(1);+ expect(secondCloseSettled).toBe(false);++ tagClose.resolve();+ await Promise.all([firstClose, secondClose]);+ expect(secondCloseSettled).toBe(true);+ });++ test('close rejects an already-admitted queued disk mutation before it can arm a timer', async () => {+ vi.useFakeTimers();+ const deferredSave = createDeferred();+ const backlinkSave = vi.spyOn(BacklinkIndex.prototype, 'saveToDisk');+ const tagSave = vi.spyOn(TagIndex.prototype, 'saveToDisk');+ const indexLog = getLogger('derived-document-index');+ const warn = vi.spyOn(indexLog, 'warn');+ const debug = vi.spyOn(indexLog, 'debug');+ const rig = createRig();+ await settleStartup(rig);+ backlinkSave.mockClear();+ tagSave.mockClear();+ backlinkSave.mockImplementationOnce(async () => deferredSave.promise);++ const running = rig.index.recordDirectDocument('running', '# running\n');+ await vi.waitFor(() => expect(backlinkSave).toHaveBeenCalledTimes(1));+ const queued = rig.index.recordDiskUpsert('queued', '# queued\n');+ await Promise.resolve();+ const closing = rig.index.close();+ deferredSave.resolve();++ await expect(running).resolves.toBeUndefined();+ await expect(queued).rejects.toThrow('Derived document index is closed');+ await closing;+ const tagSaveCallsAfterClose = tagSave.mock.calls.length;+ await vi.advanceTimersByTimeAsync(2000);++ expect(backlinkSave).toHaveBeenCalledTimes(1);+ expect(tagSave).toHaveBeenCalledTimes(tagSaveCallsAfterClose);+ expect(debug).toHaveBeenCalledWith(+ expect.objectContaining({ err: expect.any(Error) }),+ 'Derived document index operation rejected after close',+ );+ expect(warn).not.toHaveBeenCalledWith(+ expect.anything(),+ 'Derived document index operation failed; queue remains available',+ );+ });++ test('close drains a debounced save already enqueued behind in-flight work', async () => {+ vi.useFakeTimers();+ const deferredSave = createDeferred();+ const backlinkSave = vi.spyOn(BacklinkIndex.prototype, 'saveToDisk');+ const rig = createRig();+ await settleStartup(rig);+ backlinkSave.mockClear();++ await rig.index.recordDiskUpsert('disk-event', '# disk\n');+ backlinkSave.mockImplementationOnce(async () => deferredSave.promise);+ const running = rig.index.recordDirectDocument('running', '# running\n');+ await vi.waitFor(() => expect(backlinkSave).toHaveBeenCalledTimes(1));+ await vi.advanceTimersByTimeAsync(2000);++ const closing = rig.index.close();+ deferredSave.resolve();+ await expect(running).resolves.toBeUndefined();+ await closing;++ expect(backlinkSave).toHaveBeenCalledTimes(2);+ });+});diff --git a/packages/server/src/derived-document-index.ts b/packages/server/src/derived-document-index.tsnew file mode 100644index 000000000..6955b535c--- /dev/null+++ b/packages/server/src/derived-document-index.ts@@ -0,0 +1,751 @@+import type { DerivedViewChannel } from '@inkeep/open-knowledge-core';+import {+ type BacklinkEntry,+ BacklinkIndex,+ type GraphNode,+ type HubEntry,+ isOrphanMode,+ type OrphanMode,+} from './backlink-index.ts';+import { isLinkIndexExcludedDoc, isManagedArtifactDoc } from './cc1-broadcast.ts';+import type { ContentFilter } from './content-filter.ts';+import { docNameToRelativePath } from './doc-extensions.ts';+import { getLogger } from './logger.ts';+import { TagIndex, type TagSummaryEntry } from './tag-index.ts';++const log = getLogger('derived-document-index');++const DERIVED_INDEX_SAVE_DEBOUNCE_MS = 2000;++export type DerivedDocumentRelationChannel = Extract<+ DerivedViewChannel,+ 'backlinks' | 'graph' | 'tags'+>;++type ForwardLinkEntry = ReturnType<BacklinkIndex['getForwardLinkEntries']>[number];+type LinkGraph = ReturnType<BacklinkIndex['getLinkGraph']>;+type DeadLinkEntry = ReturnType<BacklinkIndex['getDeadLinks']>[number];+type TagDocMatch = ReturnType<TagIndex['getDocsForTagWithMatches']>[number];++export type DerivedGraphNode = GraphNode;+export type DerivedOrphanMode = OrphanMode;++export function isDerivedOrphanMode(value: unknown): value is DerivedOrphanMode {+ return typeof value === 'string' && isOrphanMode(value);+}++export interface DerivedIndexStartupBacklinksResult {+ deletedDocNames: readonly string[];+ backlinkIndexDegraded: boolean;+}++export interface DerivedIndexStartupSettlement {+ tagIndexDegraded: boolean;+}++export interface DerivedDocumentIndexBranchTransition {+ readonly branch: string;+}++export interface DerivedDocumentIndexLivePort {+ captureLiveUpdateToken(): number | null;+ recordLiveDocument(documentName: string, markdown: string, token: number): Promise<void>;+}++export interface DerivedDocumentIndexPersistencePort {+ recordDurableStore(documentName: string, markdown: string): Promise<void>;+}++export interface DerivedDocumentIndexTestPort {+ resetDocumentForTest(documentName: string): Promise<void>;+ rescanBacklinksForTest(): Promise<void>;+}++export type DerivedDocumentIndexMutation =+ | { kind: 'upsert'; documentName: string; markdown: string }+ | { kind: 'delete'; documentName: string }+ | {+ kind: 'rename';+ oldDocumentName: string;+ newDocumentName: string;+ markdown: string;+ }+ | { kind: 'link-rewrite'; documentName: string; markdown: string };++export interface DerivedDocumentIndexApiPort {+ readonly testOnly?: DerivedDocumentIndexTestPort;+ recordDirectMutations(mutations: readonly DerivedDocumentIndexMutation[]): Promise<void>;+ recordDirectDocument(documentName: string, markdown: string): Promise<void>;+ recordDirectDelete(documentName: string): Promise<void>;+ recordDirectRename(+ oldDocumentName: string,+ newDocumentName: string,+ markdown: string,+ ): Promise<void>;+ recordLinkRewrite(documentName: string, markdown: string): Promise<void>;+ getBacklinks(documentName: string): Promise<BacklinkEntry[]>;+ getBacklinkCount(documentName: string): Promise<number>;+ getBacklinkCounts(documentNames: readonly string[]): Promise<Record<string, number>>;+ getForwardLinkEntries(documentName: string): Promise<ForwardLinkEntry[]>;+ getLinkGraphNeighborhood(documentName: string, degrees: number): Promise<LinkGraph>;+ getLinkGraph(): Promise<LinkGraph>;+ getHubs(limit?: number): Promise<HubEntry[]>;+ getOrphans(documentNames: string[], mode: OrphanMode): Promise<string[]>;+ getDeadLinks(+ admittedDocuments: Iterable<string>,+ sourceDocumentNames?: readonly string[],+ ): Promise<DeadLinkEntry[]>;+ getIndexedDocNames(): Promise<string[]>;+ getAllTags(): Promise<TagSummaryEntry[]>;+ getDocsForTagWithMatches(tag: string): Promise<TagDocMatch[]>;+}++export interface DerivedDocumentIndexOptions {+ projectDir: string;+ contentDir: string;+ contentFilter: ContentFilter;+ getGlobalSkillRoots: () => string[];+ signalChannel: (channel: DerivedDocumentRelationChannel) => void;+}++interface Deferred {+ promise: Promise<void>;+ resolve: () => void;+}++interface BranchTransition extends DerivedDocumentIndexBranchTransition {+ barrier: Deferred;+}++function createDeferred(): Deferred {+ let resolve!: () => void;+ const promise = new Promise<void>((settle) => {+ resolve = settle;+ });+ return { promise, resolve };+}++class DerivedDocumentIndexClosedError extends Error {+ constructor() {+ super('Derived document index is closed');+ this.name = 'DerivedDocumentIndexClosedError';+ }+}++export function isDerivedDocumentIndexClosedError(err: unknown): boolean {+ return err instanceof DerivedDocumentIndexClosedError;+}++export class DerivedDocumentIndex+ implements+ DerivedDocumentIndexApiPort,+ DerivedDocumentIndexLivePort,+ DerivedDocumentIndexPersistencePort+{+ readonly testOnly: DerivedDocumentIndexTestPort;++ private readonly backlinkIndex: BacklinkIndex;+ private readonly tagIndex: TagIndex;+ private readonly contentFilter: ContentFilter;+ private readonly getGlobalSkillRoots: () => string[];+ private readonly signalChannel: (channel: DerivedDocumentRelationChannel) => void;+ private readonly startupAdmission = createDeferred();+ private readonly readyBarrier = createDeferred();+ private tail: Promise<void> = Promise.resolve();+ private saveTimer: ReturnType<typeof setTimeout> | null = null;+ private saveTagsOnNextDebounce = false;+ private branchTransition: BranchTransition | null = null;+ private startupBegun = false;+ private startupSettled = false;+ private liveUpdateToken = 0;+ private closed = false;+ private closePromise: Promise<void> | null = null;++ constructor(options: DerivedDocumentIndexOptions) {+ this.backlinkIndex = new BacklinkIndex({+ projectDir: options.projectDir,+ contentDir: options.contentDir,+ contentFilter: options.contentFilter,+ });+ this.tagIndex = new TagIndex({+ projectDir: options.projectDir,+ contentDir: options.contentDir,+ contentFilter: options.contentFilter,+ });+ this.contentFilter = options.contentFilter;+ this.getGlobalSkillRoots = options.getGlobalSkillRoots;+ this.signalChannel = options.signalChannel;+ this.testOnly = {+ resetDocumentForTest: (documentName) => this.resetDocumentForTest(documentName),+ rescanBacklinksForTest: () => this.rescanBacklinksForTest(),+ };+ }++ beginStartup(branch: string): { backlinksReady: Promise<DerivedIndexStartupBacklinksResult> } {+ this.assertOpen();+ if (this.startupBegun) {+ throw new Error('Derived document index startup has already begun');+ }+ this.startupBegun = true;+ this.backlinkIndex.switchBranch(branch);++ const backlinksReady = this.initializeBacklinks(branch);+ const tagsWarm = this.warmTags();+ this.tail = Promise.all([backlinksReady.then(() => undefined), tagsWarm]).then(() => undefined);+ this.startupAdmission.resolve();+ return { backlinksReady };+ }++ async settleStartupAfterWatcherSeed(): Promise<DerivedIndexStartupSettlement> {+ this.assertOpen();+ if (!this.startupBegun) {+ throw new Error('Derived document index startup has not begun');+ }+ if (this.startupSettled) {+ throw new Error('Derived document index startup has already settled');+ }+ this.startupSettled = true;++ try {+ return await this.enqueueInternal(async () => {+ try {+ const diff = await this.tagIndex.reconcileWithDisk();+ if (diff.added > 0 || diff.updated > 0 || diff.deleted > 0) {+ log.info(diff, '[tag-index] startup reconcile: offline changes applied');+ }+ await this.saveTagsLogOnly('startup');+ return { tagIndexDegraded: false };+ } catch (err) {+ log.error(+ { err },+ '[tag-index] startup reconcile failed; tag index updates incrementally via watcher events',+ );+ return { tagIndexDegraded: true };+ }+ });+ } finally {+ this.readyBarrier.resolve();+ }+ }++ async beginBranchSwitch(branch: string): Promise<DerivedDocumentIndexBranchTransition> {+ this.assertOpen();+ if (this.branchTransition) {+ throw new Error(`Derived document index branch switch already active`);+ }+ const transition: BranchTransition = { branch, barrier: createDeferred() };+ this.liveUpdateToken++;+ this.branchTransition = transition;+ this.cancelScheduledSave();++ try {+ await this.startupAdmission.promise;+ await this.readyBarrier.promise;+ this.assertOpen();+ await this.enqueueInternal(async () => {+ this.backlinkIndex.switchBranch(branch);+ });+ return transition;+ } catch (err) {+ this.releaseBranchTransition(transition);+ throw err;+ }+ }++ async settleBranchFromDisk(transition: DerivedDocumentIndexBranchTransition): Promise<void> {+ this.assertOpen();+ if (this.branchTransition !== transition) {+ throw new Error('No derived document index branch switch is active');+ }++ try {+ await this.enqueueInternal(async () => {+ const branch = transition.branch;+ const loaded = await this.backlinkIndex.loadFromDisk(branch);+ if (loaded) {+ const diff = await this.backlinkIndex.reconcileWithDisk(branch);+ if (diff.added > 0 || diff.updated > 0 || diff.deleted > 0) {+ log.info(diff, `[backlinks] branch-switch reconcile for ${branch}`);+ }+ } else {+ await this.backlinkIndex.rebuildFromDisk(branch);+ }+ let globalIngestFailed = false;+ let globalIngestFailure: unknown;+ try {+ await this.backlinkIndex.ingestGlobalSkillBundles(this.getGlobalSkillRoots(), branch);+ } catch (err) {+ globalIngestFailed = true;+ globalIngestFailure = err;+ }+ await this.saveBacklinksLogOnly('branch-switch', branch);++ try {+ const tagDiff = await this.tagIndex.reconcileWithDisk();+ if (tagDiff.added > 0 || tagDiff.updated > 0 || tagDiff.deleted > 0) {+ log.info(tagDiff, `[tag-index] branch-switch reconcile for ${branch}`);+ }+ } catch (err) {+ log.warn(+ { err, branch },+ '[tag-index] branch-switch reconcile failed; falling back to full rebuild',+ );+ await this.tagIndex.init();+ }+ await this.saveTagsLogOnly('branch-switch');+ if (globalIngestFailed) throw globalIngestFailure;+ });+ } finally {+ this.releaseBranchTransition(transition);+ }+ }++ abortBranchSwitch(transition: DerivedDocumentIndexBranchTransition | undefined): void {+ if (transition) this.releaseBranchTransition(transition);+ }++ captureLiveUpdateToken(): number | null {+ if (this.closed) return null;+ return this.liveUpdateToken;+ }++ recordLiveDocument(documentName: string, markdown: string, token: number): Promise<void> {+ if (this.closed || token !== this.liveUpdateToken) {+ return Promise.resolve();+ }+ return this.runCommand(async () => {+ if (token !== this.liveUpdateToken) return;+ this.updateBoth(documentName, markdown);+ this.signalAllRelations();+ });+ }++ recordDurableStore(documentName: string, markdown: string): Promise<void> {+ return this.runCommand(async () => {+ this.updateBacklinks(documentName, markdown);+ this.scheduleSave(false);+ });+ }++ recordDiskUpsert(documentName: string, markdown: string): Promise<void> {+ return this.runCommand(async () => {+ this.updateBoth(documentName, markdown);+ this.scheduleSave();+ this.signalAllRelations();+ });+ }++ recordDiskDelete(documentName: string): Promise<void> {+ return this.runCommand(async () => {+ this.deleteBoth(documentName);+ this.scheduleSave();+ this.signalAllRelations();+ });+ }++ recordDiskRename(+ oldDocumentName: string,+ newDocumentName: string,+ markdown: string,+ ): Promise<void> {+ return this.runCommand(async () => {+ this.renameBoth(oldDocumentName, newDocumentName, markdown);+ this.scheduleSave();+ this.signalAllRelations();+ });+ }++ recordDirectDocument(documentName: string, markdown: string): Promise<void> {+ return this.recordDirectMutations([{ kind: 'upsert', documentName, markdown }]);+ }++ recordDirectDelete(documentName: string): Promise<void> {+ return this.recordDirectMutations([{ kind: 'delete', documentName }]);+ }++ recordDirectRename(+ oldDocumentName: string,+ newDocumentName: string,+ markdown: string,+ ): Promise<void> {+ return this.recordDirectMutations([+ { kind: 'rename', oldDocumentName, newDocumentName, markdown },+ ]);+ }++ recordLinkRewrite(documentName: string, markdown: string): Promise<void> {+ return this.recordDirectMutations([{ kind: 'link-rewrite', documentName, markdown }]);+ }++ recordDirectMutations(mutations: readonly DerivedDocumentIndexMutation[]): Promise<void> {+ if (mutations.length === 0) return Promise.resolve();+ return this.runCommand(async () => {+ let tagsChanged = false;+ for (const mutation of mutations) {+ switch (mutation.kind) {+ case 'upsert':+ this.updateBoth(mutation.documentName, mutation.markdown);+ tagsChanged = true;+ break;+ case 'delete':+ this.deleteBoth(mutation.documentName);+ tagsChanged = true;+ break;+ case 'rename':+ this.renameBoth(mutation.oldDocumentName, mutation.newDocumentName, mutation.markdown);+ tagsChanged = true;+ break;+ case 'link-rewrite':+ this.updateBacklinks(mutation.documentName, mutation.markdown);+ break;+ }+ }++ if (tagsChanged) {+ await this.saveBothLogOnly('direct-batch');+ this.signalAllRelations();+ } else {+ await this.saveBacklinksLogOnly('direct-batch');+ this.signalBacklinks();+ }+ });+ }++ refreshGlobalSkillNodes(): Promise<void> {+ return this.runCommand(async () => {+ await this.backlinkIndex.ingestGlobalSkillBundles(this.getGlobalSkillRoots());+ this.signalBacklinks();+ });+ }++ refreshContentScope(): Promise<void> {+ return this.runCommand(async () => {+ const branch = this.backlinkIndex.getActiveBranch();+ await this.backlinkIndex.rebuildFromDisk(branch);+ let globalIngestFailed = false;+ let globalIngestFailure: unknown;+ try {+ await this.backlinkIndex.ingestGlobalSkillBundles(this.getGlobalSkillRoots(), branch);+ } catch (err) {+ globalIngestFailed = true;+ globalIngestFailure = err;+ }+ await this.tagIndex.init();+ await this.saveBothLogOnly('content-filter', branch);+ this.signalAllRelations();+ if (globalIngestFailed) throw globalIngestFailure;+ });+ }++ announceReadyViews(): void {+ // The server factory calls this only after startup settlement has resolved.+ this.assertOpen();+ this.signalAllRelations();+ }++ getBacklinks(documentName: string): Promise<BacklinkEntry[]> {+ return this.runQuery(() => this.backlinkIndex.getBacklinks(documentName));+ }++ getBacklinkCount(documentName: string): Promise<number> {+ return this.runQuery(() => this.backlinkIndex.getBacklinkCount(documentName));+ }++ getBacklinkCounts(documentNames: readonly string[]): Promise<Record<string, number>> {+ return this.runQuery(() =>+ Object.fromEntries(+ documentNames.map((documentName) => [+ documentName,+ this.backlinkIndex.getBacklinkCount(documentName),+ ]),+ ),+ );+ }++ getForwardLinkEntries(documentName: string): Promise<ForwardLinkEntry[]> {+ return this.runQuery(() => this.backlinkIndex.getForwardLinkEntries(documentName));+ }++ getLinkGraphNeighborhood(documentName: string, degrees: number): Promise<LinkGraph> {+ return this.runQuery(() => this.backlinkIndex.getLinkGraphNeighborhood(documentName, degrees));+ }++ getLinkGraph(): Promise<LinkGraph> {+ return this.runQuery(() => this.backlinkIndex.getLinkGraph());+ }++ getHubs(limit?: number): Promise<HubEntry[]> {+ return this.runQuery(() => this.backlinkIndex.getHubs(limit));+ }++ getOrphans(documentNames: string[], mode: OrphanMode): Promise<string[]> {+ return this.runQuery(() => this.backlinkIndex.getOrphans(documentNames, mode));+ }++ getDeadLinks(+ admittedDocuments: Iterable<string>,+ sourceDocumentNames?: readonly string[],+ ): Promise<DeadLinkEntry[]> {+ return this.runQuery(() =>+ this.backlinkIndex.getDeadLinks(admittedDocuments, sourceDocumentNames),+ );+ }++ getIndexedDocNames(): Promise<string[]> {+ return this.runQuery(() => this.backlinkIndex.getIndexedDocNames());+ }++ getAllTags(): Promise<TagSummaryEntry[]> {+ return this.runQuery(() => this.tagIndex.getAllTags());+ }++ getDocsForTagWithMatches(tag: string): Promise<TagDocMatch[]> {+ return this.runQuery(() => this.tagIndex.getDocsForTagWithMatches(tag));+ }++ close(): Promise<void> {+ if (this.closePromise) return this.closePromise;+ const pendingSave = this.cancelScheduledSave();+ if (pendingSave !== null) {+ void this.enqueueInternal(() => this.saveScheduledCache(pendingSave, 'close-flush'));+ }+ this.closed = true;+ this.startupAdmission.resolve();+ this.readyBarrier.resolve();+ this.releaseBranchTransition(this.branchTransition);+ this.closePromise = this.tail+ .then(() => this.tagIndex.close())+ .then(+ () => undefined,+ () => undefined,+ );+ return this.closePromise;+ }++ private resetDocumentForTest(documentName: string): Promise<void> {+ return this.runCommand(async () => {+ this.deleteBoth(documentName);+ await this.saveBothLogOnly('test-reset');+ this.signalAllRelations();+ });+ }++ private rescanBacklinksForTest(): Promise<void> {+ return this.runCommand(async () => {+ const branch = this.backlinkIndex.getActiveBranch();+ await this.backlinkIndex.rebuildFromDisk(branch);+ await this.backlinkIndex.ingestGlobalSkillBundles(this.getGlobalSkillRoots(), branch);+ await this.saveBacklinksLogOnly('test-rescan', branch);+ this.signalBacklinks();+ });+ }++ private async initializeBacklinks(branch: string): Promise<DerivedIndexStartupBacklinksResult> {+ try {+ let deletedDocNames: readonly string[] = [];+ if (await this.backlinkIndex.loadFromDisk(branch)) {+ const diff = await this.backlinkIndex.reconcileWithDisk(branch);+ deletedDocNames = diff.deletedDocNames;+ if (diff.added > 0 || diff.updated > 0 || diff.deleted > 0) {+ log.info(+ { added: diff.added, updated: diff.updated, deleted: diff.deleted },+ '[backlinks] startup reconcile: offline changes applied',+ );+ }+ } else {+ await this.backlinkIndex.rebuildFromDisk(branch);+ }+ await this.ingestGlobalSkillNodesLogOnly(branch, 'startup');+ await this.saveBacklinksLogOnly('startup', branch);+ return { deletedDocNames, backlinkIndexDegraded: false };+ } catch (err) {+ log.error(+ { err, branch },+ '[backlinks] startup init failed; index will populate incrementally via watcher',+ );+ return { deletedDocNames: [], backlinkIndexDegraded: true };+ }+ }++ private async warmTags(): Promise<void> {+ try {+ if (await this.tagIndex.loadFromDisk()) {+ await this.tagIndex.reconcileWithDisk();+ } else {+ await this.tagIndex.init();+ }+ } catch (err) {+ log.warn(+ { err },+ '[derived-document-index] tag-index warm boot failed; continuing with stale or empty index',+ );+ }+ }++ private updateBoth(documentName: string, markdown: string): void {+ this.updateBacklinks(documentName, markdown);+ if (this.isAdmitted(documentName)) {+ this.tagIndex.updateDocumentFromMarkdown(documentName, markdown);+ } else {+ this.tagIndex.deleteDocument(documentName);+ }+ }++ private updateBacklinks(documentName: string, markdown: string): void {+ if (this.isAdmitted(documentName)) {+ this.backlinkIndex.updateDocumentFromMarkdown(documentName, markdown);+ } else {+ this.backlinkIndex.deleteDocument(documentName);+ }+ }++ private deleteBoth(documentName: string): void {+ this.backlinkIndex.deleteDocument(documentName);+ this.tagIndex.deleteDocument(documentName);+ }++ private renameBoth(oldDocumentName: string, newDocumentName: string, markdown: string): void {+ if (this.isAdmitted(newDocumentName)) {+ this.backlinkIndex.renameDocument(oldDocumentName, newDocumentName, markdown);+ this.tagIndex.renameDocument(oldDocumentName, newDocumentName, markdown);+ } else {+ this.backlinkIndex.deleteDocument(oldDocumentName);+ this.backlinkIndex.deleteDocument(newDocumentName);+ this.tagIndex.deleteDocument(oldDocumentName);+ this.tagIndex.deleteDocument(newDocumentName);+ }+ }++ private isAdmitted(documentName: string): boolean {+ if (isLinkIndexExcludedDoc(documentName)) return false;+ if (isManagedArtifactDoc(documentName)) return true;+ return !this.contentFilter.isExcluded(docNameToRelativePath(documentName));+ }++ private signalBacklinks(): void {+ this.signalChannel('backlinks');+ this.signalChannel('graph');+ }++ private signalAllRelations(): void {+ this.signalBacklinks();+ this.signalChannel('tags');+ }++ private scheduleSave(includeTags = true): void {+ const saveTags = this.saveTagsOnNextDebounce || includeTags;+ this.cancelScheduledSave();+ this.saveTagsOnNextDebounce = saveTags;+ this.saveTimer = setTimeout(() => {+ this.saveTimer = null;+ const saveTags = this.saveTagsOnNextDebounce;+ this.saveTagsOnNextDebounce = false;+ void this.enqueueInternal(() => this.saveScheduledCache(saveTags, 'debounced')).catch(+ (err) => {+ log.warn({ err }, 'Failed to enqueue debounced derived-index cache save');+ },+ );+ }, DERIVED_INDEX_SAVE_DEBOUNCE_MS);+ }++ private cancelScheduledSave(): boolean | null {+ const pendingSave = this.saveTimer === null ? null : this.saveTagsOnNextDebounce;+ if (this.saveTimer !== null) clearTimeout(this.saveTimer);+ this.saveTimer = null;+ this.saveTagsOnNextDebounce = false;+ return pendingSave;+ }++ private saveScheduledCache(includeTags: boolean, reason: string): Promise<void> {+ return includeTags+ ? this.saveBothLogOnly(reason)+ : this.saveBacklinksLogOnly(`${reason}-backlinks`);+ }++ private releaseBranchTransition(transition: DerivedDocumentIndexBranchTransition | null): void {+ const active = this.branchTransition;+ if (!transition || active !== transition) return;+ this.branchTransition = null;+ active.barrier.resolve();+ }++ private async saveBothLogOnly(reason: string, branch?: string): Promise<void> {+ await this.saveBacklinksLogOnly(reason, branch);+ await this.saveTagsLogOnly(reason);+ }++ private async saveBacklinksLogOnly(reason: string, branch?: string): Promise<void> {+ try {+ await this.backlinkIndex.saveToDisk(branch);+ } catch (err) {+ log.warn({ err, reason, branch }, 'Failed to persist backlink cache');+ }+ }++ private async saveTagsLogOnly(reason: string): Promise<void> {+ try {+ await this.tagIndex.saveToDisk();+ } catch (err) {+ log.warn({ err, reason }, 'Failed to persist tag snapshot');+ }+ }++ private async ingestGlobalSkillNodesLogOnly(branch: string, reason: string): Promise<void> {+ try {+ await this.backlinkIndex.ingestGlobalSkillBundles(this.getGlobalSkillRoots(), branch);+ } catch (err) {+ log.warn({ err, branch, reason }, 'Global skill bundle ingest failed');+ }+ }++ private async runCommand<T>(operation: () => Promise<T> | T): Promise<T> {+ this.assertOpen();+ await this.startupAdmission.promise;+ this.assertOpen();+ return this.enqueueAfterBranchAdmission(operation);+ }++ private async runQuery<T>(query: () => Promise<T> | T): Promise<T> {+ this.assertOpen();+ await this.startupAdmission.promise;+ await this.readyBarrier.promise;+ this.assertOpen();+ return this.enqueueAfterBranchAdmission(query);+ }++ private async enqueueAfterBranchAdmission<T>(operation: () => Promise<T> | T): Promise<T> {+ while (true) {+ const transition = this.branchTransition;+ if (!transition) {+ this.assertOpen();+ return this.enqueueInternal(() => {+ this.assertOpen();+ return operation();+ });+ }+ await transition.barrier.promise;+ this.assertOpen();+ }+ }++ private enqueueInternal<T>(operation: () => Promise<T> | T): Promise<T> {+ const run = this.tail.then(operation);+ this.tail = run.then(+ () => undefined,+ (err) => {+ if (isDerivedDocumentIndexClosedError(err)) {+ log.debug({ err }, 'Derived document index operation rejected after close');+ } else {+ log.warn({ err }, 'Derived document index operation failed; queue remains available');+ }+ },+ );+ return run;+ }++ private assertOpen(): void {+ if (this.closed) {+ throw new DerivedDocumentIndexClosedError();+ }+ }+}diff --git a/packages/server/src/head-watcher.test.ts b/packages/server/src/head-watcher.test.tsindex 8900aa628..0e245784d 100644--- a/packages/server/src/head-watcher.test.ts+++ b/packages/server/src/head-watcher.test.ts@@ -3,7 +3,7 @@ import { mkdirSync, writeFileSync } from 'node:fs';import { mkdtemp, rm } from 'node:fs/promises';import { tmpdir } from 'node:os';import { resolve } from 'node:path';-import { afterEach, beforeEach, describe, expect, test } from 'vitest';+import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';import { readProjectHeadState, startHeadWatcher, watchedGitFile } from './head-watcher';let tmpDir: string;@@ -140,3 +140,68 @@ describe('startHeadWatcher chokidar fallback', () => {}});});++describe('startHeadWatcher batch serialization', () => {+ test('queues a second HEAD batch until the first branch-switch callback settles', async () => {+ const projectRoot = resolve(tmpDir, 'repo');+ mkdirSync(projectRoot, { recursive: true });+ const git = (args: string) => execSync(`git ${args}`, { cwd: projectRoot, stdio: 'ignore' });+ git('init -q');+ git('config user.email [email protected]');+ git('config user.name t');+ writeFileSync(resolve(projectRoot, 'a.md'), 'hello\n');+ git('add -A');+ git('commit -qm init');+ git('branch -M main');+ git('branch feature');+ git('branch second');++ let dispatch!: (rawPath: string) => void;+ let releaseFirst!: () => void;+ const firstCanSettle = new Promise<void>((resolve) => {+ releaseFirst = resolve;+ });+ const branches: Array<string | null> = [];+ let activeCallbacks = 0;+ let maxActiveCallbacks = 0;++ const handle = await startHeadWatcher(+ projectRoot,+ () => {},+ async (info) => {+ activeCallbacks++;+ maxActiveCallbacks = Math.max(maxActiveCallbacks, activeCallbacks);+ branches.push(info.newBranch);+ try {+ if (branches.length === 1) await firstCanSettle;+ } finally {+ activeCallbacks--;+ }+ },+ {+ subscribeForTest: async (_gitDir, nextDispatch) => {+ dispatch = nextDispatch;+ return async () => {};+ },+ },+ );++ try {+ git('checkout -q feature');+ dispatch(resolve(projectRoot, '.git', 'HEAD'));+ await vi.waitFor(() => expect(branches).toEqual(['feature']));++ git('checkout -q second');+ dispatch(resolve(projectRoot, '.git', 'HEAD'));+ await new Promise((resolveWait) => setTimeout(resolveWait, 150));+ expect(branches).toEqual(['feature']);++ releaseFirst();+ await vi.waitFor(() => expect(branches).toEqual(['feature', 'second']));+ expect(maxActiveCallbacks).toBe(1);+ } finally {+ releaseFirst();+ await handle.unsubscribe();+ }+ });+});diff --git a/packages/server/src/head-watcher.ts b/packages/server/src/head-watcher.tsindex ed6e3d33f..8726b3726 100644--- a/packages/server/src/head-watcher.ts+++ b/packages/server/src/head-watcher.ts@@ -179,7 +179,13 @@ export async function startHeadWatcher(projectRoot: string,onBatchBegin: OnBatchBegin,onBatchEnd: OnBatchEnd,- opts: { forceBackend?: 'parcel' | 'chokidar' } = {},+ opts: {+ forceBackend?: 'parcel' | 'chokidar';+ subscribeForTest?: (+ gitDir: string,+ dispatch: HeadEventDispatch,+ ) => Promise<() => Promise<void>>;+ } = {},): Promise<HeadWatcherHandle> {const inspected = discoverGitRepository(projectRoot);if (inspected.kind !== 'repository') {@@ -194,10 +200,15 @@ export async function startHeadWatcher(let timeoutTimer: ReturnType<typeof setTimeout> | null = null;let oldHead: string | null = null;let lastKnownBranch: string | null = null;+ let batchEndInFlight: Promise<void> | null = null;async function emitBatchEnd(timeout: boolean): Promise<void> {// Wait for onBatchBegin to finish before proceedingif (beginInFlight) await beginInFlight;+ if (batchEndInFlight) {+ await batchEndInFlight;+ return;+ }if (!inBatch) return;if (quietTimer) {@@ -226,24 +237,32 @@ export async function startHeadWatcher(const oldBranch = lastKnownBranch;+ const end = Promise.resolve().then(async () => {+ try {+ await onBatchEnd({+ headMoved,+ oldHead,+ newHead,+ timeout,+ batchKind,+ oldBranch,+ newBranch,+ });+ } catch (e) {+ log.error({ err: e }, 'onBatchEnd callback failed');+ } finally {+ // Set inBatch = false AFTER the async callback completes+ // so new file events stay buffered during branch-switch orchestration+ inBatch = false;+ oldHead = newHead;+ lastKnownBranch = newBranch;+ }+ });+ batchEndInFlight = end;try {- await onBatchEnd({- headMoved,- oldHead,- newHead,- timeout,- batchKind,- oldBranch,- newBranch,- });- } catch (e) {- log.error({ err: e }, 'onBatchEnd callback failed');+ await end;} finally {- // Set inBatch = false AFTER the async callback completes- // so new file events stay buffered during branch-switch orchestration- inBatch = false;- oldHead = newHead;- lastKnownBranch = newBranch;+ if (batchEndInFlight === end) batchEndInFlight = null;}}@@ -258,6 +277,9 @@ export async function startHeadWatcher(let beginInFlight: Promise<void> | null = null;async function handleGitEvent(trigger: string): Promise<void> {+ // A second HEAD event belongs to a new batch, but it must not re-enter the+ // still-running branch-switch callback for the preceding batch.+ if (batchEndInFlight) await batchEndInFlight;if (!inBatch) {inBatch = true;// Do NOT re-read HEAD here. `oldHead` already holds the last-settled HEAD@@ -297,8 +319,11 @@ export async function startHeadWatcher(// Prefer @parcel/watcher; fall back to chokidar when it can't start so HEAD// watching stays functional in packaged builds that omit the native addon.let resolvedUnsub: (() => Promise<void>) | null = null;- let backend: 'parcel' | 'chokidar' = 'chokidar';- if (opts.forceBackend !== 'chokidar') {+ let backend: 'parcel' | 'chokidar' | 'test' = 'chokidar';+ if (opts.subscribeForTest) {+ resolvedUnsub = await opts.subscribeForTest(gitDir, dispatch);+ backend = 'test';+ } else if (opts.forceBackend !== 'chokidar') {resolvedUnsub = await tryStartParcelHeadWatcher(gitDir, dispatch);if (resolvedUnsub) backend = 'parcel';}diff --git a/packages/server/src/http/error-response.ts b/packages/server/src/http/error-response.tsindex 9ceb6a9cd..c8ca42ba2 100644--- a/packages/server/src/http/error-response.ts+++ b/packages/server/src/http/error-response.ts@@ -126,6 +126,12 @@ interface ErrorResponseOptions {* line. Surfaces underlying errno / syscall on storage failures.*/cause?: unknown;+ /**+ * Override the status-derived log level for expected lifecycle responses.+ * For example, a 503 emitted while a subsystem is shutting down is useful+ * to the caller but is not an operational fault.+ */+ logLevel?: 'debug' | 'warn' | 'error';}/**@@ -299,7 +305,7 @@ export function errorResponse(// The Pino line is for grep correlation via `instance`. Separating// levels avoids drowning monitoring in routine 4xx noise (mirrors// pino-http's `customLogLevel` defaults).- const logLevel = status >= 500 ? 'error' : 'warn';+ const logLevel = options.logLevel ?? (status >= 500 ? 'error' : 'warn');log()[logLevel]({event: 'api.error',diff --git a/packages/server/src/lint/validation-audit.test.ts b/packages/server/src/lint/validation-audit.test.tsindex 2e0d44f78..e0176c164 100644--- a/packages/server/src/lint/validation-audit.test.ts+++ b/packages/server/src/lint/validation-audit.test.ts@@ -67,7 +67,7 @@ function deps(overrides: Partial<ValidationAuditDeps> = {}): ValidationAuditDepsprojectDir: root,contentDir: root,baseConfig: lintOn,- backlinkIndex: index,+ derivedDocumentIndex: index,admittedDocNames: () => admitted,docFilePathFor,...overrides,@@ -214,7 +214,9 @@ describe('runValidationAudit', () => {seedDoc('dirty', DOC_WITH_TAB);seedDoc('linker', '# Linker\n\nSee [[ghost]].\n');- const result = await runValidationAudit(createProjectValidators(deps({ backlinkIndex: null })));+ const result = await runValidationAudit(+ createProjectValidators(deps({ derivedDocumentIndex: null })),+ );expect(result.files.map((f) => f.file)).toEqual(['dirty.md']);expect(result.errorCount).toBe(0);@@ -302,7 +304,7 @@ describe('runValidationAudit', () => {const result = await runValidationAudit(createProjectValidators(deps({- backlinkIndex: fakeIndex,+ derivedDocumentIndex: fakeIndex,admittedDocNames: () => ['unsaved'],docFilePathFor: () => null,}),@@ -328,7 +330,7 @@ describe('runValidationAudit', () => {const result = await runValidationAudit(createProjectValidators(deps({- backlinkIndex: fakeIndex,+ derivedDocumentIndex: fakeIndex,admittedDocNames: () => ['legacy'],docFilePathFor: () => 'legacy.md',}),diff --git a/packages/server/src/lint/validation-audit.ts b/packages/server/src/lint/validation-audit.tsindex 763f1ee79..c78aae652 100644--- a/packages/server/src/lint/validation-audit.ts+++ b/packages/server/src/lint/validation-audit.ts@@ -4,7 +4,7 @@* "given the project (or a sub-path), what per-file diagnostics hold?" behind* one interface while owning its own execution model — the lint validator* walks the content tree via `auditProject`; the links validator reads the- * in-memory `BacklinkIndex`. The engine fans out and merges everything into+ * derived document index. The engine fans out and merges everything into* one source-tagged diagnostic plane. Backs `GET /api/audit` + MCP `audit`.*/@@ -15,10 +15,19 @@ import {SUPPORTED_DOC_EXTENSIONS,type ValidationDiagnostic,} from '@inkeep/open-knowledge-core';-import type { BacklinkIndex } from '../backlink-index.ts';+import type { DerivedDocumentIndexApiPort } from '../derived-document-index.ts';import { getLogger } from '../logger.ts';import { auditProject } from './audit.ts';+type DeadLinksResult = Awaited<ReturnType<DerivedDocumentIndexApiPort['getDeadLinks']>>;++interface ValidationDerivedIndexReader {+ getDeadLinks(+ admittedDocuments: Iterable<string>,+ sourceDocumentNames?: readonly string[],+ ): DeadLinksResult | Promise<DeadLinksResult>;+}+interface FileValidationResult {/** Path relative to `contentDir`. */file: string;@@ -60,15 +69,15 @@ export interface ValidationAuditDeps {baseConfig: LinterConfig;/** Live CRDT source overlay for loaded docs (see `AuditOptions.liveSourceFor`). */liveSourceFor?: (docRelPath: string) => string | null;- /** Null when the server booted without a backlink index; links findings degrade to a warning. */- backlinkIndex: Pick<BacklinkIndex, 'getDeadLinks'> | null;+ /** Null when the server booted without a derived index; links findings degrade to a warning. */+ derivedDocumentIndex: ValidationDerivedIndexReader | null;/*** Project posture for broken links (`validation.links`): 'off' silences the* links validator entirely, 'warning' (default) / 'error' set the severity.*/linksValidation?: LinksValidationSetting;/** Every docName that currently exists — the dead-link existence oracle. */- admittedDocNames: () => Iterable<string>;+ admittedDocNames: () => Iterable<string> | Promise<Iterable<string>>;/** docName → on-disk contentDir-relative path, null when no file exists yet. */docFilePathFor: (docName: string) => string | null;}@@ -177,14 +186,14 @@ function createLinksValidator(deps: ValidationAuditDeps): ProjectValidator {return { files: [], fileCount: 0, warnings: [] };}const severity = setting === 'error' ? 'error' : 'warning';- if (!deps.backlinkIndex) {+ if (!deps.derivedDocumentIndex) {return {files: [],fileCount: 0,warnings: ['links validation unavailable: backlink index is not configured'],};}- const admitted = [...deps.admittedDocNames()];+ const admitted = [...(await deps.admittedDocNames())];const sourceFilter = scopedSourceDocNames(admitted, scope.targetPath);// `getDeadLinks` reads an empty source filter as "no filter" — a scope// matching zero docs must short-circuit here or it would silently widen@@ -192,7 +201,7 @@ function createLinksValidator(deps: ValidationAuditDeps): ProjectValidator {if (sourceFilter !== undefined && sourceFilter.length === 0) {return { files: [], fileCount: 0, warnings: [] };}- const deadLinks = deps.backlinkIndex.getDeadLinks(admitted, sourceFilter);+ const deadLinks = await deps.derivedDocumentIndex.getDeadLinks(admitted, sourceFilter);const byFile = new Map<string, ValidationDiagnostic[]>();for (const { target, sources } of deadLinks) {diff --git a/packages/server/src/live-derived-index.test.ts b/packages/server/src/live-derived-index.test.tsindex 07891fc26..10aeaf3b9 100644--- a/packages/server/src/live-derived-index.test.ts+++ b/packages/server/src/live-derived-index.test.ts@@ -2,6 +2,7 @@ import { setTimeout as wait } from 'node:timers/promises';import { Hocuspocus } from '@hocuspocus/server';import { beforeEach, describe, expect, test, vi } from 'vitest';import type * as Y from 'yjs';+import type { DerivedDocumentIndexLivePort } from './derived-document-index.ts';import { DocumentDurabilityState } from './document-durability-state.ts';import { applyExternalChange } from './external-change.ts';import { createLiveDerivedIndexExtension } from './live-derived-index.ts';@@ -36,6 +37,13 @@ function makeOnChangePayload(};}+function createDerivedIndexPort(+ recordLiveDocument = vi.fn(async (_documentName: string, _markdown: string) => {}),+ captureLiveUpdateToken = vi.fn((): number | null => 0),+): DerivedDocumentIndexLivePort {+ return { captureLiveUpdateToken, recordLiveDocument };+}+describe('createLiveDerivedIndexExtension', () => {let hp: Hocuspocus;let durabilityState: DocumentDurabilityState;@@ -46,12 +54,10 @@ describe('createLiveDerivedIndexExtension', () => {});test('skips file-watcher origin transactions', async () => {- const updateDocumentFromMarkdown = vi.fn(() => {});- const signalChannel = vi.fn(() => {});+ const recordLiveDocument = vi.fn(async () => {});const extension = createLiveDerivedIndexExtension({- backlinkIndex: { updateDocumentFromMarkdown } as unknown as never,+ derivedDocumentIndex: createDerivedIndexPort(recordLiveDocument),debounceMs: 5,- signalChannel,});const conn = await hp.openDirectConnection('skip-file-watcher');const doc = getDoc(conn);@@ -65,18 +71,40 @@ describe('createLiveDerivedIndexExtension', () => {);await wait(20);- expect(updateDocumentFromMarkdown).not.toHaveBeenCalled();- expect(signalChannel).not.toHaveBeenCalled();+ expect(recordLiveDocument).not.toHaveBeenCalled();+ await conn.disconnect();+ });++ test('drops live changes while the coordinator withholds a branch token', async () => {+ const recordLiveDocument = vi.fn(async () => {});+ const extension = createLiveDerivedIndexExtension({+ derivedDocumentIndex: createDerivedIndexPort(+ recordLiveDocument,+ vi.fn(() => null),+ ),+ debounceMs: 5,+ });+ const conn = await hp.openDirectConnection('branch-transition');+ const doc = getDoc(conn);++ applyExternalChange(durabilityState, hp, 'branch-transition', '# Old branch\n');+ await extension.onChange?.(+ makeOnChangePayload(hp, doc, 'branch-transition', {+ source: 'local',+ context: { origin: 'branch-switch' },+ }),+ );+ await wait(20);++ expect(recordLiveDocument).not.toHaveBeenCalled();await conn.disconnect();});test('debounces rapid changes to a single update and preserves frontmatter', async () => {- const updateDocumentFromMarkdown = vi.fn(() => {});- const signalChannel = vi.fn(() => {});+ const recordLiveDocument = vi.fn(async () => {});const extension = createLiveDerivedIndexExtension({- backlinkIndex: { updateDocumentFromMarkdown } as unknown as never,+ derivedDocumentIndex: createDerivedIndexPort(recordLiveDocument),debounceMs: 5,- signalChannel,});const conn = await hp.openDirectConnection('debounced-doc');const doc = getDoc(conn);@@ -97,25 +125,20 @@ describe('createLiveDerivedIndexExtension', () => {await extension.onChange?.(payload);await wait(20);- expect(updateDocumentFromMarkdown).toHaveBeenCalledTimes(1);- expect(updateDocumentFromMarkdown).toHaveBeenCalledWith(+ expect(recordLiveDocument).toHaveBeenCalledTimes(1);+ expect(recordLiveDocument).toHaveBeenCalledWith('debounced-doc','---\ntitle: Debounced\n---\n# Hello\n\n[[beta]]\n',+ 0,);- expect(signalChannel).toHaveBeenCalledTimes(2);- expect(signalChannel.mock.calls).toEqual([['backlinks'], ['graph']]);await conn.disconnect();});- test('feeds tagIndex with the same markdown and signals the tags channel', async () => {- const updateBacklink = vi.fn(() => {});- const updateTag = vi.fn(() => {});- const signalChannel = vi.fn(() => {});+ test('hands one settled raw markdown string to the derived-index boundary', async () => {+ const recordLiveDocument = vi.fn(async () => {});const extension = createLiveDerivedIndexExtension({- backlinkIndex: { updateDocumentFromMarkdown: updateBacklink } as unknown as never,- tagIndex: { updateDocumentFromMarkdown: updateTag } as unknown as never,+ derivedDocumentIndex: createDerivedIndexPort(recordLiveDocument),debounceMs: 5,- signalChannel,});const conn = await hp.openDirectConnection('tag-derived-doc');const doc = getDoc(conn);@@ -129,17 +152,18 @@ describe('createLiveDerivedIndexExtension', () => {await extension.onChange?.(payload);await wait(20);- expect(updateBacklink).toHaveBeenCalledTimes(1);- expect(updateTag).toHaveBeenCalledTimes(1);- expect(updateTag).toHaveBeenCalledWith('tag-derived-doc', '# Hello\n\nA #typescript note.\n');- expect(signalChannel.mock.calls).toEqual([['backlinks'], ['graph'], ['tags']]);+ expect(recordLiveDocument).toHaveBeenCalledExactlyOnceWith(+ 'tag-derived-doc',+ '# Hello\n\nA #typescript note.\n',+ 0,+ );await conn.disconnect();});test('beforeUnloadDocument cancels pending timers', async () => {- const updateDocumentFromMarkdown = vi.fn(() => {});+ const recordLiveDocument = vi.fn(async () => {});const extension = createLiveDerivedIndexExtension({- backlinkIndex: { updateDocumentFromMarkdown } as unknown as never,+ derivedDocumentIndex: createDerivedIndexPort(recordLiveDocument),debounceMs: 20,});const conn = await hp.openDirectConnection('unload-doc');@@ -159,14 +183,14 @@ describe('createLiveDerivedIndexExtension', () => {});await wait(40);- expect(updateDocumentFromMarkdown).not.toHaveBeenCalled();+ expect(recordLiveDocument).not.toHaveBeenCalled();await conn.disconnect();});test('onDestroy clears pending timers across documents', async () => {- const updateDocumentFromMarkdown = vi.fn(() => {});+ const recordLiveDocument = vi.fn(async () => {});const extension = createLiveDerivedIndexExtension({- backlinkIndex: { updateDocumentFromMarkdown } as unknown as never,+ derivedDocumentIndex: createDerivedIndexPort(recordLiveDocument),debounceMs: 20,});const first = await hp.openDirectConnection('destroy-a');@@ -195,7 +219,7 @@ describe('createLiveDerivedIndexExtension', () => {});await wait(40);- expect(updateDocumentFromMarkdown).not.toHaveBeenCalled();+ expect(recordLiveDocument).not.toHaveBeenCalled();await first.disconnect();await second.disconnect();});@@ -206,9 +230,9 @@ describe('createLiveDerivedIndexExtension', () => {// serializer never emits them back). Under contract, body source is// `Y.Text('source').toString()` — CRLF survives byte-equal. Discriminating// because CRLF is not preserved through parse → serialize (parser strips).- const updateDocumentFromMarkdown = vi.fn(() => {});+ const recordLiveDocument = vi.fn(async () => {});const extension = createLiveDerivedIndexExtension({- backlinkIndex: { updateDocumentFromMarkdown } as unknown as never,+ derivedDocumentIndex: createDerivedIndexPort(recordLiveDocument),debounceMs: 5,});const conn = await hp.openDirectConnection('crlf-doc');@@ -227,8 +251,8 @@ describe('createLiveDerivedIndexExtension', () => {);await wait(20);- expect(updateDocumentFromMarkdown).toHaveBeenCalledTimes(1);- const [, bodyArg] = updateDocumentFromMarkdown.mock.calls[0] as [string, string];+ expect(recordLiveDocument).toHaveBeenCalledTimes(1);+ const [, bodyArg] = recordLiveDocument.mock.calls[0] as [string, string];// CRLF survives — ytext is the source-of-truth.expect(bodyArg).toContain('\r\n');// would have produced LF-only canonical bytes from@@ -242,9 +266,9 @@ describe('createLiveDerivedIndexExtension', () => {// doc starting with `---\n# H\n` would round-trip through// serialize(fragment) as `***\n\n# H\n`. Under contract, ytext keeps// the user's typed `---\n` byte-equal — discriminating.- const updateDocumentFromMarkdown = vi.fn(() => {});+ const recordLiveDocument = vi.fn(async () => {});const extension = createLiveDerivedIndexExtension({- backlinkIndex: { updateDocumentFromMarkdown } as unknown as never,+ derivedDocumentIndex: createDerivedIndexPort(recordLiveDocument),debounceMs: 5,});const conn = await hp.openDirectConnection('thematic-doc');@@ -259,8 +283,8 @@ describe('createLiveDerivedIndexExtension', () => {);await wait(20);- expect(updateDocumentFromMarkdown).toHaveBeenCalledTimes(1);- const [, bodyArg] = updateDocumentFromMarkdown.mock.calls[0] as [string, string];+ expect(recordLiveDocument).toHaveBeenCalledTimes(1);+ const [, bodyArg] = recordLiveDocument.mock.calls[0] as [string, string];expect(bodyArg).toBe('---\n# Title\n');expect(bodyArg).not.toContain('***');await conn.disconnect();@@ -272,9 +296,9 @@ describe('createLiveDerivedIndexExtension', () => {// paths in steady state. It still validates the spec acceptance: the// SNIPPET CONTENT (what consumers see) reflects what the user typed.// '<https://x>' in snippet text, not '[https://x](https://x)'.- const updateDocumentFromMarkdown = vi.fn(() => {});+ const recordLiveDocument = vi.fn(async () => {});const extension = createLiveDerivedIndexExtension({- backlinkIndex: { updateDocumentFromMarkdown } as unknown as never,+ derivedDocumentIndex: createDerivedIndexPort(recordLiveDocument),debounceMs: 5,});const conn = await hp.openDirectConnection('autolink-doc');@@ -294,22 +318,20 @@ describe('createLiveDerivedIndexExtension', () => {);await wait(20);- expect(updateDocumentFromMarkdown).toHaveBeenCalledTimes(1);- const [, bodyArg] = updateDocumentFromMarkdown.mock.calls[0] as [string, string];+ expect(recordLiveDocument).toHaveBeenCalledTimes(1);+ const [, bodyArg] = recordLiveDocument.mock.calls[0] as [string, string];expect(bodyArg).toContain('<https://example.com>');expect(bodyArg).not.toContain('[https://example.com](https://example.com)');await conn.disconnect();});test('logs and swallows callback errors', async () => {- const updateDocumentFromMarkdown = vi.fn(() => {+ const recordLiveDocument = vi.fn(async () => {throw new Error('boom');});- const signalChannel = vi.fn(() => {});const extension = createLiveDerivedIndexExtension({- backlinkIndex: { updateDocumentFromMarkdown } as unknown as never,+ derivedDocumentIndex: createDerivedIndexPort(recordLiveDocument),debounceMs: 5,- signalChannel,});const conn = await hp.openDirectConnection('error-doc');const doc = getDoc(conn);@@ -325,8 +347,7 @@ describe('createLiveDerivedIndexExtension', () => {);await wait(20);- expect(updateDocumentFromMarkdown).toHaveBeenCalledTimes(1);- expect(signalChannel).not.toHaveBeenCalled();+ expect(recordLiveDocument).toHaveBeenCalledTimes(1);expect(errorSpy).toHaveBeenCalled();expect(String(errorSpy.mock.calls[0]?.[1])).toContain('Failed to update derived views for error-doc',diff --git a/packages/server/src/live-derived-index.ts b/packages/server/src/live-derived-index.tsindex 8014f4c64..501c2c6ba 100644--- a/packages/server/src/live-derived-index.ts+++ b/packages/server/src/live-derived-index.ts@@ -1,20 +1,12 @@import type { Document, Extension } from '@hocuspocus/server';-import type { BacklinkIndex } from './backlink-index.ts';import { isLinkIndexExcludedDoc } from './cc1-broadcast.ts';+import type { DerivedDocumentIndexLivePort } from './derived-document-index.ts';import { getLogger } from './logger.ts';-import type { TagIndex } from './tag-index.ts';export const LIVE_DERIVED_INDEX_DEBOUNCE_MS = 100;export interface LiveDerivedIndexOptions {- backlinkIndex: BacklinkIndex;- /**- * Optional. When wired, every backlink-update tick also re-extracts tags- * for the changed doc and broadcasts the `'tags'` derived-view channel so- * tag-aware UIs can invalidate their caches alongside backlinks/graph.- */- tagIndex?: TagIndex;- signalChannel?: (channel: 'files' | 'backlinks' | 'graph' | 'tags') => void;+ derivedDocumentIndex: DerivedDocumentIndexLivePort;debounceMs?: number;}@@ -40,12 +32,7 @@ function serializeLiveDocument(document: Document): string {}export function createLiveDerivedIndexExtension(options: LiveDerivedIndexOptions): Extension {- const {- backlinkIndex,- tagIndex,- signalChannel,- debounceMs = LIVE_DERIVED_INDEX_DEBOUNCE_MS,- } = options;+ const { derivedDocumentIndex, debounceMs = LIVE_DERIVED_INDEX_DEBOUNCE_MS } = options;const pendingByDoc = new Map<string, ReturnType<typeof setTimeout>>();function clearPending(docName: string): void {@@ -56,7 +43,7 @@ export function createLiveDerivedIndexExtension(options: LiveDerivedIndexOptions}}- function schedule(docName: string, document: Document): void {+ function schedule(docName: string, document: Document, token: number): void {clearPending(docName);pendingByDoc.set(docName,@@ -64,13 +51,12 @@ export function createLiveDerivedIndexExtension(options: LiveDerivedIndexOptionspendingByDoc.delete(docName);try {const markdown = serializeLiveDocument(document);- backlinkIndex.updateDocumentFromMarkdown(docName, markdown);- signalChannel?.('backlinks');- signalChannel?.('graph');- if (tagIndex) {- tagIndex.updateDocumentFromMarkdown(docName, markdown);- signalChannel?.('tags');- }+ void derivedDocumentIndex.recordLiveDocument(docName, markdown, token).catch((err) => {+ getLogger('live-derived-index').error(+ { docName, err },+ `Failed to update derived views for ${docName}`,+ );+ });} catch (err) {getLogger('live-derived-index').error({ docName, err },@@ -93,9 +79,15 @@ export function createLiveDerivedIndexExtension(options: LiveDerivedIndexOptionsreturn;}+ const token = derivedDocumentIndex.captureLiveUpdateToken();+ if (token === null) {+ clearPending(documentName);+ return;+ }+// Give the source/tree bridge a short trailing window to converge so we// derive links from settled live document state instead of the 2s store debounce.- schedule(documentName, document);+ schedule(documentName, document, token);},async beforeUnloadDocument({ documentName }) {diff --git a/packages/server/src/persistence.test.ts b/packages/server/src/persistence.test.tsindex 8ed1fcd73..c60f87f31 100644--- a/packages/server/src/persistence.test.ts+++ b/packages/server/src/persistence.test.ts@@ -12,11 +12,22 @@ import {import { mkdir, realpath, rename, writeFile } from 'node:fs/promises';import { tmpdir } from 'node:os';import { dirname, join, resolve, sep } from 'node:path';-import { afterEach, beforeEach, describe, expect, test } from 'vitest';+import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';import * as Y from 'yjs';+import { composeAndWriteRawBody } from './bridge-intake.ts';+import { createContentFilter } from './content-filter.ts';import { isWithinContentDir, safeContentPath } from './content-path.ts';+import {+ DerivedDocumentIndex,+ type DerivedDocumentIndexPersistencePort,+} from './derived-document-index.ts';+import { DocumentDurabilityState } from './document-durability-state.ts';import { contentHash, isSelfWrite, registerWrite } from './file-watcher';-import { captureDocSnapshotForPersistence, resolveWriterFromOrigin } from './persistence';+import {+ captureDocSnapshotForPersistence,+ createPersistenceExtension,+ resolveWriterFromOrigin,+} from './persistence';import { FILE_SYSTEM_WRITER, GIT_UPSTREAM_WRITER, SERVICE_WRITER } from './shadow-repo';describe('safeContentPath', () => {@@ -221,6 +232,112 @@ describe('symlink-safe atomic write', () => {});});+describe('durable derived-index projection', () => {+ let contentDir: string;++ beforeEach(() => {+ contentDir = realpathSync(mkdtempSync(join(tmpdir(), 'ok-derived-persistence-')));+ });++ afterEach(() => {+ rmSync(contentDir, { recursive: true, force: true });+ });++ async function storeDocument(+ persistence: ReturnType<typeof createPersistenceExtension>,+ document: Y.Doc,+ documentName: string,+ ): Promise<void> {+ await persistence.extension.onStoreDocument?.({+ document,+ documentName,+ lastTransactionOrigin: {+ source: 'connection',+ connection: { context: { principalId: 'principal-test' } },+ },+ lastContext: {},+ } as never);+ }++ test('advances reconciled durability state before projecting the successful store', async () => {+ const documentName = 'projected';+ const markdown = '# Projected\n';+ const durabilityState = new DocumentDurabilityState();+ const recordDurableStore = vi.fn(async (receivedName: string, receivedMarkdown: string) => {+ expect(durabilityState.getReconciledBase(receivedName)).toBe(receivedMarkdown);+ });+ const derivedDocumentIndex: DerivedDocumentIndexPersistencePort = {+ recordDurableStore,+ };+ const persistence = createPersistenceExtension({+ contentDir,+ projectDir: contentDir,+ gitEnabled: false,+ durabilityState,+ derivedDocumentIndex,+ });+ const document = new Y.Doc();+ composeAndWriteRawBody(document, markdown, 'test');++ await storeDocument(persistence, document, documentName);++ expect(readFileSync(join(contentDir, `${documentName}.md`), 'utf-8')).toBe(markdown);+ expect(recordDurableStore).toHaveBeenCalledWith(documentName, markdown);+ document.destroy();+ });++ test('does not project a store whose durable disk write failed', async () => {+ const documentName = 'blocked';+ mkdirSync(join(contentDir, `${documentName}.md`));+ const durabilityState = new DocumentDurabilityState();+ const recordDurableStore = vi.fn();+ const persistence = createPersistenceExtension({+ contentDir,+ projectDir: contentDir,+ gitEnabled: false,+ durabilityState,+ derivedDocumentIndex: { recordDurableStore },+ });+ const document = new Y.Doc();+ composeAndWriteRawBody(document, '# Blocked\n', 'test');++ await expect(storeDocument(persistence, document, documentName)).rejects.toThrow();++ expect(recordDurableStore).not.toHaveBeenCalled();+ expect(durabilityState.getReconciledBase(documentName)).toBeUndefined();+ document.destroy();+ });++ test('keeps a successful store durable when the derived index has already closed', async () => {+ const documentName = 'closed-projection';+ const markdown = '# Closed projection\n';+ const durabilityState = new DocumentDurabilityState();+ const derivedDocumentIndex = new DerivedDocumentIndex({+ projectDir: contentDir,+ contentDir,+ contentFilter: createContentFilter({ projectDir: contentDir, contentDir }),+ getGlobalSkillRoots: () => [],+ signalChannel: () => {},+ });+ await derivedDocumentIndex.close();+ const persistence = createPersistenceExtension({+ contentDir,+ projectDir: contentDir,+ gitEnabled: false,+ durabilityState,+ derivedDocumentIndex,+ });+ const document = new Y.Doc();+ composeAndWriteRawBody(document, markdown, 'test');++ await expect(storeDocument(persistence, document, documentName)).resolves.toBeUndefined();++ expect(readFileSync(join(contentDir, `${documentName}.md`), 'utf-8')).toBe(markdown);+ expect(durabilityState.getReconciledBase(documentName)).toBe(markdown);+ document.destroy();+ });+});+// ─── resolveWriterFromOrigin ───────────────────────describe('resolveWriterFromOrigin', () => {diff --git a/packages/server/src/persistence.ts b/packages/server/src/persistence.tsindex e12b0278e..c2aad1ab1 100644--- a/packages/server/src/persistence.ts+++ b/packages/server/src/persistence.ts@@ -45,7 +45,6 @@ import type { JSONContent } from '@tiptap/core';import { updateYFragment, yXmlFragmentToProseMirrorRootNode } from '@tiptap/y-tiptap';import * as Y from 'yjs';import { LINEAGE_EPOCH_KEY } from './auth-token-schema.ts';-import type { BacklinkIndex } from './backlink-index.ts';import { type DeriveLossDetectOptions, detectPairedIntakeLoss } from './bridge-loss-detector.ts';import { getMsSinceLastUserTx, isDocQuiescent } from './bridge-quiescence.ts';import { assertBridgeInvariant, createDocCanonicalizer } from './bridge-watchdog.ts';@@ -68,6 +67,7 @@ import {restoreContributors,swapContributors,} from './contributor-tracker.ts';+import type { DerivedDocumentIndexPersistencePort } from './derived-document-index.ts';import { applyDiskContentToDoc, FILE_WATCHER_ORIGIN } from './disk-content-intake.ts';import { DocumentDurabilityState, type StoreFailure } from './document-durability-state.ts';import { contentHash, registerWrite } from './file-watcher.ts';@@ -315,7 +315,7 @@ export interface PersistenceOptions {shadowRef?: ShadowRef;/** Content root relative to project dir (e.g., 'content/docs'). Used for shadow repo staging. */contentRoot?: string;- backlinkIndex?: BacklinkIndex;+ derivedDocumentIndex?: DerivedDocumentIndexPersistencePort;/** Accessor for the current branch from the HEAD watcher. Used to scope WIP refs per branch. */getCurrentBranch?: () => string | null;/**@@ -610,7 +610,7 @@ export function createPersistenceExtension(options?: PersistenceOptions): Persis// project root must fall back to '.' — using the literal pathspec would// make `git add <fallback>` look for a non-existent subfolder.const contentRoot = options?.contentRoot ?? (toPosix(relative(projectDir, contentDir)) || '.');- const backlinkIndex = options?.backlinkIndex;+ const derivedDocumentIndex = options?.derivedDocumentIndex;const getPrincipal = options?.getPrincipal;const onAgentCommit = options?.onAgentCommit;const onFlushCommit = options?.onFlushCommit;@@ -2462,14 +2462,13 @@ export function createPersistenceExtension(options?: PersistenceOptions): PersistripwireResetFailedDocs.delete(documentName);persistenceDeferCounts.delete(documentName);- if (backlinkIndex) {- backlinkIndex.updateDocumentFromMarkdown(documentName, markdown);- void backlinkIndex.saveToDisk().catch((err) => {- log.warn(- { err, documentName },- `[backlinks] Failed to persist cache for ${documentName}`,- );- });+ try {+ await derivedDocumentIndex?.recordDurableStore(documentName, markdown);+ } catch (err) {+ log.warn(+ { err, documentName },+ '[derived-index] durable-store projection failed; disk write remains authoritative',+ );}setActiveSpanAttributes({ 'persistence.bytes': markdown.length });diff --git a/packages/server/src/server-factory.test.ts b/packages/server/src/server-factory.test.tsindex 17cffdacd..a34edb63b 100644--- a/packages/server/src/server-factory.test.ts+++ b/packages/server/src/server-factory.test.ts@@ -1,12 +1,25 @@-import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';+import {+ existsSync,+ mkdirSync,+ mkdtempSync,+ readFileSync,+ rmSync,+ unlinkSync,+ writeFileSync,+} from 'node:fs';import { mkdtemp, readFile, rm } from 'node:fs/promises';import { tmpdir } from 'node:os';import { join, resolve } from 'node:path';import { LOCAL_DIR } from '@inkeep/open-knowledge-core';+import { readConfigSafely, resolveConfigPath } from '@inkeep/open-knowledge-core/server';import shellQuote from 'shell-quote';import simpleGit from 'simple-git';-import { afterEach, beforeEach, describe, expect, test } from 'vitest';+import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';import * as Y from 'yjs';+import { BacklinkIndex } from './backlink-index.ts';+import { getBootTimings, resetBootTimingsForTest, startBootTimings } from './boot-timings.ts';+import { DerivedDocumentIndex } from './derived-document-index.ts';+import { classifyGitError } from './error-classification.ts';import { applyExternalChange } from './external-change.ts';import type {CheckPushPermissionOptions,@@ -21,8 +34,39 @@ import {writeManagedRenameJournal,} from './managed-rename-journal.ts';import { ensureProjectGit } from './project-git.ts';+import { saveRemovedDocsJournal } from './removed-docs-journal.ts';import { buildSyncCredentialArgs, createServer, type ServerInstance } from './server-factory.ts';+import { releaseServerLock } from './server-lock.ts';import { initShadowRepo, shadowGit } from './shadow-repo.ts';+import { TagIndex } from './tag-index.ts';++const watcherStartupFailures = vi.hoisted(() => ({ file: false, head: false }));++vi.mock('./file-watcher.ts', async (importOriginal) => {+ const actual = await importOriginal<typeof import('./file-watcher.ts')>();+ return {+ ...actual,+ startWatcher: async (...args: Parameters<typeof actual.startWatcher>) => {+ if (watcherStartupFailures.file) {+ throw new Error('injected file-watcher startup failure');+ }+ return actual.startWatcher(...args);+ },+ };+});++vi.mock('./head-watcher.ts', async (importOriginal) => {+ const actual = await importOriginal<typeof import('./head-watcher.ts')>();+ return {+ ...actual,+ startHeadWatcher: async (...args: Parameters<typeof actual.startHeadWatcher>) => {+ if (watcherStartupFailures.head) {+ throw new Error('injected head-watcher startup failure');+ }+ return actual.startHeadWatcher(...args);+ },+ };+});// ─── CaptureLogger infrastructure ───────────────────────────────────────────// Uses loggerFactory.configure() pattern from logger.test.ts.@@ -119,6 +163,7 @@ describe('createServer() — document durability state isolation', () => {const diskB = '# Disk B\n';let serverA: ServerInstance | null = null;let serverB: ServerInstance | null = null;+ const beginStartup = vi.spyOn(DerivedDocumentIndex.prototype, 'beginStartup');try {writeFileSync(join(projectA, `${docName}.md`), diskA, 'utf-8');@@ -137,6 +182,8 @@ describe('createServer() — document durability state isolation', () => {});await Promise.all([serverA.ready, serverB.ready]);+ expect(beginStartup).toHaveBeenCalledTimes(2);+ expect(beginStartup.mock.instances[0]).not.toBe(beginStartup.mock.instances[1]);const [connectionA, connectionB] = await Promise.all([serverA.hocuspocus.openDirectConnection(docName),serverB.hocuspocus.openDirectConnection(docName),@@ -178,10 +225,141 @@ describe('createServer() — document durability state isolation', () => {rm(projectA, { recursive: true, force: true }),rm(projectB, { recursive: true, force: true }),]);+ beginStartup.mockRestore();}});});+describe('createServer() — derived-index branch lifecycle', () => {+ let projectDir: string;+ let git: ReturnType<typeof simpleGit>;+ let server: ServerInstance | null;++ beforeEach(async () => {+ projectDir = await mkdtemp(join(tmpdir(), 'ok-derived-branch-'));+ git = simpleGit(projectDir);+ await git.init(['--initial-branch=main']);+ await git.raw('config', 'user.name', 'Test');+ await git.raw('config', 'user.email', '[email protected]');+ writeFileSync(join(projectDir, 'main.md'), '# Main\n\n#main-branch\n', 'utf-8');+ await git.add('.');+ await git.commit('main content');+ await git.checkoutLocalBranch('feature');+ unlinkSync(join(projectDir, 'main.md'));+ writeFileSync(join(projectDir, 'feature.md'), '# Feature\n\n#feature-branch\n', 'utf-8');+ await git.add(['-A']);+ await git.commit('feature content');+ await git.checkout('main');+ server = null;+ });++ afterEach(async () => {+ await server?.destroy();+ vi.restoreAllMocks();+ await rm(projectDir, { recursive: true, force: true });+ });++ test('settles target indexes before broadcasting branch-switched', async () => {+ const beginStartup = vi.spyOn(DerivedDocumentIndex.prototype, 'beginStartup');+ server = createServer({+ contentDir: projectDir,+ projectDir,+ quiet: true,+ gitEnabled: false,+ skipStateManifestCheck: true,+ });+ await server.ready;+ const coordinator = beginStartup.mock.instances[0] as DerivedDocumentIndex;+ beginStartup.mockRestore();+ const settle = vi.spyOn(DerivedDocumentIndex.prototype, 'settleBranchFromDisk');+ const emit = vi.spyOn(server.cc1Broadcaster, 'emitBranchSwitched');++ await git.checkout('feature');+ await vi.waitFor(() => expect(emit).toHaveBeenCalledWith('feature'), {+ timeout: 10_000,+ interval: 25,+ });++ expect(settle).toHaveBeenCalledTimes(1);+ expect(settle.mock.invocationCallOrder[0]).toBeLessThan(emit.mock.invocationCallOrder[0] ?? 0);+ expect(server.durabilityState.isBatchInProgress()).toBe(false);+ expect(await coordinator.getDocsForTagWithMatches('feature-branch')).toEqual([+ { docName: 'feature', matchingTags: ['feature-branch'] },+ ]);+ expect(await coordinator.getDocsForTagWithMatches('main-branch')).toEqual([]);+ }, 20_000);++ test('aborts a degraded branch settlement and releases coordinator queries', async () => {+ const beginStartup = vi.spyOn(DerivedDocumentIndex.prototype, 'beginStartup');+ server = createServer({+ contentDir: projectDir,+ projectDir,+ quiet: true,+ gitEnabled: false,+ skipStateManifestCheck: true,+ });+ await server.ready;+ const coordinator = beginStartup.mock.instances[0] as DerivedDocumentIndex;+ beginStartup.mockRestore();+ vi.spyOn(DerivedDocumentIndex.prototype, 'settleBranchFromDisk').mockRejectedValueOnce(+ new Error('injected branch settlement failure'),+ );+ const abort = vi.spyOn(DerivedDocumentIndex.prototype, 'abortBranchSwitch');+ const emit = vi.spyOn(server.cc1Broadcaster, 'emitBranchSwitched');++ await git.checkout('feature');+ await vi.waitFor(() => expect(emit).toHaveBeenCalledWith('feature'), {+ timeout: 10_000,+ interval: 25,+ });++ expect(abort).toHaveBeenCalled();+ expect(server.durabilityState.isBatchInProgress()).toBe(false);+ await expect(coordinator.getIndexedDocNames()).resolves.toBeInstanceOf(Array);+ }, 20_000);++ test('begin-branch failure always restores durability admission and releases queries', async () => {+ const beginStartup = vi.spyOn(DerivedDocumentIndex.prototype, 'beginStartup');+ server = createServer({+ contentDir: projectDir,+ projectDir,+ quiet: true,+ gitEnabled: false,+ skipStateManifestCheck: true,+ });+ await server.ready;+ const coordinator = beginStartup.mock.instances[0] as DerivedDocumentIndex;+ beginStartup.mockRestore();+ let rejectBegin!: (error: Error) => void;+ const beginBranch = vi+ .spyOn(DerivedDocumentIndex.prototype, 'beginBranchSwitch')+ .mockImplementationOnce(+ () =>+ new Promise<never>((_resolve, reject) => {+ rejectBegin = reject;+ }),+ );+ const abort = vi.spyOn(DerivedDocumentIndex.prototype, 'abortBranchSwitch');++ await git.checkout('feature');+ await vi.waitFor(() => expect(beginBranch).toHaveBeenCalledWith('feature'), {+ timeout: 10_000,+ interval: 25,+ });+ expect(server.durabilityState.isBatchInProgress()).toBe(true);++ rejectBegin(new Error('injected branch begin failure'));+ await vi.waitFor(+ () => {+ expect(abort).toHaveBeenCalled();+ expect(server?.durabilityState.isBatchInProgress()).toBe(false);+ },+ { timeout: 10_000, interval: 25 },+ );+ await expect(coordinator.getIndexedDocNames()).resolves.toBeInstanceOf(Array);+ }, 20_000);+});+describe('createServer().destroy() — graceful shutdown flush', () => {let tmpDir: string;let logCapture: ReturnType<typeof captureAllLoggers>;@@ -520,6 +698,39 @@ describe('createServer().destroy() — graceful shutdown flush', () => {expect(shutdownLogs[0].payload.documentCount).toBe(5);});+ test('destroy does not await the derived-index cache drain', async () => {+ const beginStartup = vi.spyOn(DerivedDocumentIndex.prototype, 'beginStartup');+ const server = createServer({+ contentDir: tmpDir,+ projectDir: tmpDir,+ quiet: true,+ destroyTimeoutMs: 500,+ });+ await server.ready;+ const coordinator = beginStartup.mock.instances[0] as DerivedDocumentIndex;+ beginStartup.mockRestore();+ const close = vi+ .spyOn(DerivedDocumentIndex.prototype, 'close')+ .mockImplementation(() => new Promise<void>(() => {}));+ let timeout: ReturnType<typeof setTimeout> | undefined;++ try {+ const result = await Promise.race([+ server.destroy().then(() => 'destroyed' as const),+ new Promise<'timed-out'>((resolveTimeout) => {+ timeout = setTimeout(() => resolveTimeout('timed-out'), 2_000);+ }),+ ]);++ expect(result).toBe('destroyed');+ expect(close).toHaveBeenCalledTimes(1);+ } finally {+ if (timeout) clearTimeout(timeout);+ close.mockRestore();+ await coordinator.close();+ }+ });+test('destroy() flushes multiple documents before resolving (multi-doc drain)', async () => {const server = createServer({contentDir: tmpDir,@@ -589,13 +800,8 @@ describe('createServer().destroy() — graceful shutdown flush', () => {* Failure injection:* - shadow-repo: forced via invalid path (file-as-dir). This subsystem's* init throws on invalid paths, so preferred technique works.- * - file-watcher + head-watcher: cannot be forced via invalid paths because- * startWatcher falls back from @parcel/watcher to chokidar (tolerates- * invalid paths) and startHeadWatcher returns a no-op handle on missing- * .git. The degraded.push wiring for these subsystems is verified by- * the shadow-repo test (same push pattern) + code-level assertions.- * vi.doMock was attempted but leaks across all test files in the same- * `bun test` process, breaking file-watcher.test.ts.+ * - file-watcher + head-watcher: hoisted, call-through module mocks throw+ * only when the corresponding per-test flag is enabled.*/describe('createServer() degraded signal', () => {@@ -606,6 +812,9 @@ describe('createServer() degraded signal', () => {});afterEach(() => {+ watcherStartupFailures.file = false;+ watcherStartupFailures.head = false;+ resetBootTimingsForTest();rmSync(testProjectDir, { recursive: true, force: true });});@@ -625,6 +834,54 @@ describe('createServer() degraded signal', () => {await srv.destroy();});+ test('backlink startup failure labels the server as backlink-index degraded', async () => {+ const contentDir = mkdtempSync(resolve(testProjectDir, 'content-'));+ const load = vi+ .spyOn(BacklinkIndex.prototype, 'loadFromDisk')+ .mockRejectedValueOnce(new Error('injected backlink startup failure'));+ const srv = createServer({+ contentDir,+ projectDir: testProjectDir,+ quiet: true,+ });++ try {+ await srv.ready;+ expect(load).toHaveBeenCalled();+ expect(srv.degraded).toContain('backlink-index');+ expect(srv.degraded.filter((name) => name === 'backlink-index')).toHaveLength(1);+ } finally {+ load.mockRestore();+ await srv.destroy();+ }+ });++ test('tag reconciliation failure labels the server as tag-index degraded', async () => {+ const contentDir = mkdtempSync(resolve(testProjectDir, 'content-'));+ const beginStartup = vi.spyOn(DerivedDocumentIndex.prototype, 'beginStartup');+ const reconcile = vi+ .spyOn(TagIndex.prototype, 'reconcileWithDisk')+ .mockRejectedValueOnce(new Error('injected tag reconciliation failure'));+ const srv = createServer({+ contentDir,+ projectDir: testProjectDir,+ quiet: true,+ });++ try {+ await srv.ready;+ const coordinator = beginStartup.mock.instances[0] as DerivedDocumentIndex;+ expect(reconcile).toHaveBeenCalled();+ expect(srv.degraded).toContain('tag-index');+ expect(srv.degraded.filter((name) => name === 'tag-index')).toHaveLength(1);+ await expect(coordinator.getAllTags()).resolves.toBeInstanceOf(Array);+ } finally {+ await srv.destroy();+ beginStartup.mockRestore();+ reconcile.mockRestore();+ }+ });+test('shadow-repo init failure — degraded includes "shadow-repo"', async () => {// Force shadow-repo init to fail by making `.git/ok` a file (not a dir).// `resolveShadowDir` returns `<projectDir>/.git/ok` for the directory case;@@ -650,28 +907,28 @@ describe('createServer() degraded signal', () => {await srv.destroy();});- test('degraded push wiring exists for all three subsystems', () => {- // Verify at the source level that the degraded.push calls exist in- // initAsync for file-watcher and head-watcher. This is a code-level- // assertion — not as strong as a runtime test, but vi.doMock leaks- // make runtime testing impractical without process isolation.- const dir = import.meta.dirname ?? new URL('.', import.meta.url).pathname;- const src = readFileSync(resolve(dir, 'server-factory.ts'), 'utf-8');-- // Each subsystem's catch block should push to the degraded array- expect(src).toContain("degraded.push('shadow-repo')");- expect(src).toContain("degraded.push('file-watcher')");- expect(src).toContain("degraded.push('head-watcher')");-- // The factory return should include degraded- expect(src).toMatch(/return\s*\{[^}]*degraded[^}]*\}/s);+ test.each([+ { failure: 'file' as const, label: 'file-watcher' },+ { failure: 'head' as const, label: 'head-watcher' },+ ])('$label startup failure is reported at runtime', async ({ failure, label }) => {+ watcherStartupFailures[failure] = true;+ if (failure === 'file') startBootTimings('2026-07-23T00:00:00.000Z');+ const contentDir = mkdtempSync(resolve(testProjectDir, 'content-'));+ const srv = createServer({+ contentDir,+ projectDir: testProjectDir,+ quiet: true,+ });- // The index-phase boot timing is recorded in a `finally`, not the `try`, so- // a partial/failed watcher start still yields a timing for the desktop- // startup waterfall. Same code-level-assertion rationale as above: pin the- // placement so a refactor that moves it into the try (dropping it on the- // degraded path) is caught here.- expect(src).toMatch(/finally\s*\{[\s\S]*?recordBootPhase\('indexesMs'/);+ try {+ await srv.ready;+ expect(srv.degraded).toEqual([label]);+ if (failure === 'file') {+ expect(getBootTimings()?.indexesMs).toEqual(expect.any(Number));+ }+ } finally {+ await srv.destroy();+ }});test('degraded is readonly — push and reassignment are compile-time errors', async () => {@@ -2305,23 +2562,75 @@ describe('createServer() — readProjectAutoSyncMode precedence', () => {});});-// onAutoDisable invocation requires a protected-branch git remote, which is-// expensive to set up in a unit test. The callback body is small and its-// effect (writeConfigPatch with scope: 'project-local') is fully covered by-// writeConfigPatch's own round-trip test. Pinning the call-site at the source-// level catches scope drift without provisioning git fixtures.-describe('createServer() — onAutoDisable scope pinning', () => {- test('onAutoDisable callback writes via scope: project-local', () => {- const dir = import.meta.dirname ?? new URL('.', import.meta.url).pathname;- const src = readFileSync(resolve(dir, 'server-factory.ts'), 'utf-8');- const onAutoDisableMatch = src.match(- /onAutoDisable:\s*async\s*\([^)]*\)\s*=>\s*\{[\s\S]*?\n\s{8}\},/,- );- expect(onAutoDisableMatch).not.toBeNull();- const body = onAutoDisableMatch?.[0] ?? '';- expect(body).toContain("scope: 'project-local'");- expect(body).toContain('autoSync: { enabled: false }');- expect(body).not.toMatch(/scope:\s*'project'(?!-local)/);+describe('createServer() — protected-branch auto-disable persistence', () => {+ test('persists autoSync.enabled=false to project-local config only', async () => {+ const projectDir = mkdtempSync(resolve(tmpdir(), 'ok-auto-disable-test-'));+ const homedir = mkdtempSync(resolve(tmpdir(), 'ok-auto-disable-home-'));+ const contentDir = mkdtempSync(resolve(projectDir, 'content-'));+ const projectConfigPath = resolveConfigPath('project', projectDir);+ const localConfigPath = resolveConfigPath('project-local', projectDir);+ mkdirSync(join(projectDir, '.ok', LOCAL_DIR), { recursive: true });+ writeFileSync(projectConfigPath, 'autoSync:\n default: true\n', 'utf-8');+ writeFileSync(localConfigPath, 'autoSync:\n enabled: true\n', 'utf-8');++ const server = createServer({+ contentDir,+ projectDir,+ quiet: true,+ gitEnabled: false,+ configHomedirOverride: homedir,+ destroyTimeoutMs: 1_000,+ });++ try {+ await server.ready;+ const engine = server.syncEngine;+ expect(engine).not.toBeNull();+ expect(engine?.getStatus().syncEnabled).toBe(true);++ const testEngine = engine as unknown as {+ handleError: (+ classified: ReturnType<typeof classifyGitError>,+ operation: 'push' | 'pull',+ ) => void;+ };+ testEngine.handleError(+ classifyGitError(new Error('remote: error: protected branch')),+ 'push',+ );++ expect(engine?.getStatus()).toMatchObject({+ state: 'disabled',+ syncEnabled: false,+ pausedReason: 'protected-branch',+ pushErrorCode: 'semantic-protected-branch',+ });+ await vi.waitFor(+ () => {+ const local = readConfigSafely({+ absPath: localConfigPath,+ sideline: false,+ warn: () => {},+ });+ expect(local.valid).toBe(true);+ expect(local.value.autoSync.enabled).toBe(false);+ },+ { timeout: 2_000, interval: 20 },+ );++ const project = readConfigSafely({+ absPath: projectConfigPath,+ sideline: false,+ warn: () => {},+ });+ expect(project.valid).toBe(true);+ expect(project.value.autoSync.default).toBe(true);+ expect(project.value.autoSync.enabled).toBeNull();+ } finally {+ await server.destroy();+ rmSync(projectDir, { recursive: true, force: true });+ rmSync(homedir, { recursive: true, force: true });+ }});});describe('createServer() — phantom-doc unload', () => {@@ -2647,6 +2956,93 @@ describe('createServer() — removalRedirectGuard registration', () => {await server.destroy();}});++ test('warm backlink reconciliation arms deletion rejection for a file removed offline', async () => {+ writeFileSync(join(tmpDir, 'removed-offline.md'), '# Removed offline\n', 'utf-8');+ const first = createServer({+ contentDir: tmpDir,+ projectDir: tmpDir,+ quiet: true,+ gitEnabled: false,+ });+ await first.ready;+ await first.destroy();+ releaseServerLock(first.lockDir);+ unlinkSync(join(tmpDir, 'removed-offline.md'));++ const restarted = createServer({+ contentDir: tmpDir,+ projectDir: tmpDir,+ quiet: true,+ gitEnabled: false,+ });+ try {+ await restarted.ready;+ const ext = restarted.hocuspocus.configuration.extensions.find(+ (entry) => (entry as { __kind?: string }).__kind === 'removal-redirect-guard',+ ) as { onAuthenticate: (payload: unknown) => Promise<void> } | undefined;+ if (!ext) throw new Error('removal-redirect-guard not registered');++ await expect(+ ext.onAuthenticate({+ token: undefined,+ context: {},+ documentName: 'removed-offline',+ }),+ ).rejects.toMatchObject({ reason: 'doc-deleted' });+ } finally {+ await restarted.destroy();+ }+ });++ test('warm removal journal preserves rename redirects', async () => {+ writeFileSync(join(tmpDir, 'renamed-source.md'), '# Renamed source\n', 'utf-8');+ const first = createServer({+ contentDir: tmpDir,+ projectDir: tmpDir,+ quiet: true,+ gitEnabled: false,+ });+ await first.ready;+ await first.destroy();+ releaseServerLock(first.lockDir);+ unlinkSync(join(tmpDir, 'renamed-source.md'));+ writeFileSync(join(tmpDir, 'renamed-target.md'), '# Renamed target\n', 'utf-8');+ saveRemovedDocsJournal(tmpDir, [+ [+ 'renamed-source',+ {+ kind: 'renamed',+ newDocName: 'renamed-target',+ addedAt: Date.now(),+ },+ ],+ ]);+ const server = createServer({+ contentDir: tmpDir,+ projectDir: tmpDir,+ quiet: true,+ gitEnabled: false,+ });++ try {+ await server.ready;+ const ext = server.hocuspocus.configuration.extensions.find(+ (entry) => (entry as { __kind?: string }).__kind === 'removal-redirect-guard',+ ) as { onAuthenticate: (payload: unknown) => Promise<void> } | undefined;+ if (!ext) throw new Error('removal-redirect-guard not registered');++ await expect(+ ext.onAuthenticate({+ token: undefined,+ context: {},+ documentName: 'renamed-source',+ }),+ ).rejects.toMatchObject({ reason: 'rename-redirect:renamed-target' });+ } finally {+ await server.destroy();+ }+ });});// ─── Production wiring: push-permission auth ────────────────────────────────diff --git a/packages/server/src/server-factory.ts b/packages/server/src/server-factory.tsindex a7e18d759..90a386601 100644--- a/packages/server/src/server-factory.ts+++ b/packages/server/src/server-factory.ts@@ -59,7 +59,6 @@ import { createApiExtension, isSafeDocName } from './api-extension.ts';import { assetReferencesChanged } from './asset-references.ts';import { seedBasenameIndex, seedSingleDirBasenameIndex } from './asset-walk.ts';import { HocuspocusAuthRejection, parseHocuspocusAuthToken } from './auth-token-schema.ts';-import { BacklinkIndex } from './backlink-index.ts';import { shellEscape } from './bash/shell-escape.ts';import { bootElapsedMs, recordBootPhase, setBootField } from './boot-timings.ts';import {@@ -90,6 +89,10 @@ import { resolveProjectTemplates } from './content/templates-resolver.ts';import { type ContentFilter, createContentFilter } from './content-filter.ts';import { isWithinContentDir, safeContentPath } from './content-path.ts';import { dropPendingDocs, recordContributor } from './contributor-tracker.ts';+import {+ DerivedDocumentIndex,+ type DerivedDocumentIndexBranchTransition,+} from './derived-document-index.ts';import { applyDiskContentToDoc } from './disk-content-intake.ts';import { docNameToRelativePath, getDocExtension, stripDocExtension } from './doc-extensions.ts';import { runDocLineageGuard } from './doc-lineage-guard.ts';@@ -201,7 +204,6 @@ import {import { assertCompatibleStateManifest } from './state-manifest.ts';import { SyncEngine } from './sync-engine.ts';import { createSyncHandshakeSpanExtension } from './sync-handshake-span-extension.ts';-import { TagIndex } from './tag-index.ts';import { initTelemetry, shutdownTelemetry, withSpan } from './telemetry.ts';import { trustSystemCertificates } from './trust-system-ca.ts';import { cleanupOrphanUploadTempfiles } from './upload-streaming.ts';@@ -416,7 +418,7 @@ export interface ServerInstance {* Read AFTER `await ready` for a stable list; reads before may return a partial result.* Empty array means all subsystems initialized successfully.* Possible values: `'shadow-repo'`, `'managed-rename-recovery'`, `'file-watcher'`,- * `'head-watcher'`.+ * `'head-watcher'`, `'backlink-index'`, `'tag-index'`.*/readonly degraded: readonly string[];/**@@ -974,8 +976,7 @@ export function createServer(options: ServerOptions): ServerInstance {// Synchronous init — if any constructor throws, release the lock before propagating.let contentFilter: ReturnType<typeof createContentFilter>;- let backlinkIndex: BacklinkIndex;- let tagIndex: TagIndex;+ let derivedDocumentIndex: DerivedDocumentIndex;let shadowRef: ShadowRef;let maintenanceCoordinator: MaintenanceCoordinator | undefined;let persistence: ReturnType<typeof createPersistenceExtension>;@@ -1045,27 +1046,6 @@ export function createServer(options: ServerOptions): ServerInstance {cc1Broadcaster?.signal(channel);}- // Debounced saveToDisk for watcher-event paths. Collapses bursts (e.g. a git- // clone landing many files) into a single write. Startup and branch-switch- // paths call backlinkIndex.saveToDisk() directly — those are deliberate- // full-state transitions that should not be deferred. The tag snapshot- // rides the same debounce: every scheduleSaveToDisk call site pairs with a- // tagIndex mutation on the same disk event.- const BACKLINK_SAVE_DEBOUNCE_MS = 2000;- let backlinkSaveTimer: ReturnType<typeof setTimeout> | null = null;- function scheduleSaveToDisk(): void {- if (backlinkSaveTimer !== null) clearTimeout(backlinkSaveTimer);- backlinkSaveTimer = setTimeout(() => {- backlinkSaveTimer = null;- void backlinkIndex.saveToDisk().catch((err) => {- getLogger('backlinks').warn({ err }, 'Failed to persist debounced cache');- });- void tagIndex.saveToDisk().catch((err) => {- getLogger('tag-index').warn({ err }, 'Failed to persist debounced tag snapshot');- });- }, BACKLINK_SAVE_DEBOUNCE_MS);- }-// LRU cache of docNames renamed away or deleted, durable across restarts// via the removal journal at `.ok/local/removed-docs.json`. Read by// `removalRedirectGuard` (registered below) to reject WebSocket@@ -1176,28 +1156,12 @@ export function createServer(options: ServerOptions): ServerInstance {contentDir,singleDocRelPath,onAfterRebuild: () => {- // Re-derive backlink + tag indexes against the new visible-set.- // Both indexes hold a live reference to ContentFilter, so they- // read the freshly-rebuilt state on their next call. Failures- // are logged but never roll back the rebuild — the in-memory- // ignore filter is correct; derived views may go stale until- // the next external trigger.- //- // All three re-derivations below are fire-and-forget and run- // concurrently. Rapid successive ignore-file edits can overlap- // them; that's fine — each walk reads the live ContentFilter, so- // the last rebuild's visible-set wins and the end state converges- // (TagIndex additionally serializes its init calls internally).- void backlinkIndex.rebuildFromDisk(getActiveBranch()).catch((err) => {+ // Re-derive relationship views against the freshly rebuilt admission+ // boundary. The coordinator serializes rapid successive rebuilds.+ void derivedDocumentIndex.refreshContentScope().catch((err) => {getLogger('server-factory').warn({ err },- '[content-filter] backlink-index rebuild failed after onAfterRebuild',- );- });- void tagIndex.init().catch((err) => {- getLogger('server-factory').warn(- { err },- '[content-filter] tag-index rebuild failed after onAfterRebuild',+ '[content-filter] derived-index rebuild failed after onAfterRebuild',);});// Reconcile the watcher's in-memory file/folder indexes with the@@ -1235,25 +1199,12 @@ export function createServer(options: ServerOptions): ServerInstance {});},});- backlinkIndex = new BacklinkIndex({ projectDir, contentDir, contentFilter });- tagIndex = new TagIndex({ projectDir, contentDir, contentFilter });- // Boot-time scan, fire-and-forget: the factory itself is synchronous.- // Warm start restores the persisted snapshot and mtime-reconciles offline- // changes; cold start (no/corrupt snapshot) falls back to the full scan.- // `initAsync` awaits a reconcile after the watcher starts; TagIndex- // serializes the passes internally, so by the time `ready` resolves the- // index reflects disk.- void (async () => {- if (await tagIndex.loadFromDisk()) {- await tagIndex.reconcileWithDisk();- } else {- await tagIndex.init();- }- })().catch((err) => {- getLogger('server-factory').warn(- { err },- '[server-factory] tag-index warm boot failed; continuing with empty index',- );+ derivedDocumentIndex = new DerivedDocumentIndex({+ projectDir,+ contentDir,+ contentFilter,+ getGlobalSkillRoots: () => managedArtifactSkillsRoots(persistence.managedArtifactCtx),+ signalChannel,});shadowRef = { current: shadowRepo };@@ -1305,7 +1256,7 @@ export function createServer(options: ServerOptions): ServerInstance {shadowRef,ephemeral,contentRoot,- backlinkIndex,+ derivedDocumentIndex,configHomedirOverride,getCurrentBranch: () => headWatcher?.getLastKnownBranch() ?? null,resolveEmbed,@@ -1545,9 +1496,7 @@ export function createServer(options: ServerOptions): ServerInstance {})),);const liveDerivedIndexExtension = createLiveDerivedIndexExtension({- backlinkIndex,- tagIndex,- signalChannel,+ derivedDocumentIndex,});hocuspocus.configuration.extensions.push(liveDerivedIndexExtension);@@ -1862,8 +1811,7 @@ export function createServer(options: ServerOptions): ServerInstance {// empty-object case.getDiskAckSVs: () => cc1Broadcaster?.getLatestDiskAckSVsAsBase64() ?? {},contentRoot,- backlinkIndex,- tagIndex,+ derivedDocumentIndex,signalChannel,agentFocusBroadcaster,agentPresenceBroadcaster,@@ -2220,13 +2168,8 @@ export function createServer(options: ServerOptions): ServerInstance {switch (event.kind) {case 'create': {log.info({ docName: event.docName }, `[reconcile] create: ${event.docName}`);- backlinkIndex.updateDocumentFromMarkdown(event.docName, event.content);- scheduleSaveToDisk();- tagIndex.updateDocumentFromMarkdown(event.docName, event.content);+ await derivedDocumentIndex.recordDiskUpsert(event.docName, event.content);signalChannel('files');- signalChannel('backlinks');- signalChannel('graph');- signalChannel('tags');onUpstreamAdd(event.docName);break;}@@ -2235,12 +2178,7 @@ export function createServer(options: ServerOptions): ServerInstance {const { docName, content: theirs } = event;const document = hocuspocus.documents.get(docName);if (!document) {- backlinkIndex.updateDocumentFromMarkdown(docName, theirs);- scheduleSaveToDisk();- tagIndex.updateDocumentFromMarkdown(docName, theirs);- signalChannel('backlinks');- signalChannel('graph');- signalChannel('tags');+ await derivedDocumentIndex.recordDiskUpsert(docName, theirs);return;}@@ -2261,83 +2199,76 @@ export function createServer(options: ServerOptions): ServerInstance {switch (result.kind) {case 'noop':clearLifecycleConflict(document);- backlinkIndex.updateDocumentFromMarkdown(docName, theirs);- scheduleSaveToDisk();- tagIndex.updateDocumentFromMarkdown(docName, theirs);- signalChannel('backlinks');- signalChannel('graph');- signalChannel('tags');+ await derivedDocumentIndex.recordDiskUpsert(docName, theirs);break;case 'clean':- try {- applyToDoc(docName, result.newContent);- setReconciledBase(docName, result.newContent);- incrementReconcile();- clearLifecycleConflict(document);- backlinkIndex.updateDocumentFromMarkdown(docName, theirs);- scheduleSaveToDisk();- tagIndex.updateDocumentFromMarkdown(docName, theirs);- signalChannel('backlinks');- signalChannel('graph');- signalChannel('tags');- } catch (e) {- log.error(- { err: e, docName },- `[reconcile] failed to apply clean content to Y.Doc for ${docName}`,- );- // Disk is source of truth — keep base in sync even if Y.Doc update failed- setReconciledBase(docName, theirs);- clearLifecycleConflict(document);+ {+ let applied = false;+ try {+ applyToDoc(docName, result.newContent);+ setReconciledBase(docName, result.newContent);+ incrementReconcile();+ clearLifecycleConflict(document);+ applied = true;+ } catch (e) {+ log.error(+ { err: e, docName },+ `[reconcile] failed to apply clean content to Y.Doc for ${docName}`,+ );+ // Disk is source of truth — keep base in sync even if Y.Doc update failed+ setReconciledBase(docName, theirs);+ clearLifecycleConflict(document);+ }+ if (applied) {+ await derivedDocumentIndex.recordDiskUpsert(docName, theirs);+ }}break;case 'merged':- try {- applyToDoc(docName, result.newContent);- // Base tracks the DISK bytes (theirs), not the merged content- // — the merge exists only in memory until a later store- // flushes it. A base pointing past disk makes every- // disk-vs-base comparator misread the world: the- // L1 before-agent-write reconcile would see a phantom- // divergence and clean-ingest disk (reverting this merge),- // the L3 store backstop would abort the next agent flush the- // same way, and the no-op store skip (ytext === base) would- // keep the merged content off disk indefinitely. With- // base = theirs, the next store writes the merge through and- // re-events reconcile to noop (theirs === base).- setReconciledBase(docName, theirs);- incrementReconcile();- clearLifecycleConflict(document);- backlinkIndex.updateDocumentFromMarkdown(docName, theirs);- scheduleSaveToDisk();- tagIndex.updateDocumentFromMarkdown(docName, theirs);- signalChannel('backlinks');- signalChannel('graph');- signalChannel('tags');- } catch (e) {- log.error(- { err: e, docName },- `[reconcile] failed to apply merged content to Y.Doc for ${docName}`,- );- // Disk is source of truth — keep base in sync even if Y.Doc update failed- setReconciledBase(docName, theirs);- clearLifecycleConflict(document);+ {+ let applied = false;+ try {+ applyToDoc(docName, result.newContent);+ // Base tracks the DISK bytes (theirs), not the merged content+ // — the merge exists only in memory until a later store+ // flushes it. A base pointing past disk makes every+ // disk-vs-base comparator misread the world: the+ // L1 before-agent-write reconcile would see a phantom+ // divergence and clean-ingest disk (reverting this merge),+ // the L3 store backstop would abort the next agent flush the+ // same way, and the no-op store skip (ytext === base) would+ // keep the merged content off disk indefinitely. With+ // base = theirs, the next store writes the merge through and+ // re-events reconcile to noop (theirs === base).+ setReconciledBase(docName, theirs);+ incrementReconcile();+ clearLifecycleConflict(document);+ applied = true;+ } catch (e) {+ log.error(+ { err: e, docName },+ `[reconcile] failed to apply merged content to Y.Doc for ${docName}`,+ );+ // Disk is source of truth — keep base in sync even if Y.Doc update failed+ setReconciledBase(docName, theirs);+ clearLifecycleConflict(document);+ }+ if (applied) {+ await derivedDocumentIndex.recordDiskUpsert(docName, theirs);+ }}break;case 'conflicts': {+ let applied = false;try {applyToDoc(docName, result.newContent);setReconciledBase(docName, result.newContent);incrementReconcile();incrementConflict();- backlinkIndex.updateDocumentFromMarkdown(docName, theirs);- scheduleSaveToDisk();- tagIndex.updateDocumentFromMarkdown(docName, theirs);- signalChannel('backlinks');- signalChannel('graph');- signalChannel('tags');+ applied = true;} catch (e) {log.error({ err: e, docName },@@ -2357,6 +2288,9 @@ export function createServer(options: ServerOptions): ServerInstance {lifecycleMap.set('status', 'conflict');lifecycleMap.set('reason', 'merged-with-markers');}+ if (applied) {+ await derivedDocumentIndex.recordDiskUpsert(docName, theirs);+ }break;}@@ -2375,13 +2309,8 @@ export function createServer(options: ServerOptions): ServerInstance {const { docName } = event;const document = hocuspocus.documents.get(docName);if (!document) {- backlinkIndex.deleteDocument(docName);- scheduleSaveToDisk();- tagIndex.deleteDocument(docName);+ await derivedDocumentIndex.recordDiskDelete(docName);signalChannel('files');- signalChannel('backlinks');- signalChannel('graph');- signalChannel('tags');onUpstreamDelete(docName);console.info(JSON.stringify({@@ -2435,18 +2364,13 @@ export function createServer(options: ServerOptions): ServerInstance {lifecycleMap.set('status', 'deleted-upstream');deleteReconciledBase(docName);- backlinkIndex.deleteDocument(docName);- scheduleSaveToDisk();- tagIndex.deleteDocument(docName);+ await derivedDocumentIndex.recordDiskDelete(docName);log.info({ docName, isDirty }, `[reconcile] delete: ${docName} (dirty=${isDirty})`);// Unload document to prevent re-creation on next persistence cyclehocuspocus.closeConnections(docName);await forceUnloadDocument(document);signalChannel('files');- signalChannel('backlinks');- signalChannel('graph');- signalChannel('tags');onUpstreamDelete(docName);console.info(JSON.stringify({@@ -2465,9 +2389,7 @@ export function createServer(options: ServerOptions): ServerInstance {deleteReconciledBase(oldDocName);setReconciledBase(newDocName, content);- backlinkIndex.renameDocument(oldDocName, newDocName, content);- scheduleSaveToDisk();- tagIndex.renameDocument(oldDocName, newDocName, content);+ await derivedDocumentIndex.recordDiskRename(oldDocName, newDocName, content);if (document) {const lifecycleMap = document.getMap('lifecycle');@@ -2477,9 +2399,6 @@ export function createServer(options: ServerOptions): ServerInstance {log.info({ oldDocName, newDocName }, `[reconcile] rename: ${oldDocName} → ${newDocName}`);signalChannel('files');- signalChannel('backlinks');- signalChannel('graph');- signalChannel('tags');onUpstreamRename(oldDocName, newDocName);console.info(JSON.stringify({@@ -2523,7 +2442,7 @@ export function createServer(options: ServerOptions): ServerInstance {}// Asset events update the basename index and fire CC1 'files' only.- // They do NOT touch backlinkIndex (markdown-only). They DO trigger+ // They do NOT touch relationship views (markdown-only). They DO trigger// a fallback re-render of any open doc that references the changed// basename via `[[name.ext]]` (see// `rerenderDocsReferencingAssetBasename`) — without this, a doc@@ -2550,8 +2469,8 @@ export function createServer(options: ServerOptions): ServerInstance {}// file-* events maintain the in-memory fileIndex as `kind:'file'`. Like// asset events they signal `files` (cache-invalidate /api/documents and- // the workspace search corpus) but do NOT touch backlinkIndex or- // tagIndex (relationship surfaces stay markdown-scoped). updateFileIndex+ // the workspace search corpus) but do NOT touch relationship views+ // (those surfaces stay markdown-scoped). updateFileIndex// in handleRawEvents already mutated the index by the time we arrive// here.case 'file-create':@@ -2778,24 +2697,6 @@ export function createServer(options: ServerOptions): ServerInstance {log.warn({ err }, '[server] failed to mark server.lock draining');}- // Cancel any pending debounced backlink cache write so the timer doesn't- // fire after resources are torn down.- if (backlinkSaveTimer !== null) {- clearTimeout(backlinkSaveTimer);- backlinkSaveTimer = null;- }-- // Close tag-index persistence: queued scans/saves become no-ops from- // here, so a fire-and-forget snapshot save from the boot path can't- // recreate files under `.ok/` while a caller tears the state dir down.- // Deliberately NOT awaited: this whole destroy runs under boot.ts's 5s- // per-step budget (destroyHocuspocus), and draining a boot-busy chain- // here stacks on top of the ready-wait below and blows that budget on- // slow machines. The synchronous flag flip is the load-bearing part;- // an already-in-flight write finishes long before callers reach their- // post-destroy cleanup.- void tagIndex.close();-// Flush the removal journal synchronously — a tombstone recorded just// before shutdown must survive the restart or a reconnecting stale// client resurrects the doc. Failure feeds `phaseErrors`: this flush is@@ -2954,6 +2855,14 @@ export function createServer(options: ServerOptions): ServerInstance {log.error({ err }, '[server] shutdown phase-3 flush failed');}+ // Close relationship-index persistence only after the final L1+ // stores have projected their durable markdown into backlinks.+ // close() moves any pending cache save onto the coordinator queue+ // before rejecting later work. The async drain is deliberately not+ // awaited here: this destroy runs under boot.ts's bounded per-step+ // shutdown budget.+ void derivedDocumentIndex.close();+// Phase 4: drain L2 (disk → git) — only meaningful AFTER L1 has run// Bounded to destroyTimeoutMs so a stuck git process doesn't hang shutdown.let l2TimeoutId: ReturnType<typeof setTimeout> | undefined;@@ -3432,18 +3341,9 @@ export function createServer(options: ServerOptions): ServerInstance {// so reference-only edits ride the next SKILL.md touch / restart; a SKILL.md// unlink does NOT fire onChange, so a deleted skill is pruned on restart.if (parseGlobalSkillBundleDoc(docName)) {- void backlinkIndex- .ingestGlobalSkillBundles(- managedArtifactSkillsRoots(persistence.managedArtifactCtx),- getActiveBranch(),- )- .then(() => {- signalChannel('backlinks');- signalChannel('graph');- })- .catch((err) => {- log.warn({ err, docName }, '[backlinks] global skill bundle re-ingest failed');- });+ void derivedDocumentIndex.refreshGlobalSkillNodes().catch((err) => {+ log.warn({ err, docName }, '[backlinks] global skill bundle re-ingest failed');+ });}};@@ -3643,11 +3543,12 @@ export function createServer(options: ServerOptions): ServerInstance {degraded.push('ignore-files-watcher');}- // Align this server's branch-scoped durability state and backlink index- // with the project's current HEAD before either is read or written.+ // Align branch-scoped durability and relationship views with the project's+ // current HEAD before either is read or written.const startupBranch = readProjectHeadState(projectDir).branch ?? 'main';switchReconciledBaseScope(startupBranch);- backlinkIndex.switchBranch(startupBranch);+ const derivedIndexStartup = derivedDocumentIndex.beginStartup(startupBranch);+ let derivedIndexStartupSettled = false;// Boot-timing scope for the index phases (backlink load/rebuild, tag// re-init, basename seed) plus the watcher's startup seed walk. The whole@@ -3660,85 +3561,21 @@ export function createServer(options: ServerOptions): ServerInstance {// `ok.boot.indexes` spans the whole index-building phase; the nested// `ok.boot.seed-walk` span (around `startWatcher`) is its child.await withSpan('ok.boot.indexes', undefined, async () => {- // Warm start: load the on-disk cache and reconcile (stat-only for- // unchanged files, async bounded reads for changed ones).- // Cold start (no cache): full async rebuild with bounded concurrency.- // Runs BEFORE startWatcher to avoid a race where watcher events mutate- // the backlink index while an async rebuild is in progress.- // Isolated in its own try/catch so a corrupt cache never prevents the- // watcher from starting — watcher events will populate the index- // incrementally from that point.- const branch = getActiveBranch();- try {- const cacheLoaded = await backlinkIndex.loadFromDisk(branch);- if (cacheLoaded) {- const diff = await backlinkIndex.reconcileWithDisk(branch);- if (diff.added > 0 || diff.updated > 0 || diff.deleted > 0) {- log.info(- { added: diff.added, updated: diff.updated, deleted: diff.deleted },- '[backlinks] startup reconcile: offline changes applied',- );- }- // Files deleted while the server was down leave no tombstone —- // the delete was never observed, so nothing stops a reconnecting- // client that still holds the doc's Yjs state from re-creating- // the file as a "legitimate first write". The reconcile's- // deleted set is exactly the last-known content docs whose file- // vanished during downtime; arm the removal guard for each (the- // journal hook makes the inference durable, and the guard's- // file-existence-first self-heal admits any legitimate- // re-creation).- let tombstonedOffline = 0;- for (const deletedDocName of diff.deletedDocNames) {- // Same doc-kind gate as every other populate site:- // `isReservedForUserTree` excludes synthetic, config, and- // managed-artifact docs — none may enter the removal cache.- // (Managed artifacts live under `.ok/` outside the content- // walk and have their own reconcile ownership.)- if (isReservedForUserTree(deletedDocName)) continue;- // A journaled rename-redirect is strictly more informative- // than an inferred flat delete — a rename inside the- // debounced backlink-save window leaves the persisted cache- // stale with the old name, so the inference sees it as- // deleted-while-down. Refuse the downgrade (mirrors the- // watcher unpaired-delete peek-guard above).- if (recentlyRemovedDocs.peek(deletedDocName)?.kind === 'renamed') continue;- recentlyRemovedDocs.setDeleted(deletedDocName);- tombstonedOffline++;- }- if (tombstonedOffline > 0) {- log.info(- { count: tombstonedOffline },- '[removal-guard] tombstoned docs deleted while the server was down',- );- }- } else {- await backlinkIndex.rebuildFromDisk(branch);- }- // Register GLOBAL skill bundle docs (SKILL + references/**) as graph- // nodes. They live at `<home>/.ok/skills`, OUTSIDE contentDir, so the- // content rebuild/reconcile above never touches them — this runs AFTER- // those (both replace branch state) so the nodes aren't dropped. Their- // bodies are deliberately not parsed (within-bundle-only). The cache is- // persisted after, but a warm restart re-ingests here regardless.- try {- await backlinkIndex.ingestGlobalSkillBundles(- managedArtifactSkillsRoots(persistence.managedArtifactCtx),- branch,- );- } catch (err) {- log.warn({ err, branch }, '[backlinks] global skill bundle ingest failed');- }- void backlinkIndex.saveToDisk().catch((err) => {- getLogger('backlinks').warn(- { branch, err },- `Failed to persist startup cache for ${branch}`,- );- });- } catch (err) {- log.error(- { err, branch },- '[backlinks] startup init failed; index will populate incrementally via watcher',+ const { deletedDocNames, backlinkIndexDegraded } = await derivedIndexStartup.backlinksReady;+ if (backlinkIndexDegraded) degraded.push('backlink-index');+ // Files deleted while the server was down leave no watcher tombstone.+ // Arm the removal guard from the coordinator's startup reconciliation.+ let tombstonedOffline = 0;+ for (const deletedDocName of deletedDocNames) {+ if (isReservedForUserTree(deletedDocName)) continue;+ if (recentlyRemovedDocs.peek(deletedDocName)?.kind === 'renamed') continue;+ recentlyRemovedDocs.setDeleted(deletedDocName);+ tombstonedOffline++;+ }+ if (tombstonedOffline > 0) {+ log.info(+ { count: tombstonedOffline },+ '[removal-guard] tombstoned docs deleted while the server was down',);}// `startWatcher` performs the file-watcher's startup seed walk — the@@ -3781,30 +3618,9 @@ export function createServer(options: ServerOptions): ServerInstance {);}}- // Reconcile tagIndex once the watcher has settled. The warm-boot pass- // in the synchronous boot path covers the snapshot restore / cold- // scan; this second pass is a cheap stat-only sweep that picks up any- // disk content that landed between constructor time and watcher- // startup (rare, but cheap to cover), then persists the snapshot for- // the next boot. Isolated in its own try/catch so a failed tag scan- // degrades tag search only — letting it reach the outer catch would- // skip the basename-index seed below and misreport a healthy watcher- // as `file-watcher` in `degraded[]`.- try {- const tagDiff = await tagIndex.reconcileWithDisk();- if (tagDiff.added > 0 || tagDiff.updated > 0 || tagDiff.deleted > 0) {- log.info(tagDiff, '[tag-index] startup reconcile: offline changes applied');- }- void tagIndex.saveToDisk().catch((err) => {- getLogger('tag-index').warn({ err }, 'Failed to persist startup tag snapshot');- });- } catch (err) {- log.error(- { err },- '[tag-index] startup reconcile failed; tag index updates incrementally via watcher events',- );- degraded.push('tag-index');- }+ const derivedIndexSettlement = await derivedDocumentIndex.settleStartupAfterWatcherSeed();+ derivedIndexStartupSettled = true;+ if (derivedIndexSettlement.tagIndexDegraded) degraded.push('tag-index');// Seed the basename index from disk once the watcher's startup walk// has finished. The watcher's fileIndex is markdown-only, so we walk// the contentDir directly for assets.@@ -3869,6 +3685,11 @@ export function createServer(options: ServerOptions): ServerInstance {} catch (err) {log.error({ err }, '[server] disk bridge watcher failed to start');degraded.push('file-watcher');+ if (!derivedIndexStartupSettled) {+ const settlement = await derivedDocumentIndex.settleStartupAfterWatcherSeed();+ derivedIndexStartupSettled = true;+ if (settlement.tagIndexDegraded) degraded.push('tag-index');+ }} finally {// Record the index-phase duration + final markdown file count even on a// partial/failed watcher start, so the waterfall still has a value. The@@ -4012,312 +3833,271 @@ export function createServer(options: ServerOptions): ServerInstance {// Cross-branch or detached-head — discard buffered events (wrong branch state)incrementBranchSwitch();eventBuffer.splice(0, eventBuffer.length);-- // Switch reconciledBase scope to target branch- switchReconciledBaseScope(newBranch);- // Cancel any pending debounced save before switching branches.- // Without this, the timer fires after activeBranch is updated and- // saves the old branch's graph state into the new branch's cache.- if (backlinkSaveTimer !== null) {- clearTimeout(backlinkSaveTimer);- backlinkSaveTimer = null;- }- backlinkIndex.switchBranch(newBranch);-- // Rebuild `ContentFilter`'s sibling-asset refcount BEFORE the- // basenameIndex reseed. ContentFilter's `dirCount` is normally- // maintained incrementally via `incrementMdDir` /- // `decrementMdDir` calls fired by the file watcher's create /- // delete events, but the cross-branch path discarded those- // events above (`eventBuffer.splice`). Without a rebuild, the- // refcount holds the previous branch's directory shape and- // legitimate sibling-asset pairs on the new branch- // (`assets/cover.md` next to `assets/photo.png`) are rejected- // by `seedBasenameIndex`'s admission check, leaving the asset- // unresolved.- contentFilter.rebuildDirCount();-- // Reseed `basenameIndex` BEFORE the doc-reset loop. The reset- // calls `applyToDoc` → `applyExternalChange` → mdast→PM with- // `resolveEmbed`, which resolves `![[photo.png]]` against the- // basename index. With the previous (stale) branch's paths- // still in the index, the PM image `src` carries the- // pre-switch resolution until the next user edit — disk- // markdown round-trips fine, but the rendered preview is- // wrong.- //- // Asset DiskEvents from the switch itself are discarded- // (`eventBuffer.splice` above) and `basenameIndex` is a flat- // Map without branch scope, so the explicit walk is the only- // mechanism by which post-switch paths enter the index.- // Mirror backlinkIndex's branch-scoped reset: drop the index,- // walk the new branch's disk, re-seed.- //- // `onSkip` wiring is symmetric with the boot path — a mid-- // session permission flip (EACCES), fd exhaustion (EMFILE),- // or root-scope read failure during the reseed walk surfaces- // the same `basename-index-partial` degraded indicator the- // boot path uses.+ let deferredStoresFlushed = false;+ let branchTransition: DerivedDocumentIndexBranchTransition | undefined;try {- let reseedSkipCount = 0;- basenameIndex.clear();- await seedBasenameIndex({- contentDir,- contentFilter,- basenameIndex,- onSkip: (reason, code, path) => {- reseedSkipCount++;+ // Switch reconciledBase scope to target branch+ switchReconciledBaseScope(newBranch);+ branchTransition = await derivedDocumentIndex.beginBranchSwitch(newBranch);++ // Rebuild `ContentFilter`'s sibling-asset refcount BEFORE the+ // basenameIndex reseed. ContentFilter's `dirCount` is normally+ // maintained incrementally via `incrementMdDir` /+ // `decrementMdDir` calls fired by the file watcher's create /+ // delete events, but the cross-branch path discarded those+ // events above (`eventBuffer.splice`). Without a rebuild, the+ // refcount holds the previous branch's directory shape and+ // legitimate sibling-asset pairs on the new branch+ // (`assets/cover.md` next to `assets/photo.png`) are rejected+ // by `seedBasenameIndex`'s admission check, leaving the asset+ // unresolved.+ contentFilter.rebuildDirCount();++ // Reseed `basenameIndex` BEFORE the doc-reset loop. The reset+ // calls `applyToDoc` → `applyExternalChange` → mdast→PM with+ // `resolveEmbed`, which resolves `![[photo.png]]` against the+ // basename index. With the previous (stale) branch's paths+ // still in the index, the PM image `src` carries the+ // pre-switch resolution until the next user edit — disk+ // markdown round-trips fine, but the rendered preview is+ // wrong.+ //+ // Asset DiskEvents from the switch itself are discarded+ // (`eventBuffer.splice` above) and `basenameIndex` is a flat+ // Map without branch scope, so the explicit walk is the only+ // mechanism by which post-switch paths enter the index.+ // Mirror the relationship coordinator's branch-scoped reset: drop+ // the basename index, walk the new branch's disk, and re-seed.+ //+ // `onSkip` wiring is symmetric with the boot path — a mid-+ // session permission flip (EACCES), fd exhaustion (EMFILE),+ // or root-scope read failure during the reseed walk surfaces+ // the same `basename-index-partial` degraded indicator the+ // boot path uses.+ try {+ let reseedSkipCount = 0;+ basenameIndex.clear();+ await seedBasenameIndex({+ contentDir,+ contentFilter,+ basenameIndex,+ onSkip: (reason, code, path) => {+ reseedSkipCount++;+ log.warn(+ { reason, code, path, branch: newBranch },+ `[basename-index] skipped entry during branch-switch reseed (${reason}${code ? ` ${code}` : ''})`,+ );+ },+ });+ if (reseedSkipCount > 0) {log.warn(- { reason, code, path, branch: newBranch },- `[basename-index] skipped entry during branch-switch reseed (${reason}${code ? ` ${code}` : ''})`,+ { count: reseedSkipCount, branch: newBranch },+ `[basename-index] branch-switch reseed completed with ${reseedSkipCount} skipped entries — embeds under inaccessible subtrees will not resolve on this branch`,);- },- });- if (reseedSkipCount > 0) {- log.warn(- { count: reseedSkipCount, branch: newBranch },- `[basename-index] branch-switch reseed completed with ${reseedSkipCount} skipped entries — embeds under inaccessible subtrees will not resolve on this branch`,- );- if (!degraded.includes('basename-index-partial')) {- degraded.push('basename-index-partial');+ if (!degraded.includes('basename-index-partial')) {+ degraded.push('basename-index-partial');+ }}+ } catch (err) {+ log.error(+ { err, branch: newBranch },+ '[basename-index] branch-switch reseed failed',+ );}- } catch (err) {- log.error({ err, branch: newBranch }, '[basename-index] branch-switch reseed failed');- }- // Reset all open Y.Docs from the target branch's disk content- for (const [docName, document] of hocuspocus.documents) {- if (isReservedForUserTree(docName)) continue;- try {- const filePath = safeContentPath(docName, contentDir);- if (!existsSync(filePath)) {- // File doesn't exist on target branch — tombstone- const base = getReconciledBase(docName) ?? '';- const ours = serializeDoc(docName) ?? '';- const isDirty = ours !== base;-- if (isDirty && shadowRef.current) {- // Silent rescue checkpoint on branch-switch tombstone- // Same pattern as reconcile-delete above.- const shadowForCheckpoint = shadowRef.current;- queueMicrotask(() => {- saveInMemoryCheckpoint(shadowForCheckpoint, contentRoot ?? '', {- kind: 'external-change-rescue',- docName,- contents: ours,- label: `External change recovered @ ${new Date().toISOString()}`,- branch: newBranch,- metadata: { incomingDiskSha: '' },- })- .then(() => {- incrementRescueBuffer();- log.info(- { docName },- `[reconcile] rescue checkpoint saved on branch switch: ${docName}`,- );+ // Reset all open Y.Docs from the target branch's disk content+ for (const [docName, document] of hocuspocus.documents) {+ if (isReservedForUserTree(docName)) continue;+ try {+ const filePath = safeContentPath(docName, contentDir);+ if (!existsSync(filePath)) {+ // File doesn't exist on target branch — tombstone+ const base = getReconciledBase(docName) ?? '';+ const ours = serializeDoc(docName) ?? '';+ const isDirty = ours !== base;++ if (isDirty && shadowRef.current) {+ // Silent rescue checkpoint on branch-switch tombstone+ // Same pattern as reconcile-delete above.+ const shadowForCheckpoint = shadowRef.current;+ queueMicrotask(() => {+ saveInMemoryCheckpoint(shadowForCheckpoint, contentRoot ?? '', {+ kind: 'external-change-rescue',+ docName,+ contents: ours,+ label: `External change recovered @ ${new Date().toISOString()}`,+ branch: newBranch,+ metadata: { incomingDiskSha: '' },})- .catch((e: unknown) => {- log.error(- { docName, err: e },- `[reconcile] rescue checkpoint write failed: ${docName}`,- );- });- });+ .then(() => {+ incrementRescueBuffer();+ log.info(+ { docName },+ `[reconcile] rescue checkpoint saved on branch switch: ${docName}`,+ );+ })+ .catch((e: unknown) => {+ log.error(+ { docName, err: e },+ `[reconcile] rescue checkpoint write failed: ${docName}`,+ );+ });+ });+ }++ const lifecycleMap = document.getMap('lifecycle');+ lifecycleMap.set('status', 'deleted-upstream');+ log.info(+ { docName, branch: newBranch },+ `[branch-switch] tombstone: ${docName} (not on ${newBranch})`,+ );+ continue;}- const lifecycleMap = document.getMap('lifecycle');- lifecycleMap.set('status', 'deleted-upstream');- log.info(- { docName, branch: newBranch },- `[branch-switch] tombstone: ${docName} (not on ${newBranch})`,- );- continue;+ // Reset Y.Doc from disk+ const diskContent = readFileSync(filePath, 'utf-8');+ applyToDoc(docName, diskContent);+ setReconciledBase(docName, diskContent);+ log.info({ docName }, `[branch-switch] reset: ${docName}`);+ } catch (e) {+ log.error({ err: e, docName }, `[branch-switch] failed to reset ${docName}`);}-- // Reset Y.Doc from disk- const diskContent = readFileSync(filePath, 'utf-8');- applyToDoc(docName, diskContent);- setReconciledBase(docName, diskContent);- log.info({ docName }, `[branch-switch] reset: ${docName}`);- } catch (e) {- log.error({ err: e, docName }, `[branch-switch] failed to reset ${docName}`);}- }- log.info(- { branch: newBranch, docCount: hocuspocus.documents.size },- `[branch-switch] loaded branch ${newBranch} (${hocuspocus.documents.size} docs)`,- );- try {- const branchCacheLoaded = await backlinkIndex.loadFromDisk(newBranch);- if (branchCacheLoaded) {- const diff = await backlinkIndex.reconcileWithDisk(newBranch);- if (diff.added > 0 || diff.updated > 0 || diff.deleted > 0) {- log.info(diff, `[backlinks] branch-switch reconcile for ${newBranch}`);- }- } else {- await backlinkIndex.rebuildFromDisk(newBranch);- }- // Global skill bundle nodes are user-global, not branch-scoped — the- // freshly-built branch state lacks them, so re-ingest for this branch- // (same node-only, within-bundle-only path as boot).+ log.info(+ { branch: newBranch, docCount: hocuspocus.documents.size },+ `[branch-switch] loaded branch ${newBranch} (${hocuspocus.documents.size} docs)`,+ );try {- await backlinkIndex.ingestGlobalSkillBundles(- managedArtifactSkillsRoots(persistence.managedArtifactCtx),- newBranch,- );+ await derivedDocumentIndex.settleBranchFromDisk(branchTransition);} catch (err) {- log.warn(+ derivedDocumentIndex.abortBranchSwitch(branchTransition);+ log.error({ err, branch: newBranch },- '[backlinks] branch-switch global skill bundle ingest failed',+ '[derived-index] branch-switch rebuild failed; relationship views may be stale',);}- void backlinkIndex.saveToDisk(newBranch).catch((err) => {- getLogger('backlinks').warn(- { branch: newBranch, err },- `Failed to persist branch cache for ${newBranch}`,- );- });- } catch (err) {- log.error(- { err, branch: newBranch },- '[backlinks] branch-switch rebuild failed; backlinks may be stale',- );- }- // TagIndex is branch-agnostic but its source-of-truth is the- // contentDir that just changed underneath it. The checkout- // rewrote changed files' mtimes, so a reconcile re-parses exactly- // the docs that differ between branches instead of the whole- // corpus; a failed reconcile falls back to the full scan so the- // index never silently keeps the old branch's tags.- try {- const tagDiff = await tagIndex.reconcileWithDisk();- if (tagDiff.added > 0 || tagDiff.updated > 0 || tagDiff.deleted > 0) {- log.info(tagDiff, `[tag-index] branch-switch reconcile for ${newBranch}`);- }- } catch (err) {- log.warn(- { err, branch: newBranch },- '[tag-index] branch-switch reconcile failed; falling back to full rebuild',- );- await tagIndex.init();- }- void tagIndex.saveToDisk().catch((err) => {- getLogger('tag-index').warn(- { branch: newBranch, err },- 'Failed to persist tag snapshot after branch switch',- );- });- // Restore parked WIP if exists (three-way merge parked state against current disk)- if (shadowRef.current && info.batchKind === 'cross-branch') {- let restoredCount = 0;- for (const [docName] of hocuspocus.documents) {- if (isReservedForUserTree(docName)) continue;- try {- const parked = await readParkedState(- shadowRef.current,- newBranch,- SERVICE_WRITER.id,- docName,- );- if (!parked) continue;- // Skip if no in-flight edits were parked- if (parked.markdown === parked.diskSnapshot) continue;-- const currentDisk = getReconciledBase(docName);- if (!currentDisk) continue;+ // Restore parked WIP if exists (three-way merge parked state against current disk)+ if (shadowRef.current && info.batchKind === 'cross-branch') {+ let restoredCount = 0;+ for (const [docName] of hocuspocus.documents) {+ if (isReservedForUserTree(docName)) continue;+ try {+ const parked = await readParkedState(+ shadowRef.current,+ newBranch,+ SERVICE_WRITER.id,+ docName,+ );+ if (!parked) continue;+ // Skip if no in-flight edits were parked+ if (parked.markdown === parked.diskSnapshot) continue;++ const currentDisk = getReconciledBase(docName);+ if (!currentDisk) continue;++ const outcome = reconcile({+ docName,+ base: parked.diskSnapshot,+ ours: parked.markdown,+ theirs: currentDisk,+ });- const outcome = reconcile({- docName,- base: parked.diskSnapshot,- ours: parked.markdown,- theirs: currentDisk,- });-- switch (outcome.kind) {- case 'merged':- case 'clean':- applyToDoc(docName, outcome.newContent);- setReconciledBase(docName, outcome.newContent);- restoredCount++;- break;- case 'conflicts': {- applyToDoc(docName, outcome.newContent);- setReconciledBase(docName, outcome.newContent);- incrementConflict();- restoredCount++;- // Mirror the file-watcher `case 'conflicts'` lifecycle set- // so block-level reconcile failures during branch-switch- // WIP restore also fire the UI swap + mutating-handler- // refusal gate. Raw Y.Map.set, no transact — matches the- // sibling convention.- {- const restoredDoc = hocuspocus.documents.get(docName);- if (restoredDoc) {- const lifecycleMap = restoredDoc.getMap('lifecycle');- lifecycleMap.set('status', 'conflict');- lifecycleMap.set('reason', 'merged-with-markers');+ switch (outcome.kind) {+ case 'merged':+ case 'clean':+ applyToDoc(docName, outcome.newContent);+ setReconciledBase(docName, outcome.newContent);+ restoredCount++;+ break;+ case 'conflicts': {+ applyToDoc(docName, outcome.newContent);+ setReconciledBase(docName, outcome.newContent);+ incrementConflict();+ restoredCount++;+ // Mirror the file-watcher `case 'conflicts'` lifecycle set+ // so block-level reconcile failures during branch-switch+ // WIP restore also fire the UI swap + mutating-handler+ // refusal gate. Raw Y.Map.set, no transact — matches the+ // sibling convention.+ {+ const restoredDoc = hocuspocus.documents.get(docName);+ if (restoredDoc) {+ const lifecycleMap = restoredDoc.getMap('lifecycle');+ lifecycleMap.set('status', 'conflict');+ lifecycleMap.set('reason', 'merged-with-markers');+ }}+ break;}- break;+ case 'noop':+ case 'refused':+ break;}- case 'noop':- case 'refused':- break;+ } catch (e) {+ log.error(+ { err: e, docName },+ `[branch-switch] restore WIP failed for ${docName}`,+ );}- } catch (e) {- log.error(- { err: e, docName },- `[branch-switch] restore WIP failed for ${docName}`,+ }+ if (restoredCount > 0) {+ log.info(+ { count: restoredCount, branch: newBranch },+ `[branch-switch] restored ${restoredCount} parked docs on ${newBranch}`,);}}- if (restoredCount > 0) {- log.info(- { count: restoredCount, branch: newBranch },- `[branch-switch] restored ${restoredCount} parked docs on ${newBranch}`,- );- }- }- // Clean up detached HEAD context if switching FROM detached TO named branch- if (info.oldBranch?.startsWith('detached-') && shadowRef.current) {- try {- const sg = shadowGit(shadowRef.current);- // List refs under the detached context- const refs = (- await sg.raw('for-each-ref', `refs/wip/${info.oldBranch}/`, '--format=%(refname)')- ).trim();- if (refs) {- // Ref deletion is a shadow mutation — take the op gate so it- // cannot interleave with a maintenance gc run.- await shadowOpGateFor(shadowRef.current).withMutator(async () => {- for (const ref of refs.split('\n')) {- if (ref) {- await sg.raw('update-ref', '-d', ref);+ // Clean up detached HEAD context if switching FROM detached TO named branch+ if (info.oldBranch?.startsWith('detached-') && shadowRef.current) {+ try {+ const sg = shadowGit(shadowRef.current);+ // List refs under the detached context+ const refs = (+ await sg.raw(+ 'for-each-ref',+ `refs/wip/${info.oldBranch}/`,+ '--format=%(refname)',+ )+ ).trim();+ if (refs) {+ // Ref deletion is a shadow mutation — take the op gate so it+ // cannot interleave with a maintenance gc run.+ await shadowOpGateFor(shadowRef.current).withMutator(async () => {+ for (const ref of refs.split('\n')) {+ if (ref) {+ await sg.raw('update-ref', '-d', ref);+ }}- }- });- log.info(- { context: info.oldBranch },- `[branch-switch] cleaned up detached context ${info.oldBranch}`,- );+ });+ log.info(+ { context: info.oldBranch },+ `[branch-switch] cleaned up detached context ${info.oldBranch}`,+ );+ }+ } catch (e) {+ log.error({ err: e }, '[branch-switch] detached cleanup failed');}- } catch (e) {- log.error({ err: e }, '[branch-switch] detached cleanup failed');}- }- // Notify connected clients that the branch scope changed so they can- // invalidate their IDB persistence caches. Emit AFTER all server-side- // state transitions (Y.Doc reset, backlink rebuild, WIP restore,- // detached-ref cleanup) so a client's recycle-triggered reconnect- // synchronizes against the new branch's fully-settled state.- setBatchInProgress(false);- await persistence.flushDeferredStores('discard-stale');- cc1Broadcaster?.emitBranchSwitched(newBranch);+ // Notify connected clients that the branch scope changed so they can+ // invalidate their IDB persistence caches. Emit AFTER all server-side+ // state transitions (Y.Doc reset, backlink rebuild, WIP restore,+ // detached-ref cleanup) so a client's recycle-triggered reconnect+ // synchronizes against the new branch's fully-settled state.+ setBatchInProgress(false);+ await persistence.flushDeferredStores('discard-stale');+ deferredStoresFlushed = true;+ cc1Broadcaster?.emitBranchSwitched(newBranch);+ } finally {+ derivedDocumentIndex.abortBranchSwitch(branchTransition);+ if (!deferredStoresFlushed) {+ setBatchInProgress(false);+ await persistence.flushDeferredStores('discard-stale');+ }+ }}// Record upstream import if HEAD moved AND content files were affected.@@ -4522,9 +4302,7 @@ export function createServer(options: ServerOptions): ServerInstance {// nothing else would trigger a refresh until the next focus /// visibilitychange.signalChannel('files');- signalChannel('backlinks');- signalChannel('graph');- signalChannel('tags');+ derivedDocumentIndex.announceReadyViews();// initAsync has reached the point where `resolveReady` will fire — record// the elapsed-from-boot-start so the waterfall has a server-ready mark.@@ -4539,7 +4317,13 @@ export function createServer(options: ServerOptions): ServerInstance {// (so it could be passed into createApiExtension before initAsync ran).// Settle it now from initAsync's completion. Errors propagate through the// same channel callers awaited before.- initAsync().then(resolveReady, rejectReady);+ initAsync().then(resolveReady, (err) => {+ // A pre-index startup failure would otherwise strand coordinator callers+ // behind unresolved admission/readiness barriers. Failed server startup+ // has no recovery path, so close releases every waiter with a rejection.+ void derivedDocumentIndex.close();+ rejectReady(err);+ });return {hocuspocus,