Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

fix(open-knowledge): unify doc-existence oracle so file-index lag stops causing stale link/title results (PRD-7201) (#2255)

mike-inkeep2b750aabdca2
diff --git a/.changeset/unify-doc-existence-oracle.md b/.changeset/unify-doc-existence-oracle.md
new file mode 100644
index 000000000..c2b24f51f
--- /dev/null
+++ b/.changeset/unify-doc-existence-oracle.md
@@ -0,0 +1,6 @@
+---
+"@inkeep/open-knowledge-server": patch
+"@inkeep/open-knowledge": patch
+---
+
+Stop the file-watcher's indexing lag from making freshly-written docs look missing across link, title, and dead-link results. The file index the watcher maintains can lag an in-session write — or permanently drop the create event for a file written into a just-created subfolder — which made a brand-new doc render as a raw red-link name (instead of its title) and could flag a valid link to it as dead, until a server restart. Two complementary fixes close the gap: doc-existence now also trusts the link graph (updated in-process on every write), and the agent write/edit/frontmatter-patch handlers register the doc they just wrote into the file index immediately, mirroring page creation. Both paths honor content-scope exclusions, so a gitignore/okignore'd doc never leaks its title through these endpoints.
diff --git a/packages/server/src/api-agent-write-file-index.test.ts b/packages/server/src/api-agent-write-file-index.test.ts
new file mode 100644
index 000000000..60077348d
--- /dev/null
+++ b/packages/server/src/api-agent-write-file-index.test.ts
@@ -0,0 +1,257 @@
+import { describe, expect, test } from 'bun:test';
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import type { IncomingMessage, ServerResponse } from 'node:http';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { Readable } from 'node:stream';
+import { Hocuspocus } from '@hocuspocus/server';
+import {
+ AGENT_WRITE_ORIGIN,
+ AgentSessionManager,
+ applyAgentMarkdownWrite,
+} from './agent-sessions.ts';
+import { createApiExtension } from './api-extension.ts';
+import { createContentFilter } from './content-filter.ts';
+import type { DiskEvent, FileIndexEntry } from './file-watcher.ts';
+
+function makeJsonPostReq(url: string, body: unknown): IncomingMessage {
+ const readable = Readable.from(Buffer.from(JSON.stringify(body))) as unknown as IncomingMessage;
+ readable.method = 'POST';
+ readable.url = url;
+ readable.headers = { host: 'localhost', 'content-type': 'application/json' };
+ return readable;
+}
+
+function makeRes(): { res: ServerResponse; captured: { status: number; body: string } } {
+ const captured = { status: 0, body: '' };
+ const res = {
+ writeHead(status: number) {
+ captured.status = status;
+ },
+ end(body?: string) {
+ captured.body = body ?? '';
+ },
+ } as unknown as ServerResponse;
+ return { res, captured };
+}
+
+async function post(
+ ext: {
+ onRequest: (ctx: { request: IncomingMessage; response: ServerResponse }) => Promise<void>;
+ },
+ url: string,
+ body: unknown,
+): Promise<{ status: number; body: string }> {
+ const req = makeJsonPostReq(url, body);
+ const { res, captured } = makeRes();
+ await ext.onRequest({ request: req, response: res });
+ return captured;
+}
+
+describe('agent write self-registers the file index (PRD-7201)', () => {
+ test('agent-write-md registers the just-written doc into the file index', async () => {
+ const projectDir = mkdtempSync(join(tmpdir(), 'ok-write-file-index-'));
+ const contentDir = join(projectDir, 'content');
+ mkdirSync(contentDir, { recursive: true });
+ const hocuspocus = new Hocuspocus({ quiet: true });
+ const sessionManager = new AgentSessionManager(hocuspocus);
+ const events: DiskEvent[] = [];
+
+ try {
+ const ext = createApiExtension({
+ hocuspocus,
+ sessionManager,
+ contentDir,
+ getFileIndex: () => new Map(),
+ mutateFileIndex: (event) => events.push(event),
+ }) as {
+ onRequest: (ctx: { request: IncomingMessage; response: ServerResponse }) => Promise<void>;
+ };
+
+ const captured = await post(ext, '/api/agent-write-md', {
+ docName: 'evidence/new-target',
+ markdown: '# New Target\n',
+ position: 'replace',
+ });
+
+ expect(captured.status).toBe(200);
+ const docEvents = events.filter(
+ (e) => 'docName' in e && (e as { docName?: string }).docName === 'evidence/new-target',
+ );
+ expect(docEvents).toHaveLength(1);
+ expect(docEvents[0].kind).toBe('create');
+ } finally {
+ await sessionManager.closeAll();
+ rmSync(projectDir, { recursive: true, force: true });
+ }
+ });
+
+ test('a write to a doc already in the file index registers as kind:update', async () => {
+ const projectDir = mkdtempSync(join(tmpdir(), 'ok-write-file-index-'));
+ const contentDir = join(projectDir, 'content');
+ mkdirSync(contentDir, { recursive: true });
+ const hocuspocus = new Hocuspocus({ quiet: true });
+ const sessionManager = new AgentSessionManager(hocuspocus);
+ const events: DiskEvent[] = [];
+ const fileIndex = new Map<string, FileIndexEntry>([
+ [
+ 'notes',
+ {
+ size: 1,
+ modified: new Date(0).toISOString(),
+ canonicalPath: '',
+ inode: 0,
+ aliases: [],
+ kind: 'markdown',
+ },
+ ],
+ ]);
+
+ try {
+ const ext = createApiExtension({
+ hocuspocus,
+ sessionManager,
+ contentDir,
+ getFileIndex: () => fileIndex,
+ mutateFileIndex: (event) => events.push(event),
+ }) as {
+ onRequest: (ctx: { request: IncomingMessage; response: ServerResponse }) => Promise<void>;
+ };
+
+ const captured = await post(ext, '/api/agent-write-md', {
+ docName: 'notes',
+ markdown: '# Notes\n',
+ position: 'replace',
+ });
+
+ expect(captured.status).toBe(200);
+ const docEvents = events.filter(
+ (e) => 'docName' in e && (e as { docName?: string }).docName === 'notes',
+ );
+ expect(docEvents).toHaveLength(1);
+ expect(docEvents[0].kind).toBe('update');
+ } finally {
+ await sessionManager.closeAll();
+ rmSync(projectDir, { recursive: true, force: true });
+ }
+ });
+
+ test('agent-patch registers the just-edited doc into the file index', async () => {
+ const projectDir = mkdtempSync(join(tmpdir(), 'ok-write-file-index-'));
+ const contentDir = join(projectDir, 'content');
+ mkdirSync(contentDir, { recursive: true });
+ const hocuspocus = new Hocuspocus({ quiet: true });
+ const sessionManager = new AgentSessionManager(hocuspocus);
+ const events: DiskEvent[] = [];
+
+ try {
+ const session = await sessionManager.getSession('notes');
+ session.dc.document.transact(() => {
+ applyAgentMarkdownWrite(session.dc.document, '# Notes\n\nalpha\n', 'replace');
+ }, AGENT_WRITE_ORIGIN);
+
+ const ext = createApiExtension({
+ hocuspocus,
+ sessionManager,
+ contentDir,
+ getFileIndex: () => new Map(),
+ mutateFileIndex: (event) => events.push(event),
+ }) as {
+ onRequest: (ctx: { request: IncomingMessage; response: ServerResponse }) => Promise<void>;
+ };
+
+ const captured = await post(ext, '/api/agent-patch', {
+ docName: 'notes',
+ find: 'alpha',
+ replace: 'beta',
+ });
+
+ expect(captured.status).toBe(200);
+ const docEvents = events.filter(
+ (e) => 'docName' in e && (e as { docName?: string }).docName === 'notes',
+ );
+ expect(docEvents).toHaveLength(1);
+ } finally {
+ await sessionManager.closeAll();
+ rmSync(projectDir, { recursive: true, force: true });
+ }
+ });
+
+ test('frontmatter-patch registers the patched doc into the file index', async () => {
+ const projectDir = mkdtempSync(join(tmpdir(), 'ok-write-file-index-'));
+ const contentDir = join(projectDir, 'content');
+ mkdirSync(contentDir, { recursive: true });
+ const hocuspocus = new Hocuspocus({ quiet: true });
+ const sessionManager = new AgentSessionManager(hocuspocus);
+ const events: DiskEvent[] = [];
+
+ try {
+ const session = await sessionManager.getSession('notes');
+ session.dc.document.transact(() => {
+ applyAgentMarkdownWrite(session.dc.document, '# Notes\n\nbody text\n', 'replace');
+ }, AGENT_WRITE_ORIGIN);
+
+ const ext = createApiExtension({
+ hocuspocus,
+ sessionManager,
+ contentDir,
+ getFileIndex: () => new Map(),
+ mutateFileIndex: (event) => events.push(event),
+ }) as {
+ onRequest: (ctx: { request: IncomingMessage; response: ServerResponse }) => Promise<void>;
+ };
+
+ const captured = await post(ext, '/api/frontmatter-patch', {
+ docName: 'notes',
+ patch: { addedkey: 'v1' },
+ });
+
+ expect(captured.status).toBe(200);
+ const docEvents = events.filter(
+ (e) => 'docName' in e && (e as { docName?: string }).docName === 'notes',
+ );
+ expect(docEvents).toHaveLength(1);
+ } finally {
+ await sessionManager.closeAll();
+ rmSync(projectDir, { recursive: true, force: true });
+ }
+ });
+
+ test('does NOT register a content-scope-excluded doc (mirrors the watcher gate)', async () => {
+ const projectDir = mkdtempSync(join(tmpdir(), 'ok-write-file-index-excluded-'));
+ const contentDir = join(projectDir, 'content');
+ mkdirSync(contentDir, { recursive: true });
+ writeFileSync(join(contentDir, '.okignore'), 'secret.md\n', 'utf-8');
+ const hocuspocus = new Hocuspocus({ quiet: true });
+ const sessionManager = new AgentSessionManager(hocuspocus);
+ const events: DiskEvent[] = [];
+
+ try {
+ const ext = createApiExtension({
+ hocuspocus,
+ sessionManager,
+ contentDir,
+ getFileIndex: () => new Map(),
+ contentFilter: createContentFilter({ projectDir, contentDir }),
+ mutateFileIndex: (event) => events.push(event),
+ }) as {
+ onRequest: (ctx: { request: IncomingMessage; response: ServerResponse }) => Promise<void>;
+ };
+
+ const captured = await post(ext, '/api/agent-write-md', {
+ docName: 'secret',
+ markdown: '# Top Secret\n',
+ position: 'replace',
+ });
+
+ expect(captured.status).toBe(200);
+ const docEvents = events.filter(
+ (e) => 'docName' in e && (e as { docName?: string }).docName === 'secret',
+ );
+ expect(docEvents).toHaveLength(0);
+ } finally {
+ await sessionManager.closeAll();
+ rmSync(projectDir, { recursive: true, force: true });
+ }
+ });
+});
diff --git a/packages/server/src/api-backlinks.test.ts b/packages/server/src/api-backlinks.test.ts
index 04fcc9fae..576f6b161 100644
--- a/packages/server/src/api-backlinks.test.ts
+++ b/packages/server/src/api-backlinks.test.ts
@@ -6,6 +6,7 @@ import { join } from 'node:path';
import { Readable } from 'node:stream';
import { createApiExtension } from './api-extension.ts';
import { BacklinkIndex } from './backlink-index.ts';
+import { type ContentFilter, createContentFilter } from './content-filter.ts';
import type { FileIndexEntry } from './file-watcher.ts';
interface CapturedResponse {
@@ -41,7 +42,7 @@ async function callRoute(
url: string,
fileIndex: ReadonlyMap<string, FileIndexEntry>,
backlinkIndex?: BacklinkIndex,
- options?: { method?: string; enableTestRoutes?: boolean },
+ options?: { method?: string; enableTestRoutes?: boolean; contentFilter?: ContentFilter },
): Promise<CapturedResponse> {
const ext = createApiExtension({
hocuspocus: {} as never,
@@ -50,6 +51,7 @@ async function callRoute(
getFileIndex: () => fileIndex,
backlinkIndex,
enableTestRoutes: options?.enableTestRoutes,
+ ...(options?.contentFilter ? { contentFilter: options.contentFilter } : {}),
});
const req = makeReq(url, options?.method ?? 'GET');
const { res, captured } = makeRes();
@@ -386,6 +388,51 @@ describe('graph endpoints', () => {
}
});
+ test('link/title consumers resolve a graph-indexed doc the file index is missing (PRD-7201)', async () => {
+ const projectDir = mkdtempSync(join(tmpdir(), 'ok-graph-api-prd7201-'));
+ const contentDir = join(projectDir, 'content');
+ mkdirSync(join(contentDir, 'evidence'), { recursive: true });
+ try {
+ writeFileSync(
+ join(contentDir, 'alpha.md'),
+ '# Alpha\n\nLinks to [[evidence/beta]].\n',
+ 'utf-8',
+ );
+ writeFileSync(join(contentDir, 'evidence', 'beta.md'), '# Beta\n\nBody.\n', 'utf-8');
+
+ const backlinkIndex = new BacklinkIndex({ projectDir, contentDir });
+ await backlinkIndex.rebuildFromDisk();
+
+ const fileIndex = new Map<string, FileIndexEntry>([
+ [
+ 'alpha',
+ {
+ size: 10,
+ modified: new Date(0).toISOString(),
+ canonicalPath: '',
+ inode: 0,
+ aliases: [],
+ },
+ ],
+ ]);
+
+ const forward = JSON.parse(
+ (await callRoute(contentDir, '/api/forward-links?docName=alpha', fileIndex, backlinkIndex))
+ .body,
+ ) as { forwardLinks: Array<{ kind: string; docName: string; title: string }> };
+ expect(forward.forwardLinks).toEqual([
+ expect.objectContaining({ kind: 'doc', docName: 'evidence/beta', title: 'Beta' }),
+ ]);
+
+ const dead = JSON.parse(
+ (await callRoute(contentDir, '/api/dead-links', fileIndex, backlinkIndex)).body,
+ ) as { deadLinks: Array<{ target: string }> };
+ expect(dead.deadLinks).toEqual([]);
+ } finally {
+ rmSync(projectDir, { recursive: true, force: true });
+ }
+ });
+
test('returns 503 when the backlink index is unavailable', async () => {
const projectDir = mkdtempSync(join(tmpdir(), 'ok-dead-links-unavailable-'));
const contentDir = join(projectDir, 'content');
@@ -407,6 +454,7 @@ describe('graph endpoints', () => {
const contentDir = join(projectDir, 'content');
mkdirSync(contentDir, { recursive: true });
try {
+ writeFileSync(join(contentDir, '.okignore'), 'secret.md\n', 'utf-8');
writeFileSync(join(contentDir, 'public.md'), '# Public\n\nLinks to [[secret]].\n', 'utf-8');
writeFileSync(
join(contentDir, 'secret.md'),
@@ -429,9 +477,20 @@ describe('graph endpoints', () => {
const backlinkIndex = new BacklinkIndex({ projectDir, contentDir });
await backlinkIndex.rebuildFromDisk();
+ const contentFilter = createContentFilter({ projectDir, contentDir });
+
const forward = JSON.parse(
- (await callRoute(contentDir, '/api/forward-links?docName=public', fileIndex, backlinkIndex))
- .body,
+ (
+ await callRoute(
+ contentDir,
+ '/api/forward-links?docName=public',
+ fileIndex,
+ backlinkIndex,
+ {
+ contentFilter,
+ },
+ )
+ ).body,
) as {
forwardLinks: Array<{ kind: 'doc'; docName: string; title: string }>;
};
@@ -440,12 +499,17 @@ describe('graph endpoints', () => {
expect(forward.forwardLinks[0].title).toBe('secret');
const hubs = JSON.parse(
- (await callRoute(contentDir, '/api/hubs', fileIndex, backlinkIndex)).body,
+ (await callRoute(contentDir, '/api/hubs', fileIndex, backlinkIndex, { contentFilter }))
+ .body,
) as { hubs: Array<{ docName: string; title: string; count: number }> };
expect(hubs.hubs).toContainEqual({ docName: 'secret', title: 'secret', count: 1 });
const linkGraph = JSON.parse(
- (await callRoute(contentDir, '/api/link-graph', fileIndex, backlinkIndex)).body,
+ (
+ await callRoute(contentDir, '/api/link-graph', fileIndex, backlinkIndex, {
+ contentFilter,
+ })
+ ).body,
) as {
nodes: Array<{
id: string;
diff --git a/packages/server/src/api-extension.ts b/packages/server/src/api-extension.ts
index 2ac46d7f0..48cc5a373 100644
--- a/packages/server/src/api-extension.ts
+++ b/packages/server/src/api-extension.ts
@@ -2172,6 +2172,12 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
};
}
+ function isDocNameContentExcluded(docName: string): boolean {
+ if (!contentFilter) return false;
+ const relPath = isSupportedDocFile(docName) ? docName : `${docName}${getDocExtension(docName)}`;
+ return contentFilter.isExcluded(relPath);
+ }
+
function collectAdmittedDocNames(): Set<string> {
const admitted = new Set<string>();
for (const [docName, entry] of getFileIndex()) {
@@ -2196,12 +2202,26 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
} catch (err) {
log.warn({ err }, '[collectAdmittedDocNames] managed-artifact enumeration failed');
}
+ for (const docName of backlinkIndex?.getIndexedDocNames() ?? []) {
+ if (admitted.has(docName)) continue;
+ if (!isDocNameContentExcluded(docName)) admitted.add(docName);
+ }
return admitted;
}
const linkedFileExists = (contentRootRelativePath: string): boolean =>
existsSync(resolve(contentDir, contentRootRelativePath));
+ function registerWrittenDocInFileIndex(docName: string, content: string): void {
+ if (isDocNameContentExcluded(docName)) return;
+ mutateFileIndex?.({
+ kind: getFileIndex().has(docName) ? 'update' : 'create',
+ path: resolveContentEntryPath(contentDir, 'file', docName),
+ docName,
+ content,
+ });
+ }
+
function createSerializedRunner() {
let pending = Promise.resolve();
return async function runSerialized<T>(task: () => Promise<T>): Promise<T> {
@@ -3724,6 +3744,8 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
const writtenSource = session.dc.document.getText('source').toString();
+ registerWrittenDocInFileIndex(resolvedDocName, writtenSource);
+
const renderWarnings = await validateMermaidFences(writtenSource, resolvedDocName);
const admittedForLinks = collectAdmittedDocNames();
@@ -3999,6 +4021,11 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
const fmWarning = buildReconcileWarning(fmReconcile);
+ registerWrittenDocInFileIndex(
+ resolvedDocName,
+ session.dc.document.getText('source').toString(),
+ );
+
const admittedForLinks = collectAdmittedDocNames();
admittedForLinks.add(resolvedDocName);
const brokenLinks = computeBrokenOutboundLinks(
@@ -5194,6 +5221,8 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
const patchedSource = session.dc.document.getText('source').toString();
+ registerWrittenDocInFileIndex(docName, patchedSource);
+
const renderWarnings = await validateMermaidFences(patchedSource, docName);
const admittedForLinks = collectAdmittedDocNames();
diff --git a/packages/server/src/backlink-index.test.ts b/packages/server/src/backlink-index.test.ts
index 0d3bb4ce8..92367ef93 100644
--- a/packages/server/src/backlink-index.test.ts
+++ b/packages/server/src/backlink-index.test.ts
@@ -373,6 +373,30 @@ describe('BacklinkIndex', () => {
}
});
+ test('getIndexedDocNames returns one entry per indexed doc and never a referenced-but-missing target', () => {
+ const projectDir = mkdtempSync(join(tmpdir(), 'ok-backlinks-indexed-names-'));
+ const contentDir = join(projectDir, 'content');
+ mkdirSync(contentDir, { recursive: true });
+ try {
+ const index = new BacklinkIndex({ projectDir, contentDir });
+ index.updateDocument('report', [
+ { target: 'evidence/new-target', anchor: null, snippet: 'see new target' },
+ { target: 'evidence/ghost', anchor: null, snippet: 'see ghost' },
+ ]);
+ index.updateDocument('evidence/new-target', []);
+
+ expect(new Set(index.getIndexedDocNames())).toEqual(
+ new Set(['report', 'evidence/new-target']),
+ );
+ expect(index.getIndexedDocNames()).not.toContain('evidence/ghost');
+
+ index.deleteDocument('evidence/new-target');
+ expect(index.getIndexedDocNames()).toEqual(['report']);
+ } finally {
+ rmSync(projectDir, { recursive: true, force: true });
+ }
+ });
+
test('getDeadLinks reports a target as dead again after deleteDocument removes its forward node', () => {
const projectDir = mkdtempSync(join(tmpdir(), 'ok-backlinks-dead-after-delete-'));
const contentDir = join(projectDir, 'content');
diff --git a/packages/server/src/backlink-index.ts b/packages/server/src/backlink-index.ts
index 0ee06a95b..2d805dd81 100644
--- a/packages/server/src/backlink-index.ts
+++ b/packages/server/src/backlink-index.ts
@@ -1163,6 +1163,10 @@ export class BacklinkIndex {
.slice(0, limit);
}
+ getIndexedDocNames(branch = this.activeBranch): string[] {
+ return [...this.getState(branch).forward.keys()];
+ }
+
getDeadLinks(
admittedDocs: Iterable<string>,
sourceDocNames?: readonly string[],