Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
fix(open-knowledge): unify doc-existence oracle so file-index lag stops causing stale link/title results (PRD-7201) (#2255)
mike-inkeep2b750aabdca2
diff --git a/.changeset/unify-doc-existence-oracle.md b/.changeset/unify-doc-existence-oracle.mdnew file mode 100644index 000000000..c2b24f51f--- /dev/null+++ b/.changeset/unify-doc-existence-oracle.md@@ -0,0 +1,6 @@+---+"@inkeep/open-knowledge-server": patch+"@inkeep/open-knowledge": patch+---++Stop the file-watcher's indexing lag from making freshly-written docs look missing across link, title, and dead-link results. The file index the watcher maintains can lag an in-session write — or permanently drop the create event for a file written into a just-created subfolder — which made a brand-new doc render as a raw red-link name (instead of its title) and could flag a valid link to it as dead, until a server restart. Two complementary fixes close the gap: doc-existence now also trusts the link graph (updated in-process on every write), and the agent write/edit/frontmatter-patch handlers register the doc they just wrote into the file index immediately, mirroring page creation. Both paths honor content-scope exclusions, so a gitignore/okignore'd doc never leaks its title through these endpoints.diff --git a/packages/server/src/api-agent-write-file-index.test.ts b/packages/server/src/api-agent-write-file-index.test.tsnew file mode 100644index 000000000..60077348d--- /dev/null+++ b/packages/server/src/api-agent-write-file-index.test.ts@@ -0,0 +1,257 @@+import { describe, expect, test } from 'bun:test';+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';+import type { IncomingMessage, ServerResponse } from 'node:http';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { Readable } from 'node:stream';+import { Hocuspocus } from '@hocuspocus/server';+import {+ AGENT_WRITE_ORIGIN,+ AgentSessionManager,+ applyAgentMarkdownWrite,+} from './agent-sessions.ts';+import { createApiExtension } from './api-extension.ts';+import { createContentFilter } from './content-filter.ts';+import type { DiskEvent, FileIndexEntry } from './file-watcher.ts';++function makeJsonPostReq(url: string, body: unknown): IncomingMessage {+ const readable = Readable.from(Buffer.from(JSON.stringify(body))) as unknown as IncomingMessage;+ readable.method = 'POST';+ readable.url = url;+ readable.headers = { host: 'localhost', 'content-type': 'application/json' };+ return readable;+}++function makeRes(): { res: ServerResponse; captured: { status: number; body: string } } {+ const captured = { status: 0, body: '' };+ const res = {+ writeHead(status: number) {+ captured.status = status;+ },+ end(body?: string) {+ captured.body = body ?? '';+ },+ } as unknown as ServerResponse;+ return { res, captured };+}++async function post(+ ext: {+ onRequest: (ctx: { request: IncomingMessage; response: ServerResponse }) => Promise<void>;+ },+ url: string,+ body: unknown,+): Promise<{ status: number; body: string }> {+ const req = makeJsonPostReq(url, body);+ const { res, captured } = makeRes();+ await ext.onRequest({ request: req, response: res });+ return captured;+}++describe('agent write self-registers the file index (PRD-7201)', () => {+ test('agent-write-md registers the just-written doc into the file index', async () => {+ const projectDir = mkdtempSync(join(tmpdir(), 'ok-write-file-index-'));+ const contentDir = join(projectDir, 'content');+ mkdirSync(contentDir, { recursive: true });+ const hocuspocus = new Hocuspocus({ quiet: true });+ const sessionManager = new AgentSessionManager(hocuspocus);+ const events: DiskEvent[] = [];++ try {+ const ext = createApiExtension({+ hocuspocus,+ sessionManager,+ contentDir,+ getFileIndex: () => new Map(),+ mutateFileIndex: (event) => events.push(event),+ }) as {+ onRequest: (ctx: { request: IncomingMessage; response: ServerResponse }) => Promise<void>;+ };++ const captured = await post(ext, '/api/agent-write-md', {+ docName: 'evidence/new-target',+ markdown: '# New Target\n',+ position: 'replace',+ });++ expect(captured.status).toBe(200);+ const docEvents = events.filter(+ (e) => 'docName' in e && (e as { docName?: string }).docName === 'evidence/new-target',+ );+ expect(docEvents).toHaveLength(1);+ expect(docEvents[0].kind).toBe('create');+ } finally {+ await sessionManager.closeAll();+ rmSync(projectDir, { recursive: true, force: true });+ }+ });++ test('a write to a doc already in the file index registers as kind:update', async () => {+ const projectDir = mkdtempSync(join(tmpdir(), 'ok-write-file-index-'));+ const contentDir = join(projectDir, 'content');+ mkdirSync(contentDir, { recursive: true });+ const hocuspocus = new Hocuspocus({ quiet: true });+ const sessionManager = new AgentSessionManager(hocuspocus);+ const events: DiskEvent[] = [];+ const fileIndex = new Map<string, FileIndexEntry>([+ [+ 'notes',+ {+ size: 1,+ modified: new Date(0).toISOString(),+ canonicalPath: '',+ inode: 0,+ aliases: [],+ kind: 'markdown',+ },+ ],+ ]);++ try {+ const ext = createApiExtension({+ hocuspocus,+ sessionManager,+ contentDir,+ getFileIndex: () => fileIndex,+ mutateFileIndex: (event) => events.push(event),+ }) as {+ onRequest: (ctx: { request: IncomingMessage; response: ServerResponse }) => Promise<void>;+ };++ const captured = await post(ext, '/api/agent-write-md', {+ docName: 'notes',+ markdown: '# Notes\n',+ position: 'replace',+ });++ expect(captured.status).toBe(200);+ const docEvents = events.filter(+ (e) => 'docName' in e && (e as { docName?: string }).docName === 'notes',+ );+ expect(docEvents).toHaveLength(1);+ expect(docEvents[0].kind).toBe('update');+ } finally {+ await sessionManager.closeAll();+ rmSync(projectDir, { recursive: true, force: true });+ }+ });++ test('agent-patch registers the just-edited doc into the file index', async () => {+ const projectDir = mkdtempSync(join(tmpdir(), 'ok-write-file-index-'));+ const contentDir = join(projectDir, 'content');+ mkdirSync(contentDir, { recursive: true });+ const hocuspocus = new Hocuspocus({ quiet: true });+ const sessionManager = new AgentSessionManager(hocuspocus);+ const events: DiskEvent[] = [];++ try {+ const session = await sessionManager.getSession('notes');+ session.dc.document.transact(() => {+ applyAgentMarkdownWrite(session.dc.document, '# Notes\n\nalpha\n', 'replace');+ }, AGENT_WRITE_ORIGIN);++ const ext = createApiExtension({+ hocuspocus,+ sessionManager,+ contentDir,+ getFileIndex: () => new Map(),+ mutateFileIndex: (event) => events.push(event),+ }) as {+ onRequest: (ctx: { request: IncomingMessage; response: ServerResponse }) => Promise<void>;+ };++ const captured = await post(ext, '/api/agent-patch', {+ docName: 'notes',+ find: 'alpha',+ replace: 'beta',+ });++ expect(captured.status).toBe(200);+ const docEvents = events.filter(+ (e) => 'docName' in e && (e as { docName?: string }).docName === 'notes',+ );+ expect(docEvents).toHaveLength(1);+ } finally {+ await sessionManager.closeAll();+ rmSync(projectDir, { recursive: true, force: true });+ }+ });++ test('frontmatter-patch registers the patched doc into the file index', async () => {+ const projectDir = mkdtempSync(join(tmpdir(), 'ok-write-file-index-'));+ const contentDir = join(projectDir, 'content');+ mkdirSync(contentDir, { recursive: true });+ const hocuspocus = new Hocuspocus({ quiet: true });+ const sessionManager = new AgentSessionManager(hocuspocus);+ const events: DiskEvent[] = [];++ try {+ const session = await sessionManager.getSession('notes');+ session.dc.document.transact(() => {+ applyAgentMarkdownWrite(session.dc.document, '# Notes\n\nbody text\n', 'replace');+ }, AGENT_WRITE_ORIGIN);++ const ext = createApiExtension({+ hocuspocus,+ sessionManager,+ contentDir,+ getFileIndex: () => new Map(),+ mutateFileIndex: (event) => events.push(event),+ }) as {+ onRequest: (ctx: { request: IncomingMessage; response: ServerResponse }) => Promise<void>;+ };++ const captured = await post(ext, '/api/frontmatter-patch', {+ docName: 'notes',+ patch: { addedkey: 'v1' },+ });++ expect(captured.status).toBe(200);+ const docEvents = events.filter(+ (e) => 'docName' in e && (e as { docName?: string }).docName === 'notes',+ );+ expect(docEvents).toHaveLength(1);+ } finally {+ await sessionManager.closeAll();+ rmSync(projectDir, { recursive: true, force: true });+ }+ });++ test('does NOT register a content-scope-excluded doc (mirrors the watcher gate)', async () => {+ const projectDir = mkdtempSync(join(tmpdir(), 'ok-write-file-index-excluded-'));+ const contentDir = join(projectDir, 'content');+ mkdirSync(contentDir, { recursive: true });+ writeFileSync(join(contentDir, '.okignore'), 'secret.md\n', 'utf-8');+ const hocuspocus = new Hocuspocus({ quiet: true });+ const sessionManager = new AgentSessionManager(hocuspocus);+ const events: DiskEvent[] = [];++ try {+ const ext = createApiExtension({+ hocuspocus,+ sessionManager,+ contentDir,+ getFileIndex: () => new Map(),+ contentFilter: createContentFilter({ projectDir, contentDir }),+ mutateFileIndex: (event) => events.push(event),+ }) as {+ onRequest: (ctx: { request: IncomingMessage; response: ServerResponse }) => Promise<void>;+ };++ const captured = await post(ext, '/api/agent-write-md', {+ docName: 'secret',+ markdown: '# Top Secret\n',+ position: 'replace',+ });++ expect(captured.status).toBe(200);+ const docEvents = events.filter(+ (e) => 'docName' in e && (e as { docName?: string }).docName === 'secret',+ );+ expect(docEvents).toHaveLength(0);+ } finally {+ await sessionManager.closeAll();+ rmSync(projectDir, { recursive: true, force: true });+ }+ });+});diff --git a/packages/server/src/api-backlinks.test.ts b/packages/server/src/api-backlinks.test.tsindex 04fcc9fae..576f6b161 100644--- a/packages/server/src/api-backlinks.test.ts+++ b/packages/server/src/api-backlinks.test.ts@@ -6,6 +6,7 @@ import { join } from 'node:path';import { Readable } from 'node:stream';import { createApiExtension } from './api-extension.ts';import { BacklinkIndex } from './backlink-index.ts';+import { type ContentFilter, createContentFilter } from './content-filter.ts';import type { FileIndexEntry } from './file-watcher.ts';interface CapturedResponse {@@ -41,7 +42,7 @@ async function callRoute(url: string,fileIndex: ReadonlyMap<string, FileIndexEntry>,backlinkIndex?: BacklinkIndex,- options?: { method?: string; enableTestRoutes?: boolean },+ options?: { method?: string; enableTestRoutes?: boolean; contentFilter?: ContentFilter },): Promise<CapturedResponse> {const ext = createApiExtension({hocuspocus: {} as never,@@ -50,6 +51,7 @@ async function callRoute(getFileIndex: () => fileIndex,backlinkIndex,enableTestRoutes: options?.enableTestRoutes,+ ...(options?.contentFilter ? { contentFilter: options.contentFilter } : {}),});const req = makeReq(url, options?.method ?? 'GET');const { res, captured } = makeRes();@@ -386,6 +388,51 @@ describe('graph endpoints', () => {}});+ test('link/title consumers resolve a graph-indexed doc the file index is missing (PRD-7201)', async () => {+ const projectDir = mkdtempSync(join(tmpdir(), 'ok-graph-api-prd7201-'));+ const contentDir = join(projectDir, 'content');+ mkdirSync(join(contentDir, 'evidence'), { recursive: true });+ try {+ writeFileSync(+ join(contentDir, 'alpha.md'),+ '# Alpha\n\nLinks to [[evidence/beta]].\n',+ 'utf-8',+ );+ writeFileSync(join(contentDir, 'evidence', 'beta.md'), '# Beta\n\nBody.\n', 'utf-8');++ const backlinkIndex = new BacklinkIndex({ projectDir, contentDir });+ await backlinkIndex.rebuildFromDisk();++ const fileIndex = new Map<string, FileIndexEntry>([+ [+ 'alpha',+ {+ size: 10,+ modified: new Date(0).toISOString(),+ canonicalPath: '',+ inode: 0,+ aliases: [],+ },+ ],+ ]);++ const forward = JSON.parse(+ (await callRoute(contentDir, '/api/forward-links?docName=alpha', fileIndex, backlinkIndex))+ .body,+ ) as { forwardLinks: Array<{ kind: string; docName: string; title: string }> };+ expect(forward.forwardLinks).toEqual([+ expect.objectContaining({ kind: 'doc', docName: 'evidence/beta', title: 'Beta' }),+ ]);++ const dead = JSON.parse(+ (await callRoute(contentDir, '/api/dead-links', fileIndex, backlinkIndex)).body,+ ) as { deadLinks: Array<{ target: string }> };+ expect(dead.deadLinks).toEqual([]);+ } finally {+ rmSync(projectDir, { recursive: true, force: true });+ }+ });+test('returns 503 when the backlink index is unavailable', async () => {const projectDir = mkdtempSync(join(tmpdir(), 'ok-dead-links-unavailable-'));const contentDir = join(projectDir, 'content');@@ -407,6 +454,7 @@ describe('graph endpoints', () => {const contentDir = join(projectDir, 'content');mkdirSync(contentDir, { recursive: true });try {+ writeFileSync(join(contentDir, '.okignore'), 'secret.md\n', 'utf-8');writeFileSync(join(contentDir, 'public.md'), '# Public\n\nLinks to [[secret]].\n', 'utf-8');writeFileSync(join(contentDir, 'secret.md'),@@ -429,9 +477,20 @@ describe('graph endpoints', () => {const backlinkIndex = new BacklinkIndex({ projectDir, contentDir });await backlinkIndex.rebuildFromDisk();+ const contentFilter = createContentFilter({ projectDir, contentDir });+const forward = JSON.parse(- (await callRoute(contentDir, '/api/forward-links?docName=public', fileIndex, backlinkIndex))- .body,+ (+ await callRoute(+ contentDir,+ '/api/forward-links?docName=public',+ fileIndex,+ backlinkIndex,+ {+ contentFilter,+ },+ )+ ).body,) as {forwardLinks: Array<{ kind: 'doc'; docName: string; title: string }>;};@@ -440,12 +499,17 @@ describe('graph endpoints', () => {expect(forward.forwardLinks[0].title).toBe('secret');const hubs = JSON.parse(- (await callRoute(contentDir, '/api/hubs', fileIndex, backlinkIndex)).body,+ (await callRoute(contentDir, '/api/hubs', fileIndex, backlinkIndex, { contentFilter }))+ .body,) as { hubs: Array<{ docName: string; title: string; count: number }> };expect(hubs.hubs).toContainEqual({ docName: 'secret', title: 'secret', count: 1 });const linkGraph = JSON.parse(- (await callRoute(contentDir, '/api/link-graph', fileIndex, backlinkIndex)).body,+ (+ await callRoute(contentDir, '/api/link-graph', fileIndex, backlinkIndex, {+ contentFilter,+ })+ ).body,) as {nodes: Array<{id: string;diff --git a/packages/server/src/api-extension.ts b/packages/server/src/api-extension.tsindex 2ac46d7f0..48cc5a373 100644--- a/packages/server/src/api-extension.ts+++ b/packages/server/src/api-extension.ts@@ -2172,6 +2172,12 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {};}+ function isDocNameContentExcluded(docName: string): boolean {+ if (!contentFilter) return false;+ const relPath = isSupportedDocFile(docName) ? docName : `${docName}${getDocExtension(docName)}`;+ return contentFilter.isExcluded(relPath);+ }+function collectAdmittedDocNames(): Set<string> {const admitted = new Set<string>();for (const [docName, entry] of getFileIndex()) {@@ -2196,12 +2202,26 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {} catch (err) {log.warn({ err }, '[collectAdmittedDocNames] managed-artifact enumeration failed');}+ for (const docName of backlinkIndex?.getIndexedDocNames() ?? []) {+ if (admitted.has(docName)) continue;+ if (!isDocNameContentExcluded(docName)) admitted.add(docName);+ }return admitted;}const linkedFileExists = (contentRootRelativePath: string): boolean =>existsSync(resolve(contentDir, contentRootRelativePath));+ function registerWrittenDocInFileIndex(docName: string, content: string): void {+ if (isDocNameContentExcluded(docName)) return;+ mutateFileIndex?.({+ kind: getFileIndex().has(docName) ? 'update' : 'create',+ path: resolveContentEntryPath(contentDir, 'file', docName),+ docName,+ content,+ });+ }+function createSerializedRunner() {let pending = Promise.resolve();return async function runSerialized<T>(task: () => Promise<T>): Promise<T> {@@ -3724,6 +3744,8 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {const writtenSource = session.dc.document.getText('source').toString();+ registerWrittenDocInFileIndex(resolvedDocName, writtenSource);+const renderWarnings = await validateMermaidFences(writtenSource, resolvedDocName);const admittedForLinks = collectAdmittedDocNames();@@ -3999,6 +4021,11 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {const fmWarning = buildReconcileWarning(fmReconcile);+ registerWrittenDocInFileIndex(+ resolvedDocName,+ session.dc.document.getText('source').toString(),+ );+const admittedForLinks = collectAdmittedDocNames();admittedForLinks.add(resolvedDocName);const brokenLinks = computeBrokenOutboundLinks(@@ -5194,6 +5221,8 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {const patchedSource = session.dc.document.getText('source').toString();+ registerWrittenDocInFileIndex(docName, patchedSource);+const renderWarnings = await validateMermaidFences(patchedSource, docName);const admittedForLinks = collectAdmittedDocNames();diff --git a/packages/server/src/backlink-index.test.ts b/packages/server/src/backlink-index.test.tsindex 0d3bb4ce8..92367ef93 100644--- a/packages/server/src/backlink-index.test.ts+++ b/packages/server/src/backlink-index.test.ts@@ -373,6 +373,30 @@ describe('BacklinkIndex', () => {}});+ test('getIndexedDocNames returns one entry per indexed doc and never a referenced-but-missing target', () => {+ const projectDir = mkdtempSync(join(tmpdir(), 'ok-backlinks-indexed-names-'));+ const contentDir = join(projectDir, 'content');+ mkdirSync(contentDir, { recursive: true });+ try {+ const index = new BacklinkIndex({ projectDir, contentDir });+ index.updateDocument('report', [+ { target: 'evidence/new-target', anchor: null, snippet: 'see new target' },+ { target: 'evidence/ghost', anchor: null, snippet: 'see ghost' },+ ]);+ index.updateDocument('evidence/new-target', []);++ expect(new Set(index.getIndexedDocNames())).toEqual(+ new Set(['report', 'evidence/new-target']),+ );+ expect(index.getIndexedDocNames()).not.toContain('evidence/ghost');++ index.deleteDocument('evidence/new-target');+ expect(index.getIndexedDocNames()).toEqual(['report']);+ } finally {+ rmSync(projectDir, { recursive: true, force: true });+ }+ });+test('getDeadLinks reports a target as dead again after deleteDocument removes its forward node', () => {const projectDir = mkdtempSync(join(tmpdir(), 'ok-backlinks-dead-after-delete-'));const contentDir = join(projectDir, 'content');diff --git a/packages/server/src/backlink-index.ts b/packages/server/src/backlink-index.tsindex 0ee06a95b..2d805dd81 100644--- a/packages/server/src/backlink-index.ts+++ b/packages/server/src/backlink-index.ts@@ -1163,6 +1163,10 @@ export class BacklinkIndex {.slice(0, limit);}+ getIndexedDocNames(branch = this.activeBranch): string[] {+ return [...this.getState(branch).forward.keys()];+ }+getDeadLinks(admittedDocs: Iterable<string>,sourceDocNames?: readonly string[],