Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
PRD-9586 Reduce backlink-cache disk writes (#7459)
Andrew Mikofalvy2534085210e7
diff --git a/.changeset/backlink-cache-write-volume.md b/.changeset/backlink-cache-write-volume.mdnew file mode 100644index 000000000..75e373d62--- /dev/null+++ b/.changeset/backlink-cache-write-volume.md@@ -0,0 +1,6 @@+---+'@inkeep/open-knowledge': patch+'@inkeep/open-knowledge-server': patch+---++Reduce disk writes from backlink caching during sustained editing.diff --git a/packages/server/src/backlink-index.test.ts b/packages/server/src/backlink-index.test.tsindex 0fe73b841..2f3727f92 100644--- a/packages/server/src/backlink-index.test.ts+++ b/packages/server/src/backlink-index.test.ts@@ -774,6 +774,11 @@ describe('BacklinkIndex', () => {const index = new BacklinkIndex({ projectDir, contentDir });index.updateDocumentFromMarkdown('alpha', '# Alpha\n\nSee [[ghost]].\n');await index.saveToDisk();+ const raw = readFileSync(+ join(projectDir, '.ok', LOCAL_DIR, 'cache', 'main', 'backlinks.json'),+ 'utf-8',+ );+ expect(raw).toBe(JSON.stringify(JSON.parse(raw)));const reloaded = new BacklinkIndex({ projectDir, contentDir });expect(await reloaded.loadFromDisk()).toBe(true);diff --git a/packages/server/src/backlink-index.ts b/packages/server/src/backlink-index.tsindex 11b5a6340..0258f8108 100644--- a/packages/server/src/backlink-index.ts+++ b/packages/server/src/backlink-index.ts@@ -2158,7 +2158,7 @@ export class BacklinkIndex {...serializeState(state),...(mtimes ? { mtimes: Object.fromEntries(mtimes) } : {}),};- await atomicWriteFile(filePath, JSON.stringify(data, null, 2), { fs: tracedAtomicFs });+ await atomicWriteFile(filePath, JSON.stringify(data), { fs: tracedAtomicFs });}async loadFromDisk(branch = this.activeBranch): Promise<boolean> {diff --git a/packages/server/src/derived-document-index.test.ts b/packages/server/src/derived-document-index.test.tsindex 68e8f6d26..10024855e 100644--- a/packages/server/src/derived-document-index.test.ts+++ b/packages/server/src/derived-document-index.test.ts@@ -76,6 +76,152 @@ afterEach(async () => {});describe('DerivedDocumentIndex', () => {+ test('normal cache saves stay bounded during sustained edits and flush latest state on close', async () => {+ vi.useFakeTimers();+ const backlinkSave = vi.spyOn(BacklinkIndex.prototype, 'saveToDisk');+ const tagSave = vi.spyOn(TagIndex.prototype, 'saveToDisk');+ const rig = createRig();+ await settleStartup(rig);+ backlinkSave.mockClear();+ tagSave.mockClear();++ for (let i = 0; i < 75; i++) {+ const markdown = `See [[target-${i}]]. #stable\n`;+ writeDoc(rig, 'source.md', markdown);+ switch (i % 4) {+ case 0:+ await rig.index.recordDirectDocument('source', markdown);+ break;+ case 1:+ await rig.index.recordDiskUpsert('source', markdown);+ break;+ case 2:+ await rig.index.recordDurableStore('source', markdown);+ break;+ case 3:+ await rig.index.recordLinkRewrite('source', markdown);+ break;+ }+ expect(await rig.index.getBacklinks(`target-${i}`)).toHaveLength(1);+ await vi.advanceTimersByTimeAsync(1000);+ await rig.index.getIndexedDocNames();+ }++ expect(backlinkSave).toHaveBeenCalledTimes(3);+ expect(tagSave).toHaveBeenCalledTimes(3);+ const warm = new BacklinkIndex(rig);+ expect(await warm.loadFromDisk()).toBe(true);+ expect(warm.getBacklinks('target-59')).toHaveLength(1);+ expect(warm.getBacklinks('target-74')).toEqual([]);++ await vi.advanceTimersByTimeAsync(15_000);+ await rig.index.getIndexedDocNames();+ expect(backlinkSave).toHaveBeenCalledTimes(4);+ const quiet = new BacklinkIndex(rig);+ expect(await quiet.loadFromDisk()).toBe(true);+ expect(quiet.getBacklinks('target-74')).toHaveLength(1);++ writeDoc(rig, 'late.md', '#late-tag\n');+ await rig.index.recordDiskUpsert('late', '#late-tag\n');+ writeDoc(rig, 'source.md', 'See [[close-target]]. #stable\n');+ await rig.index.recordDurableStore('source', 'See [[close-target]]. #stable\n');+ await rig.index.close();+ expect(backlinkSave).toHaveBeenCalledTimes(5);+ expect(tagSave).toHaveBeenCalledTimes(5);+ await vi.advanceTimersByTimeAsync(60_000);+ expect(backlinkSave).toHaveBeenCalledTimes(5);+ expect(tagSave).toHaveBeenCalledTimes(5);++ const closed = new BacklinkIndex(rig);+ expect(await closed.loadFromDisk()).toBe(true);+ expect(closed.getBacklinks('close-target')).toHaveLength(1);+ expect(closed.getBacklinks('target-74')).toEqual([]);+ const tags = new TagIndex(rig);+ expect(await tags.loadFromDisk()).toBe(true);+ expect(tags.getDocsForTagWithMatches('late-tag')).toEqual([+ { docName: 'late', matchingTags: ['late-tag'] },+ ]);+ await tags.close();+ });++ test('a queued cache save checks the minimum interval when it actually executes', async () => {+ vi.useFakeTimers();+ const originalSave = BacklinkIndex.prototype.saveToDisk;+ const backlinkSave = vi.spyOn(BacklinkIndex.prototype, 'saveToDisk');+ const rig = createRig();+ await settleStartup(rig);+ backlinkSave.mockClear();+ const started = Promise.withResolvers<void>();+ const release = Promise.withResolvers<void>();+ backlinkSave.mockImplementationOnce(async function (this: BacklinkIndex, branch) {+ started.resolve();+ await release.promise;+ await originalSave.call(this, branch);+ });++ writeDoc(rig, 'source.md', 'See [[queued-target]].\n');+ await rig.index.recordDiskUpsert('source', 'See [[queued-target]].\n');+ const running = rig.index.recordDirectDocument('running', '# Running\n');+ await started.promise;+ await vi.advanceTimersByTimeAsync(5000);+ release.resolve();+ await running;+ await rig.index.getIndexedDocNames();++ expect(backlinkSave).toHaveBeenCalledTimes(1);+ await vi.advanceTimersByTimeAsync(29_999);+ await rig.index.getIndexedDocNames();+ expect(backlinkSave).toHaveBeenCalledTimes(1);+ await vi.advanceTimersByTimeAsync(1);+ await rig.index.getIndexedDocNames();+ expect(backlinkSave).toHaveBeenCalledTimes(2);+ const cached = new BacklinkIndex(rig);+ expect(await cached.loadFromDisk()).toBe(true);+ expect(cached.getBacklinks('queued-target')).toHaveLength(1);+ });++ test('a queued throttled save cannot rearm an old-branch timer during a switch', async () => {+ vi.useFakeTimers();+ const originalSave = BacklinkIndex.prototype.saveToDisk;+ const backlinkSave = vi.spyOn(BacklinkIndex.prototype, 'saveToDisk');+ const tagSave = vi.spyOn(TagIndex.prototype, 'saveToDisk');+ const rig = createRig();+ await settleStartup(rig);+ const started = Promise.withResolvers<void>();+ const release = Promise.withResolvers<void>();+ backlinkSave.mockImplementationOnce(async function (this: BacklinkIndex, branch) {+ started.resolve();+ await release.promise;+ await originalSave.call(this, branch);+ });++ writeDoc(rig, 'old.md', 'See [[old-target]].\n');+ await rig.index.recordDiskUpsert('old', 'See [[old-target]].\n');+ const running = rig.index.recordDirectDocument('running', '# Running\n');+ await started.promise;+ await vi.advanceTimersByTimeAsync(2000);+ const switching = rig.index.beginBranchSwitch('feature');+ release.resolve();+ await running;+ const transition = await switching;+ unlinkSync(join(rig.contentDir, 'old.md'));+ writeDoc(rig, 'feature.md', 'See [[feature-target]]. #feature\n');+ await rig.index.settleBranchFromDisk(transition);+ backlinkSave.mockClear();+ tagSave.mockClear();+ await vi.advanceTimersByTimeAsync(60_000);+ await rig.index.getIndexedDocNames();+ expect(backlinkSave).not.toHaveBeenCalled();+ expect(tagSave).not.toHaveBeenCalled();++ const cached = new BacklinkIndex(rig);+ expect(await cached.loadFromDisk('feature')).toBe(true);+ expect(cached.getBacklinks('feature-target', 'feature')).toHaveLength(1);+ expect(cached.getBacklinks('old-target', 'feature')).toEqual([]);+ expect(await cached.loadFromDisk('main')).toBe(true);+ expect(cached.getBacklinks('old-target', 'main')).toHaveLength(1);+ });+test('every body feeder refreshes the resolved graph with unchanged document inventory', async () => {const rig = createRig();writeDoc(rig, 'notes/beta.md', '# Beta');diff --git a/packages/server/src/derived-document-index.ts b/packages/server/src/derived-document-index.tsindex 98969dda9..1abd6a412 100644--- a/packages/server/src/derived-document-index.ts+++ b/packages/server/src/derived-document-index.ts@@ -30,7 +30,8 @@ import { TagIndex, type TagSummaryEntry } from './tag-index.ts';const log = getLogger('derived-document-index');const SLOW_STARTUP_LOG_THRESHOLD_MS = 10_000;-const DERIVED_INDEX_SAVE_DEBOUNCE_MS = 2000;+const DERIVED_INDEX_SAVE_COALESCE_MS = 2000;+const DERIVED_INDEX_SAVE_MIN_INTERVAL_MS = 30_000;const LOCAL_TARGET_REBUILD_RETRY_BASE_MS = 1000;const LOCAL_TARGET_REBUILD_RETRY_MAX_MS = 30_000;const LOCAL_TARGET_FILE_EXISTENCE_SWEEP_MS = 5000;@@ -181,6 +182,7 @@ export class DerivedDocumentIndexprivate readonly readyBarrier = Promise.withResolvers<void>();private tail: Promise<void> = Promise.resolve();private saveTimer: ReturnType<typeof setTimeout> | null = null;+ private lastNormalSaveAt: number | null = null;private localTargetRebuildRetryTimer: ReturnType<typeof setTimeout> | null = null;private readonly localTargetFileExistenceTimer: ReturnType<typeof setInterval>;private localTargetRebuildRetryAttempt = 0;@@ -188,7 +190,7 @@ export class DerivedDocumentIndexprivate graphFileTargets: TargetNamespace<'file'> | null = null;private startupBranch: string | null = null;private graphBuiltWithoutFileOracle = false;- private saveTagsOnNextDebounce = false;+ private saveTagsOnNextFlush = false;private branchTransition: BranchTransition | null = null;private startupBegun = false;private startupBegunAt = 0;@@ -519,10 +521,10 @@ export class DerivedDocumentIndexif (tagsChanged) {this.reconcileLocalTargetInventory();- await this.saveBothLogOnly('direct-batch');+ await this.saveMutationCache(true, 'direct-batch');this.signalAllRelations();} else {- await this.saveBacklinksLogOnly('direct-batch');+ await this.saveMutationCache(false, 'direct-batch');this.signalBacklinks();}});@@ -684,7 +686,7 @@ export class DerivedDocumentIndexif (this.closePromise) return this.closePromise;const pendingSave = this.cancelScheduledSave();if (pendingSave !== null) {- void this.enqueueInternal(() => this.saveScheduledCache(pendingSave, 'close-flush'));+ void this.enqueueInternal(() => this.saveMutationCache(pendingSave, 'close-flush'));}this.closed = true;clearInterval(this.localTargetFileExistenceTimer);@@ -949,33 +951,51 @@ export class DerivedDocumentIndex}private scheduleSave(includeTags = true): void {- const saveTags = this.saveTagsOnNextDebounce || includeTags;- this.cancelScheduledSave();- this.saveTagsOnNextDebounce = saveTags;+ if (this.closed || this.branchTransition !== null) return;+ this.saveTagsOnNextFlush ||= includeTags;+ if (this.saveTimer !== null) return;+ const delay =+ this.lastNormalSaveAt === null+ ? DERIVED_INDEX_SAVE_COALESCE_MS+ : Math.max(+ DERIVED_INDEX_SAVE_COALESCE_MS,+ DERIVED_INDEX_SAVE_MIN_INTERVAL_MS - (performance.now() - this.lastNormalSaveAt),+ );this.saveTimer = setTimeout(() => {this.saveTimer = null;- const saveTags = this.saveTagsOnNextDebounce;- this.saveTagsOnNextDebounce = false;- void this.enqueueInternal(() => this.saveScheduledCache(saveTags, 'debounced')).catch(+ const saveTags = this.saveTagsOnNextFlush;+ this.saveTagsOnNextFlush = false;+ void this.enqueueInternal(() => this.saveMutationCache(saveTags, 'coalesced')).catch((err) => {- log.warn({ err }, 'Failed to enqueue debounced derived-index cache save');+ log.warn({ err }, 'Failed to enqueue coalesced derived-index cache save');},);- }, DERIVED_INDEX_SAVE_DEBOUNCE_MS);+ }, delay);}private cancelScheduledSave(): boolean | null {- const pendingSave = this.saveTimer === null ? null : this.saveTagsOnNextDebounce;+ const pendingSave = this.saveTimer === null ? null : this.saveTagsOnNextFlush;if (this.saveTimer !== null) clearTimeout(this.saveTimer);this.saveTimer = null;- this.saveTagsOnNextDebounce = false;+ this.saveTagsOnNextFlush = false;return pendingSave;}- private saveScheduledCache(includeTags: boolean, reason: string): Promise<void> {- return includeTags- ? this.saveBothLogOnly(reason)- : this.saveBacklinksLogOnly(`${reason}-backlinks`);+ private async saveMutationCache(includeTags: boolean, reason: string): Promise<void> {+ if (+ !this.closed &&+ this.lastNormalSaveAt !== null &&+ performance.now() - this.lastNormalSaveAt < DERIVED_INDEX_SAVE_MIN_INTERVAL_MS+ ) {+ this.scheduleSave(includeTags);+ return;+ }+ if (includeTags) {+ await this.saveBothLogOnly(reason);+ } else {+ await this.saveBacklinksLogOnly(`${reason}-backlinks`);+ }+ this.lastNormalSaveAt = performance.now();}private releaseBranchTransition(transition: DerivedDocumentIndexBranchTransition | null): void {diff --git a/packages/server/src/tag-index.persistence.test.ts b/packages/server/src/tag-index.persistence.test.tsindex f6ccf755e..461430603 100644--- a/packages/server/src/tag-index.persistence.test.ts+++ b/packages/server/src/tag-index.persistence.test.ts@@ -64,6 +64,8 @@ describe('TagIndex persistence', () => {const first = newIndex(rig);await first.init();await first.saveToDisk();+ const raw = await readFile(rig.snapshotPath, 'utf-8');+ expect(raw).toBe(JSON.stringify(JSON.parse(raw)));const second = newIndex(rig);expect(await second.loadFromDisk()).toBe(true);diff --git a/packages/server/src/tag-index.ts b/packages/server/src/tag-index.tsindex eb1a55fcf..14ee6d2bf 100644--- a/packages/server/src/tag-index.ts+++ b/packages/server/src/tag-index.ts@@ -371,7 +371,7 @@ export class TagIndex {files: Object.fromEntries(this.fileMeta),};await tracedMkdir(dirname(filePath), { recursive: true });- await atomicWriteFile(filePath, JSON.stringify(snapshot, null, 2), { fs: tracedAtomicFs });+ await atomicWriteFile(filePath, JSON.stringify(snapshot), { fs: tracedAtomicFs });}reconcileWithDisk(): Promise<{ added: number; updated: number; deleted: number }> {