Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

PRD-9586 Reduce backlink-cache disk writes (#7459)

Andrew Mikofalvy2534085210e7
diff --git a/.changeset/backlink-cache-write-volume.md b/.changeset/backlink-cache-write-volume.md
new file mode 100644
index 000000000..75e373d62
--- /dev/null
+++ b/.changeset/backlink-cache-write-volume.md
@@ -0,0 +1,6 @@
+---
+'@inkeep/open-knowledge': patch
+'@inkeep/open-knowledge-server': patch
+---
+
+Reduce disk writes from backlink caching during sustained editing.
diff --git a/packages/server/src/backlink-index.test.ts b/packages/server/src/backlink-index.test.ts
index 0fe73b841..2f3727f92 100644
--- a/packages/server/src/backlink-index.test.ts
+++ b/packages/server/src/backlink-index.test.ts
@@ -774,6 +774,11 @@ describe('BacklinkIndex', () => {
const index = new BacklinkIndex({ projectDir, contentDir });
index.updateDocumentFromMarkdown('alpha', '# Alpha\n\nSee [[ghost]].\n');
await index.saveToDisk();
+ const raw = readFileSync(
+ join(projectDir, '.ok', LOCAL_DIR, 'cache', 'main', 'backlinks.json'),
+ 'utf-8',
+ );
+ expect(raw).toBe(JSON.stringify(JSON.parse(raw)));
const reloaded = new BacklinkIndex({ projectDir, contentDir });
expect(await reloaded.loadFromDisk()).toBe(true);
diff --git a/packages/server/src/backlink-index.ts b/packages/server/src/backlink-index.ts
index 11b5a6340..0258f8108 100644
--- a/packages/server/src/backlink-index.ts
+++ b/packages/server/src/backlink-index.ts
@@ -2158,7 +2158,7 @@ export class BacklinkIndex {
...serializeState(state),
...(mtimes ? { mtimes: Object.fromEntries(mtimes) } : {}),
};
- await atomicWriteFile(filePath, JSON.stringify(data, null, 2), { fs: tracedAtomicFs });
+ await atomicWriteFile(filePath, JSON.stringify(data), { fs: tracedAtomicFs });
}
async loadFromDisk(branch = this.activeBranch): Promise<boolean> {
diff --git a/packages/server/src/derived-document-index.test.ts b/packages/server/src/derived-document-index.test.ts
index 68e8f6d26..10024855e 100644
--- a/packages/server/src/derived-document-index.test.ts
+++ b/packages/server/src/derived-document-index.test.ts
@@ -76,6 +76,152 @@ afterEach(async () => {
});
describe('DerivedDocumentIndex', () => {
+ test('normal cache saves stay bounded during sustained edits and flush latest state on close', async () => {
+ vi.useFakeTimers();
+ const backlinkSave = vi.spyOn(BacklinkIndex.prototype, 'saveToDisk');
+ const tagSave = vi.spyOn(TagIndex.prototype, 'saveToDisk');
+ const rig = createRig();
+ await settleStartup(rig);
+ backlinkSave.mockClear();
+ tagSave.mockClear();
+
+ for (let i = 0; i < 75; i++) {
+ const markdown = `See [[target-${i}]]. #stable\n`;
+ writeDoc(rig, 'source.md', markdown);
+ switch (i % 4) {
+ case 0:
+ await rig.index.recordDirectDocument('source', markdown);
+ break;
+ case 1:
+ await rig.index.recordDiskUpsert('source', markdown);
+ break;
+ case 2:
+ await rig.index.recordDurableStore('source', markdown);
+ break;
+ case 3:
+ await rig.index.recordLinkRewrite('source', markdown);
+ break;
+ }
+ expect(await rig.index.getBacklinks(`target-${i}`)).toHaveLength(1);
+ await vi.advanceTimersByTimeAsync(1000);
+ await rig.index.getIndexedDocNames();
+ }
+
+ expect(backlinkSave).toHaveBeenCalledTimes(3);
+ expect(tagSave).toHaveBeenCalledTimes(3);
+ const warm = new BacklinkIndex(rig);
+ expect(await warm.loadFromDisk()).toBe(true);
+ expect(warm.getBacklinks('target-59')).toHaveLength(1);
+ expect(warm.getBacklinks('target-74')).toEqual([]);
+
+ await vi.advanceTimersByTimeAsync(15_000);
+ await rig.index.getIndexedDocNames();
+ expect(backlinkSave).toHaveBeenCalledTimes(4);
+ const quiet = new BacklinkIndex(rig);
+ expect(await quiet.loadFromDisk()).toBe(true);
+ expect(quiet.getBacklinks('target-74')).toHaveLength(1);
+
+ writeDoc(rig, 'late.md', '#late-tag\n');
+ await rig.index.recordDiskUpsert('late', '#late-tag\n');
+ writeDoc(rig, 'source.md', 'See [[close-target]]. #stable\n');
+ await rig.index.recordDurableStore('source', 'See [[close-target]]. #stable\n');
+ await rig.index.close();
+ expect(backlinkSave).toHaveBeenCalledTimes(5);
+ expect(tagSave).toHaveBeenCalledTimes(5);
+ await vi.advanceTimersByTimeAsync(60_000);
+ expect(backlinkSave).toHaveBeenCalledTimes(5);
+ expect(tagSave).toHaveBeenCalledTimes(5);
+
+ const closed = new BacklinkIndex(rig);
+ expect(await closed.loadFromDisk()).toBe(true);
+ expect(closed.getBacklinks('close-target')).toHaveLength(1);
+ expect(closed.getBacklinks('target-74')).toEqual([]);
+ const tags = new TagIndex(rig);
+ expect(await tags.loadFromDisk()).toBe(true);
+ expect(tags.getDocsForTagWithMatches('late-tag')).toEqual([
+ { docName: 'late', matchingTags: ['late-tag'] },
+ ]);
+ await tags.close();
+ });
+
+ test('a queued cache save checks the minimum interval when it actually executes', async () => {
+ vi.useFakeTimers();
+ const originalSave = BacklinkIndex.prototype.saveToDisk;
+ const backlinkSave = vi.spyOn(BacklinkIndex.prototype, 'saveToDisk');
+ const rig = createRig();
+ await settleStartup(rig);
+ backlinkSave.mockClear();
+ const started = Promise.withResolvers<void>();
+ const release = Promise.withResolvers<void>();
+ backlinkSave.mockImplementationOnce(async function (this: BacklinkIndex, branch) {
+ started.resolve();
+ await release.promise;
+ await originalSave.call(this, branch);
+ });
+
+ writeDoc(rig, 'source.md', 'See [[queued-target]].\n');
+ await rig.index.recordDiskUpsert('source', 'See [[queued-target]].\n');
+ const running = rig.index.recordDirectDocument('running', '# Running\n');
+ await started.promise;
+ await vi.advanceTimersByTimeAsync(5000);
+ release.resolve();
+ await running;
+ await rig.index.getIndexedDocNames();
+
+ expect(backlinkSave).toHaveBeenCalledTimes(1);
+ await vi.advanceTimersByTimeAsync(29_999);
+ await rig.index.getIndexedDocNames();
+ expect(backlinkSave).toHaveBeenCalledTimes(1);
+ await vi.advanceTimersByTimeAsync(1);
+ await rig.index.getIndexedDocNames();
+ expect(backlinkSave).toHaveBeenCalledTimes(2);
+ const cached = new BacklinkIndex(rig);
+ expect(await cached.loadFromDisk()).toBe(true);
+ expect(cached.getBacklinks('queued-target')).toHaveLength(1);
+ });
+
+ test('a queued throttled save cannot rearm an old-branch timer during a switch', async () => {
+ vi.useFakeTimers();
+ const originalSave = BacklinkIndex.prototype.saveToDisk;
+ const backlinkSave = vi.spyOn(BacklinkIndex.prototype, 'saveToDisk');
+ const tagSave = vi.spyOn(TagIndex.prototype, 'saveToDisk');
+ const rig = createRig();
+ await settleStartup(rig);
+ const started = Promise.withResolvers<void>();
+ const release = Promise.withResolvers<void>();
+ backlinkSave.mockImplementationOnce(async function (this: BacklinkIndex, branch) {
+ started.resolve();
+ await release.promise;
+ await originalSave.call(this, branch);
+ });
+
+ writeDoc(rig, 'old.md', 'See [[old-target]].\n');
+ await rig.index.recordDiskUpsert('old', 'See [[old-target]].\n');
+ const running = rig.index.recordDirectDocument('running', '# Running\n');
+ await started.promise;
+ await vi.advanceTimersByTimeAsync(2000);
+ const switching = rig.index.beginBranchSwitch('feature');
+ release.resolve();
+ await running;
+ const transition = await switching;
+ unlinkSync(join(rig.contentDir, 'old.md'));
+ writeDoc(rig, 'feature.md', 'See [[feature-target]]. #feature\n');
+ await rig.index.settleBranchFromDisk(transition);
+ backlinkSave.mockClear();
+ tagSave.mockClear();
+ await vi.advanceTimersByTimeAsync(60_000);
+ await rig.index.getIndexedDocNames();
+ expect(backlinkSave).not.toHaveBeenCalled();
+ expect(tagSave).not.toHaveBeenCalled();
+
+ const cached = new BacklinkIndex(rig);
+ expect(await cached.loadFromDisk('feature')).toBe(true);
+ expect(cached.getBacklinks('feature-target', 'feature')).toHaveLength(1);
+ expect(cached.getBacklinks('old-target', 'feature')).toEqual([]);
+ expect(await cached.loadFromDisk('main')).toBe(true);
+ expect(cached.getBacklinks('old-target', 'main')).toHaveLength(1);
+ });
+
test('every body feeder refreshes the resolved graph with unchanged document inventory', async () => {
const rig = createRig();
writeDoc(rig, 'notes/beta.md', '# Beta');
diff --git a/packages/server/src/derived-document-index.ts b/packages/server/src/derived-document-index.ts
index 98969dda9..1abd6a412 100644
--- a/packages/server/src/derived-document-index.ts
+++ b/packages/server/src/derived-document-index.ts
@@ -30,7 +30,8 @@ import { TagIndex, type TagSummaryEntry } from './tag-index.ts';
const log = getLogger('derived-document-index');
const SLOW_STARTUP_LOG_THRESHOLD_MS = 10_000;
-const DERIVED_INDEX_SAVE_DEBOUNCE_MS = 2000;
+const DERIVED_INDEX_SAVE_COALESCE_MS = 2000;
+const DERIVED_INDEX_SAVE_MIN_INTERVAL_MS = 30_000;
const LOCAL_TARGET_REBUILD_RETRY_BASE_MS = 1000;
const LOCAL_TARGET_REBUILD_RETRY_MAX_MS = 30_000;
const LOCAL_TARGET_FILE_EXISTENCE_SWEEP_MS = 5000;
@@ -181,6 +182,7 @@ export class DerivedDocumentIndex
private readonly readyBarrier = Promise.withResolvers<void>();
private tail: Promise<void> = Promise.resolve();
private saveTimer: ReturnType<typeof setTimeout> | null = null;
+ private lastNormalSaveAt: number | null = null;
private localTargetRebuildRetryTimer: ReturnType<typeof setTimeout> | null = null;
private readonly localTargetFileExistenceTimer: ReturnType<typeof setInterval>;
private localTargetRebuildRetryAttempt = 0;
@@ -188,7 +190,7 @@ export class DerivedDocumentIndex
private graphFileTargets: TargetNamespace<'file'> | null = null;
private startupBranch: string | null = null;
private graphBuiltWithoutFileOracle = false;
- private saveTagsOnNextDebounce = false;
+ private saveTagsOnNextFlush = false;
private branchTransition: BranchTransition | null = null;
private startupBegun = false;
private startupBegunAt = 0;
@@ -519,10 +521,10 @@ export class DerivedDocumentIndex
if (tagsChanged) {
this.reconcileLocalTargetInventory();
- await this.saveBothLogOnly('direct-batch');
+ await this.saveMutationCache(true, 'direct-batch');
this.signalAllRelations();
} else {
- await this.saveBacklinksLogOnly('direct-batch');
+ await this.saveMutationCache(false, 'direct-batch');
this.signalBacklinks();
}
});
@@ -684,7 +686,7 @@ export class DerivedDocumentIndex
if (this.closePromise) return this.closePromise;
const pendingSave = this.cancelScheduledSave();
if (pendingSave !== null) {
- void this.enqueueInternal(() => this.saveScheduledCache(pendingSave, 'close-flush'));
+ void this.enqueueInternal(() => this.saveMutationCache(pendingSave, 'close-flush'));
}
this.closed = true;
clearInterval(this.localTargetFileExistenceTimer);
@@ -949,33 +951,51 @@ export class DerivedDocumentIndex
}
private scheduleSave(includeTags = true): void {
- const saveTags = this.saveTagsOnNextDebounce || includeTags;
- this.cancelScheduledSave();
- this.saveTagsOnNextDebounce = saveTags;
+ if (this.closed || this.branchTransition !== null) return;
+ this.saveTagsOnNextFlush ||= includeTags;
+ if (this.saveTimer !== null) return;
+ const delay =
+ this.lastNormalSaveAt === null
+ ? DERIVED_INDEX_SAVE_COALESCE_MS
+ : Math.max(
+ DERIVED_INDEX_SAVE_COALESCE_MS,
+ DERIVED_INDEX_SAVE_MIN_INTERVAL_MS - (performance.now() - this.lastNormalSaveAt),
+ );
this.saveTimer = setTimeout(() => {
this.saveTimer = null;
- const saveTags = this.saveTagsOnNextDebounce;
- this.saveTagsOnNextDebounce = false;
- void this.enqueueInternal(() => this.saveScheduledCache(saveTags, 'debounced')).catch(
+ const saveTags = this.saveTagsOnNextFlush;
+ this.saveTagsOnNextFlush = false;
+ void this.enqueueInternal(() => this.saveMutationCache(saveTags, 'coalesced')).catch(
(err) => {
- log.warn({ err }, 'Failed to enqueue debounced derived-index cache save');
+ log.warn({ err }, 'Failed to enqueue coalesced derived-index cache save');
},
);
- }, DERIVED_INDEX_SAVE_DEBOUNCE_MS);
+ }, delay);
}
private cancelScheduledSave(): boolean | null {
- const pendingSave = this.saveTimer === null ? null : this.saveTagsOnNextDebounce;
+ const pendingSave = this.saveTimer === null ? null : this.saveTagsOnNextFlush;
if (this.saveTimer !== null) clearTimeout(this.saveTimer);
this.saveTimer = null;
- this.saveTagsOnNextDebounce = false;
+ this.saveTagsOnNextFlush = false;
return pendingSave;
}
- private saveScheduledCache(includeTags: boolean, reason: string): Promise<void> {
- return includeTags
- ? this.saveBothLogOnly(reason)
- : this.saveBacklinksLogOnly(`${reason}-backlinks`);
+ private async saveMutationCache(includeTags: boolean, reason: string): Promise<void> {
+ if (
+ !this.closed &&
+ this.lastNormalSaveAt !== null &&
+ performance.now() - this.lastNormalSaveAt < DERIVED_INDEX_SAVE_MIN_INTERVAL_MS
+ ) {
+ this.scheduleSave(includeTags);
+ return;
+ }
+ if (includeTags) {
+ await this.saveBothLogOnly(reason);
+ } else {
+ await this.saveBacklinksLogOnly(`${reason}-backlinks`);
+ }
+ this.lastNormalSaveAt = performance.now();
}
private releaseBranchTransition(transition: DerivedDocumentIndexBranchTransition | null): void {
diff --git a/packages/server/src/tag-index.persistence.test.ts b/packages/server/src/tag-index.persistence.test.ts
index f6ccf755e..461430603 100644
--- a/packages/server/src/tag-index.persistence.test.ts
+++ b/packages/server/src/tag-index.persistence.test.ts
@@ -64,6 +64,8 @@ describe('TagIndex persistence', () => {
const first = newIndex(rig);
await first.init();
await first.saveToDisk();
+ const raw = await readFile(rig.snapshotPath, 'utf-8');
+ expect(raw).toBe(JSON.stringify(JSON.parse(raw)));
const second = newIndex(rig);
expect(await second.loadFromDisk()).toBe(true);
diff --git a/packages/server/src/tag-index.ts b/packages/server/src/tag-index.ts
index eb1a55fcf..14ee6d2bf 100644
--- a/packages/server/src/tag-index.ts
+++ b/packages/server/src/tag-index.ts
@@ -371,7 +371,7 @@ export class TagIndex {
files: Object.fromEntries(this.fileMeta),
};
await tracedMkdir(dirname(filePath), { recursive: true });
- await atomicWriteFile(filePath, JSON.stringify(snapshot, null, 2), { fs: tracedAtomicFs });
+ await atomicWriteFile(filePath, JSON.stringify(snapshot), { fs: tracedAtomicFs });
}
reconcileWithDisk(): Promise<{ added: number; updated: number; deleted: number }> {