Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
PRD-7400 Write link and tag caches atomically (#5789)
Andrew Mikofalvyf62ebf9b273a
diff --git a/.changeset/atomic-link-tag-caches.md b/.changeset/atomic-link-tag-caches.mdnew file mode 100644index 000000000..c8dd74b20--- /dev/null+++ b/.changeset/atomic-link-tag-caches.md@@ -0,0 +1,5 @@+---+"@inkeep/open-knowledge": patch+---++A crash while saving the link or tag index cache no longer leaves a damaged cache behind.diff --git a/packages/server/src/backlink-index.interrupted-save.test.ts b/packages/server/src/backlink-index.interrupted-save.test.tsnew file mode 100644index 000000000..fbfe44dc5--- /dev/null+++ b/packages/server/src/backlink-index.interrupted-save.test.ts@@ -0,0 +1,58 @@+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';+import { writeFile } from 'node:fs/promises';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { LOCAL_DIR } from '@inkeep/open-knowledge-core';+import { afterEach, describe, expect, test, vi } from 'vitest';+import { BacklinkIndex } from './backlink-index.ts';++const crash = vi.hoisted(() => ({ armed: false }));++vi.mock('./fs-traced.ts', async (importOriginal) => {+ const actual = await importOriginal<typeof import('./fs-traced.ts')>();+ return {+ ...actual,+ tracedAtomicFs: {+ writeFile: async (path: string, content: string, opts: { encoding: 'utf-8' }) => {+ if (!crash.armed) return actual.tracedAtomicFs.writeFile(path, content, opts);+ await writeFile(path, content.slice(0, Math.floor(content.length / 2)), opts);+ throw new Error('simulated crash mid-write');+ },+ rename: (from: string, to: string) => actual.tracedAtomicFs.rename(from, to),+ },+ };+});++afterEach(() => {+ crash.armed = false;+});++describe('BacklinkIndex cache save interrupted mid-write', () => {+ test('leaves the previous good snapshot intact and loadable', async () => {+ const projectDir = mkdtempSync(join(tmpdir(), 'ok-backlinks-interrupted-'));+ const contentDir = join(projectDir, 'content');+ const cachePath = join(projectDir, '.ok', LOCAL_DIR, 'cache', 'main', 'backlinks.json');+ mkdirSync(contentDir, { recursive: true });+ try {+ writeFileSync(join(contentDir, 'alpha.md'), 'See [[beta]].\n');+ writeFileSync(join(contentDir, 'beta.md'), '# Beta\n');+ const first = new BacklinkIndex({ projectDir, contentDir });+ await first.rebuildFromDisk();+ await first.saveToDisk();+ const good = readFileSync(cachePath, 'utf-8');++ writeFileSync(join(contentDir, 'gamma.md'), 'See [[beta]] too.\n');+ await first.rebuildFromDisk();+ crash.armed = true;+ await expect(first.saveToDisk()).rejects.toThrow('simulated crash mid-write');+ crash.armed = false;++ expect(readFileSync(cachePath, 'utf-8')).toBe(good);+ const second = new BacklinkIndex({ projectDir, contentDir });+ expect(await second.loadFromDisk()).toBe(true);+ expect(second.getBacklinks('beta').map((link) => link.source)).toEqual(['alpha']);+ } finally {+ rmSync(projectDir, { recursive: true, force: true });+ }+ });+});diff --git a/packages/server/src/backlink-index.test.ts b/packages/server/src/backlink-index.test.tsindex 7aa38435e..0fe73b841 100644--- a/packages/server/src/backlink-index.test.ts+++ b/packages/server/src/backlink-index.test.ts@@ -1,4 +1,5 @@import {+ existsSync,mkdirSync,mkdtempSync,readFileSync,@@ -900,13 +901,14 @@ describe('BacklinkIndex', () => {});test.each([- { version: 3, sourceLinks: undefined, expectedWarnings: [] },+ { version: 3, sourceLinks: undefined, expectedWarnings: [], kept: true },{version: 4,sourceLinks: {},expectedWarnings: [[{ branch: 'main' }, 'Incomplete backlink cache snapshot for main; rebuilding from disk'],],+ kept: false,},{version: 4,@@ -914,6 +916,7 @@ describe('BacklinkIndex', () => {expectedWarnings: [[{ branch: 'main' }, 'Incomplete backlink cache snapshot for main; rebuilding from disk'],],+ kept: false,},])('rejects incomplete source-link snapshots and rebuilds from disk: $version', async (cache) => {const projectDir = mkdtempSync(join(tmpdir(), 'ok-backlinks-source-links-guard-'));@@ -939,6 +942,7 @@ describe('BacklinkIndex', () => {try {expect(await index.loadFromDisk()).toBe(false);expect(warn.mock.calls).toEqual(cache.expectedWarnings);+ expect(existsSync(join(cacheDir, 'backlinks.json'))).toBe(cache.kept);} finally {warn.mockRestore();}@@ -952,6 +956,43 @@ describe('BacklinkIndex', () => {}});+ test.each([+ {+ label: 'truncated JSON',+ corrupt: (good: string) => good.slice(0, Math.floor(good.length / 2)),+ },+ { label: 'NUL-padded file', corrupt: (good: string) => '\0'.repeat(good.length) },+ ])(+ 'a torn cache snapshot ($label) is discarded once and replaced on the next save',+ async ({ corrupt }) => {+ const projectDir = mkdtempSync(join(tmpdir(), 'ok-backlinks-torn-cache-'));+ const contentDir = join(projectDir, 'content');+ const cachePath = join(projectDir, '.ok', LOCAL_DIR, 'cache', 'main', 'backlinks.json');+ mkdirSync(contentDir, { recursive: true });+ try {+ writeFileSync(join(contentDir, 'alpha.md'), 'See [[beta]].\n');+ writeFileSync(join(contentDir, 'beta.md'), '# Beta\n');+ const first = new BacklinkIndex({ projectDir, contentDir });+ await first.rebuildFromDisk();+ await first.saveToDisk();+ writeFileSync(cachePath, corrupt(readFileSync(cachePath, 'utf-8')));++ const second = new BacklinkIndex({ projectDir, contentDir });+ expect(await second.loadFromDisk()).toBe(false);+ expect(existsSync(cachePath)).toBe(false);+ await second.rebuildFromDisk();+ expect(second.getBacklinks('beta').map((link) => link.source)).toEqual(['alpha']);+ await second.saveToDisk();++ const third = new BacklinkIndex({ projectDir, contentDir });+ expect(await third.loadFromDisk()).toBe(true);+ expect(third.getBacklinks('beta').map((link) => link.source)).toEqual(['alpha']);+ } finally {+ rmSync(projectDir, { recursive: true, force: true });+ }+ },+ );+test('a versionless pre-upgrade cache is rejected so boot cold-rebuilds instead of serving stale keys', async () => {const projectDir = mkdtempSync(join(tmpdir(), 'ok-backlinks-version-guard-'));const contentDir = join(projectDir, 'content');diff --git a/packages/server/src/backlink-index.ts b/packages/server/src/backlink-index.tsindex 6e8f55ed1..11b5a6340 100644--- a/packages/server/src/backlink-index.ts+++ b/packages/server/src/backlink-index.ts@@ -1,5 +1,5 @@-import { type Dirent, existsSync, mkdirSync } from 'node:fs';-import { readdir, readFile, stat, writeFile } from 'node:fs/promises';+import { type Dirent, existsSync } from 'node:fs';+import { readdir, readFile, stat } from 'node:fs/promises';import { dirname, join, relative, resolve } from 'node:path';import {addDocumentFolders,@@ -36,10 +36,12 @@ import {toWikiLinkSlug,type WikiLinkLookupIndex,} from '@inkeep/open-knowledge-core';+import { atomicWriteFile } from '@inkeep/open-knowledge-core/server';import { isLinkIndexExcludedDoc } from './cc1-broadcast.ts';import { getLocalDir } from './config/paths.ts';import type { ContentFilter } from './content-filter.ts';import { isSupportedDocFile, linkNamesDocumentFile, stripDocExtension } from './doc-extensions.ts';+import { tracedAtomicFs, tracedMkdir, tracedRm } from './fs-traced.ts';import { instrumentIndexRebuild } from './index-telemetry.ts';import {createJsxSrcAttrRe,@@ -2148,7 +2150,7 @@ export class BacklinkIndex {async saveToDisk(branch = this.activeBranch): Promise<void> {const filePath = this.cachePath(branch);- mkdirSync(dirname(filePath), { recursive: true });+ await tracedMkdir(dirname(filePath), { recursive: true });const state = this.getState(branch);const mtimes = this.mtimesByBranch.get(branch);const data: SerializedBranchGraphState = {@@ -2156,14 +2158,20 @@ export class BacklinkIndex {...serializeState(state),...(mtimes ? { mtimes: Object.fromEntries(mtimes) } : {}),};- await writeFile(filePath, JSON.stringify(data, null, 2), 'utf-8');+ await atomicWriteFile(filePath, JSON.stringify(data, null, 2), { fs: tracedAtomicFs });}async loadFromDisk(branch = this.activeBranch): Promise<boolean> {const filePath = this.cachePath(branch);if (!existsSync(filePath)) return false;+ let raw: string;+ try {+ raw = await readFile(filePath, 'utf-8');+ } catch (err) {+ log.warn({ branch, err }, `Failed to load cache for ${branch}`);+ return false;+ }try {- const raw = await readFile(filePath, 'utf-8');const parsed = JSON.parse(raw) as SerializedBranchGraphState;if (parsed.version !== SNAPSHOT_VERSION) return false;if (@@ -2177,6 +2185,7 @@ export class BacklinkIndex {{ branch },`Incomplete backlink cache snapshot for ${branch}; rebuilding from disk`,);+ await this.discardCorruptSnapshot(filePath, branch, 'incomplete');return false;}this.states.set(branch, deserializeState(parsed));@@ -2187,11 +2196,27 @@ export class BacklinkIndex {}return true;} catch (err) {- log.warn({ branch, err }, `Failed to load cache for ${branch}`);+ log.warn(+ { branch, err },+ `Corrupt backlink cache snapshot for ${branch}; rebuilding from disk`,+ );+ await this.discardCorruptSnapshot(filePath, branch, 'malformed');return false;}}+ private async discardCorruptSnapshot(+ filePath: string,+ branch: string,+ reason: 'incomplete' | 'malformed',+ ): Promise<void> {+ try {+ await tracedRm(filePath, { force: true });+ } catch (err) {+ log.warn({ branch, reason, err }, 'Failed to discard corrupt backlink cache snapshot');+ }+ }+clear(branch = this.activeBranch): void {this.states.set(branch, createEmptyState());this.mtimesByBranch.delete(branch);diff --git a/packages/server/src/server-factory.test.ts b/packages/server/src/server-factory.test.tsindex c74498905..221ba4e39 100644--- a/packages/server/src/server-factory.test.ts+++ b/packages/server/src/server-factory.test.ts@@ -511,7 +511,7 @@ describe('createServer().destroy() — graceful shutdown flush', () => {afterEach(async () => {loggerFactory.reset();- await rm(tmpDir, { recursive: true, force: true });+ await rm(tmpDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });});test('flushes L1 markdown writes before destroy() resolves + emits shutdown log', async () => {diff --git a/packages/server/src/tag-index.interrupted-save.test.ts b/packages/server/src/tag-index.interrupted-save.test.tsnew file mode 100644index 000000000..38424382f--- /dev/null+++ b/packages/server/src/tag-index.interrupted-save.test.ts@@ -0,0 +1,57 @@+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';+import { writeFile } from 'node:fs/promises';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { afterEach, describe, expect, test, vi } from 'vitest';+import { TagIndex } from './tag-index.ts';++const crash = vi.hoisted(() => ({ armed: false }));++vi.mock('./fs-traced.ts', async (importOriginal) => {+ const actual = await importOriginal<typeof import('./fs-traced.ts')>();+ return {+ ...actual,+ tracedAtomicFs: {+ writeFile: async (path: string, content: string, opts: { encoding: 'utf-8' }) => {+ if (!crash.armed) return actual.tracedAtomicFs.writeFile(path, content, opts);+ await writeFile(path, content.slice(0, Math.floor(content.length / 2)), opts);+ throw new Error('simulated crash mid-write');+ },+ rename: (from: string, to: string) => actual.tracedAtomicFs.rename(from, to),+ },+ };+});++afterEach(() => {+ crash.armed = false;+});++describe('TagIndex snapshot save interrupted mid-write', () => {+ test('leaves the previous good snapshot intact and loadable', async () => {+ const projectDir = mkdtempSync(join(tmpdir(), 'ok-tag-interrupted-'));+ const contentDir = join(projectDir, 'content');+ const snapshotPath = join(projectDir, '.ok', 'local', 'cache', 'tags.json');+ mkdirSync(contentDir, { recursive: true });+ try {+ writeFileSync(join(contentDir, 'alpha.md'), 'Tag #before.\n');+ const first = new TagIndex({ projectDir, contentDir });+ await first.init();+ await first.saveToDisk();+ const good = readFileSync(snapshotPath, 'utf-8');++ writeFileSync(join(contentDir, 'beta.md'), 'Tag #after.\n');+ await first.reconcileWithDisk();+ crash.armed = true;+ await expect(first.saveToDisk()).rejects.toThrow('simulated crash mid-write');+ crash.armed = false;++ expect(readFileSync(snapshotPath, 'utf-8')).toBe(good);+ const second = new TagIndex({ projectDir, contentDir });+ expect(await second.loadFromDisk()).toBe(true);+ expect(second.getDocsForTag('before')).toEqual(['alpha']);+ expect(second.getDocsForTag('after')).toEqual([]);+ } finally {+ rmSync(projectDir, { recursive: true, force: true });+ }+ });+});diff --git a/packages/server/src/tag-index.persistence.test.ts b/packages/server/src/tag-index.persistence.test.tsindex 456e68f88..c4e283ef7 100644--- a/packages/server/src/tag-index.persistence.test.ts+++ b/packages/server/src/tag-index.persistence.test.ts@@ -169,21 +169,54 @@ describe('TagIndex persistence', () => {}});- test('off-shape snapshot (valid JSON, wrong structure) also reports a miss', async () => {+ test.each([+ {+ label: 'truncated JSON',+ corrupt: (good: string) => good.slice(0, Math.floor(good.length / 2)),+ },+ { label: 'NUL-padded file', corrupt: (good: string) => '\0'.repeat(good.length) },+ ])(+ 'a torn snapshot ($label) is discarded once and replaced on the next save',+ async ({ corrupt }) => {+ const rig = tempRig();+ try {+ writeDoc(rig.contentDir, 'alpha.md', 'Tag #survivor.\n', 1_000);+ const first = newIndex(rig);+ await first.init();+ await first.saveToDisk();+ writeFileSync(rig.snapshotPath, corrupt(await readFile(rig.snapshotPath, 'utf-8')));++ const second = newIndex(rig);+ expect(await second.loadFromDisk()).toBe(false);+ expect(existsSync(rig.snapshotPath)).toBe(false);+ await second.init();+ expect(second.getDocsForTag('survivor')).toEqual(['alpha']);+ await second.saveToDisk();++ const third = newIndex(rig);+ expect(await third.loadFromDisk()).toBe(true);+ expect(third.getDocsForTag('survivor')).toEqual(['alpha']);+ expect(await third.reconcileWithDisk()).toEqual({ added: 0, updated: 0, deleted: 0 });+ } finally {+ rig.cleanup();+ }+ },+ );++ test.each([+ { snapshot: '[]', kept: false },+ { snapshot: '{"version":1,"docs":{"a":"not-an-array"},"files":{}}', kept: false },+ { snapshot: '{"version":1,"docs":{"a":[""]},"files":{}}', kept: false },+ { snapshot: '{"version":1,"docs":{},"files":{"a":{"mtimeMs":"nan","size":1}}}', kept: false },+ { snapshot: '{"version":2,"docs":{},"files":{}}', kept: true },+ ])('off-shape snapshot $snapshot reports a miss (kept: $kept)', async ({ snapshot, kept }) => {const rig = tempRig();try {mkdirSync(join(rig.projectDir, '.ok', 'local', 'cache'), { recursive: true });- for (const bad of [- '[]',- '{"version":2,"docs":{},"files":{}}',- '{"version":1,"docs":{"a":"not-an-array"},"files":{}}',- '{"version":1,"docs":{"a":[""]},"files":{}}',- '{"version":1,"docs":{},"files":{"a":{"mtimeMs":"nan","size":1}}}',- ]) {- writeFileSync(rig.snapshotPath, bad);- const idx = newIndex(rig);- expect(await idx.loadFromDisk()).toBe(false);- }+ writeFileSync(rig.snapshotPath, snapshot);+ const idx = newIndex(rig);+ expect(await idx.loadFromDisk()).toBe(false);+ expect(existsSync(rig.snapshotPath)).toBe(kept);} finally {rig.cleanup();}diff --git a/packages/server/src/tag-index.ts b/packages/server/src/tag-index.tsindex 3d258e8ba..ba3a55037 100644--- a/packages/server/src/tag-index.ts+++ b/packages/server/src/tag-index.ts@@ -9,11 +9,12 @@ import {tagsMatchingPrefix,unwrapFrontmatterFences,} from '@inkeep/open-knowledge-core';+import { atomicWriteFile } from '@inkeep/open-knowledge-core/server';import { isLinkIndexExcludedDoc } from './cc1-broadcast.ts';import { getLocalDir } from './config/paths.ts';import type { ContentFilter } from './content-filter.ts';import { isSupportedDocFile, stripDocExtension } from './doc-extensions.ts';-import { tracedMkdir, tracedWriteFile } from './fs-traced.ts';+import { tracedAtomicFs, tracedMkdir, tracedRm } from './fs-traced.ts';import { instrumentIndexRebuild } from './index-telemetry.ts';import { getLogger } from './logger.ts';import { toPosix } from './path-utils.ts';@@ -47,6 +48,12 @@ interface SerializedTagIndexSnapshot {const SNAPSHOT_VERSION = 1;+function isOtherSnapshotVersion(data: unknown): boolean {+ if (typeof data !== 'object' || data === null) return false;+ const { version } = data as Record<string, unknown>;+ return typeof version === 'number' && version !== SNAPSHOT_VERSION;+}+function isValidSnapshot(data: unknown): data is SerializedTagIndexSnapshot {if (typeof data !== 'object' || data === null) return false;const snapshot = data as Record<string, unknown>;@@ -297,15 +304,25 @@ export class TagIndex {if (this.closed) return false;const filePath = this.snapshotPath();if (!filePath || !existsSync(filePath)) return false;+ let raw: string;+ try {+ raw = await readFile(filePath, 'utf-8');+ } catch (err) {+ log.warn({ err }, 'Failed to read tag snapshot; falling back to full rebuild');+ return false;+ }let parsed: unknown;try {- parsed = JSON.parse(await readFile(filePath, 'utf-8'));+ parsed = JSON.parse(raw);} catch (err) {- log.warn({ err }, 'Failed to load tag snapshot; falling back to full rebuild');+ log.warn({ err }, 'Corrupt tag snapshot; rebuilding from disk');+ await this.discardCorruptSnapshot(filePath, 'malformed');return false;}+ if (isOtherSnapshotVersion(parsed)) return false;if (!isValidSnapshot(parsed)) {- log.warn({}, 'Tag snapshot failed validation; falling back to full rebuild');+ log.warn({}, 'Incomplete tag snapshot; rebuilding from disk');+ await this.discardCorruptSnapshot(filePath, 'incomplete');return false;}this.state = createEmptyState();@@ -324,6 +341,17 @@ export class TagIndex {return true;}+ private async discardCorruptSnapshot(+ filePath: string,+ reason: 'malformed' | 'incomplete',+ ): Promise<void> {+ try {+ await tracedRm(filePath, { force: true });+ } catch (err) {+ log.warn({ reason, err }, 'Failed to discard corrupt tag snapshot');+ }+ }+saveToDisk(): Promise<void> {return this.enqueue(() => this.saveOnce());}@@ -343,7 +371,7 @@ export class TagIndex {files: Object.fromEntries(this.fileMeta),};await tracedMkdir(dirname(filePath), { recursive: true });- await tracedWriteFile(filePath, JSON.stringify(snapshot, null, 2), 'utf-8');+ await atomicWriteFile(filePath, JSON.stringify(snapshot, null, 2), { fs: tracedAtomicFs });}reconcileWithDisk(): Promise<{ added: number; updated: number; deleted: number }> {