Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

fix(open-knowledge): add persistence staleness watchdog (PRD-7492) (#2783)

Andrew Mikofalvyd873fe6333d1
diff --git a/.changeset/prd-7492-persistence-staleness.md b/.changeset/prd-7492-persistence-staleness.md
new file mode 100644
index 000000000..dc50f77a0
--- /dev/null
+++ b/.changeset/prd-7492-persistence-staleness.md
@@ -0,0 +1,5 @@
+---
+'@inkeep/open-knowledge': patch
+---
+
+Add a persistence staleness watchdog that bounds how long CRDT edits can sit unflushed to disk. If a document's in-memory edits ever fail to reach the on-disk markdown (for example after a transient disk error, since the store debounce only re-arms on the next edit), the server now detects the stale file within minutes and re-runs the store through the normal persistence path instead of leaving disk outdated until the next edit. The watchdog never overwrites external edits: any on-disk state the persistence layer has not reconciled makes it stand down and log instead. Three new counters (`persistenceStalenessDetected`, `persistenceStalenessForcedStores`, `persistenceStalenessStoodDown`) surface these events alongside the existing persistence queue metrics; a sustained non-zero `persistenceStalenessStoodDown` rate is the alertable one (unflushed edits pinned in memory would be lost on a restart).
diff --git a/packages/app/tests/integration/persistence-staleness-watchdog.test.ts b/packages/app/tests/integration/persistence-staleness-watchdog.test.ts
new file mode 100644
index 000000000..e3d6828ce
--- /dev/null
+++ b/packages/app/tests/integration/persistence-staleness-watchdog.test.ts
@@ -0,0 +1,146 @@
+/**
+ * Persistence staleness watchdog — end-to-end coverage of the days-stale
+ * durability wedge: a store cycle that fails after a session's last edit has
+ * no natural retry (Hocuspocus re-arms the store debounce only on a new
+ * Y.Doc update, and user docs stay resident for the server lifetime), so the
+ * live doc stays newer than disk indefinitely. The watchdog must re-flush
+ * such a doc through the normal store spine once its grace window passes —
+ * and must never overwrite disk state the persistence layer has not
+ * accounted for (external native edits stay authoritative).
+ *
+ * Uses `OK_TEST_STORE_FAULT=<docName>` (a synthetic ENOSPC on the atomic
+ * write, matched by exact docName) to wedge a store deterministically, then
+ * clears it so the watchdog's forced retry can land.
+ */
+
+import { randomUUID } from 'node:crypto';
+import { existsSync, readFileSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { getMetrics } from '@inkeep/open-knowledge-server';
+import { afterEach, describe, expect, test } from 'vitest';
+import { createTestServer, type TestServer, wait } from './test-harness.ts';
+
+let server: TestServer | undefined;
+
+afterEach(async () => {
+ delete process.env.OK_TEST_STORE_FAULT;
+ if (server) {
+ await server.cleanup();
+ server = undefined;
+ }
+});
+
+async function writeMd(port: number, docName: string, markdown: string): Promise<Response> {
+ return fetch(`http://127.0.0.1:${port}/api/agent-write-md`, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({ docName, markdown, position: 'replace' }),
+ });
+}
+
+async function pollUntil(predicate: () => boolean, timeoutMs: number): Promise<boolean> {
+ const deadline = Date.now() + timeoutMs;
+ while (Date.now() < deadline) {
+ if (predicate()) return true;
+ await wait(50);
+ }
+ return predicate();
+}
+
+describe('persistence staleness watchdog (integration)', () => {
+ test('re-flushes a doc whose store failed with no retry pending', async () => {
+ server = await createTestServer({ stalenessGraceMs: 250, stalenessSweepIntervalMs: 100 });
+ const docName = `staleness-rescue-${randomUUID()}`;
+ const filePath = join(server.contentDir, `${docName}.md`);
+ const forcedBefore = getMetrics().persistenceStalenessForcedStores;
+ const detectedBefore = getMetrics().persistenceStalenessDetected;
+
+ process.env.OK_TEST_STORE_FAULT = docName;
+ const res = await writeMd(server.port, docName, '# persisted content\n');
+ expect(res.status).toBe(507);
+ delete process.env.OK_TEST_STORE_FAULT;
+
+ // The wedge shape: content lives only in server memory, nothing on disk,
+ // and no store is scheduled (the handler's force-flush consumed it).
+ expect(existsSync(filePath)).toBe(false);
+
+ const rescued = await pollUntil(
+ () => existsSync(filePath) && readFileSync(filePath, 'utf-8') === '# persisted content\n',
+ 10_000,
+ );
+ expect(rescued).toBe(true);
+ expect(getMetrics().persistenceStalenessDetected).toBeGreaterThan(detectedBefore);
+ expect(getMetrics().persistenceStalenessForcedStores).toBeGreaterThan(forcedBefore);
+ });
+
+ test('production-shaped grace window does not fire inside a test run (negative control)', async () => {
+ server = await createTestServer();
+ const docName = `staleness-control-${randomUUID()}`;
+ const filePath = join(server.contentDir, `${docName}.md`);
+ const detectedBefore = getMetrics().persistenceStalenessDetected;
+ const forcedBefore = getMetrics().persistenceStalenessForcedStores;
+
+ process.env.OK_TEST_STORE_FAULT = docName;
+ const res = await writeMd(server.port, docName, '# never flushed here\n');
+ expect(res.status).toBe(507);
+ delete process.env.OK_TEST_STORE_FAULT;
+
+ // With the default 5-minute grace, nothing rescues the doc in-test —
+ // proving the rescue above is the watchdog, not some other retry path.
+ // The unchanged counters pin that the watchdog itself never fired, so
+ // the absent file isn't just "nothing else happened to write it".
+ await wait(1_500);
+ expect(existsSync(filePath)).toBe(false);
+ expect(getMetrics().persistenceStalenessDetected).toBe(detectedBefore);
+ expect(getMetrics().persistenceStalenessForcedStores).toBe(forcedBefore);
+ });
+
+ test('stands down instead of clobbering an external native edit', async () => {
+ server = await createTestServer({ stalenessGraceMs: 250, stalenessSweepIntervalMs: 100 });
+ const docName = `staleness-external-${randomUUID()}`;
+ const filePath = join(server.contentDir, `${docName}.md`);
+ const forcedBefore = getMetrics().persistenceStalenessForcedStores;
+
+ const first = await writeMd(server.port, docName, '# version one\n');
+ expect(first.status).toBe(200);
+ const flushed = await pollUntil(
+ () => existsSync(filePath) && readFileSync(filePath, 'utf-8') === '# version one\n',
+ 10_000,
+ );
+ expect(flushed).toBe(true);
+
+ // Wedge a second version in memory (store fails), then land an external
+ // native edit on disk before the grace window elapses.
+ const stoodDownBefore = getMetrics().persistenceStalenessStoodDown;
+ process.env.OK_TEST_STORE_FAULT = docName;
+ const second = await writeMd(server.port, docName, '# version two\n');
+ expect(second.status).toBe(507);
+ delete process.env.OK_TEST_STORE_FAULT;
+ writeFileSync(filePath, '# external native edit\n', 'utf-8');
+
+ // Positive proof the WATCHDOG ran and chose to stand down (not merely
+ // that the file-watcher reconciled first): the stood-down counter must
+ // advance. If the watcher's ingest wins the race instead, memory
+ // converges to the external bytes and no stand-down fires — accept
+ // that interleaving only when the doc has genuinely converged.
+ const watchdogStoodDown = await pollUntil(
+ () => getMetrics().persistenceStalenessStoodDown > stoodDownBefore,
+ 2_000,
+ );
+ if (!watchdogStoodDown) {
+ const state = await fetch(
+ `http://127.0.0.1:${server.port}/api/document?docName=${encodeURIComponent(docName)}`,
+ );
+ // Watcher-reconciled interleaving: server must now hold the external
+ // bytes (so there was nothing for the watchdog to stand down on).
+ expect(state.ok).toBe(true);
+ expect(await state.text()).toContain('external native edit');
+ }
+
+ // Either way, the external bytes must win on disk and no forced store
+ // may ever have overwritten them.
+ await wait(500);
+ expect(readFileSync(filePath, 'utf-8')).toBe('# external native edit\n');
+ expect(getMetrics().persistenceStalenessForcedStores).toBe(forcedBefore);
+ });
+});
diff --git a/packages/app/tests/integration/test-harness.ts b/packages/app/tests/integration/test-harness.ts
index b07a4bd2f..84d497fac 100644
--- a/packages/app/tests/integration/test-harness.ts
+++ b/packages/app/tests/integration/test-harness.ts
@@ -102,6 +102,14 @@ export interface TestServer {
export interface CreateTestServerOptions {
debounce?: ServerOptions['debounce'];
maxDebounce?: ServerOptions['maxDebounce'];
+ /**
+ * Persistence staleness watchdog tuning. Production defaults (5 min grace,
+ * 60 s sweep) never fire inside a test run; pass small values (e.g.
+ * `{ stalenessGraceMs: 250, stalenessSweepIntervalMs: 100 }`) to exercise
+ * the forced-store rescue path.
+ */
+ stalenessGraceMs?: ServerOptions['stalenessGraceMs'];
+ stalenessSweepIntervalMs?: ServerOptions['stalenessSweepIntervalMs'];
/** Reuse an existing content directory (for server-restart tests that need
* persistence to load canonical state written by a prior test-server instance).
* When provided, the caller owns directory lifecycle — cleanup() will not
@@ -248,6 +256,8 @@ export async function createTestServer(options: CreateTestServerOptions = {}): P
quiet: true,
debounce: options.debounce ?? 200,
maxDebounce: options.maxDebounce ?? 1000,
+ stalenessGraceMs: options.stalenessGraceMs,
+ stalenessSweepIntervalMs: options.stalenessSweepIntervalMs,
gitEnabled: ephemeral ? false : (options.gitEnabled ?? false),
commitDebounceMs: options.commitDebounceMs ?? 200,
// When gitEnabled: true, the test-harness contentDir IS the tmpdir root (no
diff --git a/packages/server/src/cc1-broadcast.ts b/packages/server/src/cc1-broadcast.ts
index ad3417afb..5e4481daa 100644
--- a/packages/server/src/cc1-broadcast.ts
+++ b/packages/server/src/cc1-broadcast.ts
@@ -123,6 +123,26 @@ export function isLinkIndexExcludedDoc(documentName: string): boolean {
return isSystemDoc(documentName) || isConfigDoc(documentName);
}
+/**
+ * True when a doc name is excluded from the markdown L1 store spine
+ * (`storeDocumentNow`) because its persistence flows through a dedicated
+ * store/load path: system docs (never persisted), config docs (config
+ * persistence + validation hook), managed artifacts (skill/template
+ * persistence), and Mermaid docs (Y.Text-only persistence). Shared by
+ * `PersistenceHandle.forceStore` and the staleness watchdog so the two
+ * gates cannot drift. The debounced `onStoreDocument` hook dispatches each
+ * class to its dedicated path individually, so it cannot collapse onto
+ * this predicate.
+ */
+export function isPersistenceExcludedDoc(documentName: string): boolean {
+ return (
+ isSystemDoc(documentName) ||
+ isConfigDoc(documentName) ||
+ isManagedArtifactDoc(documentName) ||
+ isMermaidDoc(documentName)
+ );
+}
+
export class CC1Broadcaster {
private readonly hocuspocus: Hocuspocus;
private readonly seqs = new Map<string, number>();
diff --git a/packages/server/src/conflict-errors.ts b/packages/server/src/conflict-errors.ts
index 135edf189..8eeaa0a15 100644
--- a/packages/server/src/conflict-errors.ts
+++ b/packages/server/src/conflict-errors.ts
@@ -31,6 +31,7 @@
import type { ServerResponse } from 'node:http';
import type { Document } from '@hocuspocus/server';
+import type * as Y from 'yjs';
import type { ResolveStrategy } from './conflict-storage.ts';
import { stripDocExtension } from './doc-extensions.ts';
import { errorResponse } from './http/error-response.ts';
@@ -71,12 +72,41 @@ const _exhaustiveResolveStrategy: _ExhaustiveResolveStrategy = true;
* `Y.Map('lifecycle').get('status') === 'conflict'`. Every mutating write
* surface reads this via the helper rather than re-deriving the literal.
*
- * Centralized so the lifecycle Y.Map contract has exactly one named accessor
- * — adding new lifecycle discriminators (`deleted-upstream`, `renamed`,
- * future block-conflict shadows) goes through this one site.
+ * Delegates to `frozenDocLifecycleStatus` — the single `lifecycle.status`
+ * read site — so adding new lifecycle discriminators (`deleted-upstream`,
+ * `renamed`, future block-conflict shadows) goes through the
+ * `FROZEN_LIFECYCLE_STATUSES` registry below, never a scattered compare.
*/
export function isDocInConflict(document: Document): boolean {
- return document.getMap('lifecycle').get('status') === 'conflict';
+ return frozenDocLifecycleStatus(document) === 'conflict';
+}
+
+/**
+ * Lifecycle statuses under which the persistence spine intentionally holds
+ * memory != disk and skips stores: externally deleted / renamed docs must
+ * not be rewritten at their old path, and conflict docs keep merge markers
+ * on disk for the resolver UI. Registry-first (mirroring
+ * `RESOLUTION_OPTIONS` above) so a new frozen status is added in exactly
+ * one place and every reader — `storeDocumentNow`'s guard, the staleness
+ * watchdog, `isDocInConflict` — picks it up together.
+ */
+export const FROZEN_LIFECYCLE_STATUSES = ['deleted-upstream', 'renamed', 'conflict'] as const;
+export type FrozenLifecycleStatus = (typeof FROZEN_LIFECYCLE_STATUSES)[number];
+
+function isFrozenLifecycleStatus(value: unknown): value is FrozenLifecycleStatus {
+ return (FROZEN_LIFECYCLE_STATUSES as readonly unknown[]).includes(value);
+}
+
+/**
+ * The frozen lifecycle status a doc currently carries, or null when the
+ * doc is writable. Together with `isDocInConflict` (which delegates here)
+ * this is the single read site for `lifecycle.status` discriminators —
+ * adding a new lifecycle status goes through `FROZEN_LIFECYCLE_STATUSES`
+ * above, never a scattered string compare.
+ */
+export function frozenDocLifecycleStatus(document: Y.Doc): FrozenLifecycleStatus | null {
+ const status = document.getMap('lifecycle').get('status');
+ return isFrozenLifecycleStatus(status) ? status : null;
}
/**
diff --git a/packages/server/src/metrics.ts b/packages/server/src/metrics.ts
index bfb7f0036..837f2a74d 100644
--- a/packages/server/src/metrics.ts
+++ b/packages/server/src/metrics.ts
@@ -119,6 +119,33 @@ export interface ReconciliationMetrics {
* work undurable. The matching fragment-reconciliation queues
* unconditionally on the next settlement after the force-flush. */
persistenceForceFlushDuringBurst: number;
+ /** Staleness watchdog — count of loaded docs first observed with
+ * unpersisted in-memory edits older than the grace window and no pending
+ * store (memory diverged from the persistence reconciled base). Counted
+ * once per distinct stale content per doc, whether or not the watchdog
+ * then forced a store or stood down for disk safety. A healthy server
+ * stays at 0; any growth means a store cycle was lost after a session's
+ * last edit. */
+ persistenceStalenessDetected: number;
+ /** Staleness watchdog — count of forced store dispatch attempts through
+ * the normal persistence spine (retries after a failed attempt included,
+ * so this can exceed `persistenceStalenessDetected`, which counts once
+ * per distinct stale content). Stand-down detections don't dispatch and
+ * never increment this. A healthy server stays at 0; growth means lost
+ * store cycles are being rescued — investigate the underlying store
+ * failures even though durability recovered. */
+ persistenceStalenessForcedStores: number;
+ /** Staleness watchdog — count of stand-down events: a stale doc was
+ * detected but NOT re-flushed because disk holds state the reconciled
+ * base does not account for (unreconciled external edit or delete,
+ * never-loaded file) or the disk read failed. Verified-external-state
+ * stand-downs AND structural read refusals (symlink-escape, symlink
+ * cycle, oversized file — additionally logged at error) count once per
+ * distinct stale content, then suppress until the doc's content
+ * changes; only transient disk-read failures recount on each retry
+ * (one per grace window). Any sustained non-zero rate is alertable —
+ * unflushed edits are pinned in memory and lost on restart. */
+ persistenceStalenessStoodDown: number;
/** Collab WebSocket upgrade sockets emitting EPIPE from `ws.send()` AFTER
* the call returned control — kernel-level TCP race against a peer that
* has sent FIN. Filtered at the socket-boundary listener per precedent
@@ -424,6 +451,9 @@ const counters: ReconciliationMetrics = {
bridgeInvariantViolationsSuppressed: 0,
persistenceSkipNonQuiescent: 0,
persistenceForceFlushDuringBurst: 0,
+ persistenceStalenessDetected: 0,
+ persistenceStalenessForcedStores: 0,
+ persistenceStalenessStoodDown: 0,
collabSocketEpipeCount: 0,
collabSocketEconnresetCount: 0,
collabMessageTooLargeCount: 0,
@@ -578,6 +608,18 @@ export function incrementPersistenceForceFlushDuringBurst(): void {
counters.persistenceForceFlushDuringBurst++;
}
+export function incrementPersistenceStalenessDetected(): void {
+ counters.persistenceStalenessDetected++;
+}
+
+export function incrementPersistenceStalenessForcedStores(): void {
+ counters.persistenceStalenessForcedStores++;
+}
+
+export function incrementPersistenceStalenessStoodDown(): void {
+ counters.persistenceStalenessStoodDown++;
+}
+
export function incrementAgentPatchFindMismatches(): void {
counters.agentPatchFindMismatches++;
}
@@ -806,6 +848,9 @@ export function resetMetrics(): void {
counters.bridgeInvariantViolationsSuppressed = 0;
counters.persistenceSkipNonQuiescent = 0;
counters.persistenceForceFlushDuringBurst = 0;
+ counters.persistenceStalenessDetected = 0;
+ counters.persistenceStalenessForcedStores = 0;
+ counters.persistenceStalenessStoodDown = 0;
counters.collabSocketEpipeCount = 0;
counters.collabSocketEconnresetCount = 0;
counters.collabMessageTooLargeCount = 0;
diff --git a/packages/server/src/persistence-deferred-store.test.ts b/packages/server/src/persistence-deferred-store.test.ts
index f87b609b1..98d797a20 100644
--- a/packages/server/src/persistence-deferred-store.test.ts
+++ b/packages/server/src/persistence-deferred-store.test.ts
@@ -1,8 +1,8 @@
-import { afterEach, beforeEach, describe, expect, spyOn, test } from 'bun:test';
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { BridgeInvariantViolationError } from '@inkeep/open-knowledge-core';
+import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';
import * as Y from 'yjs';
import { composeAndWriteRawBody } from './bridge-intake.ts';
import { __setQuiescentOverrideForTests } from './bridge-quiescence.ts';
@@ -564,7 +564,7 @@ describe('Y.Text-is-truth wiring (FR-33 / FR-35)', () => {
*/
describe('FR-9 — deferred-store-failed event + counter', () => {
let tmpDir: string;
- let warnSpy: ReturnType<typeof spyOn>;
+ let warnSpy: ReturnType<typeof vi.spyOn>;
let warnings: string[];
function findEventLines(eventName: string): Array<Record<string, unknown>> {
@@ -588,7 +588,7 @@ describe('FR-9 — deferred-store-failed event + counter', () => {
switchReconciledBaseScope('main');
resetMetrics();
warnings = [];
- warnSpy = spyOn(console, 'warn').mockImplementation((...args: unknown[]) => {
+ warnSpy = vi.spyOn(console, 'warn').mockImplementation((...args: unknown[]) => {
warnings.push(args.map(String).join(' '));
});
});
@@ -709,7 +709,7 @@ describe('FR-9 — deferred-store-failed event + counter', () => {
// must absorb the throw and emit `deferred-store-classifier-failed`
// alongside the outer `deferred-store-failed` event with
// errorClass='unknown'.
- const renameSpy = spyOn(fsTraced, 'tracedRename').mockImplementation(async () => {
+ const renameSpy = vi.spyOn(fsTraced, 'tracedRename').mockImplementation(async () => {
const malformed = Object.create(Error.prototype) as Error & { name: string };
Object.defineProperty(malformed, 'message', { value: 'malformed-error', enumerable: true });
Object.defineProperty(malformed, 'name', { value: 'MalformedError', enumerable: true });
@@ -784,7 +784,7 @@ describe('FR-9 — deferred-store-failed event + counter', () => {
await storeDocument(persistence, document, docName);
setBatchInProgress(false);
- const renameSpy = spyOn(fsTraced, 'tracedRename').mockImplementation(async () => {
+ const renameSpy = vi.spyOn(fsTraced, 'tracedRename').mockImplementation(async () => {
const malformed = Object.create(Error.prototype) as Error & { name: string };
Object.defineProperty(malformed, 'name', { value: 'MalformedError', enumerable: true });
Object.defineProperty(malformed, 'message', {
@@ -835,7 +835,7 @@ describe('FR-9 — deferred-store-failed event + counter', () => {
await storeDocument(persistence, document, docName);
setBatchInProgress(false);
- const renameSpy = spyOn(fsTraced, 'tracedRename').mockImplementation(async () => {
+ const renameSpy = vi.spyOn(fsTraced, 'tracedRename').mockImplementation(async () => {
const malformed = Object.create(Error.prototype) as Error & { name: string };
Object.defineProperty(malformed, 'message', {
value: 'malformed-error-verbose',
@@ -943,3 +943,94 @@ describe('FR-9 — classifyDeferredStoreError behavior', () => {
expect(classifyDeferredStoreError(42)).toBe('unknown');
});
});
+
+describe('forceStore dispatch (staleness watchdog entry point)', () => {
+ let tmpDir: string;
+
+ beforeEach(() => {
+ tmpDir = mkdtempSync(join(tmpdir(), 'ok-force-store-'));
+ mkdirSync(tmpDir, { recursive: true });
+ setBatchInProgress(false);
+ switchReconciledBaseScope('main');
+ });
+
+ afterEach(() => {
+ setBatchInProgress(false);
+ switchReconciledBaseScope('main');
+ rmSync(tmpDir, { recursive: true, force: true });
+ });
+
+ test('flushes a doc whose debounced store never landed', async () => {
+ const docName = 'wedged-doc';
+ const docPath = join(tmpDir, `${docName}.md`);
+ writeFileSync(docPath, 'initial\n', 'utf-8');
+ const persistence = createPersistenceExtension({
+ contentDir: tmpDir,
+ projectDir: tmpDir,
+ gitEnabled: false,
+ });
+ const document = new Y.Doc();
+
+ await loadDocument(persistence, document, docName);
+ document.transact(() => replaceDocParagraph(document, 'rescued edit'), BROWSER_ORIGIN);
+ // No storeDocument call — the wedge shape: an edit with no store cycle left.
+
+ await persistence.forceStore(document, docName);
+
+ expect(readFileSync(docPath, 'utf-8')).toBe('rescued edit\n');
+ expect(getReconciledBase(docName)).toBe('rescued edit\n');
+
+ document.destroy();
+ });
+
+ test('defers during a coordinated batch and replays on drain', async () => {
+ const docName = 'batched-force';
+ const docPath = join(tmpDir, `${docName}.md`);
+ writeFileSync(docPath, 'initial\n', 'utf-8');
+ const persistence = createPersistenceExtension({
+ contentDir: tmpDir,
+ projectDir: tmpDir,
+ gitEnabled: false,
+ });
+ const document = new Y.Doc();
+
+ await loadDocument(persistence, document, docName);
+ document.transact(() => replaceDocParagraph(document, 'parked edit'), BROWSER_ORIGIN);
+
+ setBatchInProgress(true);
+ await persistence.forceStore(document, docName);
+ expect(readFileSync(docPath, 'utf-8')).toBe('initial\n');
+ expect(persistence.getQueueDepths().branchDeferred).toBe(1);
+
+ setBatchInProgress(false);
+ await persistence.flushDeferredStores('within-branch');
+ expect(readFileSync(docPath, 'utf-8')).toBe('parked edit\n');
+
+ document.destroy();
+ });
+
+ test('short-circuits synthetic docs without touching queues', async () => {
+ const persistence = createPersistenceExtension({
+ contentDir: tmpDir,
+ projectDir: tmpDir,
+ gitEnabled: false,
+ });
+ const document = new Y.Doc();
+ document.getText('source').insert(0, 'anything\n');
+
+ setBatchInProgress(true);
+ for (const name of [
+ '__system__',
+ '__config__/project',
+ '__skill__/project/foo',
+ 'diagram.mmd',
+ 'assets/flow.mermaid',
+ ]) {
+ await persistence.forceStore(document, name);
+ }
+
+ expect(persistence.getQueueDepths().branchDeferred).toBe(0);
+
+ document.destroy();
+ });
+});
diff --git a/packages/server/src/persistence-staleness-watchdog.test.ts b/packages/server/src/persistence-staleness-watchdog.test.ts
new file mode 100644
index 000000000..a5b7a41d4
--- /dev/null
+++ b/packages/server/src/persistence-staleness-watchdog.test.ts
@@ -0,0 +1,513 @@
+import { afterEach, beforeEach, describe, expect, test } from 'vitest';
+import * as Y from 'yjs';
+import { isPersistenceExcludedDoc } from './cc1-broadcast.ts';
+import { FROZEN_LIFECYCLE_STATUSES } from './conflict-errors.ts';
+import { getMetrics, resetMetrics } from './metrics.ts';
+import {
+ createPersistenceStalenessWatchdog,
+ type StalenessWatchdogHandle,
+ type StalenessWatchdogOptions,
+ StructuralDiskReadError,
+} from './persistence-staleness-watchdog.ts';
+
+const GRACE_MS = 1_000;
+
+interface Rig {
+ watchdog: StalenessWatchdogHandle;
+ docs: Map<string, Y.Doc>;
+ bases: Map<string, string>;
+ disk: Map<string, string>;
+ /** Per-doc ms-since-last-user-tx; missing entry → null (never observed). */
+ txAges: Map<string, number>;
+ forceCalls: string[];
+ clock: { nowMs: number };
+ batchActive: { value: boolean };
+ inFlight: Map<string, string>;
+ /** Behavior of the injected forceStore. */
+ forceBehavior: {
+ mode: 'advance-base' | 'no-op' | 'throw';
+ };
+ diskReadFault: { kind: 'transient' | 'structural' | null };
+}
+
+function makeDoc(source: string): Y.Doc {
+ const doc = new Y.Doc();
+ doc.getText('source').insert(0, source);
+ return doc;
+}
+
+function makeRig(overrides: Partial<StalenessWatchdogOptions> = {}): Rig {
+ const docs = new Map<string, Y.Doc>();
+ const bases = new Map<string, string>();
+ const disk = new Map<string, string>();
+ const txAges = new Map<string, number>();
+ const forceCalls: string[] = [];
+ const clock = { nowMs: 100_000 };
+ const batchActive = { value: false };
+ const inFlight = new Map<string, string>();
+ const forceBehavior: Rig['forceBehavior'] = { mode: 'advance-base' };
+ const diskReadFault: Rig['diskReadFault'] = { kind: null };
+
+ const watchdog = createPersistenceStalenessWatchdog({
+ getLoadedDocuments: () => docs,
+ forceStore: async (document, documentName) => {
+ forceCalls.push(documentName);
+ if (forceBehavior.mode === 'throw') throw new Error('injected store failure');
+ if (forceBehavior.mode === 'advance-base') {
+ const bytes = document.getText('source').toString();
+ bases.set(documentName, bytes);
+ disk.set(documentName, bytes);
+ }
+ },
+ readDiskBytes: (documentName) => {
+ if (diskReadFault.kind === 'structural') {
+ throw new StructuralDiskReadError('injected structural refusal');
+ }
+ if (diskReadFault.kind === 'transient') throw new Error('injected disk read failure');
+ return disk.get(documentName) ?? null;
+ },
+ graceMs: GRACE_MS,
+ // Interval never fires inside a test; sweeps are driven manually.
+ sweepIntervalMs: 3_600_000,
+ now: () => clock.nowMs,
+ getBase: (documentName) => bases.get(documentName),
+ isBatchActive: () => batchActive.value,
+ peekInFlight: (documentName) => inFlight.get(documentName),
+ msSinceLastUserTx: (doc) => {
+ for (const [name, d] of docs) {
+ if (d === doc) return txAges.get(name) ?? null;
+ }
+ return null;
+ },
+ ...overrides,
+ });
+
+ return {
+ watchdog,
+ docs,
+ bases,
+ disk,
+ txAges,
+ forceCalls,
+ clock,
+ batchActive,
+ inFlight,
+ forceBehavior,
+ diskReadFault,
+ };
+}
+
+/** A doc whose memory diverged from base+disk long ago — the wedge shape. */
+function seedWedgedDoc(rig: Rig, name: string): Y.Doc {
+ const doc = makeDoc('# edited in memory\n');
+ rig.docs.set(name, doc);
+ rig.bases.set(name, '# old on disk\n');
+ rig.disk.set(name, '# old on disk\n');
+ rig.txAges.set(name, GRACE_MS * 10);
+ return doc;
+}
+
+let rig: Rig;
+
+beforeEach(() => {
+ resetMetrics();
+ rig = makeRig();
+});
+
+afterEach(async () => {
+ await rig.watchdog.dispose();
+});
+
+describe('staleness detection and forced store', () => {
+ test('force-stores a doc divergent past the grace window and counts it', async () => {
+ seedWedgedDoc(rig, 'wedged-doc');
+
+ await rig.watchdog.sweep();
+
+ expect(rig.forceCalls).toEqual(['wedged-doc']);
+ expect(getMetrics().persistenceStalenessDetected).toBe(1);
+ expect(getMetrics().persistenceStalenessForcedStores).toBe(1);
+ // Base advanced by the store; the next sweep sees convergence and stays quiet.
+ await rig.watchdog.sweep();
+ expect(rig.forceCalls).toEqual(['wedged-doc']);
+ });
+
+ test('does not force while the last user transaction is younger than the grace window', async () => {
+ seedWedgedDoc(rig, 'active-doc');
+ rig.txAges.set('active-doc', GRACE_MS - 1);
+
+ await rig.watchdog.sweep();
+
+ expect(rig.forceCalls).toEqual([]);
+ expect(getMetrics().persistenceStalenessDetected).toBe(0);
+ });
+
+ test('treats a null transaction age as old enough', async () => {
+ seedWedgedDoc(rig, 'no-tx-doc');
+ rig.txAges.delete('no-tx-doc');
+
+ await rig.watchdog.sweep();
+
+ expect(rig.forceCalls).toEqual(['no-tx-doc']);
+ });
+
+ test('ignores docs whose memory matches the reconciled base', async () => {
+ const doc = makeDoc('# same\n');
+ rig.docs.set('clean-doc', doc);
+ rig.bases.set('clean-doc', '# same\n');
+ rig.disk.set('clean-doc', '# same\n');
+ rig.txAges.set('clean-doc', GRACE_MS * 10);
+
+ await rig.watchdog.sweep();
+
+ expect(rig.forceCalls).toEqual([]);
+ });
+
+ test('classifies frontmatter-carrying docs with the store spine comparator', async () => {
+ // Byte-unequal but normalize-equal (trailing newlines): the shared
+ // `normalizedSourceForm` derivation must read this as converged even
+ // through the frontmatter strip/prepend round-trip — the same verdict
+ // the store's no-op skip would reach.
+ const fmBase = '---\ntitle: x\n---\n\n# body\n\n\n';
+ const doc = makeDoc('---\ntitle: x\n---\n\n# body\n');
+ rig.docs.set('fm-doc', doc);
+ rig.bases.set('fm-doc', fmBase);
+ rig.disk.set('fm-doc', fmBase);
+ rig.txAges.set('fm-doc', GRACE_MS * 10);
+
+ await rig.watchdog.sweep();
+ expect(rig.forceCalls).toEqual([]);
+
+ // A frontmatter-only edit IS a divergence and must be rescued.
+ doc.getText('source').delete(0, doc.getText('source').length);
+ doc.getText('source').insert(0, '---\ntitle: y\n---\n\n# body\n');
+ await rig.watchdog.sweep();
+ expect(rig.forceCalls).toEqual(['fm-doc']);
+ });
+
+ test('materializes a never-persisted doc with content and no disk file', async () => {
+ const doc = makeDoc('# brand new\n');
+ rig.docs.set('new-doc', doc);
+ rig.txAges.set('new-doc', GRACE_MS * 10);
+
+ await rig.watchdog.sweep();
+
+ expect(rig.forceCalls).toEqual(['new-doc']);
+ });
+});
+
+describe('exclusions', () => {
+ test('isPersistenceExcludedDoc admits exactly the dedicated-store-path doc classes', () => {
+ // One representative per dedicated-store-path class dispatched in
+ // persistence's onLoadDocument/onStoreDocument. A fifth doc class
+ // added to that dispatch must be added here (and to the predicate) or
+ // the watchdog would force it through the markdown L1 spine.
+ for (const name of [
+ '__system__',
+ '__config__/project',
+ '__user__/config.yml',
+ '__local__/project',
+ '__skill__/project/foo',
+ '__template__/notes/weekly',
+ 'diagram.mmd',
+ 'assets/flow.mermaid',
+ ]) {
+ expect(isPersistenceExcludedDoc(name)).toBe(true);
+ }
+ for (const name of ['notes', 'folder/doc', 'README', 'docs/getting-started.mdx']) {
+ expect(isPersistenceExcludedDoc(name)).toBe(false);
+ }
+ });
+
+ test('skips system, config, managed-artifact, and mermaid docs', async () => {
+ for (const name of [
+ '__system__',
+ '__config__/project',
+ '__user__/config.yml',
+ '__local__/project',
+ '__skill__/project/foo',
+ '__template__/notes/weekly',
+ 'diagram.mmd',
+ ]) {
+ const doc = makeDoc('# divergent\n');
+ rig.docs.set(name, doc);
+ rig.bases.set(name, '# other\n');
+ rig.txAges.set(name, GRACE_MS * 10);
+ }
+
+ await rig.watchdog.sweep();
+
+ expect(rig.forceCalls).toEqual([]);
+ expect(getMetrics().persistenceStalenessDetected).toBe(0);
+ });
+
+ test('skips docs frozen by lifecycle status', async () => {
+ // Driven from the registry so a newly added frozen status is covered
+ // here automatically.
+ for (const status of FROZEN_LIFECYCLE_STATUSES) {
+ const doc = seedWedgedDoc(rig, `lifecycle-${status}`);
+ doc.getMap('lifecycle').set('status', status);
+ }
+
+ await rig.watchdog.sweep();
+
+ expect(rig.forceCalls).toEqual([]);
+ });
+
+ test('skips the whole sweep while a coordinated batch is active', async () => {
+ seedWedgedDoc(rig, 'batch-doc');
+ rig.batchActive.value = true;
+
+ await rig.watchdog.sweep();
+ expect(rig.forceCalls).toEqual([]);
+
+ rig.batchActive.value = false;
+ await rig.watchdog.sweep();
+ expect(rig.forceCalls).toEqual(['batch-doc']);
+ });
+
+ test('a batch activating mid-sweep does not decline the parked store', async () => {
+ // A batch starting inside the forceStore await means persistence PARKED
+ // the store (deferStore) rather than running it — the doc stays
+ // divergent with unchanged content, which must not be classified as a
+ // declined no-op (that would suppress retries even if the batch replay
+ // later drops the parked entry).
+ const parkingRig = makeRig({
+ forceStore: async (_document, documentName) => {
+ parkingRig.forceCalls.push(documentName);
+ parkingRig.batchActive.value = true;
+ },
+ });
+ try {
+ seedWedgedDoc(parkingRig, 'parked-doc');
+
+ await parkingRig.watchdog.sweep();
+ expect(parkingRig.forceCalls).toEqual(['parked-doc']);
+
+ // Batch over, base still stale, content unchanged: the next grace
+ // window must retry rather than treat the doc as suppressed.
+ parkingRig.batchActive.value = false;
+ parkingRig.clock.nowMs += GRACE_MS + 1;
+ await parkingRig.watchdog.sweep();
+ expect(parkingRig.forceCalls).toEqual(['parked-doc', 'parked-doc']);
+ } finally {
+ await parkingRig.watchdog.dispose();
+ }
+ });
+
+ test('skips a doc whose flush is currently mid-commit', async () => {
+ seedWedgedDoc(rig, 'inflight-doc');
+ rig.inFlight.set('inflight-doc', 'anything');
+
+ await rig.watchdog.sweep();
+
+ expect(rig.forceCalls).toEqual([]);
+ });
+
+ test('is a no-op after dispose', async () => {
+ seedWedgedDoc(rig, 'late-doc');
+ await rig.watchdog.dispose();
+
+ await rig.watchdog.sweep();
+
+ expect(rig.forceCalls).toEqual([]);
+ });
+
+ test('dispose drains an in-flight sweep and stops it before the next doc', async () => {
+ // Two wedged docs; block the first forceStore so dispose lands mid-sweep.
+ let releaseFirstStore: (() => void) | undefined;
+ const gate = new Promise<void>((resolve) => {
+ releaseFirstStore = resolve;
+ });
+ const blockingRig = makeRig({
+ forceStore: async (_document, documentName) => {
+ blockingRig.forceCalls.push(documentName);
+ await gate;
+ },
+ });
+ try {
+ seedWedgedDoc(blockingRig, 'doc-a');
+ seedWedgedDoc(blockingRig, 'doc-b');
+
+ const sweepPromise = blockingRig.watchdog.sweep();
+ const disposePromise = blockingRig.watchdog.dispose();
+ releaseFirstStore?.();
+ await disposePromise;
+ await sweepPromise;
+
+ // The in-flight sweep finished its current store but never reached
+ // the second doc.
+ expect(blockingRig.forceCalls).toEqual(['doc-a']);
+ } finally {
+ releaseFirstStore?.();
+ await blockingRig.watchdog.dispose();
+ }
+ });
+});
+
+describe('external-edit stand-down (disk authority)', () => {
+ test('does not overwrite disk bytes the base does not account for', async () => {
+ seedWedgedDoc(rig, 'external-doc');
+ rig.disk.set('external-doc', '# external native edit\n');
+
+ await rig.watchdog.sweep();
+
+ expect(rig.forceCalls).toEqual([]);
+ // Detected and counted as a stand-down, but not forced.
+ expect(getMetrics().persistenceStalenessDetected).toBe(1);
+ expect(getMetrics().persistenceStalenessForcedStores).toBe(0);
+ expect(getMetrics().persistenceStalenessStoodDown).toBe(1);
+ expect(rig.disk.get('external-doc')).toBe('# external native edit\n');
+
+ // Verified external state suppresses until content changes — a second
+ // sweep must not re-count the same stand-down.
+ await rig.watchdog.sweep();
+ expect(getMetrics().persistenceStalenessStoodDown).toBe(1);
+ });
+
+ test('does not resurrect a file deleted out-of-band', async () => {
+ seedWedgedDoc(rig, 'deleted-doc');
+ rig.disk.delete('deleted-doc');
+
+ await rig.watchdog.sweep();
+
+ expect(rig.forceCalls).toEqual([]);
+ expect(getMetrics().persistenceStalenessStoodDown).toBe(1);
+ });
+
+ test('does not overwrite an on-disk file the doc never loaded', async () => {
+ const doc = makeDoc('# memory only\n');
+ rig.docs.set('never-loaded', doc);
+ rig.disk.set('never-loaded', '# unread disk bytes\n');
+ rig.txAges.set('never-loaded', GRACE_MS * 10);
+
+ await rig.watchdog.sweep();
+
+ expect(rig.forceCalls).toEqual([]);
+ });
+
+ test('stands down when the disk read fails, then retries after another grace window', async () => {
+ seedWedgedDoc(rig, 'unreadable-doc');
+ rig.diskReadFault.kind = 'transient';
+
+ await rig.watchdog.sweep();
+ expect(rig.forceCalls).toEqual([]);
+
+ // Still failing, still inside the retry pace: no hammering.
+ await rig.watchdog.sweep();
+ expect(rig.forceCalls).toEqual([]);
+
+ // The read fault clears (transient EMFILE/EACCES class): unlike the
+ // verified external-state stand-downs, the doc must NOT stay suppressed.
+ rig.diskReadFault.kind = null;
+ rig.clock.nowMs += GRACE_MS + 1;
+ await rig.watchdog.sweep();
+ expect(rig.forceCalls).toEqual(['unreadable-doc']);
+ });
+
+ test('a structural read refusal declines until content changes', async () => {
+ const doc = seedWedgedDoc(rig, 'refused-doc');
+ rig.diskReadFault.kind = 'structural';
+
+ await rig.watchdog.sweep();
+ expect(rig.forceCalls).toEqual([]);
+ expect(getMetrics().persistenceStalenessStoodDown).toBe(1);
+
+ // A refusal that never clears (symlink-escape / oversized file) must
+ // not inflate the alertable counter or retry across grace windows.
+ rig.clock.nowMs += GRACE_MS * 3;
+ await rig.watchdog.sweep();
+ rig.clock.nowMs += GRACE_MS * 3;
+ await rig.watchdog.sweep();
+ expect(rig.forceCalls).toEqual([]);
+ expect(getMetrics().persistenceStalenessStoodDown).toBe(1);
+
+ // Only a content change re-arms the doc.
+ rig.diskReadFault.kind = null;
+ doc.getText('source').insert(0, 'more ');
+ rig.clock.nowMs += GRACE_MS + 1;
+ await rig.watchdog.sweep();
+ expect(rig.forceCalls).toEqual(['refused-doc']);
+ });
+
+ test('a stand-down re-arms when the memory content changes', async () => {
+ const doc = seedWedgedDoc(rig, 'rearm-doc');
+ rig.disk.set('rearm-doc', '# external native edit\n');
+
+ await rig.watchdog.sweep();
+ await rig.watchdog.sweep();
+ expect(rig.forceCalls).toEqual([]);
+ expect(getMetrics().persistenceStalenessDetected).toBe(1);
+
+ // External state reconciled (watcher caught up: base = disk), then a new
+ // memory edit wedges again.
+ rig.bases.set('rearm-doc', '# external native edit\n');
+ doc.getText('source').delete(0, doc.getText('source').length);
+ doc.getText('source').insert(0, '# newer memory edit\n');
+
+ await rig.watchdog.sweep();
+
+ expect(rig.forceCalls).toEqual(['rearm-doc']);
+ expect(getMetrics().persistenceStalenessDetected).toBe(2);
+ });
+});
+
+describe('retry and suppression discipline', () => {
+ test('a store that completes without clearing divergence suppresses until content changes', async () => {
+ rig.forceBehavior.mode = 'no-op';
+ const doc = seedWedgedDoc(rig, 'noop-doc');
+
+ await rig.watchdog.sweep();
+ expect(rig.forceCalls).toEqual(['noop-doc']);
+
+ // Same content, later sweeps — even past another grace window: no retry.
+ rig.clock.nowMs += GRACE_MS * 5;
+ await rig.watchdog.sweep();
+ expect(rig.forceCalls).toEqual(['noop-doc']);
+
+ // Content changed — re-armed.
+ doc.getText('source').insert(0, 'more ');
+ rig.clock.nowMs += GRACE_MS * 5;
+ await rig.watchdog.sweep();
+ expect(rig.forceCalls).toEqual(['noop-doc', 'noop-doc']);
+ });
+
+ test('a failed store retries only after another full grace window', async () => {
+ rig.forceBehavior.mode = 'throw';
+ seedWedgedDoc(rig, 'failing-doc');
+
+ await rig.watchdog.sweep();
+ expect(rig.forceCalls).toEqual(['failing-doc']);
+
+ // Immediately after: no hammering.
+ await rig.watchdog.sweep();
+ expect(rig.forceCalls).toEqual(['failing-doc']);
+
+ // A grace window later: retried; a recovered disk then clears it.
+ rig.clock.nowMs += GRACE_MS + 1;
+ rig.forceBehavior.mode = 'advance-base';
+ await rig.watchdog.sweep();
+ expect(rig.forceCalls).toEqual(['failing-doc', 'failing-doc']);
+ await rig.watchdog.sweep();
+ expect(rig.forceCalls).toEqual(['failing-doc', 'failing-doc']);
+ });
+
+ test('drops bookkeeping for docs that unloaded', async () => {
+ rig.forceBehavior.mode = 'no-op';
+ seedWedgedDoc(rig, 'transient-doc');
+
+ await rig.watchdog.sweep();
+ expect(rig.forceCalls).toEqual(['transient-doc']);
+
+ // Unload, then reload in the same wedged shape: suppression must not
+ // survive the unload (a fresh load re-reads disk and re-bases anyway).
+ const doc = rig.docs.get('transient-doc');
+ rig.docs.delete('transient-doc');
+ await rig.watchdog.sweep();
+ if (doc) rig.docs.set('transient-doc', doc);
+ rig.clock.nowMs += GRACE_MS * 2;
+
+ await rig.watchdog.sweep();
+ expect(rig.forceCalls).toEqual(['transient-doc', 'transient-doc']);
+ });
+});
diff --git a/packages/server/src/persistence-staleness-watchdog.ts b/packages/server/src/persistence-staleness-watchdog.ts
new file mode 100644
index 000000000..b6c6bd527
--- /dev/null
+++ b/packages/server/src/persistence-staleness-watchdog.ts
@@ -0,0 +1,421 @@
+/**
+ * Persistence staleness watchdog — a durability backstop for the L1
+ * CRDT→disk write spine.
+ *
+ * Hocuspocus re-arms the persistence store debounce only when a NEW Y.Doc
+ * update arrives, and user docs stay resident for the whole server
+ * lifetime. Any store cycle that fails or is skipped after the last edit
+ * of a session (a transient disk-write error, a batch deferral dropped on
+ * branch mismatch, a lifecycle race) therefore leaves the live doc newer
+ * than disk with no retry scheduled — silently, for as long as the server
+ * runs. The watchdog sweeps loaded docs on an interval and force-stores
+ * any doc whose in-memory bytes have diverged from the persistence layer's
+ * own reconciled base for longer than a generous grace window.
+ *
+ * Why a general sweep rather than a retry at each failure site: the
+ * lost-cycle classes have no common hook point with a retry OWNER. A
+ * throwing debounced store is recorded out-of-band (`storeFailures`) but
+ * nothing on the debounced path ever re-drives it; a branch-mismatched
+ * deferral is dropped inside `flushDeferredStores` with no owner to
+ * notify; a lifecycle-race skip is by construction invisible to the
+ * writer. One anti-entropy sweep over the divergence signal covers every
+ * class, including ones not yet catalogued.
+ *
+ * Safety posture (additive; never bypasses the write spine):
+ * - The staleness predicate is memory vs `reconciledBase` — the
+ * persistence layer's own last-agreed-disk baseline — never memory vs
+ * raw disk. An external disk edit the file-watcher has ingested
+ * advances both sides, so it can never trip the predicate.
+ * - Before forcing, disk is re-read and compared to the base. Bytes on
+ * disk the base doesn't account for mean an out-of-band edit the
+ * watcher hasn't reconciled yet; the watchdog stands down (warn only)
+ * rather than clobber it. Disk stays authoritative for unseen edits,
+ * including out-of-band deletes (base set, file gone → stand down, so
+ * a forced store can't resurrect a deleted file). Residual risk: an
+ * external edit landing in the few-ms window between this pre-check
+ * and the store's atomic write is overwritten — the same window every
+ * debounce-triggered non-agent store has today (only agent-marked
+ * stores get the store-time re-check), so the watchdog adds no new
+ * clobber class; the file-watcher reconcile remains the designed
+ * recovery for that race.
+ * - The force routes through the persistence handle's `forceStore`,
+ * i.e. the exact store spine the debounced hook uses — every existing
+ * guard (lifecycle, quiescence gate, no-op skip, duplication
+ * tripwire, writeTracker registration) applies unchanged.
+ */
+
+import { normalizeBridge } from '@inkeep/open-knowledge-core';
+import type * as Y from 'yjs';
+import { getMsSinceLastUserTx } from './bridge-quiescence.ts';
+import { isPersistenceExcludedDoc } from './cc1-broadcast.ts';
+import { frozenDocLifecycleStatus } from './conflict-errors.ts';
+import { getLogger } from './logger.ts';
+import {
+ incrementPersistenceStalenessDetected,
+ incrementPersistenceStalenessForcedStores,
+ incrementPersistenceStalenessStoodDown,
+} from './metrics.ts';
+import {
+ getReconciledBase,
+ isBatchInProgress,
+ normalizedSourceForm,
+ peekInFlightFlush,
+} from './persistence.ts';
+
+const log = getLogger('persistence-staleness');
+
+/**
+ * Throw from `readDiskBytes` for structurally permanent read refusals — a
+ * content path replaced by a symlink resolving outside the content dir, or
+ * an out-of-band file grown past the open byte limit. The watchdog then
+ * declines the doc until its content changes (instead of retrying every
+ * grace window and inflating the alertable stand-down counter forever) and
+ * logs at error — a symlink-escape is security-relevant, not transient
+ * I/O. Any other throw is treated as transient and retried.
+ */
+export class StructuralDiskReadError extends Error {
+ constructor(message: string) {
+ super(message);
+ this.name = 'StructuralDiskReadError';
+ }
+}
+
+/**
+ * Generous by design: the natural flush path settles within seconds
+ * (2 s debounce, 10 s maxDebounce, ~16 s quiescence-defer ceiling), so
+ * anything divergent this long after its last transaction has no pending
+ * store left to wait for.
+ */
+const DEFAULT_STALENESS_GRACE_MS = 5 * 60_000;
+const DEFAULT_STALENESS_SWEEP_INTERVAL_MS = 60_000;
+
+export interface StalenessWatchdogOptions {
+ /** Live loaded-docs view, typically `() => hocuspocus.documents`. */
+ getLoadedDocuments: () => Iterable<readonly [string, Y.Doc]>;
+ /**
+ * Force one store cycle through the persistence handle
+ * (`PersistenceHandle.forceStore`). Rejections are logged and retried on
+ * a later sweep.
+ */
+ forceStore: (document: Y.Doc, documentName: string) => Promise<void>;
+ /**
+ * Raw disk bytes for a docName, or `null` when the file does not exist.
+ * Throws make the watchdog stand down for that doc — an unreadable file
+ * must never be overwritten on a guess. A {@link StructuralDiskReadError}
+ * declines the doc until its content changes (the refusal won't clear on
+ * its own); any other throw is a transient inability to check and
+ * retries a grace window later.
+ */
+ readDiskBytes: (documentName: string) => string | null;
+ /** Minimum age of the newest user transaction before a doc counts as stale. */
+ graceMs?: number;
+ /** Sweep cadence. The interval timer is unref'd. */
+ sweepIntervalMs?: number;
+ /** Test seam for deterministic clocks. */
+ now?: () => number;
+ /** Test seams — default to the real persistence/quiescence surfaces. */
+ getBase?: (documentName: string) => string | undefined;
+ isBatchActive?: () => boolean;
+ peekInFlight?: (documentName: string) => string | undefined;
+ msSinceLastUserTx?: (doc: Y.Doc, nowMs: number) => number | null;
+}
+
+export interface StalenessWatchdogHandle {
+ /** Run one sweep now. The interval skips while a sweep is in flight. */
+ sweep: () => Promise<void>;
+ /**
+ * Stop the interval and drain: resolves after any in-flight sweep has
+ * finished, so a caller tearing the server down can be sure no forced
+ * store fires against a draining write spine afterwards. An in-flight
+ * sweep also short-circuits at its next between-docs checkpoint.
+ */
+ dispose: () => Promise<void>;
+}
+
+interface AttemptRecord {
+ fingerprint: string;
+ atMs: number;
+ /**
+ * When this doc's current staleness episode was first observed, carried
+ * across retries (unlike `atMs`, which paces them) so logs can report
+ * how long the doc has been stale — "1 doc failing for an hour" and
+ * "100 docs rescued once" are different incidents.
+ */
+ firstDetectedAtMs: number;
+ /**
+ * Retrying these exact bytes is provably futile: the store ran to
+ * completion but chose not to write them (no-op classification, tripwire
+ * breaker, lifecycle transition), or disk verifiably holds external
+ * state the base doesn't account for (divergent bytes, out-of-band
+ * delete, never-loaded file). Re-arming requires the content to change.
+ * NOT set for transient conditions — a failed store, a failed disk
+ * read, or a store parked by a mid-sweep batch keeps `declined: false`
+ * so the next grace window retries.
+ */
+ declined: boolean;
+}
+
+export function createPersistenceStalenessWatchdog(
+ options: StalenessWatchdogOptions,
+): StalenessWatchdogHandle {
+ const graceMs = options.graceMs ?? DEFAULT_STALENESS_GRACE_MS;
+ const sweepIntervalMs = options.sweepIntervalMs ?? DEFAULT_STALENESS_SWEEP_INTERVAL_MS;
+ const now = options.now ?? Date.now;
+ const getBase = options.getBase ?? getReconciledBase;
+ const isBatchActive = options.isBatchActive ?? isBatchInProgress;
+ const peekInFlight = options.peekInFlight ?? peekInFlightFlush;
+ const msSinceLastUserTx = options.msSinceLastUserTx ?? getMsSinceLastUserTx;
+
+ const attempts = new Map<string, AttemptRecord>();
+ let disposed = false;
+ let sweepInFlight: Promise<void> | null = null;
+
+ /**
+ * The store's own no-op comparator form, via the SAME derivation
+ * (`normalizedSourceForm`) `storeDocumentNow`'s
+ * `markdownSemanticallyUnchanged` skip uses — the two classifiers agree
+ * by construction. The store's extra ephemeral-canonical branch is NOT
+ * replicated: ephemeral single-file servers don't construct a watchdog
+ * at all (see the `!ephemeral` gate in server-factory), precisely
+ * because this classifier would read a round-trip-unstable file at rest
+ * as divergent.
+ */
+ function candidateFromRaw(rawYText: string): string {
+ return normalizedSourceForm(rawYText);
+ }
+
+ function normalizedBaseFor(base: string | undefined): string | undefined {
+ return base === undefined ? undefined : normalizeBridge(base);
+ }
+
+ function isDivergent(candidate: string, normalizedBase: string | undefined): boolean {
+ return normalizedBase === undefined ? candidate !== '' : candidate !== normalizedBase;
+ }
+
+ async function sweepOnce(): Promise<void> {
+ if (disposed) return;
+ // A coordinated git operation owns the write spine right now; bail so
+ // the sweep can't interleave with it. Within-branch deferral replay
+ // covers most pending work when the batch ends; anything it drops
+ // (branch-mismatched entries, discard-stale mode) re-surfaces as
+ // divergence on a later sweep — this sweep is that recovery path.
+ if (isBatchActive()) return;
+
+ const startedMs = now();
+ let scanned = 0;
+ let divergent = 0;
+ let forced = 0;
+ let stoodDown = 0;
+ const seen = new Set<string>();
+ for (const [documentName, document] of options.getLoadedDocuments()) {
+ // dispose() during a previous iteration's await must stop the sweep
+ // before it touches another doc.
+ if (disposed) return;
+ seen.add(documentName);
+ if (isPersistenceExcludedDoc(documentName)) continue;
+ if (frozenDocLifecycleStatus(document) !== null) {
+ // These docs intentionally hold memory != disk; the store spine
+ // skips them for the same reason.
+ continue;
+ }
+ if (peekInFlight(documentName) !== undefined) continue;
+
+ scanned++;
+ const rawYText = document.getText('source').toString();
+ const base = getBase(documentName);
+ // Byte-equal fast path: after any successful store or file-watcher
+ // ingest the raw Y.Text equals the base verbatim, so resident clean
+ // docs skip the strip/normalize comparator work entirely.
+ if (base !== undefined && rawYText === base) {
+ attempts.delete(documentName);
+ continue;
+ }
+ const candidate = candidateFromRaw(rawYText);
+ const normalizedBase = normalizedBaseFor(base);
+ if (!isDivergent(candidate, normalizedBase)) {
+ attempts.delete(documentName);
+ continue;
+ }
+ divergent++;
+
+ const nowMs = now();
+ const ageMs = msSinceLastUserTx(document, nowMs);
+ if (ageMs !== null && ageMs < graceMs) continue;
+
+ const previous = attempts.get(documentName);
+ if (previous && previous.fingerprint === candidate) {
+ if (previous.declined) continue;
+ if (nowMs - previous.atMs < graceMs) continue;
+ }
+ const firstSighting = previous?.fingerprint !== candidate;
+ if (firstSighting) incrementPersistenceStalenessDetected();
+ const firstDetectedAtMs = !firstSighting && previous ? previous.firstDetectedAtMs : nowMs;
+ const staleForMs = nowMs - firstDetectedAtMs;
+
+ // External-edit stand-down. Disk bytes the base doesn't account for
+ // mean the file-watcher hasn't reconciled an out-of-band change yet;
+ // writing memory over them would clobber it. A missing file with a
+ // known base is an unreconciled out-of-band delete — same stand-down
+ // (a forced store would resurrect the file). Only two shapes are
+ // safe to write: disk still matches the base, or the doc has never
+ // had an on-disk form at all.
+ let standDownReason: string | null = null;
+ let standDownRetryable = false;
+ try {
+ const diskBytes = options.readDiskBytes(documentName);
+ if (normalizedBase === undefined) {
+ if (diskBytes !== null) standDownReason = 'disk-file-never-loaded';
+ } else if (diskBytes === null) {
+ standDownReason = 'disk-file-missing';
+ } else if (normalizeBridge(diskBytes) !== normalizedBase) {
+ standDownReason = 'disk-diverged-from-base';
+ }
+ } catch (err) {
+ if (err instanceof StructuralDiskReadError) {
+ // The refusal (symlink-escape, oversized file) won't clear on its
+ // own — decline until the doc's content changes so the alertable
+ // stand-down counter doesn't grow forever on one bad path.
+ standDownReason = 'disk-read-refused';
+ log.error(
+ { err, docName: documentName },
+ '[persistence-staleness] disk read refused; standing down until content changes',
+ );
+ } else {
+ // Can't verify disk, so don't write — but a transient read fault
+ // says nothing about external state, so retry after the next
+ // grace window rather than suppressing the doc.
+ standDownReason = 'disk-read-failed';
+ standDownRetryable = true;
+ log.warn(
+ { err, docName: documentName },
+ '[persistence-staleness] disk read failed; standing down',
+ );
+ }
+ }
+
+ if (standDownReason !== null) {
+ stoodDown++;
+ incrementPersistenceStalenessStoodDown();
+ attempts.set(documentName, {
+ fingerprint: candidate,
+ atMs: nowMs,
+ firstDetectedAtMs,
+ declined: !standDownRetryable,
+ });
+ log.warn(
+ {
+ docName: documentName,
+ ageMs,
+ staleForMs,
+ candidateBytes: candidate.length,
+ baseBytes: base?.length ?? 0,
+ action: 'stood-down',
+ reason: standDownReason,
+ },
+ `[persistence-staleness] Unflushed edits detected for ${documentName} but disk state is unverified or unreconciled; not overwriting`,
+ );
+ continue;
+ }
+
+ log.warn(
+ {
+ docName: documentName,
+ ageMs,
+ staleForMs,
+ candidateBytes: candidate.length,
+ baseBytes: base?.length ?? 0,
+ action: 'forced-store',
+ },
+ `[persistence-staleness] Doc ${documentName} has unpersisted edits past the grace window with no pending store; forcing a store`,
+ );
+ attempts.set(documentName, {
+ fingerprint: candidate,
+ atMs: nowMs,
+ firstDetectedAtMs,
+ declined: false,
+ });
+ incrementPersistenceStalenessForcedStores();
+ forced++;
+ try {
+ await options.forceStore(document, documentName);
+ } catch (err) {
+ // The store spine already recorded the failure out-of-band; keep
+ // declined=false so the next grace window retries (a transient disk
+ // condition may have cleared).
+ log.error(
+ { err, docName: documentName, staleForMs },
+ `[persistence-staleness] Forced store failed for ${documentName}; will retry`,
+ );
+ continue;
+ }
+ if (disposed) return;
+ // A batch that activated inside the forceStore await means the store
+ // was PARKED (deferStore), not run — classifying the still-divergent
+ // doc as declined here would permanently suppress it even if the
+ // batch's replay later drops the parked entry. Bail the sweep (the
+ // batch owns the spine now); the entry stays declined:false so the
+ // next grace window re-checks.
+ if (isBatchActive()) return;
+
+ // Re-read both sides: a store that completed yet left the same bytes
+ // divergent chose not to write them (no-op class) — mark declined so
+ // we don't loop on it. Convergence (or content that moved on) re-arms.
+ const afterCandidate = candidateFromRaw(document.getText('source').toString());
+ const afterNormalizedBase = normalizedBaseFor(getBase(documentName));
+ if (isDivergent(afterCandidate, afterNormalizedBase) && afterCandidate === candidate) {
+ attempts.set(documentName, {
+ fingerprint: candidate,
+ atMs: nowMs,
+ firstDetectedAtMs,
+ declined: true,
+ });
+ log.info(
+ { docName: documentName },
+ `[persistence-staleness] Store completed without clearing divergence for ${documentName}; suppressing until content changes`,
+ );
+ } else {
+ attempts.delete(documentName);
+ }
+ }
+
+ // Docs that unloaded since the last sweep drop their bookkeeping.
+ for (const name of attempts.keys()) {
+ if (!seen.has(name)) attempts.delete(name);
+ }
+
+ // Sweep-health signal: the outcome counters say nothing about sweep
+ // cost or a wedged sweep; this line does, without per-doc volume.
+ // Eventful sweeps log at info so production log levels see them; the
+ // steady-state all-clean sweep stays at debug to keep logs quiet.
+ const summary = { scanned, divergent, forced, stoodDown, elapsedMs: now() - startedMs };
+ if (divergent > 0 || forced > 0 || stoodDown > 0) {
+ log.info(summary, '[persistence-staleness] sweep complete');
+ } else {
+ log.debug(summary, '[persistence-staleness] sweep complete');
+ }
+ }
+
+ function sweep(): Promise<void> {
+ if (sweepInFlight) return sweepInFlight;
+ sweepInFlight = sweepOnce().finally(() => {
+ sweepInFlight = null;
+ });
+ return sweepInFlight;
+ }
+
+ const timer = setInterval(() => {
+ void sweep().catch((err) => {
+ log.error({ err }, '[persistence-staleness] sweep failed');
+ });
+ }, sweepIntervalMs);
+ timer.unref?.();
+
+ return {
+ sweep,
+ dispose: () => {
+ disposed = true;
+ clearInterval(timer);
+ return sweepInFlight ?? Promise.resolve();
+ },
+ };
+}
diff --git a/packages/server/src/persistence.ts b/packages/server/src/persistence.ts
index 196d8cfd0..6e6775d93 100644
--- a/packages/server/src/persistence.ts
+++ b/packages/server/src/persistence.ts
@@ -45,8 +45,15 @@ import { LINEAGE_EPOCH_KEY } from './auth-token-schema.ts';
import type { BacklinkIndex } from './backlink-index.ts';
import { getMsSinceLastUserTx, isDocQuiescent } from './bridge-quiescence.ts';
import { assertBridgeInvariant, createDocCanonicalizer } from './bridge-watchdog.ts';
-import { isConfigDoc, isManagedArtifactDoc, isMermaidDoc, isSystemDoc } from './cc1-broadcast.ts';
+import {
+ isConfigDoc,
+ isManagedArtifactDoc,
+ isMermaidDoc,
+ isPersistenceExcludedDoc,
+ isSystemDoc,
+} from './cc1-broadcast.ts';
import { type ConfigPersistenceCtx, loadConfigDoc, storeConfigDoc } from './config-persistence.ts';
+import { frozenDocLifecycleStatus } from './conflict-errors.ts';
import type { ContributorEntry } from './contributor-tracker.ts';
import {
contributorCount,
@@ -430,6 +437,22 @@ export function captureDocSnapshotForPersistence(document: Y.Doc): {
};
}
+/**
+ * The store spine's canonical "what would land on disk" form of raw
+ * Y.Text bytes: frontmatter re-composed onto the body, then
+ * `normalizeBridge`d. This is the LEFT side of `storeDocumentNow`'s
+ * no-op skip (`markdownSemanticallyUnchanged`) AND the staleness
+ * watchdog's divergence predicate — both compare it against
+ * `normalizeBridge(reconciledBase)`. Single derivation site so the two
+ * classifiers cannot drift: a doc the store would no-op must never read
+ * as stale to the watchdog, and a doc the watchdog reads as converged
+ * must never be one the store would rewrite.
+ */
+export function normalizedSourceForm(rawYText: string): string {
+ const { frontmatter, body } = stripFrontmatter(rawYText);
+ return normalizeBridge(prependFrontmatter(frontmatter, body));
+}
+
export function safeContentPath(documentName: string, contentDir: string): string {
if (documentName.includes('\x00')) {
throw new Error(`Invalid document name: ${documentName}`);
@@ -636,6 +659,19 @@ export interface PersistenceHandle {
* mutates state.
*/
getQueueDepths: () => PersistenceQueueDepths;
+ /**
+ * Force one immediate L1 store cycle for a loaded doc through the exact
+ * dispatch the debounced `onStoreDocument` hook uses: synthetic docs
+ * (system / config / managed-artifact / mermaid) short-circuit, a
+ * coordinated batch defers, and everything else runs `storeDocumentNow`
+ * with all of its guards (lifecycle, quiescence, no-op skip, tripwire,
+ * writeTracker registration). Exists for the staleness watchdog —
+ * Hocuspocus only re-arms its store debounce on a new Y.Doc update, so a
+ * store cycle that failed or was skipped after a session's last edit has
+ * no natural retry. Rejections mirror `storeDocumentNow`'s (the failure
+ * is also recorded for `takeStoreFailure`).
+ */
+ forceStore: (document: Y.Doc, documentName: string) => Promise<void>;
/**
* Config-doc persistence context. Exposed so the file-watcher
* orchestration in `server-factory.ts` can call `applyExternalConfigChange`
@@ -1296,12 +1332,8 @@ export function createPersistenceExtension(options?: PersistenceOptions): Persis
// the conflict window stay in Y.Text; on resolution the
// case 'update' reconcile in server-factory.ts runs against
// them as `ours`.
- const lifecycleStatus = document.getMap('lifecycle').get('status');
- if (
- lifecycleStatus === 'deleted-upstream' ||
- lifecycleStatus === 'renamed' ||
- lifecycleStatus === 'conflict'
- ) {
+ const lifecycleStatus = frozenDocLifecycleStatus(document);
+ if (lifecycleStatus !== null) {
log.info(
{ documentName, lifecycleStatus },
`[persistence] Skipped store for ${documentName}: lifecycle=${lifecycleStatus}`,
@@ -1491,7 +1523,9 @@ export function createPersistenceExtension(options?: PersistenceOptions): Persis
// safety-net below, preventing phantom commits attributed to the
// browser's principal when a later agent write triggers the L2 fan-out.
const currentBase = getReconciledBase(documentName);
- const normalizedMarkdown = normalizeBridge(markdown);
+ // Routed through the shared helper so this compare and the
+ // staleness watchdog's divergence predicate stay one derivation.
+ const normalizedMarkdown = normalizedSourceForm(ytextSnapshot);
let markdownSemanticallyUnchanged =
currentBase !== undefined && normalizedMarkdown === normalizeBridge(currentBase);
// G8 (ephemeral single-file mode only): also treat the candidate as a
@@ -2388,6 +2422,22 @@ export function createPersistenceExtension(options?: PersistenceOptions): Persis
};
}
+ async function forceStore(document: Y.Doc, documentName: string): Promise<void> {
+ if (isPersistenceExcludedDoc(documentName)) {
+ return;
+ }
+ if (isBatchInProgress()) {
+ // Same parking the debounced hook applies; the batch's own
+ // `flushDeferredStores` replays it when the coordinated operation ends.
+ deferStore({ document, documentName, lastTransactionOrigin: null });
+ return;
+ }
+ // Origin null: the triggering write's contributor entry (if any) was
+ // recorded when the edit happened; a forced re-flush must not invent a
+ // new writer identity.
+ return storeDocumentNow({ document, documentName, lastTransactionOrigin: null });
+ }
+
return {
extension,
flushDeferredStores,
@@ -2398,6 +2448,7 @@ export function createPersistenceExtension(options?: PersistenceOptions): Persis
takeStoreDivergence,
markAgentWriteStore,
getQueueDepths,
+ forceStore,
configPersistenceCtx,
managedArtifactCtx,
};
diff --git a/packages/server/src/server-factory.ts b/packages/server/src/server-factory.ts
index b9a347245..17fcf4e82 100644
--- a/packages/server/src/server-factory.ts
+++ b/packages/server/src/server-factory.ts
@@ -1,6 +1,13 @@
import { type SpawnSyncReturns, spawnSync } from 'node:child_process';
import { randomUUID } from 'node:crypto';
-import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
+import {
+ existsSync,
+ mkdirSync,
+ readFileSync,
+ realpathSync,
+ statSync,
+ writeFileSync,
+} from 'node:fs';
import { basename, dirname, join, relative, resolve } from 'node:path';
import type { Document, Extension } from '@hocuspocus/server';
import { Hocuspocus, IncomingMessage, MessageType } from '@hocuspocus/server';
@@ -15,6 +22,7 @@ import {
createBasenameIndex,
DEFAULT_ATTACHMENT_FOLDER_PATH,
DEFAULT_LINTER_CONFIG,
+ DOCUMENT_OPEN_BYTE_LIMIT,
humanFormat,
isKnownConfigError,
type LinterConfig,
@@ -141,6 +149,11 @@ import {
setReconciledBase,
switchReconciledBaseScope,
} from './persistence.ts';
+import {
+ createPersistenceStalenessWatchdog,
+ type StalenessWatchdogHandle,
+ StructuralDiskReadError,
+} from './persistence-staleness-watchdog.ts';
import { loadPrincipal } from './principal.ts';
import { RecentlyRemovedDocs } from './recently-removed-docs.ts';
import { reconcile } from './reconciliation.ts';
@@ -189,6 +202,14 @@ export interface ServerOptions {
quiet?: boolean;
debounce?: number;
maxDebounce?: number;
+ /**
+ * Persistence staleness watchdog tuning. Production defaults (5 min
+ * grace, 60 s sweep) live in `persistence-staleness-watchdog.ts`; tests
+ * pass small values to exercise the forced-store rescue path without
+ * wall-clock waits.
+ */
+ stalenessGraceMs?: number;
+ stalenessSweepIntervalMs?: number;
gitEnabled?: boolean;
commitDebounceMs?: number;
wipRef?: string;
@@ -862,6 +883,7 @@ export function createServer(options: ServerOptions): ServerInstance {
let agentFocusBroadcaster: AgentFocusBroadcaster | null = null;
let agentPresenceBroadcaster: AgentPresenceBroadcaster | null = null;
let invalidateReferencedAssetsCache: (() => void) | null = null;
+ let stalenessWatchdog: StalenessWatchdogHandle | null = null;
// Semantic-search service. Always constructed (cheap, inert until enabled +
// keyed; the loader makes no network call). The api-extension drives the lazy
@@ -1226,6 +1248,74 @@ export function createServer(options: ServerOptions): ServerInstance {
);
installServerWorkloadGauges();
+ // Durability backstop: Hocuspocus re-arms the store debounce only on a
+ // new Y.Doc update and user docs stay resident for the server lifetime,
+ // so a store cycle lost after a session's last edit (transient disk
+ // error, dropped deferral) would otherwise leave disk stale until the
+ // next edit. The watchdog re-flushes through `persistence.forceStore`
+ // (the normal store spine) and never overwrites disk state the
+ // reconciled base doesn't account for.
+ //
+ // Ephemeral single-file mode opts out: the watchdog's divergence
+ // classifier deliberately lacks the store's ephemeral-canonical no-op
+ // branch, so a round-trip-unstable file at rest would read as stale
+ // and pollute the staleness counters on a healthy session. Those
+ // sessions are short-lived and user-attended, not the days-wedged
+ // resident-doc class this backstop exists for.
+ if (!ephemeral) {
+ stalenessWatchdog = createPersistenceStalenessWatchdog({
+ getLoadedDocuments: () => hp.documents,
+ forceStore: (document, documentName) => persistence.forceStore(document, documentName),
+ // Realpath symlink-escape gate + open-byte-limit cap, matching the
+ // load-path read discipline. Not atomic — three syscalls — but
+ // ENOENT from any of them maps to `null` (out-of-band delete)
+ // rather than a transient read fault, so a delete landing between
+ // syscalls classifies the same as one landing before the first.
+ // The two refusals are structurally permanent (they won't clear
+ // without an on-disk change), so they throw the typed error that
+ // makes the watchdog decline instead of retrying every window.
+ readDiskBytes: (documentName) => {
+ const requestedPath = safeContentPath(documentName, contentDir);
+ // The structural throws live OUTSIDE the errno-filtering try
+ // blocks so a future widening of the errno filter can never
+ // swallow a security-relevant refusal.
+ let canonical: string;
+ let size: number;
+ try {
+ canonical = realpathSync(requestedPath);
+ size = statSync(canonical).size;
+ } catch (err) {
+ const code = (err as NodeJS.ErrnoException).code;
+ if (code === 'ENOENT') return null;
+ if (code === 'ELOOP') {
+ // A symlink cycle is structurally permanent, matching the
+ // load path's explicit ELOOP refusal.
+ throw new StructuralDiskReadError(`symlink cycle at content path: ${documentName}`);
+ }
+ throw err;
+ }
+ if (!isWithinContentDir(canonical, contentDir)) {
+ throw new StructuralDiskReadError(
+ `symlink-escape: ${requestedPath} resolves outside the content dir`,
+ );
+ }
+ if (size > DOCUMENT_OPEN_BYTE_LIMIT) {
+ throw new StructuralDiskReadError(
+ `document exceeds the open byte limit: ${documentName}`,
+ );
+ }
+ try {
+ return readFileSync(canonical, 'utf-8');
+ } catch (err) {
+ if ((err as NodeJS.ErrnoException).code === 'ENOENT') return null;
+ throw err;
+ }
+ },
+ graceMs: options.stalenessGraceMs,
+ sweepIntervalMs: options.stalenessSweepIntervalMs,
+ });
+ }
+
// Hocuspocus unloads documents as soon as the last WebSocket disconnects.
// That is unsafe with client-side y-indexeddb: a browser refresh leaves a
// durable client copy of the same Yjs items, while the server rebuilds a
@@ -1693,10 +1783,15 @@ export function createServer(options: ServerOptions): ServerInstance {
// here too — otherwise their closures (capturing `hocuspocus`/`persistence`)
// leak into the process-global registries and keep summing off a
// half-torn-down object. Idempotent via splice(0), so destroy()'s later
- // drain stays a no-op.
+ // drain stays a no-op. The staleness watchdog's interval holds the same
+ // closures, so it gets the same treatment.
for (const unregister of unregisterWorkloadProviders.splice(0)) {
unregister();
}
+ // Sync context — can't await the drain; the disposed flag makes any
+ // in-flight sweep short-circuit at its next between-docs checkpoint.
+ void stalenessWatchdog?.dispose();
+ stalenessWatchdog = null;
releaseServerLock(lockDir);
throw err;
}
@@ -2431,10 +2526,19 @@ export function createServer(options: ServerOptions): ServerInstance {
// Stop contributing to the process-wide workload gauges before any
// teardown phase runs — a mid-destroy sample would otherwise read
- // half-dismantled structures.
+ // half-dismantled structures. Same for the staleness watchdog: a sweep
+ // firing mid-destroy could force a store against a draining spine.
for (const unregister of unregisterWorkloadProviders.splice(0)) {
unregister();
}
+ // Awaited drain: dispose resolves only after any in-flight sweep has
+ // finished, so no forced store can land once teardown proceeds.
+ try {
+ await stalenessWatchdog?.dispose();
+ } catch (err) {
+ log.warn({ err }, '[server] staleness watchdog drain failed during destroy');
+ }
+ stalenessWatchdog = null;
// Advertise teardown FIRST — before any flush work. Readers (MCP
// discovery, desktop attach, spawners) see `draining: true` and stop