Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
fix(open-knowledge): add persistence staleness watchdog (PRD-7492) (#2783)
Andrew Mikofalvyd873fe6333d1
diff --git a/.changeset/prd-7492-persistence-staleness.md b/.changeset/prd-7492-persistence-staleness.mdnew file mode 100644index 000000000..dc50f77a0--- /dev/null+++ b/.changeset/prd-7492-persistence-staleness.md@@ -0,0 +1,5 @@+---+'@inkeep/open-knowledge': patch+---++Add a persistence staleness watchdog that bounds how long CRDT edits can sit unflushed to disk. If a document's in-memory edits ever fail to reach the on-disk markdown (for example after a transient disk error, since the store debounce only re-arms on the next edit), the server now detects the stale file within minutes and re-runs the store through the normal persistence path instead of leaving disk outdated until the next edit. The watchdog never overwrites external edits: any on-disk state the persistence layer has not reconciled makes it stand down and log instead. Three new counters (`persistenceStalenessDetected`, `persistenceStalenessForcedStores`, `persistenceStalenessStoodDown`) surface these events alongside the existing persistence queue metrics; a sustained non-zero `persistenceStalenessStoodDown` rate is the alertable one (unflushed edits pinned in memory would be lost on a restart).diff --git a/packages/app/tests/integration/persistence-staleness-watchdog.test.ts b/packages/app/tests/integration/persistence-staleness-watchdog.test.tsnew file mode 100644index 000000000..e3d6828ce--- /dev/null+++ b/packages/app/tests/integration/persistence-staleness-watchdog.test.ts@@ -0,0 +1,146 @@+/**+ * Persistence staleness watchdog — end-to-end coverage of the days-stale+ * durability wedge: a store cycle that fails after a session's last edit has+ * no natural retry (Hocuspocus re-arms the store debounce only on a new+ * Y.Doc update, and user docs stay resident for the server lifetime), so the+ * live doc stays newer than disk indefinitely. The watchdog must re-flush+ * such a doc through the normal store spine once its grace window passes —+ * and must never overwrite disk state the persistence layer has not+ * accounted for (external native edits stay authoritative).+ *+ * Uses `OK_TEST_STORE_FAULT=<docName>` (a synthetic ENOSPC on the atomic+ * write, matched by exact docName) to wedge a store deterministically, then+ * clears it so the watchdog's forced retry can land.+ */++import { randomUUID } from 'node:crypto';+import { existsSync, readFileSync, writeFileSync } from 'node:fs';+import { join } from 'node:path';+import { getMetrics } from '@inkeep/open-knowledge-server';+import { afterEach, describe, expect, test } from 'vitest';+import { createTestServer, type TestServer, wait } from './test-harness.ts';++let server: TestServer | undefined;++afterEach(async () => {+ delete process.env.OK_TEST_STORE_FAULT;+ if (server) {+ await server.cleanup();+ server = undefined;+ }+});++async function writeMd(port: number, docName: string, markdown: string): Promise<Response> {+ return fetch(`http://127.0.0.1:${port}/api/agent-write-md`, {+ method: 'POST',+ headers: { 'Content-Type': 'application/json' },+ body: JSON.stringify({ docName, markdown, position: 'replace' }),+ });+}++async function pollUntil(predicate: () => boolean, timeoutMs: number): Promise<boolean> {+ const deadline = Date.now() + timeoutMs;+ while (Date.now() < deadline) {+ if (predicate()) return true;+ await wait(50);+ }+ return predicate();+}++describe('persistence staleness watchdog (integration)', () => {+ test('re-flushes a doc whose store failed with no retry pending', async () => {+ server = await createTestServer({ stalenessGraceMs: 250, stalenessSweepIntervalMs: 100 });+ const docName = `staleness-rescue-${randomUUID()}`;+ const filePath = join(server.contentDir, `${docName}.md`);+ const forcedBefore = getMetrics().persistenceStalenessForcedStores;+ const detectedBefore = getMetrics().persistenceStalenessDetected;++ process.env.OK_TEST_STORE_FAULT = docName;+ const res = await writeMd(server.port, docName, '# persisted content\n');+ expect(res.status).toBe(507);+ delete process.env.OK_TEST_STORE_FAULT;++ // The wedge shape: content lives only in server memory, nothing on disk,+ // and no store is scheduled (the handler's force-flush consumed it).+ expect(existsSync(filePath)).toBe(false);++ const rescued = await pollUntil(+ () => existsSync(filePath) && readFileSync(filePath, 'utf-8') === '# persisted content\n',+ 10_000,+ );+ expect(rescued).toBe(true);+ expect(getMetrics().persistenceStalenessDetected).toBeGreaterThan(detectedBefore);+ expect(getMetrics().persistenceStalenessForcedStores).toBeGreaterThan(forcedBefore);+ });++ test('production-shaped grace window does not fire inside a test run (negative control)', async () => {+ server = await createTestServer();+ const docName = `staleness-control-${randomUUID()}`;+ const filePath = join(server.contentDir, `${docName}.md`);+ const detectedBefore = getMetrics().persistenceStalenessDetected;+ const forcedBefore = getMetrics().persistenceStalenessForcedStores;++ process.env.OK_TEST_STORE_FAULT = docName;+ const res = await writeMd(server.port, docName, '# never flushed here\n');+ expect(res.status).toBe(507);+ delete process.env.OK_TEST_STORE_FAULT;++ // With the default 5-minute grace, nothing rescues the doc in-test —+ // proving the rescue above is the watchdog, not some other retry path.+ // The unchanged counters pin that the watchdog itself never fired, so+ // the absent file isn't just "nothing else happened to write it".+ await wait(1_500);+ expect(existsSync(filePath)).toBe(false);+ expect(getMetrics().persistenceStalenessDetected).toBe(detectedBefore);+ expect(getMetrics().persistenceStalenessForcedStores).toBe(forcedBefore);+ });++ test('stands down instead of clobbering an external native edit', async () => {+ server = await createTestServer({ stalenessGraceMs: 250, stalenessSweepIntervalMs: 100 });+ const docName = `staleness-external-${randomUUID()}`;+ const filePath = join(server.contentDir, `${docName}.md`);+ const forcedBefore = getMetrics().persistenceStalenessForcedStores;++ const first = await writeMd(server.port, docName, '# version one\n');+ expect(first.status).toBe(200);+ const flushed = await pollUntil(+ () => existsSync(filePath) && readFileSync(filePath, 'utf-8') === '# version one\n',+ 10_000,+ );+ expect(flushed).toBe(true);++ // Wedge a second version in memory (store fails), then land an external+ // native edit on disk before the grace window elapses.+ const stoodDownBefore = getMetrics().persistenceStalenessStoodDown;+ process.env.OK_TEST_STORE_FAULT = docName;+ const second = await writeMd(server.port, docName, '# version two\n');+ expect(second.status).toBe(507);+ delete process.env.OK_TEST_STORE_FAULT;+ writeFileSync(filePath, '# external native edit\n', 'utf-8');++ // Positive proof the WATCHDOG ran and chose to stand down (not merely+ // that the file-watcher reconciled first): the stood-down counter must+ // advance. If the watcher's ingest wins the race instead, memory+ // converges to the external bytes and no stand-down fires — accept+ // that interleaving only when the doc has genuinely converged.+ const watchdogStoodDown = await pollUntil(+ () => getMetrics().persistenceStalenessStoodDown > stoodDownBefore,+ 2_000,+ );+ if (!watchdogStoodDown) {+ const state = await fetch(+ `http://127.0.0.1:${server.port}/api/document?docName=${encodeURIComponent(docName)}`,+ );+ // Watcher-reconciled interleaving: server must now hold the external+ // bytes (so there was nothing for the watchdog to stand down on).+ expect(state.ok).toBe(true);+ expect(await state.text()).toContain('external native edit');+ }++ // Either way, the external bytes must win on disk and no forced store+ // may ever have overwritten them.+ await wait(500);+ expect(readFileSync(filePath, 'utf-8')).toBe('# external native edit\n');+ expect(getMetrics().persistenceStalenessForcedStores).toBe(forcedBefore);+ });+});diff --git a/packages/app/tests/integration/test-harness.ts b/packages/app/tests/integration/test-harness.tsindex b07a4bd2f..84d497fac 100644--- a/packages/app/tests/integration/test-harness.ts+++ b/packages/app/tests/integration/test-harness.ts@@ -102,6 +102,14 @@ export interface TestServer {export interface CreateTestServerOptions {debounce?: ServerOptions['debounce'];maxDebounce?: ServerOptions['maxDebounce'];+ /**+ * Persistence staleness watchdog tuning. Production defaults (5 min grace,+ * 60 s sweep) never fire inside a test run; pass small values (e.g.+ * `{ stalenessGraceMs: 250, stalenessSweepIntervalMs: 100 }`) to exercise+ * the forced-store rescue path.+ */+ stalenessGraceMs?: ServerOptions['stalenessGraceMs'];+ stalenessSweepIntervalMs?: ServerOptions['stalenessSweepIntervalMs'];/** Reuse an existing content directory (for server-restart tests that need* persistence to load canonical state written by a prior test-server instance).* When provided, the caller owns directory lifecycle — cleanup() will not@@ -248,6 +256,8 @@ export async function createTestServer(options: CreateTestServerOptions = {}): Pquiet: true,debounce: options.debounce ?? 200,maxDebounce: options.maxDebounce ?? 1000,+ stalenessGraceMs: options.stalenessGraceMs,+ stalenessSweepIntervalMs: options.stalenessSweepIntervalMs,gitEnabled: ephemeral ? false : (options.gitEnabled ?? false),commitDebounceMs: options.commitDebounceMs ?? 200,// When gitEnabled: true, the test-harness contentDir IS the tmpdir root (nodiff --git a/packages/server/src/cc1-broadcast.ts b/packages/server/src/cc1-broadcast.tsindex ad3417afb..5e4481daa 100644--- a/packages/server/src/cc1-broadcast.ts+++ b/packages/server/src/cc1-broadcast.ts@@ -123,6 +123,26 @@ export function isLinkIndexExcludedDoc(documentName: string): boolean {return isSystemDoc(documentName) || isConfigDoc(documentName);}+/**+ * True when a doc name is excluded from the markdown L1 store spine+ * (`storeDocumentNow`) because its persistence flows through a dedicated+ * store/load path: system docs (never persisted), config docs (config+ * persistence + validation hook), managed artifacts (skill/template+ * persistence), and Mermaid docs (Y.Text-only persistence). Shared by+ * `PersistenceHandle.forceStore` and the staleness watchdog so the two+ * gates cannot drift. The debounced `onStoreDocument` hook dispatches each+ * class to its dedicated path individually, so it cannot collapse onto+ * this predicate.+ */+export function isPersistenceExcludedDoc(documentName: string): boolean {+ return (+ isSystemDoc(documentName) ||+ isConfigDoc(documentName) ||+ isManagedArtifactDoc(documentName) ||+ isMermaidDoc(documentName)+ );+}+export class CC1Broadcaster {private readonly hocuspocus: Hocuspocus;private readonly seqs = new Map<string, number>();diff --git a/packages/server/src/conflict-errors.ts b/packages/server/src/conflict-errors.tsindex 135edf189..8eeaa0a15 100644--- a/packages/server/src/conflict-errors.ts+++ b/packages/server/src/conflict-errors.ts@@ -31,6 +31,7 @@import type { ServerResponse } from 'node:http';import type { Document } from '@hocuspocus/server';+import type * as Y from 'yjs';import type { ResolveStrategy } from './conflict-storage.ts';import { stripDocExtension } from './doc-extensions.ts';import { errorResponse } from './http/error-response.ts';@@ -71,12 +72,41 @@ const _exhaustiveResolveStrategy: _ExhaustiveResolveStrategy = true;* `Y.Map('lifecycle').get('status') === 'conflict'`. Every mutating write* surface reads this via the helper rather than re-deriving the literal.*- * Centralized so the lifecycle Y.Map contract has exactly one named accessor- * — adding new lifecycle discriminators (`deleted-upstream`, `renamed`,- * future block-conflict shadows) goes through this one site.+ * Delegates to `frozenDocLifecycleStatus` — the single `lifecycle.status`+ * read site — so adding new lifecycle discriminators (`deleted-upstream`,+ * `renamed`, future block-conflict shadows) goes through the+ * `FROZEN_LIFECYCLE_STATUSES` registry below, never a scattered compare.*/export function isDocInConflict(document: Document): boolean {- return document.getMap('lifecycle').get('status') === 'conflict';+ return frozenDocLifecycleStatus(document) === 'conflict';+}++/**+ * Lifecycle statuses under which the persistence spine intentionally holds+ * memory != disk and skips stores: externally deleted / renamed docs must+ * not be rewritten at their old path, and conflict docs keep merge markers+ * on disk for the resolver UI. Registry-first (mirroring+ * `RESOLUTION_OPTIONS` above) so a new frozen status is added in exactly+ * one place and every reader — `storeDocumentNow`'s guard, the staleness+ * watchdog, `isDocInConflict` — picks it up together.+ */+export const FROZEN_LIFECYCLE_STATUSES = ['deleted-upstream', 'renamed', 'conflict'] as const;+export type FrozenLifecycleStatus = (typeof FROZEN_LIFECYCLE_STATUSES)[number];++function isFrozenLifecycleStatus(value: unknown): value is FrozenLifecycleStatus {+ return (FROZEN_LIFECYCLE_STATUSES as readonly unknown[]).includes(value);+}++/**+ * The frozen lifecycle status a doc currently carries, or null when the+ * doc is writable. Together with `isDocInConflict` (which delegates here)+ * this is the single read site for `lifecycle.status` discriminators —+ * adding a new lifecycle status goes through `FROZEN_LIFECYCLE_STATUSES`+ * above, never a scattered string compare.+ */+export function frozenDocLifecycleStatus(document: Y.Doc): FrozenLifecycleStatus | null {+ const status = document.getMap('lifecycle').get('status');+ return isFrozenLifecycleStatus(status) ? status : null;}/**diff --git a/packages/server/src/metrics.ts b/packages/server/src/metrics.tsindex bfb7f0036..837f2a74d 100644--- a/packages/server/src/metrics.ts+++ b/packages/server/src/metrics.ts@@ -119,6 +119,33 @@ export interface ReconciliationMetrics {* work undurable. The matching fragment-reconciliation queues* unconditionally on the next settlement after the force-flush. */persistenceForceFlushDuringBurst: number;+ /** Staleness watchdog — count of loaded docs first observed with+ * unpersisted in-memory edits older than the grace window and no pending+ * store (memory diverged from the persistence reconciled base). Counted+ * once per distinct stale content per doc, whether or not the watchdog+ * then forced a store or stood down for disk safety. A healthy server+ * stays at 0; any growth means a store cycle was lost after a session's+ * last edit. */+ persistenceStalenessDetected: number;+ /** Staleness watchdog — count of forced store dispatch attempts through+ * the normal persistence spine (retries after a failed attempt included,+ * so this can exceed `persistenceStalenessDetected`, which counts once+ * per distinct stale content). Stand-down detections don't dispatch and+ * never increment this. A healthy server stays at 0; growth means lost+ * store cycles are being rescued — investigate the underlying store+ * failures even though durability recovered. */+ persistenceStalenessForcedStores: number;+ /** Staleness watchdog — count of stand-down events: a stale doc was+ * detected but NOT re-flushed because disk holds state the reconciled+ * base does not account for (unreconciled external edit or delete,+ * never-loaded file) or the disk read failed. Verified-external-state+ * stand-downs AND structural read refusals (symlink-escape, symlink+ * cycle, oversized file — additionally logged at error) count once per+ * distinct stale content, then suppress until the doc's content+ * changes; only transient disk-read failures recount on each retry+ * (one per grace window). Any sustained non-zero rate is alertable —+ * unflushed edits are pinned in memory and lost on restart. */+ persistenceStalenessStoodDown: number;/** Collab WebSocket upgrade sockets emitting EPIPE from `ws.send()` AFTER* the call returned control — kernel-level TCP race against a peer that* has sent FIN. Filtered at the socket-boundary listener per precedent@@ -424,6 +451,9 @@ const counters: ReconciliationMetrics = {bridgeInvariantViolationsSuppressed: 0,persistenceSkipNonQuiescent: 0,persistenceForceFlushDuringBurst: 0,+ persistenceStalenessDetected: 0,+ persistenceStalenessForcedStores: 0,+ persistenceStalenessStoodDown: 0,collabSocketEpipeCount: 0,collabSocketEconnresetCount: 0,collabMessageTooLargeCount: 0,@@ -578,6 +608,18 @@ export function incrementPersistenceForceFlushDuringBurst(): void {counters.persistenceForceFlushDuringBurst++;}+export function incrementPersistenceStalenessDetected(): void {+ counters.persistenceStalenessDetected++;+}++export function incrementPersistenceStalenessForcedStores(): void {+ counters.persistenceStalenessForcedStores++;+}++export function incrementPersistenceStalenessStoodDown(): void {+ counters.persistenceStalenessStoodDown++;+}+export function incrementAgentPatchFindMismatches(): void {counters.agentPatchFindMismatches++;}@@ -806,6 +848,9 @@ export function resetMetrics(): void {counters.bridgeInvariantViolationsSuppressed = 0;counters.persistenceSkipNonQuiescent = 0;counters.persistenceForceFlushDuringBurst = 0;+ counters.persistenceStalenessDetected = 0;+ counters.persistenceStalenessForcedStores = 0;+ counters.persistenceStalenessStoodDown = 0;counters.collabSocketEpipeCount = 0;counters.collabSocketEconnresetCount = 0;counters.collabMessageTooLargeCount = 0;diff --git a/packages/server/src/persistence-deferred-store.test.ts b/packages/server/src/persistence-deferred-store.test.tsindex f87b609b1..98d797a20 100644--- a/packages/server/src/persistence-deferred-store.test.ts+++ b/packages/server/src/persistence-deferred-store.test.ts@@ -1,8 +1,8 @@-import { afterEach, beforeEach, describe, expect, spyOn, test } from 'bun:test';import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';import { tmpdir } from 'node:os';import { join } from 'node:path';import { BridgeInvariantViolationError } from '@inkeep/open-knowledge-core';+import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';import * as Y from 'yjs';import { composeAndWriteRawBody } from './bridge-intake.ts';import { __setQuiescentOverrideForTests } from './bridge-quiescence.ts';@@ -564,7 +564,7 @@ describe('Y.Text-is-truth wiring (FR-33 / FR-35)', () => {*/describe('FR-9 — deferred-store-failed event + counter', () => {let tmpDir: string;- let warnSpy: ReturnType<typeof spyOn>;+ let warnSpy: ReturnType<typeof vi.spyOn>;let warnings: string[];function findEventLines(eventName: string): Array<Record<string, unknown>> {@@ -588,7 +588,7 @@ describe('FR-9 — deferred-store-failed event + counter', () => {switchReconciledBaseScope('main');resetMetrics();warnings = [];- warnSpy = spyOn(console, 'warn').mockImplementation((...args: unknown[]) => {+ warnSpy = vi.spyOn(console, 'warn').mockImplementation((...args: unknown[]) => {warnings.push(args.map(String).join(' '));});});@@ -709,7 +709,7 @@ describe('FR-9 — deferred-store-failed event + counter', () => {// must absorb the throw and emit `deferred-store-classifier-failed`// alongside the outer `deferred-store-failed` event with// errorClass='unknown'.- const renameSpy = spyOn(fsTraced, 'tracedRename').mockImplementation(async () => {+ const renameSpy = vi.spyOn(fsTraced, 'tracedRename').mockImplementation(async () => {const malformed = Object.create(Error.prototype) as Error & { name: string };Object.defineProperty(malformed, 'message', { value: 'malformed-error', enumerable: true });Object.defineProperty(malformed, 'name', { value: 'MalformedError', enumerable: true });@@ -784,7 +784,7 @@ describe('FR-9 — deferred-store-failed event + counter', () => {await storeDocument(persistence, document, docName);setBatchInProgress(false);- const renameSpy = spyOn(fsTraced, 'tracedRename').mockImplementation(async () => {+ const renameSpy = vi.spyOn(fsTraced, 'tracedRename').mockImplementation(async () => {const malformed = Object.create(Error.prototype) as Error & { name: string };Object.defineProperty(malformed, 'name', { value: 'MalformedError', enumerable: true });Object.defineProperty(malformed, 'message', {@@ -835,7 +835,7 @@ describe('FR-9 — deferred-store-failed event + counter', () => {await storeDocument(persistence, document, docName);setBatchInProgress(false);- const renameSpy = spyOn(fsTraced, 'tracedRename').mockImplementation(async () => {+ const renameSpy = vi.spyOn(fsTraced, 'tracedRename').mockImplementation(async () => {const malformed = Object.create(Error.prototype) as Error & { name: string };Object.defineProperty(malformed, 'message', {value: 'malformed-error-verbose',@@ -943,3 +943,94 @@ describe('FR-9 — classifyDeferredStoreError behavior', () => {expect(classifyDeferredStoreError(42)).toBe('unknown');});});++describe('forceStore dispatch (staleness watchdog entry point)', () => {+ let tmpDir: string;++ beforeEach(() => {+ tmpDir = mkdtempSync(join(tmpdir(), 'ok-force-store-'));+ mkdirSync(tmpDir, { recursive: true });+ setBatchInProgress(false);+ switchReconciledBaseScope('main');+ });++ afterEach(() => {+ setBatchInProgress(false);+ switchReconciledBaseScope('main');+ rmSync(tmpDir, { recursive: true, force: true });+ });++ test('flushes a doc whose debounced store never landed', async () => {+ const docName = 'wedged-doc';+ const docPath = join(tmpDir, `${docName}.md`);+ writeFileSync(docPath, 'initial\n', 'utf-8');+ const persistence = createPersistenceExtension({+ contentDir: tmpDir,+ projectDir: tmpDir,+ gitEnabled: false,+ });+ const document = new Y.Doc();++ await loadDocument(persistence, document, docName);+ document.transact(() => replaceDocParagraph(document, 'rescued edit'), BROWSER_ORIGIN);+ // No storeDocument call — the wedge shape: an edit with no store cycle left.++ await persistence.forceStore(document, docName);++ expect(readFileSync(docPath, 'utf-8')).toBe('rescued edit\n');+ expect(getReconciledBase(docName)).toBe('rescued edit\n');++ document.destroy();+ });++ test('defers during a coordinated batch and replays on drain', async () => {+ const docName = 'batched-force';+ const docPath = join(tmpDir, `${docName}.md`);+ writeFileSync(docPath, 'initial\n', 'utf-8');+ const persistence = createPersistenceExtension({+ contentDir: tmpDir,+ projectDir: tmpDir,+ gitEnabled: false,+ });+ const document = new Y.Doc();++ await loadDocument(persistence, document, docName);+ document.transact(() => replaceDocParagraph(document, 'parked edit'), BROWSER_ORIGIN);++ setBatchInProgress(true);+ await persistence.forceStore(document, docName);+ expect(readFileSync(docPath, 'utf-8')).toBe('initial\n');+ expect(persistence.getQueueDepths().branchDeferred).toBe(1);++ setBatchInProgress(false);+ await persistence.flushDeferredStores('within-branch');+ expect(readFileSync(docPath, 'utf-8')).toBe('parked edit\n');++ document.destroy();+ });++ test('short-circuits synthetic docs without touching queues', async () => {+ const persistence = createPersistenceExtension({+ contentDir: tmpDir,+ projectDir: tmpDir,+ gitEnabled: false,+ });+ const document = new Y.Doc();+ document.getText('source').insert(0, 'anything\n');++ setBatchInProgress(true);+ for (const name of [+ '__system__',+ '__config__/project',+ '__skill__/project/foo',+ 'diagram.mmd',+ 'assets/flow.mermaid',+ ]) {+ await persistence.forceStore(document, name);+ }++ expect(persistence.getQueueDepths().branchDeferred).toBe(0);++ document.destroy();+ });+});diff --git a/packages/server/src/persistence-staleness-watchdog.test.ts b/packages/server/src/persistence-staleness-watchdog.test.tsnew file mode 100644index 000000000..a5b7a41d4--- /dev/null+++ b/packages/server/src/persistence-staleness-watchdog.test.ts@@ -0,0 +1,513 @@+import { afterEach, beforeEach, describe, expect, test } from 'vitest';+import * as Y from 'yjs';+import { isPersistenceExcludedDoc } from './cc1-broadcast.ts';+import { FROZEN_LIFECYCLE_STATUSES } from './conflict-errors.ts';+import { getMetrics, resetMetrics } from './metrics.ts';+import {+ createPersistenceStalenessWatchdog,+ type StalenessWatchdogHandle,+ type StalenessWatchdogOptions,+ StructuralDiskReadError,+} from './persistence-staleness-watchdog.ts';++const GRACE_MS = 1_000;++interface Rig {+ watchdog: StalenessWatchdogHandle;+ docs: Map<string, Y.Doc>;+ bases: Map<string, string>;+ disk: Map<string, string>;+ /** Per-doc ms-since-last-user-tx; missing entry → null (never observed). */+ txAges: Map<string, number>;+ forceCalls: string[];+ clock: { nowMs: number };+ batchActive: { value: boolean };+ inFlight: Map<string, string>;+ /** Behavior of the injected forceStore. */+ forceBehavior: {+ mode: 'advance-base' | 'no-op' | 'throw';+ };+ diskReadFault: { kind: 'transient' | 'structural' | null };+}++function makeDoc(source: string): Y.Doc {+ const doc = new Y.Doc();+ doc.getText('source').insert(0, source);+ return doc;+}++function makeRig(overrides: Partial<StalenessWatchdogOptions> = {}): Rig {+ const docs = new Map<string, Y.Doc>();+ const bases = new Map<string, string>();+ const disk = new Map<string, string>();+ const txAges = new Map<string, number>();+ const forceCalls: string[] = [];+ const clock = { nowMs: 100_000 };+ const batchActive = { value: false };+ const inFlight = new Map<string, string>();+ const forceBehavior: Rig['forceBehavior'] = { mode: 'advance-base' };+ const diskReadFault: Rig['diskReadFault'] = { kind: null };++ const watchdog = createPersistenceStalenessWatchdog({+ getLoadedDocuments: () => docs,+ forceStore: async (document, documentName) => {+ forceCalls.push(documentName);+ if (forceBehavior.mode === 'throw') throw new Error('injected store failure');+ if (forceBehavior.mode === 'advance-base') {+ const bytes = document.getText('source').toString();+ bases.set(documentName, bytes);+ disk.set(documentName, bytes);+ }+ },+ readDiskBytes: (documentName) => {+ if (diskReadFault.kind === 'structural') {+ throw new StructuralDiskReadError('injected structural refusal');+ }+ if (diskReadFault.kind === 'transient') throw new Error('injected disk read failure');+ return disk.get(documentName) ?? null;+ },+ graceMs: GRACE_MS,+ // Interval never fires inside a test; sweeps are driven manually.+ sweepIntervalMs: 3_600_000,+ now: () => clock.nowMs,+ getBase: (documentName) => bases.get(documentName),+ isBatchActive: () => batchActive.value,+ peekInFlight: (documentName) => inFlight.get(documentName),+ msSinceLastUserTx: (doc) => {+ for (const [name, d] of docs) {+ if (d === doc) return txAges.get(name) ?? null;+ }+ return null;+ },+ ...overrides,+ });++ return {+ watchdog,+ docs,+ bases,+ disk,+ txAges,+ forceCalls,+ clock,+ batchActive,+ inFlight,+ forceBehavior,+ diskReadFault,+ };+}++/** A doc whose memory diverged from base+disk long ago — the wedge shape. */+function seedWedgedDoc(rig: Rig, name: string): Y.Doc {+ const doc = makeDoc('# edited in memory\n');+ rig.docs.set(name, doc);+ rig.bases.set(name, '# old on disk\n');+ rig.disk.set(name, '# old on disk\n');+ rig.txAges.set(name, GRACE_MS * 10);+ return doc;+}++let rig: Rig;++beforeEach(() => {+ resetMetrics();+ rig = makeRig();+});++afterEach(async () => {+ await rig.watchdog.dispose();+});++describe('staleness detection and forced store', () => {+ test('force-stores a doc divergent past the grace window and counts it', async () => {+ seedWedgedDoc(rig, 'wedged-doc');++ await rig.watchdog.sweep();++ expect(rig.forceCalls).toEqual(['wedged-doc']);+ expect(getMetrics().persistenceStalenessDetected).toBe(1);+ expect(getMetrics().persistenceStalenessForcedStores).toBe(1);+ // Base advanced by the store; the next sweep sees convergence and stays quiet.+ await rig.watchdog.sweep();+ expect(rig.forceCalls).toEqual(['wedged-doc']);+ });++ test('does not force while the last user transaction is younger than the grace window', async () => {+ seedWedgedDoc(rig, 'active-doc');+ rig.txAges.set('active-doc', GRACE_MS - 1);++ await rig.watchdog.sweep();++ expect(rig.forceCalls).toEqual([]);+ expect(getMetrics().persistenceStalenessDetected).toBe(0);+ });++ test('treats a null transaction age as old enough', async () => {+ seedWedgedDoc(rig, 'no-tx-doc');+ rig.txAges.delete('no-tx-doc');++ await rig.watchdog.sweep();++ expect(rig.forceCalls).toEqual(['no-tx-doc']);+ });++ test('ignores docs whose memory matches the reconciled base', async () => {+ const doc = makeDoc('# same\n');+ rig.docs.set('clean-doc', doc);+ rig.bases.set('clean-doc', '# same\n');+ rig.disk.set('clean-doc', '# same\n');+ rig.txAges.set('clean-doc', GRACE_MS * 10);++ await rig.watchdog.sweep();++ expect(rig.forceCalls).toEqual([]);+ });++ test('classifies frontmatter-carrying docs with the store spine comparator', async () => {+ // Byte-unequal but normalize-equal (trailing newlines): the shared+ // `normalizedSourceForm` derivation must read this as converged even+ // through the frontmatter strip/prepend round-trip — the same verdict+ // the store's no-op skip would reach.+ const fmBase = '---\ntitle: x\n---\n\n# body\n\n\n';+ const doc = makeDoc('---\ntitle: x\n---\n\n# body\n');+ rig.docs.set('fm-doc', doc);+ rig.bases.set('fm-doc', fmBase);+ rig.disk.set('fm-doc', fmBase);+ rig.txAges.set('fm-doc', GRACE_MS * 10);++ await rig.watchdog.sweep();+ expect(rig.forceCalls).toEqual([]);++ // A frontmatter-only edit IS a divergence and must be rescued.+ doc.getText('source').delete(0, doc.getText('source').length);+ doc.getText('source').insert(0, '---\ntitle: y\n---\n\n# body\n');+ await rig.watchdog.sweep();+ expect(rig.forceCalls).toEqual(['fm-doc']);+ });++ test('materializes a never-persisted doc with content and no disk file', async () => {+ const doc = makeDoc('# brand new\n');+ rig.docs.set('new-doc', doc);+ rig.txAges.set('new-doc', GRACE_MS * 10);++ await rig.watchdog.sweep();++ expect(rig.forceCalls).toEqual(['new-doc']);+ });+});++describe('exclusions', () => {+ test('isPersistenceExcludedDoc admits exactly the dedicated-store-path doc classes', () => {+ // One representative per dedicated-store-path class dispatched in+ // persistence's onLoadDocument/onStoreDocument. A fifth doc class+ // added to that dispatch must be added here (and to the predicate) or+ // the watchdog would force it through the markdown L1 spine.+ for (const name of [+ '__system__',+ '__config__/project',+ '__user__/config.yml',+ '__local__/project',+ '__skill__/project/foo',+ '__template__/notes/weekly',+ 'diagram.mmd',+ 'assets/flow.mermaid',+ ]) {+ expect(isPersistenceExcludedDoc(name)).toBe(true);+ }+ for (const name of ['notes', 'folder/doc', 'README', 'docs/getting-started.mdx']) {+ expect(isPersistenceExcludedDoc(name)).toBe(false);+ }+ });++ test('skips system, config, managed-artifact, and mermaid docs', async () => {+ for (const name of [+ '__system__',+ '__config__/project',+ '__user__/config.yml',+ '__local__/project',+ '__skill__/project/foo',+ '__template__/notes/weekly',+ 'diagram.mmd',+ ]) {+ const doc = makeDoc('# divergent\n');+ rig.docs.set(name, doc);+ rig.bases.set(name, '# other\n');+ rig.txAges.set(name, GRACE_MS * 10);+ }++ await rig.watchdog.sweep();++ expect(rig.forceCalls).toEqual([]);+ expect(getMetrics().persistenceStalenessDetected).toBe(0);+ });++ test('skips docs frozen by lifecycle status', async () => {+ // Driven from the registry so a newly added frozen status is covered+ // here automatically.+ for (const status of FROZEN_LIFECYCLE_STATUSES) {+ const doc = seedWedgedDoc(rig, `lifecycle-${status}`);+ doc.getMap('lifecycle').set('status', status);+ }++ await rig.watchdog.sweep();++ expect(rig.forceCalls).toEqual([]);+ });++ test('skips the whole sweep while a coordinated batch is active', async () => {+ seedWedgedDoc(rig, 'batch-doc');+ rig.batchActive.value = true;++ await rig.watchdog.sweep();+ expect(rig.forceCalls).toEqual([]);++ rig.batchActive.value = false;+ await rig.watchdog.sweep();+ expect(rig.forceCalls).toEqual(['batch-doc']);+ });++ test('a batch activating mid-sweep does not decline the parked store', async () => {+ // A batch starting inside the forceStore await means persistence PARKED+ // the store (deferStore) rather than running it — the doc stays+ // divergent with unchanged content, which must not be classified as a+ // declined no-op (that would suppress retries even if the batch replay+ // later drops the parked entry).+ const parkingRig = makeRig({+ forceStore: async (_document, documentName) => {+ parkingRig.forceCalls.push(documentName);+ parkingRig.batchActive.value = true;+ },+ });+ try {+ seedWedgedDoc(parkingRig, 'parked-doc');++ await parkingRig.watchdog.sweep();+ expect(parkingRig.forceCalls).toEqual(['parked-doc']);++ // Batch over, base still stale, content unchanged: the next grace+ // window must retry rather than treat the doc as suppressed.+ parkingRig.batchActive.value = false;+ parkingRig.clock.nowMs += GRACE_MS + 1;+ await parkingRig.watchdog.sweep();+ expect(parkingRig.forceCalls).toEqual(['parked-doc', 'parked-doc']);+ } finally {+ await parkingRig.watchdog.dispose();+ }+ });++ test('skips a doc whose flush is currently mid-commit', async () => {+ seedWedgedDoc(rig, 'inflight-doc');+ rig.inFlight.set('inflight-doc', 'anything');++ await rig.watchdog.sweep();++ expect(rig.forceCalls).toEqual([]);+ });++ test('is a no-op after dispose', async () => {+ seedWedgedDoc(rig, 'late-doc');+ await rig.watchdog.dispose();++ await rig.watchdog.sweep();++ expect(rig.forceCalls).toEqual([]);+ });++ test('dispose drains an in-flight sweep and stops it before the next doc', async () => {+ // Two wedged docs; block the first forceStore so dispose lands mid-sweep.+ let releaseFirstStore: (() => void) | undefined;+ const gate = new Promise<void>((resolve) => {+ releaseFirstStore = resolve;+ });+ const blockingRig = makeRig({+ forceStore: async (_document, documentName) => {+ blockingRig.forceCalls.push(documentName);+ await gate;+ },+ });+ try {+ seedWedgedDoc(blockingRig, 'doc-a');+ seedWedgedDoc(blockingRig, 'doc-b');++ const sweepPromise = blockingRig.watchdog.sweep();+ const disposePromise = blockingRig.watchdog.dispose();+ releaseFirstStore?.();+ await disposePromise;+ await sweepPromise;++ // The in-flight sweep finished its current store but never reached+ // the second doc.+ expect(blockingRig.forceCalls).toEqual(['doc-a']);+ } finally {+ releaseFirstStore?.();+ await blockingRig.watchdog.dispose();+ }+ });+});++describe('external-edit stand-down (disk authority)', () => {+ test('does not overwrite disk bytes the base does not account for', async () => {+ seedWedgedDoc(rig, 'external-doc');+ rig.disk.set('external-doc', '# external native edit\n');++ await rig.watchdog.sweep();++ expect(rig.forceCalls).toEqual([]);+ // Detected and counted as a stand-down, but not forced.+ expect(getMetrics().persistenceStalenessDetected).toBe(1);+ expect(getMetrics().persistenceStalenessForcedStores).toBe(0);+ expect(getMetrics().persistenceStalenessStoodDown).toBe(1);+ expect(rig.disk.get('external-doc')).toBe('# external native edit\n');++ // Verified external state suppresses until content changes — a second+ // sweep must not re-count the same stand-down.+ await rig.watchdog.sweep();+ expect(getMetrics().persistenceStalenessStoodDown).toBe(1);+ });++ test('does not resurrect a file deleted out-of-band', async () => {+ seedWedgedDoc(rig, 'deleted-doc');+ rig.disk.delete('deleted-doc');++ await rig.watchdog.sweep();++ expect(rig.forceCalls).toEqual([]);+ expect(getMetrics().persistenceStalenessStoodDown).toBe(1);+ });++ test('does not overwrite an on-disk file the doc never loaded', async () => {+ const doc = makeDoc('# memory only\n');+ rig.docs.set('never-loaded', doc);+ rig.disk.set('never-loaded', '# unread disk bytes\n');+ rig.txAges.set('never-loaded', GRACE_MS * 10);++ await rig.watchdog.sweep();++ expect(rig.forceCalls).toEqual([]);+ });++ test('stands down when the disk read fails, then retries after another grace window', async () => {+ seedWedgedDoc(rig, 'unreadable-doc');+ rig.diskReadFault.kind = 'transient';++ await rig.watchdog.sweep();+ expect(rig.forceCalls).toEqual([]);++ // Still failing, still inside the retry pace: no hammering.+ await rig.watchdog.sweep();+ expect(rig.forceCalls).toEqual([]);++ // The read fault clears (transient EMFILE/EACCES class): unlike the+ // verified external-state stand-downs, the doc must NOT stay suppressed.+ rig.diskReadFault.kind = null;+ rig.clock.nowMs += GRACE_MS + 1;+ await rig.watchdog.sweep();+ expect(rig.forceCalls).toEqual(['unreadable-doc']);+ });++ test('a structural read refusal declines until content changes', async () => {+ const doc = seedWedgedDoc(rig, 'refused-doc');+ rig.diskReadFault.kind = 'structural';++ await rig.watchdog.sweep();+ expect(rig.forceCalls).toEqual([]);+ expect(getMetrics().persistenceStalenessStoodDown).toBe(1);++ // A refusal that never clears (symlink-escape / oversized file) must+ // not inflate the alertable counter or retry across grace windows.+ rig.clock.nowMs += GRACE_MS * 3;+ await rig.watchdog.sweep();+ rig.clock.nowMs += GRACE_MS * 3;+ await rig.watchdog.sweep();+ expect(rig.forceCalls).toEqual([]);+ expect(getMetrics().persistenceStalenessStoodDown).toBe(1);++ // Only a content change re-arms the doc.+ rig.diskReadFault.kind = null;+ doc.getText('source').insert(0, 'more ');+ rig.clock.nowMs += GRACE_MS + 1;+ await rig.watchdog.sweep();+ expect(rig.forceCalls).toEqual(['refused-doc']);+ });++ test('a stand-down re-arms when the memory content changes', async () => {+ const doc = seedWedgedDoc(rig, 'rearm-doc');+ rig.disk.set('rearm-doc', '# external native edit\n');++ await rig.watchdog.sweep();+ await rig.watchdog.sweep();+ expect(rig.forceCalls).toEqual([]);+ expect(getMetrics().persistenceStalenessDetected).toBe(1);++ // External state reconciled (watcher caught up: base = disk), then a new+ // memory edit wedges again.+ rig.bases.set('rearm-doc', '# external native edit\n');+ doc.getText('source').delete(0, doc.getText('source').length);+ doc.getText('source').insert(0, '# newer memory edit\n');++ await rig.watchdog.sweep();++ expect(rig.forceCalls).toEqual(['rearm-doc']);+ expect(getMetrics().persistenceStalenessDetected).toBe(2);+ });+});++describe('retry and suppression discipline', () => {+ test('a store that completes without clearing divergence suppresses until content changes', async () => {+ rig.forceBehavior.mode = 'no-op';+ const doc = seedWedgedDoc(rig, 'noop-doc');++ await rig.watchdog.sweep();+ expect(rig.forceCalls).toEqual(['noop-doc']);++ // Same content, later sweeps — even past another grace window: no retry.+ rig.clock.nowMs += GRACE_MS * 5;+ await rig.watchdog.sweep();+ expect(rig.forceCalls).toEqual(['noop-doc']);++ // Content changed — re-armed.+ doc.getText('source').insert(0, 'more ');+ rig.clock.nowMs += GRACE_MS * 5;+ await rig.watchdog.sweep();+ expect(rig.forceCalls).toEqual(['noop-doc', 'noop-doc']);+ });++ test('a failed store retries only after another full grace window', async () => {+ rig.forceBehavior.mode = 'throw';+ seedWedgedDoc(rig, 'failing-doc');++ await rig.watchdog.sweep();+ expect(rig.forceCalls).toEqual(['failing-doc']);++ // Immediately after: no hammering.+ await rig.watchdog.sweep();+ expect(rig.forceCalls).toEqual(['failing-doc']);++ // A grace window later: retried; a recovered disk then clears it.+ rig.clock.nowMs += GRACE_MS + 1;+ rig.forceBehavior.mode = 'advance-base';+ await rig.watchdog.sweep();+ expect(rig.forceCalls).toEqual(['failing-doc', 'failing-doc']);+ await rig.watchdog.sweep();+ expect(rig.forceCalls).toEqual(['failing-doc', 'failing-doc']);+ });++ test('drops bookkeeping for docs that unloaded', async () => {+ rig.forceBehavior.mode = 'no-op';+ seedWedgedDoc(rig, 'transient-doc');++ await rig.watchdog.sweep();+ expect(rig.forceCalls).toEqual(['transient-doc']);++ // Unload, then reload in the same wedged shape: suppression must not+ // survive the unload (a fresh load re-reads disk and re-bases anyway).+ const doc = rig.docs.get('transient-doc');+ rig.docs.delete('transient-doc');+ await rig.watchdog.sweep();+ if (doc) rig.docs.set('transient-doc', doc);+ rig.clock.nowMs += GRACE_MS * 2;++ await rig.watchdog.sweep();+ expect(rig.forceCalls).toEqual(['transient-doc', 'transient-doc']);+ });+});diff --git a/packages/server/src/persistence-staleness-watchdog.ts b/packages/server/src/persistence-staleness-watchdog.tsnew file mode 100644index 000000000..b6c6bd527--- /dev/null+++ b/packages/server/src/persistence-staleness-watchdog.ts@@ -0,0 +1,421 @@+/**+ * Persistence staleness watchdog — a durability backstop for the L1+ * CRDT→disk write spine.+ *+ * Hocuspocus re-arms the persistence store debounce only when a NEW Y.Doc+ * update arrives, and user docs stay resident for the whole server+ * lifetime. Any store cycle that fails or is skipped after the last edit+ * of a session (a transient disk-write error, a batch deferral dropped on+ * branch mismatch, a lifecycle race) therefore leaves the live doc newer+ * than disk with no retry scheduled — silently, for as long as the server+ * runs. The watchdog sweeps loaded docs on an interval and force-stores+ * any doc whose in-memory bytes have diverged from the persistence layer's+ * own reconciled base for longer than a generous grace window.+ *+ * Why a general sweep rather than a retry at each failure site: the+ * lost-cycle classes have no common hook point with a retry OWNER. A+ * throwing debounced store is recorded out-of-band (`storeFailures`) but+ * nothing on the debounced path ever re-drives it; a branch-mismatched+ * deferral is dropped inside `flushDeferredStores` with no owner to+ * notify; a lifecycle-race skip is by construction invisible to the+ * writer. One anti-entropy sweep over the divergence signal covers every+ * class, including ones not yet catalogued.+ *+ * Safety posture (additive; never bypasses the write spine):+ * - The staleness predicate is memory vs `reconciledBase` — the+ * persistence layer's own last-agreed-disk baseline — never memory vs+ * raw disk. An external disk edit the file-watcher has ingested+ * advances both sides, so it can never trip the predicate.+ * - Before forcing, disk is re-read and compared to the base. Bytes on+ * disk the base doesn't account for mean an out-of-band edit the+ * watcher hasn't reconciled yet; the watchdog stands down (warn only)+ * rather than clobber it. Disk stays authoritative for unseen edits,+ * including out-of-band deletes (base set, file gone → stand down, so+ * a forced store can't resurrect a deleted file). Residual risk: an+ * external edit landing in the few-ms window between this pre-check+ * and the store's atomic write is overwritten — the same window every+ * debounce-triggered non-agent store has today (only agent-marked+ * stores get the store-time re-check), so the watchdog adds no new+ * clobber class; the file-watcher reconcile remains the designed+ * recovery for that race.+ * - The force routes through the persistence handle's `forceStore`,+ * i.e. the exact store spine the debounced hook uses — every existing+ * guard (lifecycle, quiescence gate, no-op skip, duplication+ * tripwire, writeTracker registration) applies unchanged.+ */++import { normalizeBridge } from '@inkeep/open-knowledge-core';+import type * as Y from 'yjs';+import { getMsSinceLastUserTx } from './bridge-quiescence.ts';+import { isPersistenceExcludedDoc } from './cc1-broadcast.ts';+import { frozenDocLifecycleStatus } from './conflict-errors.ts';+import { getLogger } from './logger.ts';+import {+ incrementPersistenceStalenessDetected,+ incrementPersistenceStalenessForcedStores,+ incrementPersistenceStalenessStoodDown,+} from './metrics.ts';+import {+ getReconciledBase,+ isBatchInProgress,+ normalizedSourceForm,+ peekInFlightFlush,+} from './persistence.ts';++const log = getLogger('persistence-staleness');++/**+ * Throw from `readDiskBytes` for structurally permanent read refusals — a+ * content path replaced by a symlink resolving outside the content dir, or+ * an out-of-band file grown past the open byte limit. The watchdog then+ * declines the doc until its content changes (instead of retrying every+ * grace window and inflating the alertable stand-down counter forever) and+ * logs at error — a symlink-escape is security-relevant, not transient+ * I/O. Any other throw is treated as transient and retried.+ */+export class StructuralDiskReadError extends Error {+ constructor(message: string) {+ super(message);+ this.name = 'StructuralDiskReadError';+ }+}++/**+ * Generous by design: the natural flush path settles within seconds+ * (2 s debounce, 10 s maxDebounce, ~16 s quiescence-defer ceiling), so+ * anything divergent this long after its last transaction has no pending+ * store left to wait for.+ */+const DEFAULT_STALENESS_GRACE_MS = 5 * 60_000;+const DEFAULT_STALENESS_SWEEP_INTERVAL_MS = 60_000;++export interface StalenessWatchdogOptions {+ /** Live loaded-docs view, typically `() => hocuspocus.documents`. */+ getLoadedDocuments: () => Iterable<readonly [string, Y.Doc]>;+ /**+ * Force one store cycle through the persistence handle+ * (`PersistenceHandle.forceStore`). Rejections are logged and retried on+ * a later sweep.+ */+ forceStore: (document: Y.Doc, documentName: string) => Promise<void>;+ /**+ * Raw disk bytes for a docName, or `null` when the file does not exist.+ * Throws make the watchdog stand down for that doc — an unreadable file+ * must never be overwritten on a guess. A {@link StructuralDiskReadError}+ * declines the doc until its content changes (the refusal won't clear on+ * its own); any other throw is a transient inability to check and+ * retries a grace window later.+ */+ readDiskBytes: (documentName: string) => string | null;+ /** Minimum age of the newest user transaction before a doc counts as stale. */+ graceMs?: number;+ /** Sweep cadence. The interval timer is unref'd. */+ sweepIntervalMs?: number;+ /** Test seam for deterministic clocks. */+ now?: () => number;+ /** Test seams — default to the real persistence/quiescence surfaces. */+ getBase?: (documentName: string) => string | undefined;+ isBatchActive?: () => boolean;+ peekInFlight?: (documentName: string) => string | undefined;+ msSinceLastUserTx?: (doc: Y.Doc, nowMs: number) => number | null;+}++export interface StalenessWatchdogHandle {+ /** Run one sweep now. The interval skips while a sweep is in flight. */+ sweep: () => Promise<void>;+ /**+ * Stop the interval and drain: resolves after any in-flight sweep has+ * finished, so a caller tearing the server down can be sure no forced+ * store fires against a draining write spine afterwards. An in-flight+ * sweep also short-circuits at its next between-docs checkpoint.+ */+ dispose: () => Promise<void>;+}++interface AttemptRecord {+ fingerprint: string;+ atMs: number;+ /**+ * When this doc's current staleness episode was first observed, carried+ * across retries (unlike `atMs`, which paces them) so logs can report+ * how long the doc has been stale — "1 doc failing for an hour" and+ * "100 docs rescued once" are different incidents.+ */+ firstDetectedAtMs: number;+ /**+ * Retrying these exact bytes is provably futile: the store ran to+ * completion but chose not to write them (no-op classification, tripwire+ * breaker, lifecycle transition), or disk verifiably holds external+ * state the base doesn't account for (divergent bytes, out-of-band+ * delete, never-loaded file). Re-arming requires the content to change.+ * NOT set for transient conditions — a failed store, a failed disk+ * read, or a store parked by a mid-sweep batch keeps `declined: false`+ * so the next grace window retries.+ */+ declined: boolean;+}++export function createPersistenceStalenessWatchdog(+ options: StalenessWatchdogOptions,+): StalenessWatchdogHandle {+ const graceMs = options.graceMs ?? DEFAULT_STALENESS_GRACE_MS;+ const sweepIntervalMs = options.sweepIntervalMs ?? DEFAULT_STALENESS_SWEEP_INTERVAL_MS;+ const now = options.now ?? Date.now;+ const getBase = options.getBase ?? getReconciledBase;+ const isBatchActive = options.isBatchActive ?? isBatchInProgress;+ const peekInFlight = options.peekInFlight ?? peekInFlightFlush;+ const msSinceLastUserTx = options.msSinceLastUserTx ?? getMsSinceLastUserTx;++ const attempts = new Map<string, AttemptRecord>();+ let disposed = false;+ let sweepInFlight: Promise<void> | null = null;++ /**+ * The store's own no-op comparator form, via the SAME derivation+ * (`normalizedSourceForm`) `storeDocumentNow`'s+ * `markdownSemanticallyUnchanged` skip uses — the two classifiers agree+ * by construction. The store's extra ephemeral-canonical branch is NOT+ * replicated: ephemeral single-file servers don't construct a watchdog+ * at all (see the `!ephemeral` gate in server-factory), precisely+ * because this classifier would read a round-trip-unstable file at rest+ * as divergent.+ */+ function candidateFromRaw(rawYText: string): string {+ return normalizedSourceForm(rawYText);+ }++ function normalizedBaseFor(base: string | undefined): string | undefined {+ return base === undefined ? undefined : normalizeBridge(base);+ }++ function isDivergent(candidate: string, normalizedBase: string | undefined): boolean {+ return normalizedBase === undefined ? candidate !== '' : candidate !== normalizedBase;+ }++ async function sweepOnce(): Promise<void> {+ if (disposed) return;+ // A coordinated git operation owns the write spine right now; bail so+ // the sweep can't interleave with it. Within-branch deferral replay+ // covers most pending work when the batch ends; anything it drops+ // (branch-mismatched entries, discard-stale mode) re-surfaces as+ // divergence on a later sweep — this sweep is that recovery path.+ if (isBatchActive()) return;++ const startedMs = now();+ let scanned = 0;+ let divergent = 0;+ let forced = 0;+ let stoodDown = 0;+ const seen = new Set<string>();+ for (const [documentName, document] of options.getLoadedDocuments()) {+ // dispose() during a previous iteration's await must stop the sweep+ // before it touches another doc.+ if (disposed) return;+ seen.add(documentName);+ if (isPersistenceExcludedDoc(documentName)) continue;+ if (frozenDocLifecycleStatus(document) !== null) {+ // These docs intentionally hold memory != disk; the store spine+ // skips them for the same reason.+ continue;+ }+ if (peekInFlight(documentName) !== undefined) continue;++ scanned++;+ const rawYText = document.getText('source').toString();+ const base = getBase(documentName);+ // Byte-equal fast path: after any successful store or file-watcher+ // ingest the raw Y.Text equals the base verbatim, so resident clean+ // docs skip the strip/normalize comparator work entirely.+ if (base !== undefined && rawYText === base) {+ attempts.delete(documentName);+ continue;+ }+ const candidate = candidateFromRaw(rawYText);+ const normalizedBase = normalizedBaseFor(base);+ if (!isDivergent(candidate, normalizedBase)) {+ attempts.delete(documentName);+ continue;+ }+ divergent++;++ const nowMs = now();+ const ageMs = msSinceLastUserTx(document, nowMs);+ if (ageMs !== null && ageMs < graceMs) continue;++ const previous = attempts.get(documentName);+ if (previous && previous.fingerprint === candidate) {+ if (previous.declined) continue;+ if (nowMs - previous.atMs < graceMs) continue;+ }+ const firstSighting = previous?.fingerprint !== candidate;+ if (firstSighting) incrementPersistenceStalenessDetected();+ const firstDetectedAtMs = !firstSighting && previous ? previous.firstDetectedAtMs : nowMs;+ const staleForMs = nowMs - firstDetectedAtMs;++ // External-edit stand-down. Disk bytes the base doesn't account for+ // mean the file-watcher hasn't reconciled an out-of-band change yet;+ // writing memory over them would clobber it. A missing file with a+ // known base is an unreconciled out-of-band delete — same stand-down+ // (a forced store would resurrect the file). Only two shapes are+ // safe to write: disk still matches the base, or the doc has never+ // had an on-disk form at all.+ let standDownReason: string | null = null;+ let standDownRetryable = false;+ try {+ const diskBytes = options.readDiskBytes(documentName);+ if (normalizedBase === undefined) {+ if (diskBytes !== null) standDownReason = 'disk-file-never-loaded';+ } else if (diskBytes === null) {+ standDownReason = 'disk-file-missing';+ } else if (normalizeBridge(diskBytes) !== normalizedBase) {+ standDownReason = 'disk-diverged-from-base';+ }+ } catch (err) {+ if (err instanceof StructuralDiskReadError) {+ // The refusal (symlink-escape, oversized file) won't clear on its+ // own — decline until the doc's content changes so the alertable+ // stand-down counter doesn't grow forever on one bad path.+ standDownReason = 'disk-read-refused';+ log.error(+ { err, docName: documentName },+ '[persistence-staleness] disk read refused; standing down until content changes',+ );+ } else {+ // Can't verify disk, so don't write — but a transient read fault+ // says nothing about external state, so retry after the next+ // grace window rather than suppressing the doc.+ standDownReason = 'disk-read-failed';+ standDownRetryable = true;+ log.warn(+ { err, docName: documentName },+ '[persistence-staleness] disk read failed; standing down',+ );+ }+ }++ if (standDownReason !== null) {+ stoodDown++;+ incrementPersistenceStalenessStoodDown();+ attempts.set(documentName, {+ fingerprint: candidate,+ atMs: nowMs,+ firstDetectedAtMs,+ declined: !standDownRetryable,+ });+ log.warn(+ {+ docName: documentName,+ ageMs,+ staleForMs,+ candidateBytes: candidate.length,+ baseBytes: base?.length ?? 0,+ action: 'stood-down',+ reason: standDownReason,+ },+ `[persistence-staleness] Unflushed edits detected for ${documentName} but disk state is unverified or unreconciled; not overwriting`,+ );+ continue;+ }++ log.warn(+ {+ docName: documentName,+ ageMs,+ staleForMs,+ candidateBytes: candidate.length,+ baseBytes: base?.length ?? 0,+ action: 'forced-store',+ },+ `[persistence-staleness] Doc ${documentName} has unpersisted edits past the grace window with no pending store; forcing a store`,+ );+ attempts.set(documentName, {+ fingerprint: candidate,+ atMs: nowMs,+ firstDetectedAtMs,+ declined: false,+ });+ incrementPersistenceStalenessForcedStores();+ forced++;+ try {+ await options.forceStore(document, documentName);+ } catch (err) {+ // The store spine already recorded the failure out-of-band; keep+ // declined=false so the next grace window retries (a transient disk+ // condition may have cleared).+ log.error(+ { err, docName: documentName, staleForMs },+ `[persistence-staleness] Forced store failed for ${documentName}; will retry`,+ );+ continue;+ }+ if (disposed) return;+ // A batch that activated inside the forceStore await means the store+ // was PARKED (deferStore), not run — classifying the still-divergent+ // doc as declined here would permanently suppress it even if the+ // batch's replay later drops the parked entry. Bail the sweep (the+ // batch owns the spine now); the entry stays declined:false so the+ // next grace window re-checks.+ if (isBatchActive()) return;++ // Re-read both sides: a store that completed yet left the same bytes+ // divergent chose not to write them (no-op class) — mark declined so+ // we don't loop on it. Convergence (or content that moved on) re-arms.+ const afterCandidate = candidateFromRaw(document.getText('source').toString());+ const afterNormalizedBase = normalizedBaseFor(getBase(documentName));+ if (isDivergent(afterCandidate, afterNormalizedBase) && afterCandidate === candidate) {+ attempts.set(documentName, {+ fingerprint: candidate,+ atMs: nowMs,+ firstDetectedAtMs,+ declined: true,+ });+ log.info(+ { docName: documentName },+ `[persistence-staleness] Store completed without clearing divergence for ${documentName}; suppressing until content changes`,+ );+ } else {+ attempts.delete(documentName);+ }+ }++ // Docs that unloaded since the last sweep drop their bookkeeping.+ for (const name of attempts.keys()) {+ if (!seen.has(name)) attempts.delete(name);+ }++ // Sweep-health signal: the outcome counters say nothing about sweep+ // cost or a wedged sweep; this line does, without per-doc volume.+ // Eventful sweeps log at info so production log levels see them; the+ // steady-state all-clean sweep stays at debug to keep logs quiet.+ const summary = { scanned, divergent, forced, stoodDown, elapsedMs: now() - startedMs };+ if (divergent > 0 || forced > 0 || stoodDown > 0) {+ log.info(summary, '[persistence-staleness] sweep complete');+ } else {+ log.debug(summary, '[persistence-staleness] sweep complete');+ }+ }++ function sweep(): Promise<void> {+ if (sweepInFlight) return sweepInFlight;+ sweepInFlight = sweepOnce().finally(() => {+ sweepInFlight = null;+ });+ return sweepInFlight;+ }++ const timer = setInterval(() => {+ void sweep().catch((err) => {+ log.error({ err }, '[persistence-staleness] sweep failed');+ });+ }, sweepIntervalMs);+ timer.unref?.();++ return {+ sweep,+ dispose: () => {+ disposed = true;+ clearInterval(timer);+ return sweepInFlight ?? Promise.resolve();+ },+ };+}diff --git a/packages/server/src/persistence.ts b/packages/server/src/persistence.tsindex 196d8cfd0..6e6775d93 100644--- a/packages/server/src/persistence.ts+++ b/packages/server/src/persistence.ts@@ -45,8 +45,15 @@ import { LINEAGE_EPOCH_KEY } from './auth-token-schema.ts';import type { BacklinkIndex } from './backlink-index.ts';import { getMsSinceLastUserTx, isDocQuiescent } from './bridge-quiescence.ts';import { assertBridgeInvariant, createDocCanonicalizer } from './bridge-watchdog.ts';-import { isConfigDoc, isManagedArtifactDoc, isMermaidDoc, isSystemDoc } from './cc1-broadcast.ts';+import {+ isConfigDoc,+ isManagedArtifactDoc,+ isMermaidDoc,+ isPersistenceExcludedDoc,+ isSystemDoc,+} from './cc1-broadcast.ts';import { type ConfigPersistenceCtx, loadConfigDoc, storeConfigDoc } from './config-persistence.ts';+import { frozenDocLifecycleStatus } from './conflict-errors.ts';import type { ContributorEntry } from './contributor-tracker.ts';import {contributorCount,@@ -430,6 +437,22 @@ export function captureDocSnapshotForPersistence(document: Y.Doc): {};}+/**+ * The store spine's canonical "what would land on disk" form of raw+ * Y.Text bytes: frontmatter re-composed onto the body, then+ * `normalizeBridge`d. This is the LEFT side of `storeDocumentNow`'s+ * no-op skip (`markdownSemanticallyUnchanged`) AND the staleness+ * watchdog's divergence predicate — both compare it against+ * `normalizeBridge(reconciledBase)`. Single derivation site so the two+ * classifiers cannot drift: a doc the store would no-op must never read+ * as stale to the watchdog, and a doc the watchdog reads as converged+ * must never be one the store would rewrite.+ */+export function normalizedSourceForm(rawYText: string): string {+ const { frontmatter, body } = stripFrontmatter(rawYText);+ return normalizeBridge(prependFrontmatter(frontmatter, body));+}+export function safeContentPath(documentName: string, contentDir: string): string {if (documentName.includes('\x00')) {throw new Error(`Invalid document name: ${documentName}`);@@ -636,6 +659,19 @@ export interface PersistenceHandle {* mutates state.*/getQueueDepths: () => PersistenceQueueDepths;+ /**+ * Force one immediate L1 store cycle for a loaded doc through the exact+ * dispatch the debounced `onStoreDocument` hook uses: synthetic docs+ * (system / config / managed-artifact / mermaid) short-circuit, a+ * coordinated batch defers, and everything else runs `storeDocumentNow`+ * with all of its guards (lifecycle, quiescence, no-op skip, tripwire,+ * writeTracker registration). Exists for the staleness watchdog —+ * Hocuspocus only re-arms its store debounce on a new Y.Doc update, so a+ * store cycle that failed or was skipped after a session's last edit has+ * no natural retry. Rejections mirror `storeDocumentNow`'s (the failure+ * is also recorded for `takeStoreFailure`).+ */+ forceStore: (document: Y.Doc, documentName: string) => Promise<void>;/*** Config-doc persistence context. Exposed so the file-watcher* orchestration in `server-factory.ts` can call `applyExternalConfigChange`@@ -1296,12 +1332,8 @@ export function createPersistenceExtension(options?: PersistenceOptions): Persis// the conflict window stay in Y.Text; on resolution the// case 'update' reconcile in server-factory.ts runs against// them as `ours`.- const lifecycleStatus = document.getMap('lifecycle').get('status');- if (- lifecycleStatus === 'deleted-upstream' ||- lifecycleStatus === 'renamed' ||- lifecycleStatus === 'conflict'- ) {+ const lifecycleStatus = frozenDocLifecycleStatus(document);+ if (lifecycleStatus !== null) {log.info({ documentName, lifecycleStatus },`[persistence] Skipped store for ${documentName}: lifecycle=${lifecycleStatus}`,@@ -1491,7 +1523,9 @@ export function createPersistenceExtension(options?: PersistenceOptions): Persis// safety-net below, preventing phantom commits attributed to the// browser's principal when a later agent write triggers the L2 fan-out.const currentBase = getReconciledBase(documentName);- const normalizedMarkdown = normalizeBridge(markdown);+ // Routed through the shared helper so this compare and the+ // staleness watchdog's divergence predicate stay one derivation.+ const normalizedMarkdown = normalizedSourceForm(ytextSnapshot);let markdownSemanticallyUnchanged =currentBase !== undefined && normalizedMarkdown === normalizeBridge(currentBase);// G8 (ephemeral single-file mode only): also treat the candidate as a@@ -2388,6 +2422,22 @@ export function createPersistenceExtension(options?: PersistenceOptions): Persis};}+ async function forceStore(document: Y.Doc, documentName: string): Promise<void> {+ if (isPersistenceExcludedDoc(documentName)) {+ return;+ }+ if (isBatchInProgress()) {+ // Same parking the debounced hook applies; the batch's own+ // `flushDeferredStores` replays it when the coordinated operation ends.+ deferStore({ document, documentName, lastTransactionOrigin: null });+ return;+ }+ // Origin null: the triggering write's contributor entry (if any) was+ // recorded when the edit happened; a forced re-flush must not invent a+ // new writer identity.+ return storeDocumentNow({ document, documentName, lastTransactionOrigin: null });+ }+return {extension,flushDeferredStores,@@ -2398,6 +2448,7 @@ export function createPersistenceExtension(options?: PersistenceOptions): PersistakeStoreDivergence,markAgentWriteStore,getQueueDepths,+ forceStore,configPersistenceCtx,managedArtifactCtx,};diff --git a/packages/server/src/server-factory.ts b/packages/server/src/server-factory.tsindex b9a347245..17fcf4e82 100644--- a/packages/server/src/server-factory.ts+++ b/packages/server/src/server-factory.ts@@ -1,6 +1,13 @@import { type SpawnSyncReturns, spawnSync } from 'node:child_process';import { randomUUID } from 'node:crypto';-import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';+import {+ existsSync,+ mkdirSync,+ readFileSync,+ realpathSync,+ statSync,+ writeFileSync,+} from 'node:fs';import { basename, dirname, join, relative, resolve } from 'node:path';import type { Document, Extension } from '@hocuspocus/server';import { Hocuspocus, IncomingMessage, MessageType } from '@hocuspocus/server';@@ -15,6 +22,7 @@ import {createBasenameIndex,DEFAULT_ATTACHMENT_FOLDER_PATH,DEFAULT_LINTER_CONFIG,+ DOCUMENT_OPEN_BYTE_LIMIT,humanFormat,isKnownConfigError,type LinterConfig,@@ -141,6 +149,11 @@ import {setReconciledBase,switchReconciledBaseScope,} from './persistence.ts';+import {+ createPersistenceStalenessWatchdog,+ type StalenessWatchdogHandle,+ StructuralDiskReadError,+} from './persistence-staleness-watchdog.ts';import { loadPrincipal } from './principal.ts';import { RecentlyRemovedDocs } from './recently-removed-docs.ts';import { reconcile } from './reconciliation.ts';@@ -189,6 +202,14 @@ export interface ServerOptions {quiet?: boolean;debounce?: number;maxDebounce?: number;+ /**+ * Persistence staleness watchdog tuning. Production defaults (5 min+ * grace, 60 s sweep) live in `persistence-staleness-watchdog.ts`; tests+ * pass small values to exercise the forced-store rescue path without+ * wall-clock waits.+ */+ stalenessGraceMs?: number;+ stalenessSweepIntervalMs?: number;gitEnabled?: boolean;commitDebounceMs?: number;wipRef?: string;@@ -862,6 +883,7 @@ export function createServer(options: ServerOptions): ServerInstance {let agentFocusBroadcaster: AgentFocusBroadcaster | null = null;let agentPresenceBroadcaster: AgentPresenceBroadcaster | null = null;let invalidateReferencedAssetsCache: (() => void) | null = null;+ let stalenessWatchdog: StalenessWatchdogHandle | null = null;// Semantic-search service. Always constructed (cheap, inert until enabled +// keyed; the loader makes no network call). The api-extension drives the lazy@@ -1226,6 +1248,74 @@ export function createServer(options: ServerOptions): ServerInstance {);installServerWorkloadGauges();+ // Durability backstop: Hocuspocus re-arms the store debounce only on a+ // new Y.Doc update and user docs stay resident for the server lifetime,+ // so a store cycle lost after a session's last edit (transient disk+ // error, dropped deferral) would otherwise leave disk stale until the+ // next edit. The watchdog re-flushes through `persistence.forceStore`+ // (the normal store spine) and never overwrites disk state the+ // reconciled base doesn't account for.+ //+ // Ephemeral single-file mode opts out: the watchdog's divergence+ // classifier deliberately lacks the store's ephemeral-canonical no-op+ // branch, so a round-trip-unstable file at rest would read as stale+ // and pollute the staleness counters on a healthy session. Those+ // sessions are short-lived and user-attended, not the days-wedged+ // resident-doc class this backstop exists for.+ if (!ephemeral) {+ stalenessWatchdog = createPersistenceStalenessWatchdog({+ getLoadedDocuments: () => hp.documents,+ forceStore: (document, documentName) => persistence.forceStore(document, documentName),+ // Realpath symlink-escape gate + open-byte-limit cap, matching the+ // load-path read discipline. Not atomic — three syscalls — but+ // ENOENT from any of them maps to `null` (out-of-band delete)+ // rather than a transient read fault, so a delete landing between+ // syscalls classifies the same as one landing before the first.+ // The two refusals are structurally permanent (they won't clear+ // without an on-disk change), so they throw the typed error that+ // makes the watchdog decline instead of retrying every window.+ readDiskBytes: (documentName) => {+ const requestedPath = safeContentPath(documentName, contentDir);+ // The structural throws live OUTSIDE the errno-filtering try+ // blocks so a future widening of the errno filter can never+ // swallow a security-relevant refusal.+ let canonical: string;+ let size: number;+ try {+ canonical = realpathSync(requestedPath);+ size = statSync(canonical).size;+ } catch (err) {+ const code = (err as NodeJS.ErrnoException).code;+ if (code === 'ENOENT') return null;+ if (code === 'ELOOP') {+ // A symlink cycle is structurally permanent, matching the+ // load path's explicit ELOOP refusal.+ throw new StructuralDiskReadError(`symlink cycle at content path: ${documentName}`);+ }+ throw err;+ }+ if (!isWithinContentDir(canonical, contentDir)) {+ throw new StructuralDiskReadError(+ `symlink-escape: ${requestedPath} resolves outside the content dir`,+ );+ }+ if (size > DOCUMENT_OPEN_BYTE_LIMIT) {+ throw new StructuralDiskReadError(+ `document exceeds the open byte limit: ${documentName}`,+ );+ }+ try {+ return readFileSync(canonical, 'utf-8');+ } catch (err) {+ if ((err as NodeJS.ErrnoException).code === 'ENOENT') return null;+ throw err;+ }+ },+ graceMs: options.stalenessGraceMs,+ sweepIntervalMs: options.stalenessSweepIntervalMs,+ });+ }+// Hocuspocus unloads documents as soon as the last WebSocket disconnects.// That is unsafe with client-side y-indexeddb: a browser refresh leaves a// durable client copy of the same Yjs items, while the server rebuilds a@@ -1693,10 +1783,15 @@ export function createServer(options: ServerOptions): ServerInstance {// here too — otherwise their closures (capturing `hocuspocus`/`persistence`)// leak into the process-global registries and keep summing off a// half-torn-down object. Idempotent via splice(0), so destroy()'s later- // drain stays a no-op.+ // drain stays a no-op. The staleness watchdog's interval holds the same+ // closures, so it gets the same treatment.for (const unregister of unregisterWorkloadProviders.splice(0)) {unregister();}+ // Sync context — can't await the drain; the disposed flag makes any+ // in-flight sweep short-circuit at its next between-docs checkpoint.+ void stalenessWatchdog?.dispose();+ stalenessWatchdog = null;releaseServerLock(lockDir);throw err;}@@ -2431,10 +2526,19 @@ export function createServer(options: ServerOptions): ServerInstance {// Stop contributing to the process-wide workload gauges before any// teardown phase runs — a mid-destroy sample would otherwise read- // half-dismantled structures.+ // half-dismantled structures. Same for the staleness watchdog: a sweep+ // firing mid-destroy could force a store against a draining spine.for (const unregister of unregisterWorkloadProviders.splice(0)) {unregister();}+ // Awaited drain: dispose resolves only after any in-flight sweep has+ // finished, so no forced store can land once teardown proceeds.+ try {+ await stalenessWatchdog?.dispose();+ } catch (err) {+ log.warn({ err }, '[server] staleness watchdog drain failed during destroy');+ }+ stalenessWatchdog = null;// Advertise teardown FIRST — before any flush work. Readers (MCP// discovery, desktop attach, spawners) see `draining: true` and stop