Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
fix(ok): stop false conflicts from overlapping own disk flushes (#4098)
Nick Gomezc8fc36738ff5
diff --git a/.changeset/reconcile-overlapping-own-flush.md b/.changeset/reconcile-overlapping-own-flush.mdnew file mode 100644index 000000000..5ffa52683--- /dev/null+++ b/.changeset/reconcile-overlapping-own-flush.md@@ -0,0 +1,11 @@+---+"@inkeep/open-knowledge": patch+---++Fixes a bug where a document could drop into a merge-conflict state, replacing the editor with the conflict diff view, even though nothing outside Open Knowledge had touched the file.++Before writing to a document, the server compares what is on disk against what it believes it last wrote, so that a write from another program is detected but its own recent save is not mistaken for one. That check only remembered a single save per document. When an agent write and your typing kept a document busy enough that two saves were in flight at once, the second save displaced the record of the first, the still-settling first save on disk no longer matched anything the server recognized, and it was treated as a foreign edit. The three-way merge that followed compared the document against its own earlier save and, when both had changed the same paragraph, reported a conflict that never existed. The editor was then swapped for the conflict resolution view and later agent writes were refused with a 409 until the phantom conflict was resolved.++The server now tracks every save still in flight for a document rather than only the most recent one, the way the file watcher already recognizes its own writes. Those records expire after a minute and are treated as absent by everything that reads them, so a save that never reports completion cannot quietly disable the background durability check for that document. Detection of genuine external edits is unchanged: a write from another program is still reconciled unless it is byte-for-byte a save Open Knowledge is already making.++Separately, the background check that rescues documents whose edits never reached disk no longer waits forever on a single save. If one document's write stops responding, that document is set aside and reported instead of stalling the check for every other open document until the app is restarted. It is picked up again as soon as that write finishes, whichever way it finishes, so a save that hangs and then fails on a network or cloud-synced folder no longer leaves that document unprotected for the rest of the session.diff --git a/packages/server/src/document-durability-state.test.ts b/packages/server/src/document-durability-state.test.tsindex 23d758e6b..4af8ccced 100644--- a/packages/server/src/document-durability-state.test.ts+++ b/packages/server/src/document-durability-state.test.ts@@ -1,5 +1,6 @@-import { describe, expect, test } from 'vitest';+import { describe, expect, test, vi } from 'vitest';import { DocumentDurabilityState } from './document-durability-state.ts';+import { getMetrics, resetMetrics } from './metrics.ts';describe('DocumentDurabilityState', () => {test('starts with an empty main scope and no transient coordination state', () => {@@ -90,3 +91,133 @@ describe('DocumentDurabilityState', () => {expect(state.takeStoreFailure('retained')).toEqual({ message: 'readonly' });});});++describe('in-flight flush queue semantics', () => {+ test('finishing one of two identical snapshots leaves the other pending', () => {+ const state = new DocumentDurabilityState();+ state.beginInFlightFlush('doc', 'same');+ state.beginInFlightFlush('doc', 'same');+ state.finishInFlightFlush('doc', 'same');+ expect(state.inFlightFlushCount('doc')).toBe(1);+ expect(state.hasInFlightFlush('doc', 'same')).toBe(true);+ state.finishInFlightFlush('doc', 'same');+ expect(state.peekInFlightFlush('doc')).toBeUndefined();+ expect(state.inFlightFlushCount('doc')).toBe(0);+ });++ test('finishing a snapshot that was never begun leaves the queue untouched', () => {+ const state = new DocumentDurabilityState();+ state.beginInFlightFlush('doc', 'real');+ state.finishInFlightFlush('doc', 'never-begun');+ expect(state.inFlightFlushCount('doc')).toBe(1);+ expect(state.hasInFlightFlush('doc', 'real')).toBe(true);+ });++ test('a middle entry can finish out of order without disturbing its neighbours', () => {+ const state = new DocumentDurabilityState();+ state.beginInFlightFlush('doc', 'a');+ state.beginInFlightFlush('doc', 'b');+ state.beginInFlightFlush('doc', 'c');+ state.finishInFlightFlush('doc', 'b');+ expect(state.hasInFlightFlush('doc', 'a')).toBe(true);+ expect(state.hasInFlightFlush('doc', 'b')).toBe(false);+ expect(state.hasInFlightFlush('doc', 'c')).toBe(true);+ expect(state.peekInFlightFlush('doc')).toBe('c');+ });++ test('content absent from a populated queue does not match', () => {+ const state = new DocumentDurabilityState();+ state.beginInFlightFlush('doc', 'a');+ state.beginInFlightFlush('doc', 'b');+ expect(state.hasInFlightFlush('doc', 'absent')).toBe(false);+ expect(state.hasInFlightFlush('other-doc', 'a')).toBe(false);+ });++ test('a leaked entry expires on read, with no later flush needed to sweep it', () => {+ vi.useFakeTimers();+ try {+ const state = new DocumentDurabilityState();+ state.beginInFlightFlush('doc', 'leaked');+ expect(state.peekInFlightFlush('doc')).toBe('leaked');++ vi.advanceTimersByTime(61_000);++ expect(state.peekInFlightFlush('doc')).toBeUndefined();+ expect(state.inFlightFlushCount('doc')).toBe(0);+ expect(state.hasInFlightFlush('doc', 'leaked')).toBe(false);+ } finally {+ vi.useRealTimers();+ }+ });++ test('deleting a doc clears its in-flight flush records alongside its reconciled base', () => {+ const state = new DocumentDurabilityState();+ state.setReconciledBase('doc', 'base');+ state.beginInFlightFlush('doc', 'pending');+ state.beginInFlightFlush('other', 'pending');++ state.deleteReconciledBase('doc');++ expect(state.getReconciledBase('doc')).toBeUndefined();+ expect(state.inFlightFlushCount('doc')).toBe(0);+ expect(state.inFlightFlushCount('other')).toBe(1);+ });++ test('switching branch scope leaves in-flight flush records alone, since a write in flight is not branch-scoped', () => {+ const state = new DocumentDurabilityState();+ state.beginInFlightFlush('doc', 'pending');++ state.switchReconciledBaseScope('feature');++ expect(state.inFlightFlushCount('doc')).toBe(1);+ expect(state.hasInFlightFlush('doc', 'pending')).toBe(true);+ });++ test('the expiry counter reports how many records were dropped, not how many prunes ran', () => {+ vi.useFakeTimers();+ try {+ resetMetrics();+ const state = new DocumentDurabilityState();+ state.beginInFlightFlush('doc', 'first');+ state.beginInFlightFlush('doc', 'second');++ vi.advanceTimersByTime(61_000);+ expect(state.inFlightFlushCount('doc')).toBe(0);++ expect(getMetrics().inFlightFlushExpired).toBe(2);+ } finally {+ vi.useRealTimers();+ }+ });++ test('a stale entry is also discarded when the next flush for the same document begins', () => {+ vi.useFakeTimers();+ try {+ const state = new DocumentDurabilityState();+ state.beginInFlightFlush('doc', 'leaked');+ vi.advanceTimersByTime(61_000);+ state.beginInFlightFlush('doc', 'fresh');++ expect(state.hasInFlightFlush('doc', 'leaked')).toBe(false);+ expect(state.inFlightFlushCount('doc')).toBe(1);+ state.finishInFlightFlush('doc', 'fresh');+ expect(state.peekInFlightFlush('doc')).toBeUndefined();+ } finally {+ vi.useRealTimers();+ }+ });++ test('a flush still in flight is not aged out while it keeps company with newer flushes', () => {+ vi.useFakeTimers();+ try {+ const state = new DocumentDurabilityState();+ state.beginInFlightFlush('doc', 'slow');+ vi.advanceTimersByTime(5_000);+ state.beginInFlightFlush('doc', 'quick');+ expect(state.hasInFlightFlush('doc', 'slow')).toBe(true);+ expect(state.inFlightFlushCount('doc')).toBe(2);+ } finally {+ vi.useRealTimers();+ }+ });+});diff --git a/packages/server/src/document-durability-state.ts b/packages/server/src/document-durability-state.tsindex 67de2628d..236e72faa 100644--- a/packages/server/src/document-durability-state.ts+++ b/packages/server/src/document-durability-state.ts@@ -1,11 +1,20 @@+import { incrementInFlightFlushExpired } from './metrics.ts';+export interface StoreFailure {code?: string;message: string;}+const IN_FLIGHT_FLUSH_TTL_MS = 60_000;++interface PendingFlush {+ value: string;+ at: number;+}+export class DocumentDurabilityState {private readonly reconciledBaseByBranch = new Map<string, Map<string, string>>();- private readonly inFlightFlushByDoc = new Map<string, string>();+ private readonly inFlightFlushByDoc = new Map<string, PendingFlush[]>();private readonly agentWriteStores = new Set<string>();private readonly storeFailures = new Map<string, StoreFailure>();private readonly storeDivergences = new Set<string>();@@ -41,21 +50,58 @@ export class DocumentDurabilityState {bases.set(docName, content);}+ /**+ * STOP: at the moment of this call the doc must already be outside the staleness sweep's+ * candidate set — either not loaded, or holding a frozen lifecycle status. The sweep iterates+ * loaded documents and continues on an excluded or frozen doc before it reads+ * inFlightFlushCount, which is what makes dropping the in-flight flush records here inert for+ * it. Call this with the doc still loaded and unfrozen and the sweep can force a store for a+ * doc whose own write is still physically outstanding. The reconcile guard, the other reader,+ * reads that same base and returns early without it.+ */deleteReconciledBase(docName: string): void {this.reconciledBaseByBranch.get(this.activeBranch)?.delete(docName);+ this.inFlightFlushByDoc.delete(docName);+ }++ private freshInFlightFlushes(docName: string): PendingFlush[] {+ const pending = this.inFlightFlushByDoc.get(docName);+ if (!pending) return [];+ const now = Date.now();+ const fresh = pending.filter((entry) => now - entry.at <= IN_FLIGHT_FLUSH_TTL_MS);+ if (fresh.length === pending.length) return pending;+ incrementInFlightFlushExpired(pending.length - fresh.length);+ if (fresh.length === 0) this.inFlightFlushByDoc.delete(docName);+ else this.inFlightFlushByDoc.set(docName, fresh);+ return fresh;}beginInFlightFlush(docName: string, normalizedMarkdown: string): void {- this.inFlightFlushByDoc.set(docName, normalizedMarkdown);+ const fresh = this.freshInFlightFlushes(docName);+ fresh.push({ value: normalizedMarkdown, at: Date.now() });+ this.inFlightFlushByDoc.set(docName, fresh);}peekInFlightFlush(docName: string): string | undefined {- return this.inFlightFlushByDoc.get(docName);+ const pending = this.freshInFlightFlushes(docName);+ return pending[pending.length - 1]?.value;+ }++ inFlightFlushCount(docName: string): number {+ return this.freshInFlightFlushes(docName).length;+ }++ hasInFlightFlush(docName: string, normalizedMarkdown: string): boolean {+ return this.freshInFlightFlushes(docName).some((entry) => entry.value === normalizedMarkdown);}finishInFlightFlush(docName: string, expectedNormalizedMarkdown: string): void {- if (this.inFlightFlushByDoc.get(docName) === expectedNormalizedMarkdown) {- this.inFlightFlushByDoc.delete(docName);+ const pending = this.freshInFlightFlushes(docName);+ for (let i = pending.length - 1; i >= 0; i--) {+ if (pending[i]?.value !== expectedNormalizedMarkdown) continue;+ pending.splice(i, 1);+ if (pending.length === 0) this.inFlightFlushByDoc.delete(docName);+ return;}}diff --git a/packages/server/src/external-change.ts b/packages/server/src/external-change.tsindex eb5e33f07..8916be954 100644--- a/packages/server/src/external-change.ts+++ b/packages/server/src/external-change.ts@@ -250,22 +250,23 @@ export function reconcileDiskBeforeAgentWrite(return NOT_RECONCILED;}- if (normalizeBridge(diskContent) === normalizeBridge(base)) return NOT_RECONCILED;+ const normalizedDisk = normalizeBridge(diskContent);+ if (normalizedDisk === normalizeBridge(base)) return NOT_RECONCILED;- const inFlightFlush = durabilityState.peekInFlightFlush(docName);- if (inFlightFlush !== undefined) {- if (normalizeBridge(diskContent) === inFlightFlush) {+ const pendingFlushes = durabilityState.inFlightFlushCount(docName);+ if (pendingFlushes > 0) {+ if (durabilityState.hasInFlightFlush(docName, normalizedDisk)) {incrementReconcileOwnFlushSkips();getLogger('reconcile').debug(- { docName, diskBytes: diskContent.length },+ { docName, diskBytes: diskContent.length, pendingFlushes },`[reconcile] disk matches own in-flight flush for ${docName}; skipping reconcile`,);return NOT_RECONCILED;}incrementReconcileInFlightFallthroughs();getLogger('reconcile').warn(- { docName, diskBytes: diskContent.length },- `[reconcile] in-flight flush present but disk differs from snapshot for ${docName}; falling through to merge`,+ { docName, diskBytes: diskContent.length, pendingFlushes },+ `[reconcile] disk matches none of ${pendingFlushes} in-flight flush snapshot(s) for ${docName}; falling through to merge`,);}diff --git a/packages/server/src/metrics.ts b/packages/server/src/metrics.tsindex ec2debb5b..3a47a300d 100644--- a/packages/server/src/metrics.ts+++ b/packages/server/src/metrics.ts@@ -39,6 +39,7 @@ export interface ReconciliationMetrics {persistenceStalenessDetected: number;persistenceStalenessForcedStores: number;persistenceStalenessStoodDown: number;+ persistenceStalenessForceStoreTimeouts: number;collabSocketEpipeCount: number;collabSocketEconnresetCount: number;collabMessageTooLargeCount: number;@@ -103,6 +104,7 @@ export interface ReconciliationMetrics {externalChangeHandlerErrors: number;reconcileOwnFlushSkips: number;reconcileInFlightFallthroughs: number;+ inFlightFlushExpired: number;persistenceSanityCheckSerializeFailures: number;deferredStoreFailures: number;authRenameRedirectCount: number;@@ -148,6 +150,7 @@ const counters: ReconciliationMetrics = {persistenceStalenessDetected: 0,persistenceStalenessForcedStores: 0,persistenceStalenessStoodDown: 0,+ persistenceStalenessForceStoreTimeouts: 0,collabSocketEpipeCount: 0,collabSocketEconnresetCount: 0,collabMessageTooLargeCount: 0,@@ -191,6 +194,7 @@ const counters: ReconciliationMetrics = {externalChangeHandlerErrors: 0,reconcileOwnFlushSkips: 0,reconcileInFlightFallthroughs: 0,+ inFlightFlushExpired: 0,persistenceSanityCheckSerializeFailures: 0,deferredStoreFailures: 0,authRenameRedirectCount: 0,@@ -470,6 +474,14 @@ export function incrementReconcileInFlightFallthroughs(): void {counters.reconcileInFlightFallthroughs++;}+export function incrementInFlightFlushExpired(count: number): void {+ counters.inFlightFlushExpired += count;+}++export function incrementPersistenceStalenessForceStoreTimeouts(): void {+ counters.persistenceStalenessForceStoreTimeouts++;+}+export function incrementPersistenceSanityCheckSerializeFailures(): void {counters.persistenceSanityCheckSerializeFailures++;}@@ -614,6 +626,7 @@ export function resetMetrics(): void {counters.persistenceStalenessDetected = 0;counters.persistenceStalenessForcedStores = 0;counters.persistenceStalenessStoodDown = 0;+ counters.persistenceStalenessForceStoreTimeouts = 0;counters.collabSocketEpipeCount = 0;counters.collabSocketEconnresetCount = 0;counters.collabMessageTooLargeCount = 0;@@ -657,6 +670,7 @@ export function resetMetrics(): void {counters.externalChangeHandlerErrors = 0;counters.reconcileOwnFlushSkips = 0;counters.reconcileInFlightFallthroughs = 0;+ counters.inFlightFlushExpired = 0;counters.persistenceSanityCheckSerializeFailures = 0;counters.deferredStoreFailures = 0;counters.authRenameRedirectCount = 0;diff --git a/packages/server/src/persistence-staleness-watchdog.test.ts b/packages/server/src/persistence-staleness-watchdog.test.tsindex a33d833de..5912cc2ab 100644--- a/packages/server/src/persistence-staleness-watchdog.test.ts+++ b/packages/server/src/persistence-staleness-watchdog.test.ts@@ -1,8 +1,9 @@import { normalizeBridge } from '@inkeep/open-knowledge-core';-import { afterEach, beforeEach, describe, expect, test } from 'vitest';+import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';import * as Y from 'yjs';import { isPersistenceExcludedDoc } from './cc1-broadcast.ts';import { FROZEN_LIFECYCLE_STATUSES } from './conflict-errors.ts';+import { DocumentDurabilityState } from './document-durability-state.ts';import { getMetrics, resetMetrics } from './metrics.ts';import {createPersistenceStalenessWatchdog,@@ -22,7 +23,9 @@ interface Rig {forceCalls: string[];clock: { nowMs: number };batchActive: { value: boolean };- inFlight: Map<string, string>;+ inFlight: Set<string>;+ hangDocs: Set<string>;+ hangSettlers: Map<string, { resolve: () => void; reject: (err: unknown) => void }>;forceBehavior: {mode: 'advance-base' | 'no-op' | 'throw';};@@ -43,7 +46,9 @@ function makeRig(overrides: Partial<StalenessWatchdogOptions> = {}): Rig {const forceCalls: string[] = [];const clock = { nowMs: 100_000 };const batchActive = { value: false };- const inFlight = new Map<string, string>();+ const inFlight = new Set<string>();+ const hangDocs = new Set<string>();+ const hangSettlers = new Map<string, { resolve: () => void; reject: (err: unknown) => void }>();const forceBehavior: Rig['forceBehavior'] = { mode: 'advance-base' };const diskReadFault: Rig['diskReadFault'] = { kind: null };@@ -51,6 +56,11 @@ function makeRig(overrides: Partial<StalenessWatchdogOptions> = {}): Rig {getLoadedDocuments: () => docs,forceStore: async (document, documentName) => {forceCalls.push(documentName);+ if (hangDocs.has(documentName)) {+ await new Promise<void>((resolve, reject) => {+ hangSettlers.set(documentName, { resolve, reject });+ });+ }if (forceBehavior.mode === 'throw') throw new Error('injected store failure');if (forceBehavior.mode === 'advance-base') {const bytes = document.getText('source').toString();@@ -70,7 +80,7 @@ function makeRig(overrides: Partial<StalenessWatchdogOptions> = {}): Rig {now: () => clock.nowMs,getBase: (documentName) => bases.get(documentName),isBatchActive: () => batchActive.value,- peekInFlight: (documentName) => inFlight.get(documentName),+ hasInFlight: (documentName) => inFlight.has(documentName),msSinceLastUserTx: (doc) => {for (const [name, d] of docs) {if (d === doc) return txAges.get(name) ?? null;@@ -90,6 +100,8 @@ function makeRig(overrides: Partial<StalenessWatchdogOptions> = {}): Rig {clock,batchActive,inFlight,+ hangDocs,+ hangSettlers,forceBehavior,diskReadFault,};@@ -307,7 +319,7 @@ describe('exclusions', () => {test('skips a doc whose flush is currently mid-commit', async () => {seedWedgedDoc(rig, 'inflight-doc');- rig.inFlight.set('inflight-doc', 'anything');+ rig.inFlight.add('inflight-doc');await rig.watchdog.sweep();@@ -501,3 +513,194 @@ describe('retry and suppression discipline', () => {expect(rig.forceCalls).toEqual(['transient-doc', 'transient-doc']);});});++describe('in-flight suppression against the real durability state', () => {+ test('a fresh record suppresses the sweep and an expired one stops suppressing it', async () => {+ vi.useFakeTimers();+ const durabilityState = new DocumentDurabilityState();+ const local = makeRig({+ hasInFlight: (documentName) => durabilityState.inFlightFlushCount(documentName) > 0,+ });+ try {+ seedWedgedDoc(local, 'in-flight-doc');+ durabilityState.beginInFlightFlush('in-flight-doc', 'bytes-never-settled');++ await local.watchdog.sweep();+ expect(local.forceCalls).toEqual([]);++ vi.advanceTimersByTime(61_000);++ await local.watchdog.sweep();+ expect(local.forceCalls).toEqual(['in-flight-doc']);+ expect(getMetrics().inFlightFlushExpired).toBe(1);+ } finally {+ await local.watchdog.dispose();+ vi.useRealTimers();+ }+ });+});++describe('a stalled forced store does not wedge the sweep', () => {+ test('a store that never settles times out, is suppressed, and later documents still sweep', async () => {+ const local = makeRig({ forceStoreTimeoutMs: 50 });+ try {+ seedWedgedDoc(local, 'a-stalled-doc');+ seedWedgedDoc(local, 'b-healthy-doc');+ local.hangDocs.add('a-stalled-doc');++ await local.watchdog.sweep();++ expect(local.forceCalls).toEqual(['a-stalled-doc', 'b-healthy-doc']);+ expect(getMetrics().persistenceStalenessForceStoreTimeouts).toBe(1);+ expect(local.bases.get('b-healthy-doc')).toBe('# edited in memory\n');++ local.clock.nowMs += GRACE_MS * 2;+ await local.watchdog.sweep();++ expect(local.forceCalls).toEqual(['a-stalled-doc', 'b-healthy-doc']);+ } finally {+ await local.watchdog.dispose();+ }+ });++ test('a store that times out and then fails is retried once it settles', async () => {+ const local = makeRig({ forceStoreTimeoutMs: 50 });+ try {+ seedWedgedDoc(local, 'late-failing-doc');+ local.hangDocs.add('late-failing-doc');++ await local.watchdog.sweep();+ expect(local.forceCalls).toEqual(['late-failing-doc']);++ local.clock.nowMs += GRACE_MS * 2;+ await local.watchdog.sweep();+ expect(local.forceCalls).toEqual(['late-failing-doc']);++ local.hangDocs.delete('late-failing-doc');+ local.hangSettlers.get('late-failing-doc')?.reject(new Error('injected late EIO'));+ await new Promise((resolve) => setImmediate(resolve));++ local.clock.nowMs += GRACE_MS * 2;+ await local.watchdog.sweep();+ expect(local.forceCalls).toEqual(['late-failing-doc', 'late-failing-doc']);+ expect(getMetrics().persistenceStalenessDetected).toBe(1);+ } finally {+ await local.watchdog.dispose();+ }+ });++ test('a re-armed document still waits a grace window before it is forced again', async () => {+ const local = makeRig({ forceStoreTimeoutMs: 50 });+ try {+ seedWedgedDoc(local, 'backoff-doc');+ local.hangDocs.add('backoff-doc');++ await local.watchdog.sweep();+ expect(local.forceCalls).toEqual(['backoff-doc']);++ local.hangDocs.delete('backoff-doc');+ local.hangSettlers.get('backoff-doc')?.reject(new Error('injected late EIO'));+ await new Promise((resolve) => setImmediate(resolve));++ await local.watchdog.sweep();+ expect(local.forceCalls).toEqual(['backoff-doc']);++ local.clock.nowMs += GRACE_MS * 2;+ await local.watchdog.sweep();+ expect(local.forceCalls).toEqual(['backoff-doc', 'backoff-doc']);+ } finally {+ await local.watchdog.dispose();+ }+ });++ test('a timed-out store that later succeeds without clearing divergence is retried after a grace window', async () => {+ const local = makeRig({ forceStoreTimeoutMs: 50 });+ try {+ local.forceBehavior.mode = 'no-op';+ seedWedgedDoc(local, 'late-landing-doc');+ local.hangDocs.add('late-landing-doc');++ await local.watchdog.sweep();+ expect(local.forceCalls).toEqual(['late-landing-doc']);++ local.hangDocs.delete('late-landing-doc');+ local.hangSettlers.get('late-landing-doc')?.resolve();+ await new Promise((resolve) => setImmediate(resolve));++ await local.watchdog.sweep();+ expect(local.forceCalls).toEqual(['late-landing-doc']);++ local.clock.nowMs += GRACE_MS * 2;+ await local.watchdog.sweep();+ expect(local.forceCalls).toEqual(['late-landing-doc', 'late-landing-doc']);+ } finally {+ await local.watchdog.dispose();+ }+ });++ test('a forceStore that throws synchronously does not abort the sweep and is retried after a grace window', async () => {+ const syncThrowCalls: string[] = [];+ const syncThrowingForceStore = (_document: Y.Doc, documentName: string): Promise<void> => {+ syncThrowCalls.push(documentName);+ throw new Error('injected synchronous store failure');+ };+ const local = makeRig({ forceStore: syncThrowingForceStore });+ try {+ const throwingDoc = seedWedgedDoc(local, 'sync-throw-doc');+ seedWedgedDoc(local, 'later-doc');++ await local.watchdog.sweep();++ expect(syncThrowCalls).toEqual(['sync-throw-doc', 'later-doc']);+ expect(getMetrics().persistenceStalenessForcedStores).toBe(2);+ expect(getMetrics().persistenceStalenessForceStoreTimeouts).toBe(0);++ local.clock.nowMs += GRACE_MS * 2;+ await local.watchdog.sweep();++ expect(syncThrowCalls).toEqual([+ 'sync-throw-doc',+ 'later-doc',+ 'sync-throw-doc',+ 'later-doc',+ ]);++ expect(() => syncThrowingForceStore(throwingDoc, 'premise-probe')).toThrow(+ 'injected synchronous store failure',+ );+ } finally {+ await local.watchdog.dispose();+ }+ });++ test('a late-settling old store does not clobber a newer attempt carrying the same fingerprint', async () => {+ const local = makeRig({ forceStoreTimeoutMs: 50 });+ try {+ const doc = seedWedgedDoc(local, 'reloaded-doc');+ local.hangDocs.add('reloaded-doc');++ await local.watchdog.sweep();+ const settlerA = local.hangSettlers.get('reloaded-doc');+ expect(local.forceCalls).toEqual(['reloaded-doc']);++ local.docs.delete('reloaded-doc');+ await local.watchdog.sweep();++ local.docs.set('reloaded-doc', doc);+ local.clock.nowMs += GRACE_MS * 2;+ await local.watchdog.sweep();+ expect(local.forceCalls).toEqual(['reloaded-doc', 'reloaded-doc']);++ expect(getMetrics().persistenceStalenessForceStoreTimeouts).toBe(2);++ settlerA?.reject(new Error('injected late EIO'));+ await new Promise((resolve) => setImmediate(resolve));++ local.clock.nowMs += GRACE_MS * 2;+ await local.watchdog.sweep();+ expect(local.forceCalls).toEqual(['reloaded-doc', 'reloaded-doc']);+ } finally {+ await local.watchdog.dispose();+ }+ });+});diff --git a/packages/server/src/persistence-staleness-watchdog.ts b/packages/server/src/persistence-staleness-watchdog.tsindex 4d3c750e1..f7b9dbcb6 100644--- a/packages/server/src/persistence-staleness-watchdog.ts+++ b/packages/server/src/persistence-staleness-watchdog.ts@@ -7,6 +7,7 @@ import { getLogger } from './logger.ts';import {incrementPersistenceStalenessDetected,incrementPersistenceStalenessForcedStores,+ incrementPersistenceStalenessForceStoreTimeouts,incrementPersistenceStalenessStoodDown,} from './metrics.ts';import { normalizedSourceForm } from './persistence.ts';@@ -22,6 +23,7 @@ export class StructuralDiskReadError extends Error {const DEFAULT_STALENESS_GRACE_MS = 5 * 60_000;const DEFAULT_STALENESS_SWEEP_INTERVAL_MS = 60_000;+const DEFAULT_FORCE_STORE_TIMEOUT_MS = 30_000;export interface StalenessWatchdogOptions {getLoadedDocuments: () => Iterable<readonly [string, Y.Doc]>;@@ -32,7 +34,8 @@ export interface StalenessWatchdogOptions {now?: () => number;getBase: (documentName: string) => string | undefined;isBatchActive: () => boolean;- peekInFlight: (documentName: string) => string | undefined;+ hasInFlight: (documentName: string) => boolean;+ forceStoreTimeoutMs?: number;msSinceLastUserTx?: (doc: Y.Doc, nowMs: number) => number | null;}@@ -41,6 +44,8 @@ export interface StalenessWatchdogHandle {dispose: () => Promise<void>;}+type SettleDisposition = 'resumed' | 'superseded' | 'watchdog-disposed';+interface AttemptRecord {fingerprint: string;atMs: number;@@ -54,7 +59,8 @@ export function createPersistenceStalenessWatchdog(const graceMs = options.graceMs ?? DEFAULT_STALENESS_GRACE_MS;const sweepIntervalMs = options.sweepIntervalMs ?? DEFAULT_STALENESS_SWEEP_INTERVAL_MS;const now = options.now ?? Date.now;- const { getBase, isBatchActive, peekInFlight } = options;+ const forceStoreTimeoutMs = options.forceStoreTimeoutMs ?? DEFAULT_FORCE_STORE_TIMEOUT_MS;+ const { getBase, isBatchActive, hasInFlight } = options;const msSinceLastUserTx = options.msSinceLastUserTx ?? getMsSinceLastUserTx;const attempts = new Map<string, AttemptRecord>();@@ -86,6 +92,7 @@ export function createPersistenceStalenessWatchdog(const startedMs = now();let scanned = 0;+ let skippedInFlight = 0;let divergent = 0;let forced = 0;let stoodDown = 0;@@ -97,7 +104,10 @@ export function createPersistenceStalenessWatchdog(if (frozenDocLifecycleStatus(document) !== null) {continue;}- if (peekInFlight(documentName) !== undefined) continue;+ if (hasInFlight(documentName)) {+ skippedInFlight++;+ continue;+ }scanned++;const rawYText = document.getText('source').toString();@@ -199,13 +209,66 @@ export function createPersistenceStalenessWatchdog(});incrementPersistenceStalenessForcedStores();forced++;+ let pendingStore: Promise<void>;try {- await options.forceStore(document, documentName);+ pendingStore = options.forceStore(document, documentName);} catch (err) {+ pendingStore = Promise.reject(err);+ }+ const forceStoreOutcome = await raceForceStore(pendingStore).catch((err: unknown) => {log.error({ err, docName: documentName, staleForMs },`[persistence-staleness] Forced store failed for ${documentName}; will retry`,);+ return 'failed' as const;+ });+ if (forceStoreOutcome === 'failed') continue;+ if (forceStoreOutcome === 'timeout') {+ incrementPersistenceStalenessForceStoreTimeouts();+ log.error(+ { docName: documentName, staleForMs, forceStoreTimeoutMs },+ `[persistence-staleness] Forced store for ${documentName} did not settle within ${forceStoreTimeoutMs}ms; setting it aside until that store settles or its content changes, so the sweep stays live for other documents`,+ );+ const declinedRecord: AttemptRecord = {+ fingerprint: candidate,+ atMs: now(),+ firstDetectedAtMs,+ declined: true,+ };+ attempts.set(documentName, declinedRecord);+ const resumeAfterSettle = (): SettleDisposition => {+ if (disposed) return 'watchdog-disposed';+ if (attempts.get(documentName) !== declinedRecord) return 'superseded';+ attempts.set(documentName, { ...declinedRecord, declined: false });+ return 'resumed';+ };+ void pendingStore+ .then(+ () => {+ const disposition = resumeAfterSettle();+ log.info(+ {+ docName: documentName,+ disposition,+ settledAfterTimeoutMs: now() - declinedRecord.atMs,+ },+ `[persistence-staleness] Previously timed-out forced store for ${documentName} landed`,+ );+ },+ (err: unknown) => {+ const disposition = resumeAfterSettle();+ log.warn(+ {+ err,+ docName: documentName,+ disposition,+ settledAfterTimeoutMs: now() - declinedRecord.atMs,+ },+ `[persistence-staleness] Previously timed-out forced store for ${documentName} settled with an error`,+ );+ },+ )+ .catch(() => {});continue;}if (disposed) return;@@ -238,7 +301,14 @@ export function createPersistenceStalenessWatchdog(if (!seen.has(name)) attempts.delete(name);}- const summary = { scanned, divergent, forced, stoodDown, elapsedMs: now() - startedMs };+ const summary = {+ scanned,+ divergent,+ forced,+ stoodDown,+ skippedInFlight,+ elapsedMs: now() - startedMs,+ };if (divergent > 0 || forced > 0 || stoodDown > 0) {log.info(summary, '[persistence-staleness] sweep complete');} else {@@ -246,6 +316,17 @@ export function createPersistenceStalenessWatchdog(}}+ async function raceForceStore(pendingStore: Promise<void>): Promise<'settled' | 'timeout'> {+ let timer: ReturnType<typeof setTimeout> | undefined;+ const deadline = new Promise<'timeout'>((resolve) => {+ timer = setTimeout(() => resolve('timeout'), forceStoreTimeoutMs);+ timer.unref?.();+ });+ return Promise.race([pendingStore.then(() => 'settled' as const), deadline]).finally(() => {+ if (timer !== undefined) clearTimeout(timer);+ });+ }+function sweep(): Promise<void> {if (sweepInFlight) return sweepInFlight;sweepInFlight = sweepOnce().finally(() => {diff --git a/packages/server/src/reconcile-own-flush-window.test.ts b/packages/server/src/reconcile-own-flush-window.test.tsindex f4f92739e..56bdd067d 100644--- a/packages/server/src/reconcile-own-flush-window.test.ts+++ b/packages/server/src/reconcile-own-flush-window.test.ts@@ -1,4 +1,4 @@-import { mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from 'node:fs';+import { mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from 'node:fs';import { tmpdir } from 'node:os';import { join } from 'node:path';import type { Hocuspocus } from '@hocuspocus/server';@@ -256,3 +256,116 @@ describe('reconcileDiskBeforeAgentWrite — own persistence flush is not foreignexpect(durabilityState.peekInFlightFlush(docName)).toBeUndefined();});});++describe('reconcileDiskBeforeAgentWrite — overlapping own flushes are not foreign divergence', () => {+ let tmpDir: string;+ let document: Y.Doc;+ let durabilityState: DocumentDurabilityState;++ beforeEach(() => {+ tmpDir = realpathSync(mkdtempSync(join(tmpdir(), 'ok-own-flush-overlap-')));+ mkdirSync(tmpDir, { recursive: true });+ durabilityState = new DocumentDurabilityState();+ document = new Y.Doc();+ });++ afterEach(() => {+ document.destroy();+ rmSync(tmpDir, { recursive: true, force: true });+ });++ async function driveOverlapWindow(+ docName: string,+ options: { diskContentInWindow?: string } = {},+ ): Promise<{+ diskAtWindow: string;+ pendingAtWindow: number;+ headAtWindow: string | undefined;+ diskMatchedPendingAtWindow: boolean;+ result: ReconcileBeforeWriteResult | undefined;+ conflictAfterGuard: boolean | undefined;+ }> {+ const docPath = join(tmpDir, `${docName}.md`);+ writeFileSync(docPath, BASE_CONTENT, 'utf-8');++ const OVERLAP_PARAGRAPHS = ['alpha', 'beta gamma epsilon'];+ let windowFired = false;+ let laterFlush: Promise<void> | undefined;+ let diskAtWindow = '';+ let pendingAtWindow = 0;+ let headAtWindow: string | undefined;+ let diskMatchedPendingAtWindow = false;+ let result: ReconcileBeforeWriteResult | undefined;+ let conflictAfterGuard: boolean | undefined;++ const persistence = createPersistenceExtension({+ contentDir: tmpDir,+ projectDir: tmpDir,+ gitEnabled: false,+ durabilityState,+ onDiskFlush: (name) => {+ if (name !== docName || windowFired) return;+ windowFired = true;+ document.transact(() => replaceDocParagraphs(document, OVERLAP_PARAGRAPHS), BROWSER_ORIGIN);+ laterFlush = storeDocument(persistence, document, docName);+ if (options.diskContentInWindow !== undefined) {+ writeFileSync(docPath, options.diskContentInWindow, 'utf-8');+ }+ diskAtWindow = readFileSync(docPath, 'utf-8');+ pendingAtWindow = durabilityState.inFlightFlushCount(docName);+ headAtWindow = durabilityState.peekInFlightFlush(docName);+ diskMatchedPendingAtWindow = durabilityState.hasInFlightFlush(+ docName,+ normalizeBridge(diskAtWindow),+ );+ result = reconcileDiskBeforeAgentWrite(+ durabilityState,+ fakeHocuspocusWith(docName, document),+ docName,+ tmpDir,+ );+ conflictAfterGuard = isDocInConflict(document as never);+ },+ });++ await loadDocument(persistence, document, docName);+ document.transact(() => replaceDocParagraphs(document, FLUSHED_PARAGRAPHS), BROWSER_ORIGIN);+ await storeDocument(persistence, document, docName);+ await laterFlush;++ expect(windowFired).toBe(true);+ expect(pendingAtWindow).toBeGreaterThanOrEqual(2);+ return {+ diskAtWindow,+ pendingAtWindow,+ headAtWindow,+ diskMatchedPendingAtWindow,+ result,+ conflictAfterGuard,+ };+ }++ test('an agent write landing between two overlapping own flushes does not latch lifecycle conflict', async () => {+ const probe = await driveOverlapWindow('overlap-window-clean');++ expect(normalizeBridge(probe.diskAtWindow)).toBe(normalizeBridge(FLUSHED_CONTENT));+ expect(probe.diskMatchedPendingAtWindow).toBe(true);+ expect(probe.headAtWindow).not.toBe(normalizeBridge(probe.diskAtWindow));+ expect(probe.conflictAfterGuard).toBe(false);+ expect(isDocInConflict(document as never)).toBe(false);+ expect(document.getMap('lifecycle').get('reason')).toBeUndefined();+ expect(probe.result?.reconciled).toBe(false);+ });++ test('a FOREIGN disk edit landing in the overlap window still reconciles', async () => {+ const probe = await driveOverlapWindow('overlap-window-foreign', {+ diskContentInWindow: 'alpha FOREIGN EDIT\n\nbeta\n',+ });++ expect(probe.diskMatchedPendingAtWindow).toBe(false);+ expect(probe.result?.reconciled).toBe(true);+ expect(probe.result?.mergeOutcome).toBe('merged');+ expect(probe.conflictAfterGuard).toBe(false);+ expect(isDocInConflict(document as never)).toBe(false);+ });+});diff --git a/packages/server/src/server-factory.ts b/packages/server/src/server-factory.tsindex 311bcabd9..adfcd440b 100644--- a/packages/server/src/server-factory.ts+++ b/packages/server/src/server-factory.ts@@ -1607,7 +1607,7 @@ export function createServer(options: ServerOptions): ServerInstance {forceStore: (document, documentName) => persistence.forceStore(document, documentName),getBase: (documentName) => durabilityState.getReconciledBase(documentName),isBatchActive: () => durabilityState.isBatchInProgress(),- peekInFlight: (documentName) => durabilityState.peekInFlightFlush(documentName),+ hasInFlight: (documentName) => durabilityState.inFlightFlushCount(documentName) > 0,readDiskBytes: (documentName) => {const requestedPath = safeContentPath(documentName, contentDir);let canonical: string;@@ -2312,16 +2312,16 @@ export function createServer(options: ServerOptions): ServerInstance {const { oldDocName, newDocName, content } = event;const document = hocuspocus.documents.get(oldDocName);- deleteReconciledBase(oldDocName);- setReconciledBase(newDocName, content);- await derivedDocumentIndex.recordDiskRename(oldDocName, newDocName, content);-if (document) {const lifecycleMap = document.getMap('lifecycle');lifecycleMap.set('status', 'renamed');lifecycleMap.set('newPath', newDocName);}+ deleteReconciledBase(oldDocName);+ setReconciledBase(newDocName, content);+ await derivedDocumentIndex.recordDiskRename(oldDocName, newDocName, content);+log.info({ oldDocName, newDocName }, `[reconcile] rename: ${oldDocName} → ${newDocName}`);signalChannel('files');onUpstreamRename(oldDocName, newDocName);