Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
import { normalizeBridge } from '@inkeep/open-knowledge-core';
import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';
import * as Y from 'yjs';
import { isPersistenceExcludedDoc } from './cc1-broadcast.ts';
import type { LifecycleView } from './conflict-kinds.ts';
import { DocumentDurabilityState } from './document-durability-state.ts';
import { getMetrics, resetMetrics } from './metrics.ts';
import {
createPersistenceStalenessWatchdog,
type StalenessWatchdogHandle,
type StalenessWatchdogOptions,
StructuralDiskReadError,
} from './persistence-staleness-watchdog.ts';
const GRACE_MS = 1_000;
const MAP_FROZEN_STATUSES = ['deleted-upstream', 'renamed'] as const;
function lifecycleViewOf(document: Y.Doc): LifecycleView | null {
const status = document.getMap('lifecycle').get('status');
if (status === 'deleted-upstream') return { status: 'deleted-upstream' };
if (status === 'renamed') return { status: 'renamed', newPath: '' };
return null;
}
interface Rig {
watchdog: StalenessWatchdogHandle;
docs: Map<string, Y.Doc>;
bases: Map<string, string>;
disk: Map<string, string>;
txAges: Map<string, number>;
forceCalls: string[];
clock: { nowMs: number };
batchActive: { value: boolean };
inFlight: Set<string>;
hangDocs: Set<string>;
hangSettlers: Map<string, { resolve: () => void; reject: (err: unknown) => void }>;
forceBehavior: {
mode: 'advance-base' | 'no-op' | 'throw';
};
diskReadFault: { kind: 'transient' | 'structural' | null };
}
function makeDoc(source: string): Y.Doc {
const doc = new Y.Doc();
doc.getText('source').insert(0, source);
return doc;
}
function makeRig(overrides: Partial<StalenessWatchdogOptions> = {}): Rig {
const docs = new Map<string, Y.Doc>();
const bases = new Map<string, string>();
const disk = new Map<string, string>();
const txAges = new Map<string, number>();
const forceCalls: string[] = [];
const clock = { nowMs: 100_000 };
const batchActive = { value: false };
const inFlight = new Set<string>();
const hangDocs = new Set<string>();
const hangSettlers = new Map<string, { resolve: () => void; reject: (err: unknown) => void }>();
const forceBehavior: Rig['forceBehavior'] = { mode: 'advance-base' };
const diskReadFault: Rig['diskReadFault'] = { kind: null };
const watchdog = createPersistenceStalenessWatchdog({
getLoadedDocuments: () => docs,
lifecycleOf: (document) => lifecycleViewOf(document),
forceStore: async (document, documentName) => {
forceCalls.push(documentName);
if (hangDocs.has(documentName)) {
await new Promise<void>((resolve, reject) => {
hangSettlers.set(documentName, { resolve, reject });
});
}
if (forceBehavior.mode === 'throw') throw new Error('injected store failure');
if (forceBehavior.mode === 'advance-base') {
const bytes = document.getText('source').toString();
bases.set(documentName, bytes);
disk.set(documentName, bytes);
}
},
readDiskBytes: (documentName) => {
if (diskReadFault.kind === 'structural') {
throw new StructuralDiskReadError('injected structural refusal');
}
if (diskReadFault.kind === 'transient') throw new Error('injected disk read failure');
return disk.get(documentName) ?? null;
},
graceMs: GRACE_MS,
sweepIntervalMs: 3_600_000,
now: () => clock.nowMs,
getBase: (documentName) => bases.get(documentName),
isBatchActive: () => batchActive.value,
hasInFlight: (documentName) => inFlight.has(documentName),
msSinceLastUserTx: (doc) => {
for (const [name, d] of docs) {
if (d === doc) return txAges.get(name) ?? null;
}
return null;
},
...overrides,
});
return {
watchdog,
docs,
bases,
disk,
txAges,
forceCalls,
clock,
batchActive,
inFlight,
hangDocs,
hangSettlers,
forceBehavior,
diskReadFault,
};
}
function seedWedgedDoc(rig: Rig, name: string): Y.Doc {
const doc = makeDoc('# edited in memory\n');
rig.docs.set(name, doc);
rig.bases.set(name, '# old on disk\n');
rig.disk.set(name, '# old on disk\n');
rig.txAges.set(name, GRACE_MS * 10);
return doc;
}
let rig: Rig;
beforeEach(() => {
resetMetrics();
rig = makeRig();
});
afterEach(async () => {
await rig.watchdog.dispose();
});
describe('staleness detection and forced store', () => {
test('force-stores a doc divergent past the grace window and counts it', async () => {
seedWedgedDoc(rig, 'wedged-doc');
await rig.watchdog.sweep();
expect(rig.forceCalls).toEqual(['wedged-doc']);
expect(getMetrics().persistenceStalenessDetected).toBe(1);
expect(getMetrics().persistenceStalenessForcedStores).toBe(1);
await rig.watchdog.sweep();
expect(rig.forceCalls).toEqual(['wedged-doc']);
});
test('does not force while the last user transaction is younger than the grace window', async () => {
seedWedgedDoc(rig, 'active-doc');
rig.txAges.set('active-doc', GRACE_MS - 1);
await rig.watchdog.sweep();
expect(rig.forceCalls).toEqual([]);
expect(getMetrics().persistenceStalenessDetected).toBe(0);
});
test('treats a null transaction age as old enough', async () => {
seedWedgedDoc(rig, 'no-tx-doc');
rig.txAges.delete('no-tx-doc');
await rig.watchdog.sweep();
expect(rig.forceCalls).toEqual(['no-tx-doc']);
});
test('ignores docs whose memory matches the reconciled base', async () => {
const doc = makeDoc('# same\n');
rig.docs.set('clean-doc', doc);
rig.bases.set('clean-doc', '# same\n');
rig.disk.set('clean-doc', '# same\n');
rig.txAges.set('clean-doc', GRACE_MS * 10);
await rig.watchdog.sweep();
expect(rig.forceCalls).toEqual([]);
});
test('classifies frontmatter-carrying docs with the store spine comparator', async () => {
const fmBase = '---\ntitle: x\n---\n\n# body\n\n\n';
const doc = makeDoc('---\ntitle: x\n---\n\n# body\n');
rig.docs.set('fm-doc', doc);
rig.bases.set('fm-doc', fmBase);
rig.disk.set('fm-doc', fmBase);
rig.txAges.set('fm-doc', GRACE_MS * 10);
await rig.watchdog.sweep();
expect(rig.forceCalls).toEqual([]);
doc.getText('source').delete(0, doc.getText('source').length);
doc.getText('source').insert(0, '---\ntitle: y\n---\n\n# body\n');
await rig.watchdog.sweep();
expect(rig.forceCalls).toEqual(['fm-doc']);
});
test('rescues a blank run the normalized comparator cannot see', async () => {
const base = 'Above.\n\nBelow.\n';
const widened = 'Above.\n\n\n\nBelow.\n';
expect(normalizeBridge(base)).toBe(normalizeBridge(widened));
const doc = makeDoc(widened);
rig.docs.set('blank-run-doc', doc);
rig.bases.set('blank-run-doc', base);
rig.disk.set('blank-run-doc', base);
rig.txAges.set('blank-run-doc', GRACE_MS * 10);
await rig.watchdog.sweep();
expect(rig.forceCalls).toEqual(['blank-run-doc']);
await rig.watchdog.sweep();
expect(rig.forceCalls).toEqual(['blank-run-doc']);
});
test('leaves a doc alone when the base carries the wider run', async () => {
const doc = makeDoc('- a\n\n- b\n');
rig.docs.set('collapsed-doc', doc);
rig.bases.set('collapsed-doc', '- a\n\n\n- b\n');
rig.disk.set('collapsed-doc', '- a\n\n\n- b\n');
rig.txAges.set('collapsed-doc', GRACE_MS * 10);
await rig.watchdog.sweep();
expect(rig.forceCalls).toEqual([]);
});
test('materializes a never-persisted doc with content and no disk file', async () => {
const doc = makeDoc('# brand new\n');
rig.docs.set('new-doc', doc);
rig.txAges.set('new-doc', GRACE_MS * 10);
await rig.watchdog.sweep();
expect(rig.forceCalls).toEqual(['new-doc']);
});
});
describe('exclusions', () => {
test('isPersistenceExcludedDoc admits exactly the dedicated-store-path doc classes', () => {
for (const name of [
'__system__',
'__config__/project',
'__user__/config.yml',
'__local__/project',
'__skill__/project/foo',
'__template__/notes/weekly',
'diagram.mmd',
'assets/flow.mermaid',
'src/util.ts',
'config.json',
'styles.css',
]) {
expect(isPersistenceExcludedDoc(name)).toBe(true);
}
for (const name of ['notes', 'folder/doc', 'README', 'docs/getting-started.mdx']) {
expect(isPersistenceExcludedDoc(name)).toBe(false);
}
});
test('skips system, config, managed-artifact, and mermaid docs', async () => {
for (const name of [
'__system__',
'__config__/project',
'__user__/config.yml',
'__local__/project',
'__skill__/project/foo',
'__template__/notes/weekly',
'diagram.mmd',
]) {
const doc = makeDoc('# divergent\n');
rig.docs.set(name, doc);
rig.bases.set(name, '# other\n');
rig.txAges.set(name, GRACE_MS * 10);
}
await rig.watchdog.sweep();
expect(rig.forceCalls).toEqual([]);
expect(getMetrics().persistenceStalenessDetected).toBe(0);
});
test('retries the held save of a file it does not otherwise watch, and only a held one', async () => {
await rig.watchdog.dispose();
const held = new Set(['diagram.mmd', '__config__/project']);
const retried: string[] = [];
rig = makeRig({
isHeldSaveDue: (name) => held.has(name),
retryHeldSave: async (_document, name) => {
retried.push(name);
},
});
for (const name of ['diagram.mmd', '__config__/project', 'notes.txt', '__system__']) {
rig.docs.set(name, makeDoc('held text\n'));
}
await rig.watchdog.sweep();
expect(retried.sort()).toEqual(['__config__/project', 'diagram.mmd']);
expect(rig.forceCalls).toEqual([]);
expect(getMetrics().heldSaveRetries).toBe(2);
});
test('skips docs frozen by lifecycle status', async () => {
for (const status of MAP_FROZEN_STATUSES) {
const doc = seedWedgedDoc(rig, `lifecycle-${status}`);
doc.getMap('lifecycle').set('status', status);
}
await rig.watchdog.sweep();
expect(rig.forceCalls).toEqual([]);
});
test('skips the whole sweep while a coordinated batch is active', async () => {
seedWedgedDoc(rig, 'batch-doc');
rig.batchActive.value = true;
await rig.watchdog.sweep();
expect(rig.forceCalls).toEqual([]);
rig.batchActive.value = false;
await rig.watchdog.sweep();
expect(rig.forceCalls).toEqual(['batch-doc']);
});
test('a batch activating mid-sweep does not decline the parked store', async () => {
const parkingRig = makeRig({
forceStore: async (_document, documentName) => {
parkingRig.forceCalls.push(documentName);
parkingRig.batchActive.value = true;
},
});
try {
seedWedgedDoc(parkingRig, 'parked-doc');
await parkingRig.watchdog.sweep();
expect(parkingRig.forceCalls).toEqual(['parked-doc']);
parkingRig.batchActive.value = false;
parkingRig.clock.nowMs += GRACE_MS + 1;
await parkingRig.watchdog.sweep();
expect(parkingRig.forceCalls).toEqual(['parked-doc', 'parked-doc']);
} finally {
await parkingRig.watchdog.dispose();
}
});
test('skips a doc whose flush is currently mid-commit', async () => {
seedWedgedDoc(rig, 'inflight-doc');
rig.inFlight.add('inflight-doc');
await rig.watchdog.sweep();
expect(rig.forceCalls).toEqual([]);
});
test('is a no-op after dispose', async () => {
seedWedgedDoc(rig, 'late-doc');
await rig.watchdog.dispose();
await rig.watchdog.sweep();
expect(rig.forceCalls).toEqual([]);
});
test('dispose drains an in-flight sweep and stops it before the next doc', async () => {
const { promise: gate, resolve: releaseFirstStore } = Promise.withResolvers<void>();
const blockingRig = makeRig({
forceStore: async (_document, documentName) => {
blockingRig.forceCalls.push(documentName);
await gate;
},
});
try {
seedWedgedDoc(blockingRig, 'doc-a');
seedWedgedDoc(blockingRig, 'doc-b');
const sweepPromise = blockingRig.watchdog.sweep();
const disposePromise = blockingRig.watchdog.dispose();
releaseFirstStore();
await disposePromise;
await sweepPromise;
expect(blockingRig.forceCalls).toEqual(['doc-a']);
} finally {
releaseFirstStore();
await blockingRig.watchdog.dispose();
}
});
});
describe('external-edit stand-down (disk authority)', () => {
test('forces the store when disk differs from the base, leaving the merge to the store compare', async () => {
seedWedgedDoc(rig, 'external-doc');
rig.disk.set('external-doc', '# external native edit\n');
await rig.watchdog.sweep();
expect(rig.forceCalls).toEqual(['external-doc']);
expect(getMetrics().persistenceStalenessDetected).toBe(1);
expect(getMetrics().persistenceStalenessForcedStores).toBe(1);
expect(getMetrics().persistenceStalenessStoodDown).toBe(0);
});
test('does not resurrect a file deleted out-of-band', async () => {
seedWedgedDoc(rig, 'deleted-doc');
rig.disk.delete('deleted-doc');
await rig.watchdog.sweep();
expect(rig.forceCalls).toEqual([]);
expect(getMetrics().persistenceStalenessStoodDown).toBe(1);
});
test('does not overwrite an on-disk file the doc never loaded', async () => {
const doc = makeDoc('# memory only\n');
rig.docs.set('never-loaded', doc);
rig.disk.set('never-loaded', '# unread disk bytes\n');
rig.txAges.set('never-loaded', GRACE_MS * 10);
await rig.watchdog.sweep();
expect(rig.forceCalls).toEqual([]);
});
test('stands down when the disk read fails, then retries after another grace window', async () => {
seedWedgedDoc(rig, 'unreadable-doc');
rig.diskReadFault.kind = 'transient';
await rig.watchdog.sweep();
expect(rig.forceCalls).toEqual([]);
await rig.watchdog.sweep();
expect(rig.forceCalls).toEqual([]);
rig.diskReadFault.kind = null;
rig.clock.nowMs += GRACE_MS + 1;
await rig.watchdog.sweep();
expect(rig.forceCalls).toEqual(['unreadable-doc']);
});
test('a structural read refusal declines until content changes', async () => {
const doc = seedWedgedDoc(rig, 'refused-doc');
rig.diskReadFault.kind = 'structural';
await rig.watchdog.sweep();
expect(rig.forceCalls).toEqual([]);
expect(getMetrics().persistenceStalenessStoodDown).toBe(1);
rig.clock.nowMs += GRACE_MS * 3;
await rig.watchdog.sweep();
rig.clock.nowMs += GRACE_MS * 3;
await rig.watchdog.sweep();
expect(rig.forceCalls).toEqual([]);
expect(getMetrics().persistenceStalenessStoodDown).toBe(1);
rig.diskReadFault.kind = null;
doc.getText('source').insert(0, 'more ');
rig.clock.nowMs += GRACE_MS + 1;
await rig.watchdog.sweep();
expect(rig.forceCalls).toEqual(['refused-doc']);
});
test('a stand-down re-arms when the memory content changes', async () => {
const doc = seedWedgedDoc(rig, 'rearm-doc');
rig.disk.delete('rearm-doc');
await rig.watchdog.sweep();
await rig.watchdog.sweep();
expect(rig.forceCalls).toEqual([]);
expect(getMetrics().persistenceStalenessDetected).toBe(1);
rig.disk.set('rearm-doc', '# old on disk\n');
doc.getText('source').delete(0, doc.getText('source').length);
doc.getText('source').insert(0, '# newer memory edit\n');
await rig.watchdog.sweep();
expect(rig.forceCalls).toEqual(['rearm-doc']);
expect(getMetrics().persistenceStalenessDetected).toBe(2);
});
});
describe('retry and suppression discipline', () => {
test('a store that completes without clearing divergence suppresses until content changes', async () => {
rig.forceBehavior.mode = 'no-op';
const doc = seedWedgedDoc(rig, 'noop-doc');
await rig.watchdog.sweep();
expect(rig.forceCalls).toEqual(['noop-doc']);
rig.clock.nowMs += GRACE_MS * 5;
await rig.watchdog.sweep();
expect(rig.forceCalls).toEqual(['noop-doc']);
doc.getText('source').insert(0, 'more ');
rig.clock.nowMs += GRACE_MS * 5;
await rig.watchdog.sweep();
expect(rig.forceCalls).toEqual(['noop-doc', 'noop-doc']);
});
test('a failed store retries only after another full grace window', async () => {
rig.forceBehavior.mode = 'throw';
seedWedgedDoc(rig, 'failing-doc');
await rig.watchdog.sweep();
expect(rig.forceCalls).toEqual(['failing-doc']);
await rig.watchdog.sweep();
expect(rig.forceCalls).toEqual(['failing-doc']);
rig.clock.nowMs += GRACE_MS + 1;
rig.forceBehavior.mode = 'advance-base';
await rig.watchdog.sweep();
expect(rig.forceCalls).toEqual(['failing-doc', 'failing-doc']);
await rig.watchdog.sweep();
expect(rig.forceCalls).toEqual(['failing-doc', 'failing-doc']);
});
test('drops bookkeeping for docs that unloaded', async () => {
rig.forceBehavior.mode = 'no-op';
seedWedgedDoc(rig, 'transient-doc');
await rig.watchdog.sweep();
expect(rig.forceCalls).toEqual(['transient-doc']);
const doc = rig.docs.get('transient-doc');
rig.docs.delete('transient-doc');
await rig.watchdog.sweep();
if (doc) rig.docs.set('transient-doc', doc);
rig.clock.nowMs += GRACE_MS * 2;
await rig.watchdog.sweep();
expect(rig.forceCalls).toEqual(['transient-doc', 'transient-doc']);
});
});
describe('in-flight suppression against the real durability state', () => {
test('a fresh record suppresses the sweep and an expired one stops suppressing it', async () => {
vi.useFakeTimers();
const durabilityState = new DocumentDurabilityState();
const local = makeRig({
hasInFlight: (documentName) => durabilityState.inFlightFlushCount(documentName) > 0,
});
try {
seedWedgedDoc(local, 'in-flight-doc');
durabilityState.beginInFlightFlush('in-flight-doc', 'bytes-never-settled');
await local.watchdog.sweep();
expect(local.forceCalls).toEqual([]);
vi.advanceTimersByTime(61_000);
await local.watchdog.sweep();
expect(local.forceCalls).toEqual(['in-flight-doc']);
expect(getMetrics().inFlightFlushExpired).toBe(1);
} finally {
await local.watchdog.dispose();
vi.useRealTimers();
}
});
});
describe('a stalled held-save retry does not wedge the sweep', () => {
test('a retry that never settles times out as a failed attempt, later documents still sweep, and neither the next sweep nor dispose waits on it', async () => {
const retried: string[] = [];
const local = makeRig({
forceStoreTimeoutMs: 50,
isHeldSaveDue: (name) => name === 'stuck.mmd',
retryHeldSave: (_document, name) => {
retried.push(name);
return new Promise<void>(() => {});
},
});
local.docs.set('stuck.mmd', makeDoc('held text\n'));
seedWedgedDoc(local, 'z-healthy-doc');
await local.watchdog.sweep();
expect(retried).toEqual(['stuck.mmd']);
expect(local.bases.get('z-healthy-doc')).toBe('# edited in memory\n');
expect(getMetrics().heldSaveRetryFailures).toBe(1);
await local.watchdog.sweep();
expect(retried).toEqual(['stuck.mmd']);
await expect(local.watchdog.dispose()).resolves.toBeUndefined();
});
test('a retry that throws is counted as a failed attempt and tried again at the next sweep', async () => {
let attempts = 0;
const local = makeRig({
isHeldSaveDue: () => true,
retryHeldSave: async () => {
attempts++;
throw new Error('injected retry failure');
},
});
try {
local.docs.set('diagram.mmd', makeDoc('held text\n'));
await local.watchdog.sweep();
await local.watchdog.sweep();
expect(attempts).toBe(2);
expect(getMetrics().heldSaveRetryFailures).toBe(2);
} finally {
await local.watchdog.dispose();
}
});
});
describe('a stalled forced store does not wedge the sweep', () => {
test('a store that never settles times out, is suppressed, and later documents still sweep', async () => {
const local = makeRig({ forceStoreTimeoutMs: 50 });
try {
seedWedgedDoc(local, 'a-stalled-doc');
seedWedgedDoc(local, 'b-healthy-doc');
local.hangDocs.add('a-stalled-doc');
await local.watchdog.sweep();
expect(local.forceCalls).toEqual(['a-stalled-doc', 'b-healthy-doc']);
expect(getMetrics().persistenceStalenessForceStoreTimeouts).toBe(1);
expect(local.bases.get('b-healthy-doc')).toBe('# edited in memory\n');
local.clock.nowMs += GRACE_MS * 2;
await local.watchdog.sweep();
expect(local.forceCalls).toEqual(['a-stalled-doc', 'b-healthy-doc']);
} finally {
await local.watchdog.dispose();
}
});
test('a store that times out and then fails is retried once it settles', async () => {
const local = makeRig({ forceStoreTimeoutMs: 50 });
try {
seedWedgedDoc(local, 'late-failing-doc');
local.hangDocs.add('late-failing-doc');
await local.watchdog.sweep();
expect(local.forceCalls).toEqual(['late-failing-doc']);
local.clock.nowMs += GRACE_MS * 2;
await local.watchdog.sweep();
expect(local.forceCalls).toEqual(['late-failing-doc']);
local.hangDocs.delete('late-failing-doc');
local.hangSettlers.get('late-failing-doc')?.reject(new Error('injected late EIO'));
await new Promise((resolve) => setImmediate(resolve));
local.clock.nowMs += GRACE_MS * 2;
await local.watchdog.sweep();
expect(local.forceCalls).toEqual(['late-failing-doc', 'late-failing-doc']);
expect(getMetrics().persistenceStalenessDetected).toBe(1);
} finally {
await local.watchdog.dispose();
}
});
test('a re-armed document still waits a grace window before it is forced again', async () => {
const local = makeRig({ forceStoreTimeoutMs: 50 });
try {
seedWedgedDoc(local, 'backoff-doc');
local.hangDocs.add('backoff-doc');
await local.watchdog.sweep();
expect(local.forceCalls).toEqual(['backoff-doc']);
local.hangDocs.delete('backoff-doc');
local.hangSettlers.get('backoff-doc')?.reject(new Error('injected late EIO'));
await new Promise((resolve) => setImmediate(resolve));
await local.watchdog.sweep();
expect(local.forceCalls).toEqual(['backoff-doc']);
local.clock.nowMs += GRACE_MS * 2;
await local.watchdog.sweep();
expect(local.forceCalls).toEqual(['backoff-doc', 'backoff-doc']);
} finally {
await local.watchdog.dispose();
}
});
test('a timed-out store that later succeeds without clearing divergence is retried after a grace window', async () => {
const local = makeRig({ forceStoreTimeoutMs: 50 });
try {
local.forceBehavior.mode = 'no-op';
seedWedgedDoc(local, 'late-landing-doc');
local.hangDocs.add('late-landing-doc');
await local.watchdog.sweep();
expect(local.forceCalls).toEqual(['late-landing-doc']);
local.hangDocs.delete('late-landing-doc');
local.hangSettlers.get('late-landing-doc')?.resolve();
await new Promise((resolve) => setImmediate(resolve));
await local.watchdog.sweep();
expect(local.forceCalls).toEqual(['late-landing-doc']);
local.clock.nowMs += GRACE_MS * 2;
await local.watchdog.sweep();
expect(local.forceCalls).toEqual(['late-landing-doc', 'late-landing-doc']);
} finally {
await local.watchdog.dispose();
}
});
test('a forceStore that throws synchronously does not abort the sweep and is retried after a grace window', async () => {
const syncThrowCalls: string[] = [];
const syncThrowingForceStore = (_document: Y.Doc, documentName: string): Promise<void> => {
syncThrowCalls.push(documentName);
throw new Error('injected synchronous store failure');
};
const local = makeRig({ forceStore: syncThrowingForceStore });
try {
const throwingDoc = seedWedgedDoc(local, 'sync-throw-doc');
seedWedgedDoc(local, 'later-doc');
await local.watchdog.sweep();
expect(syncThrowCalls).toEqual(['sync-throw-doc', 'later-doc']);
expect(getMetrics().persistenceStalenessForcedStores).toBe(2);
expect(getMetrics().persistenceStalenessForceStoreTimeouts).toBe(0);
local.clock.nowMs += GRACE_MS * 2;
await local.watchdog.sweep();
expect(syncThrowCalls).toEqual([
'sync-throw-doc',
'later-doc',
'sync-throw-doc',
'later-doc',
]);
expect(() => syncThrowingForceStore(throwingDoc, 'premise-probe')).toThrow(
'injected synchronous store failure',
);
} finally {
await local.watchdog.dispose();
}
});
test('a late-settling old store does not clobber a newer attempt carrying the same fingerprint', async () => {
const local = makeRig({ forceStoreTimeoutMs: 50 });
try {
const doc = seedWedgedDoc(local, 'reloaded-doc');
local.hangDocs.add('reloaded-doc');
await local.watchdog.sweep();
const settlerA = local.hangSettlers.get('reloaded-doc');
expect(local.forceCalls).toEqual(['reloaded-doc']);
local.docs.delete('reloaded-doc');
await local.watchdog.sweep();
local.docs.set('reloaded-doc', doc);
local.clock.nowMs += GRACE_MS * 2;
await local.watchdog.sweep();
expect(local.forceCalls).toEqual(['reloaded-doc', 'reloaded-doc']);
expect(getMetrics().persistenceStalenessForceStoreTimeouts).toBe(2);
settlerA?.reject(new Error('injected late EIO'));
await new Promise((resolve) => setImmediate(resolve));
local.clock.nowMs += GRACE_MS * 2;
await local.watchdog.sweep();
expect(local.forceCalls).toEqual(['reloaded-doc', 'reloaded-doc']);
} finally {
await local.watchdog.dispose();
}
});
});