Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

fix(open-knowledge): divergent fallback must not overwrite source (#1748)

Nick Gomeze89ca31fa4e9
diff --git a/.changeset/divergent-fallback-settlement-rederive.md b/.changeset/divergent-fallback-settlement-rederive.md
new file mode 100644
index 000000000..6c78e0752
--- /dev/null
+++ b/.changeset/divergent-fallback-settlement-rederive.md
@@ -0,0 +1,5 @@
+---
+"@inkeep/open-knowledge-server": patch
+---
+
+Divergent degraded-fallback content no longer overwrites the source bytes it stands for on WYSIWYG interaction. When a parse-error fallback block's editor content diverges from its underlying source region (possible only via dependency or plugin drift producing an unknown, position-unresolvable block), typing into the fallback followed by any later edit — including a remote peer's — previously rewrote the document source wholesale to the fallback-derived text, silently destroying the original bytes. The server bridge now detects when a sync drain would settle with the WYSIWYG view and the source text divergent beyond tolerance, keeps the source bytes authoritative, and re-derives the WYSIWYG view from source in the same drain (restoring the broken-block chrome as well).
diff --git a/packages/app/tests/integration/single-file-mode.test.ts b/packages/app/tests/integration/single-file-mode.test.ts
index ae6b64b63..918df632c 100644
--- a/packages/app/tests/integration/single-file-mode.test.ts
+++ b/packages/app/tests/integration/single-file-mode.test.ts
@@ -154,8 +154,8 @@ describe('single-file mode — write-back (FR3)', () => {
});
describe('single-file mode — no rewrite on open (FR4 / G8)', () => {
- const RAW = '1. a\n1. b\n';
- const CANONICAL = '1. a\n2. b\n';
+ const RAW = '> q\nq2\n';
+ const CANONICAL = '> q\n> q2\n';
test('a reconciliation to the file’s own canonical form is suppressed; a genuine edit persists', async () => {
const contentDir = ephemeralContentDir({ 'notes.md': RAW });
diff --git a/packages/core/src/bridge/normalize.test.ts b/packages/core/src/bridge/normalize.test.ts
index 6f97e5ecc..80587e2f3 100644
--- a/packages/core/src/bridge/normalize.test.ts
+++ b/packages/core/src/bridge/normalize.test.ts
@@ -441,6 +441,61 @@ describe('list-indent canonical collapse', () => {
});
});
+describe('ordered-list-marker-number canonical collapse', () => {
+ test('lazy `1./1.` equivalent to renumbered `1./2.`', () => {
+ expect(normalizeBridge('# Todo\n\n1. first\n1. second\n')).toBe(
+ normalizeBridge('# Todo\n\n1. first\n2. second\n'),
+ );
+ });
+
+ test('three-item lazy `1./1./1.` equivalent to renumbered `1./2./3.`', () => {
+ expect(normalizeBridge('1. a\n1. b\n1. c\n')).toBe(normalizeBridge('1. a\n2. b\n3. c\n'));
+ });
+
+ test('start offset preserved: `5./5.` equivalent to `5./6.`', () => {
+ expect(normalizeBridge('5. a\n5. b\n')).toBe(normalizeBridge('5. a\n6. b\n'));
+ });
+
+ test('paren-delimiter `1)` markers renumber-tolerate too', () => {
+ expect(normalizeBridge('1) a\n1) b\n')).toBe(normalizeBridge('1) a\n2) b\n'));
+ });
+
+ test('nested ordered list renumber collapses (mirrors list-indent nesting coverage)', () => {
+ const lazy = '1. a\n 1. x\n 1. y\n1. b\n';
+ const renum = '1. a\n 1. x\n 2. y\n2. b\n';
+ expect(normalizeBridge(lazy)).toBe(normalizeBridge(renum));
+ });
+
+ test('deep-indented ordered item renumber collapses', () => {
+ expect(normalizeBridge(' 1. nested')).toBe(normalizeBridge(' 9. nested'));
+ });
+
+ test('NEGATIVE: content after the marker differing is NOT collapsed', () => {
+ expect(normalizeBridge('1. first\n')).not.toBe(normalizeBridge('1. second\n'));
+ expect(normalizeBridge('1. apple\n2. banana\n')).not.toBe(
+ normalizeBridge('1. apple\n2. cherry\n'),
+ );
+ });
+
+ test('NEGATIVE: marker-type `.` vs `)` is NOT collapsed', () => {
+ expect(normalizeBridge('1. a\n')).not.toBe(normalizeBridge('1) a\n'));
+ });
+
+ test('NEGATIVE: ordered marker vs bullet marker is NOT collapsed', () => {
+ expect(normalizeBridge('1. a\n')).not.toBe(normalizeBridge('- a\n'));
+ });
+
+ test('NEGATIVE: adding or removing a line is NOT collapsed', () => {
+ expect(normalizeBridge('1. a\n1. b\n')).not.toBe(normalizeBridge('1. a\n'));
+ expect(normalizeBridge('1. a\n')).not.toBe(normalizeBridge('1. a\n1. b\n'));
+ });
+
+ test('NEGATIVE: a digit run that is not an ordered-list marker is preserved', () => {
+ expect(normalizeBridge('Version 2 shipped\n')).toBe('Version 2 shipped');
+ expect(normalizeBridge('Version 2 shipped\n')).not.toBe(normalizeBridge('Version 3 shipped\n'));
+ });
+});
+
describe('detectAppliedToleranceClasses (FR-41)', () => {
test('exposes the class enum for bounded-cardinality emit consumers', () => {
expect(BRIDGE_TOLERANCE_CLASSES).toEqual([
@@ -453,12 +508,29 @@ describe('detectAppliedToleranceClasses (FR-41)', () => {
'block-separator-collapse',
'table-align-row-spacing',
'list-indent-canonical',
+ 'ordered-list-marker-number',
'trailing-whitespace',
'blank-line-collapse',
'trailing-newline',
]);
});
+ test('detects ordered-list-marker-number when marker digits differ across inputs', () => {
+ expect(detectAppliedToleranceClasses('1. a\n1. b', '1. a\n2. b')).toContain(
+ 'ordered-list-marker-number',
+ );
+ expect(detectAppliedToleranceClasses(' 1. x\n 1. y', ' 1. x\n 2. y')).toContain(
+ 'ordered-list-marker-number',
+ );
+ });
+
+ test('does not detect ordered-list-marker-number when no ordered marker is present', () => {
+ expect(detectAppliedToleranceClasses('- a\n- b', '- a\n- b')).not.toContain(
+ 'ordered-list-marker-number',
+ );
+ expect(detectAppliedToleranceClasses('foo', 'foo')).not.toContain('ordered-list-marker-number');
+ });
+
test('detects emphasis-around-code when either input contains the tight wrap', () => {
expect(detectAppliedToleranceClasses('**`text-indent`**', '`text-indent`')).toContain(
'emphasis-around-code',
diff --git a/packages/core/src/bridge/normalize.ts b/packages/core/src/bridge/normalize.ts
index 08fbff78c..5dea59752 100644
--- a/packages/core/src/bridge/normalize.ts
+++ b/packages/core/src/bridge/normalize.ts
@@ -4,6 +4,13 @@ const TABLE_ALIGN_ROW_RE = /^\s*\|?\s*:?-+:?\s*(\|\s*:?-+:?\s*)+\|?\s*$/;
const LIST_ITEM_INDENT_RE = /^[ \t]+([-+*]|\d+[.)]|[a-zA-Z][.)])\s/;
+const ORDERED_LIST_MARKER_RE = /^\d+([.)])(?=\s)/;
+
+/** Canonical sentinel the ordered-list marker number collapses to. Any digit
+ * run works as long as it is stable across both sides; `1` keeps the
+ * normalized form readable. */
+const ORDERED_LIST_MARKER_CANONICAL = '1';
+
const EMPHASIS_AROUND_CODE_RE = /\*\*\s*(`[^`]+`)\s*\*\*/g;
export function normalizeBridge(s: string): string {
@@ -22,10 +29,10 @@ export function normalizeBridge(s: string): string {
if (TABLE_ALIGN_ROW_RE.test(trimmed)) {
return trimmed.replace(/\s+/g, '');
}
- if (LIST_ITEM_INDENT_RE.test(trimmed)) {
- return trimmed.replace(/^[ \t]+/, '');
- }
- return trimmed;
+ const deindented = LIST_ITEM_INDENT_RE.test(trimmed)
+ ? trimmed.replace(/^[ \t]+/, '')
+ : trimmed;
+ return deindented.replace(ORDERED_LIST_MARKER_RE, `${ORDERED_LIST_MARKER_CANONICAL}$1`);
})
.join('\n')
.replace(/\n{3,}/g, '\n\n')
@@ -42,6 +49,7 @@ export const BRIDGE_TOLERANCE_CLASSES = [
'block-separator-collapse',
'table-align-row-spacing',
'list-indent-canonical',
+ 'ordered-list-marker-number',
'trailing-whitespace',
'blank-line-collapse',
'trailing-newline',
@@ -109,6 +117,22 @@ export function detectAppliedToleranceClasses(left: string, right: string): Brid
classes.push('list-indent-canonical');
}
+ const canonMarkerLine = (line: string): string =>
+ line
+ .replace(/^[ \t]+/, '')
+ .replace(ORDERED_LIST_MARKER_RE, `${ORDERED_LIST_MARKER_CANONICAL}$1`);
+ const leftLines = leftLf.split('\n');
+ const rightLines = rightLf.split('\n');
+ const markerLineCount = Math.min(leftLines.length, rightLines.length);
+ for (let i = 0; i < markerLineCount; i++) {
+ const l = leftLines[i] ?? '';
+ const r = rightLines[i] ?? '';
+ if (l !== r && canonMarkerLine(l) === canonMarkerLine(r) && /^[ \t]*\d+[.)]\s/.test(l)) {
+ classes.push('ordered-list-marker-number');
+ break;
+ }
+ }
+
if (/[ \t]\n/.test(leftLf) || /[ \t]\n/.test(rightLf)) {
classes.push('trailing-whitespace');
}
diff --git a/packages/server/src/bridge-watchdog.test.ts b/packages/server/src/bridge-watchdog.test.ts
index b574532cd..6305e5563 100644
--- a/packages/server/src/bridge-watchdog.test.ts
+++ b/packages/server/src/bridge-watchdog.test.ts
@@ -1,11 +1,14 @@
import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
import { BridgeInvariantViolationError } from '@inkeep/open-knowledge-core';
import {
+ __getSplitBrainRateTupleCountForTests,
__getViolationRateTupleCountForTests,
__resetBridgeWatchdogForTests,
assertBridgeInvariant,
+ emitBridgeSplitBrainRederive,
emitObserverAPathBFired,
shouldEmitBridgeInvariantViolation,
+ shouldEmitBridgeSplitBrainRederive,
shouldEmitBridgeToleranceApplied,
shouldEmitObserverAPathBFired,
shouldThrowOnBridgeInvariantViolation,
@@ -387,6 +390,56 @@ describe('shouldEmitObserverAPathBFired — per-doc rate-limiter', () => {
});
});
+describe('shouldEmitBridgeSplitBrainRederive — per-(site, doc) rate-limiter', () => {
+ test('first call for a (site, doc) tuple returns true', () => {
+ expect(shouldEmitBridgeSplitBrainRederive('post-merge', 'doc-1', 1000)).toBe(true);
+ });
+
+ test('repeat call inside window returns false', () => {
+ shouldEmitBridgeSplitBrainRederive('post-merge', 'doc-1', 1000);
+ expect(shouldEmitBridgeSplitBrainRederive('post-merge', 'doc-1', 1500)).toBe(false);
+ });
+
+ test('call after debounce expires returns true', () => {
+ shouldEmitBridgeSplitBrainRederive('post-merge', 'doc-1', 1000);
+ expect(shouldEmitBridgeSplitBrainRederive('post-merge', 'doc-1', 70_000)).toBe(true);
+ });
+
+ test('sites have independent windows for the same doc', () => {
+ expect(shouldEmitBridgeSplitBrainRederive('identity-gate', 'doc-1', 1000)).toBe(true);
+ expect(shouldEmitBridgeSplitBrainRederive('post-merge', 'doc-1', 1000)).toBe(true);
+ expect(shouldEmitBridgeSplitBrainRederive('identity-gate', 'doc-1', 1500)).toBe(false);
+ expect(shouldEmitBridgeSplitBrainRederive('post-merge', 'doc-1', 1500)).toBe(false);
+ });
+
+ test('different docs have independent windows', () => {
+ expect(shouldEmitBridgeSplitBrainRederive('post-merge', 'doc-1', 1000)).toBe(true);
+ expect(shouldEmitBridgeSplitBrainRederive('post-merge', 'doc-2', 1000)).toBe(true);
+ expect(shouldEmitBridgeSplitBrainRederive('post-merge', 'doc-1', 1500)).toBe(false);
+ });
+
+ test('docName=undefined uses __nodoc__ sentinel (distinct from any named doc)', () => {
+ expect(shouldEmitBridgeSplitBrainRederive('post-merge', undefined, 1000)).toBe(true);
+ expect(shouldEmitBridgeSplitBrainRederive('post-merge', 'doc-1', 1000)).toBe(true);
+ expect(shouldEmitBridgeSplitBrainRederive('post-merge', undefined, 1500)).toBe(false);
+ });
+
+ test('emitBridgeSplitBrainRederive increments suppressed counter when rate-limited', () => {
+ expect(emitBridgeSplitBrainRederive('post-merge', 'doc-1', 1000)).toBe(true);
+ expect(getMetrics().bridgeSplitBrainRederivesSuppressed).toBe(0);
+ expect(emitBridgeSplitBrainRederive('post-merge', 'doc-1', 1500)).toBe(false);
+ expect(getMetrics().bridgeSplitBrainRederivesSuppressed).toBe(1);
+ expect(emitBridgeSplitBrainRederive('post-merge', 'doc-1', 2000)).toBe(false);
+ expect(getMetrics().bridgeSplitBrainRederivesSuppressed).toBe(2);
+ });
+
+ test('emitBridgeSplitBrainRederive returns true after window resets', () => {
+ expect(emitBridgeSplitBrainRederive('post-merge', 'doc-1', 1000)).toBe(true);
+ expect(emitBridgeSplitBrainRederive('post-merge', 'doc-1', 70_000)).toBe(true);
+ expect(getMetrics().bridgeSplitBrainRederivesSuppressed).toBe(0);
+ });
+});
+
describe('bridge-invariant-violation payload redaction (OK_TELEMETRY_VERBOSE opt-in)', () => {
let originalNodeEnv: string | undefined;
let originalVerbose: string | undefined;
@@ -676,6 +729,62 @@ describe('shouldEmitBridgeInvariantViolation — lazy prune of past-window entri
});
});
+describe('shouldEmitBridgeSplitBrainRederive — lazy prune of past-window entries', () => {
+ test('grows linearly below the prune threshold', () => {
+ for (let i = 0; i < 1023; i++) {
+ shouldEmitBridgeSplitBrainRederive('post-merge', `doc-${i}`, 0);
+ }
+ expect(__getSplitBrainRateTupleCountForTests()).toBe(1023);
+ });
+
+ test('past-window entries reclaim when threshold is exceeded', () => {
+ for (let i = 0; i < 1024; i++) {
+ shouldEmitBridgeSplitBrainRederive('post-merge', `doc-${i}`, 0);
+ }
+ expect(__getSplitBrainRateTupleCountForTests()).toBe(1024);
+
+ shouldEmitBridgeSplitBrainRederive('post-merge', 'doc-new', 70_000);
+ expect(__getSplitBrainRateTupleCountForTests()).toBe(1);
+ });
+
+ test('in-window entries are preserved during prune (mixed window state)', () => {
+ for (let i = 0; i < 1023; i++) {
+ shouldEmitBridgeSplitBrainRederive('post-merge', `doc-old-${i}`, 0);
+ }
+ shouldEmitBridgeSplitBrainRederive('post-merge', 'doc-fresh', 30_000);
+ expect(__getSplitBrainRateTupleCountForTests()).toBe(1024);
+
+ shouldEmitBridgeSplitBrainRederive('post-merge', 'doc-new', 70_000);
+ expect(__getSplitBrainRateTupleCountForTests()).toBe(2);
+ expect(shouldEmitBridgeSplitBrainRederive('post-merge', 'doc-fresh', 71_000)).toBe(false);
+ });
+
+ test('threshold boundary: exactly 1023 entries does not trigger prune', () => {
+ for (let i = 0; i < 1023; i++) {
+ shouldEmitBridgeSplitBrainRederive('post-merge', `doc-${i}`, 0);
+ }
+ shouldEmitBridgeSplitBrainRederive('post-merge', 'doc-1024th', 70_000);
+ expect(__getSplitBrainRateTupleCountForTests()).toBe(1024);
+ });
+
+ test('all-in-window: prune walks but reclaims nothing (documents conditional bound)', () => {
+ for (let i = 0; i < 1024; i++) {
+ shouldEmitBridgeSplitBrainRederive('post-merge', `doc-${i}`, 1_000);
+ }
+ expect(__getSplitBrainRateTupleCountForTests()).toBe(1024);
+
+ shouldEmitBridgeSplitBrainRederive('post-merge', 'doc-new', 2_000);
+ expect(__getSplitBrainRateTupleCountForTests()).toBe(1025);
+ });
+
+ test('all three sites for the same doc occupy distinct keys (each counted)', () => {
+ shouldEmitBridgeSplitBrainRederive('post-merge', 'doc-1', 0);
+ shouldEmitBridgeSplitBrainRederive('identity-gate', 'doc-1', 0);
+ shouldEmitBridgeSplitBrainRederive('error-recovery', 'doc-1', 0);
+ expect(__getSplitBrainRateTupleCountForTests()).toBe(3);
+ });
+});
+
describe('assertBridgeInvariant — return value reflects normalize-equality', () => {
test('byte-equal inputs return true', () => {
expect(assertBridgeInvariant('# Hello\n', '# Hello\n', { site: 'observer-b' })).toBe(true);
diff --git a/packages/server/src/bridge-watchdog.ts b/packages/server/src/bridge-watchdog.ts
index 6ec25ffc2..74af45c29 100644
--- a/packages/server/src/bridge-watchdog.ts
+++ b/packages/server/src/bridge-watchdog.ts
@@ -10,6 +10,7 @@ import {
import {
incrementBridgeInvariantViolations,
incrementBridgeInvariantViolationsSuppressed,
+ incrementBridgeSplitBrainRederivesSuppressed,
incrementBridgeToleranceApplied,
incrementObserverAPathBFiresSuppressed,
} from './metrics.ts';
@@ -79,6 +80,23 @@ const lastToleranceEmitMs = new Map<string, number>();
* WARN: same module-level state caveat as `lastEmitMs` above. */
const lastPathBEmitMs = new Map<string, number>();
+/** Observer A settlement-check site that detected a drain settling
+ * split-brain. Three production sites in `server-observers.ts`: the
+ * identity gate (fragment changed but its serialization didn't move),
+ * the post-merge baseline check (after a Path A/B Y.Text write), and
+ * the error-recovery catch (sync work threw before the settlement check,
+ * so the baseline reset must not witness a divergent Y.Text). */
+export type BridgeSplitBrainSite = 'identity-gate' | 'post-merge' | 'error-recovery';
+
+/** Map<rateKey, last-emit-Unix-ms> for the bridge-split-brain-rederive
+ * event. rateKey = `${site}::${docName ?? '__nodoc__'}` — per-(site, doc)
+ * so a chatty doc can't suppress signal from quieter docs and the two
+ * detection sites surface independently. Bounded: 2 sites × docs, with
+ * the same lazy prune as `lastEmitMs`.
+ *
+ * WARN: same module-level state caveat as `lastEmitMs` above. */
+const lastSplitBrainEmitMs = new Map<string, number>();
+
function toleranceRateKey(site: BridgeInvariantSite, cls: BridgeToleranceClass): string {
return `${site}::${cls}`;
}
@@ -151,16 +169,51 @@ export function emitObserverAPathBFired(docName: string | undefined, nowMs?: num
return shouldEmit;
}
+export function shouldEmitBridgeSplitBrainRederive(
+ site: BridgeSplitBrainSite,
+ docName: string | undefined,
+ nowMs: number = Date.now(),
+): boolean {
+ const key = `${site}::${docName ?? '__nodoc__'}`;
+ const last = lastSplitBrainEmitMs.get(key);
+ const debounceMs = readDebounceMs();
+ if (last !== undefined && nowMs - last < debounceMs) return false;
+ if (lastSplitBrainEmitMs.size >= MAX_VIOLATION_RATE_TUPLES) {
+ for (const [k, lastMs] of lastSplitBrainEmitMs) {
+ if (nowMs - lastMs >= debounceMs) lastSplitBrainEmitMs.delete(k);
+ }
+ }
+ lastSplitBrainEmitMs.set(key, nowMs);
+ return true;
+}
+
+export function emitBridgeSplitBrainRederive(
+ site: BridgeSplitBrainSite,
+ docName: string | undefined,
+ nowMs?: number,
+): boolean {
+ const shouldEmit = shouldEmitBridgeSplitBrainRederive(site, docName, nowMs);
+ if (!shouldEmit) {
+ incrementBridgeSplitBrainRederivesSuppressed();
+ }
+ return shouldEmit;
+}
+
export function __resetBridgeWatchdogForTests(): void {
lastEmitMs.clear();
lastToleranceEmitMs.clear();
lastPathBEmitMs.clear();
+ lastSplitBrainEmitMs.clear();
}
export function __getViolationRateTupleCountForTests(): number {
return lastEmitMs.size;
}
+export function __getSplitBrainRateTupleCountForTests(): number {
+ return lastSplitBrainEmitMs.size;
+}
+
export function shouldThrowOnBridgeInvariantViolation(
env: NodeJS.ProcessEnv = process.env,
): boolean {
diff --git a/packages/server/src/metrics.ts b/packages/server/src/metrics.ts
index 8a2faedb2..c83d5a8f6 100644
--- a/packages/server/src/metrics.ts
+++ b/packages/server/src/metrics.ts
@@ -159,6 +159,28 @@ export interface ReconciliationMetrics {
* Path-B rate via `actual_rate = observerAPathBFires +
* observerAPathBFiresSuppressed`. */
observerAPathBFiresSuppressed: number;
+ /** Y.Text-is-truth contract (precedent #38) — count of Observer A
+ * settlement checks that detected a drain settling split-brain (Y.Text
+ * vs serialize(fragment) divergence beyond `normalizeBridge` tolerance)
+ * and enqueued a same-drain Observer B re-derive, that escaped the
+ * per-(site, doc) rate-limiter and emitted a structured
+ * `bridge-split-brain-rederive` event. No organic input produces this
+ * divergence at HEAD — producers were narrowed to dependency/plugin
+ * drift — so this firing in production is itself the drift alert: a
+ * new divergent fallback producer has appeared. Also the operator
+ * signal for a doc stuck re-deriving its fragment on every edit (the
+ * divergence is structural and persists by design; the re-derive cost
+ * recurs per drain). Counter increments only on emit; the companion
+ * suppressed counter preserves `actual_rate = fires + suppressed`. */
+ bridgeSplitBrainRederives: number;
+ /** Companion to `bridgeSplitBrainRederives` — count of split-brain
+ * re-derive detections the rate-limiter suppressed within the
+ * per-(site, doc) debounce window. On an irreducibly-divergent doc the
+ * check fires on every Observer A drain (every WYSIWYG keystroke), so
+ * the unsuppressed `console.warn` would drown the very drift signal
+ * operators need. Incremented only on suppress, mirroring
+ * `observerAPathBFiresSuppressed`. */
+ bridgeSplitBrainRederivesSuppressed: number;
/** Y.Text-is-truth contract — count of best-effort
* fragment-reconciliation attempts (`reconcileFragmentNow`) that
* threw inside persistence's pre-write sanity-check recovery path. The
@@ -314,6 +336,8 @@ const counters: ReconciliationMetrics = {
bridgeToleranceApplied: {},
observerAPathBFires: 0,
observerAPathBFiresSuppressed: 0,
+ bridgeSplitBrainRederives: 0,
+ bridgeSplitBrainRederivesSuppressed: 0,
persistenceReconciliationFailures: 0,
externalChangeHandlerErrors: 0,
persistenceSanityCheckSerializeFailures: 0,
@@ -443,6 +467,14 @@ export function incrementObserverAPathBFiresSuppressed(): void {
counters.observerAPathBFiresSuppressed++;
}
+export function incrementBridgeSplitBrainRederives(): void {
+ counters.bridgeSplitBrainRederives++;
+}
+
+export function incrementBridgeSplitBrainRederivesSuppressed(): void {
+ counters.bridgeSplitBrainRederivesSuppressed++;
+}
+
export function incrementPersistenceReconciliationFailures(): void {
counters.persistenceReconciliationFailures++;
}
@@ -570,6 +602,8 @@ export function resetMetrics(): void {
counters.bridgeToleranceApplied = {};
counters.observerAPathBFires = 0;
counters.observerAPathBFiresSuppressed = 0;
+ counters.bridgeSplitBrainRederives = 0;
+ counters.bridgeSplitBrainRederivesSuppressed = 0;
counters.persistenceReconciliationFailures = 0;
counters.externalChangeHandlerErrors = 0;
counters.persistenceSanityCheckSerializeFailures = 0;
diff --git a/packages/server/src/server-observers-divergent-fallback.test.ts b/packages/server/src/server-observers-divergent-fallback.test.ts
new file mode 100644
index 000000000..c55190bba
--- /dev/null
+++ b/packages/server/src/server-observers-divergent-fallback.test.ts
@@ -0,0 +1,356 @@
+import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
+import { MarkdownManager, sharedExtensions } from '@inkeep/open-knowledge-core';
+import { getSchema } from '@tiptap/core';
+import { updateYFragment, yXmlFragmentToProseMirrorRootNode } from '@tiptap/y-tiptap';
+import * as Y from 'yjs';
+import { __resetBridgeWatchdogForTests } from './bridge-watchdog.ts';
+import { getMetrics, resetMetrics } from './metrics.ts';
+import { type ObserverDispatchKind, setupServerObservers } from './server-observers.ts';
+
+const mdManager = new MarkdownManager({ extensions: sharedExtensions });
+const schema = getSchema(sharedExtensions);
+
+const CLIENT_ORIGIN = { client: 'simulated-y-prosemirror-client' };
+
+const SOURCE = 'Para one.\n\n<Foo>broken</Bar>\n\nPara two.\n';
+const BROKEN_BLOCK = '<Foo>broken</Bar>';
+
+const ENV_KEYS = ['NODE_ENV', 'OK_BRIDGE_THROW_ON_VIOLATION', 'OK_RETHROW_BRIDGE_LOSS'] as const;
+let savedEnv: Partial<Record<(typeof ENV_KEYS)[number], string | undefined>>;
+
+beforeEach(() => {
+ savedEnv = {};
+ for (const key of ENV_KEYS) {
+ savedEnv[key] = process.env[key];
+ }
+ process.env.NODE_ENV = 'production';
+ delete process.env.OK_BRIDGE_THROW_ON_VIOLATION;
+ delete process.env.OK_RETHROW_BRIDGE_LOSS;
+ resetMetrics();
+ __resetBridgeWatchdogForTests();
+});
+
+afterEach(() => {
+ for (const key of ENV_KEYS) {
+ const value = savedEnv[key];
+ if (value === undefined) {
+ delete process.env[key];
+ } else {
+ process.env[key] = value;
+ }
+ }
+});
+
+interface PmJson {
+ type: string;
+ content?: PmJson[];
+ text?: string;
+ attrs?: Record<string, unknown>;
+}
+
+function findFallback(node: PmJson): PmJson | null {
+ if (node.type === 'rawMdxFallback') return node;
+ for (const child of node.content ?? []) {
+ const hit = findFallback(child);
+ if (hit) return hit;
+ }
+ return null;
+}
+
+function fragmentJson(xmlFragment: Y.XmlFragment): PmJson {
+ return yXmlFragmentToProseMirrorRootNode(xmlFragment, schema).toJSON() as PmJson;
+}
+
+function writeFragment(doc: Y.Doc, xmlFragment: Y.XmlFragment, json: PmJson): void {
+ const pmNode = schema.nodeFromJSON(json);
+ doc.transact(() => {
+ const meta = { mapping: new Map(), isOMark: new Map() };
+ updateYFragment(doc, xmlFragment, pmNode, meta);
+ }, CLIENT_ORIGIN);
+}
+
+type DivergenceShape = (fallback: PmJson) => void;
+
+const guardEmptyShape: DivergenceShape = (fallback) => {
+ fallback.content = [];
+};
+
+const sentinelShape: DivergenceShape = (fallback) => {
+ fallback.content = [{ type: 'text', text: '«unknown:someFutureType»' }];
+};
+
+function makeSerializeFault() {
+ let armed = 0;
+ let fired = false;
+ return {
+ arm(times = 1) {
+ armed = times;
+ },
+ get fired() {
+ return fired;
+ },
+ maybeThrow() {
+ if (armed > 0) {
+ armed -= 1;
+ fired = true;
+ throw new Error('injected serialize failure');
+ }
+ },
+ };
+}
+type SerializeFault = ReturnType<typeof makeSerializeFault>;
+
+function makeDegradedManager(diverge: DivergenceShape, fault?: SerializeFault): MarkdownManager {
+ return new Proxy(mdManager, {
+ get(target, prop, receiver) {
+ if (prop === 'parseWithFallback') {
+ return (markdown: string, opts?: Parameters<MarkdownManager['parseWithFallback']>[1]) => {
+ const json = target.parseWithFallback(markdown, opts) as PmJson;
+ const fallback = findFallback(json);
+ if (fallback) diverge(fallback);
+ return json;
+ };
+ }
+ if (prop === 'serialize' && fault) {
+ return (json: Parameters<MarkdownManager['serialize']>[0]) => {
+ fault.maybeThrow();
+ return target.serialize(json);
+ };
+ }
+ return Reflect.get(target, prop, receiver);
+ },
+ });
+}
+
+function loadDivergentDoc(
+ diverge: DivergenceShape,
+ onDispatch?: (kind: ObserverDispatchKind) => void,
+ fault?: SerializeFault,
+) {
+ const doc = new Y.Doc();
+ const xmlFragment = doc.getXmlFragment('default');
+ const ytext = doc.getText('source');
+ const cleanup = setupServerObservers({
+ doc,
+ xmlFragment,
+ ytext,
+ mdManager: makeDegradedManager(diverge, fault),
+ schema,
+ onDispatch,
+ });
+ doc.transact(() => {
+ ytext.insert(0, SOURCE);
+ }, CLIENT_ORIGIN);
+ return { doc, xmlFragment, ytext, cleanup };
+}
+
+async function quiesce(xmlFragment: Y.XmlFragment, ytext: Y.Text): Promise<void> {
+ const snapshot = () => `${JSON.stringify(fragmentJson(xmlFragment))}\n${ytext.toString()}`;
+ let prev = snapshot();
+ const deadline = Date.now() + 250;
+ while (Date.now() < deadline) {
+ await new Promise((resolve) => setTimeout(resolve, 10));
+ const current = snapshot();
+ if (current === prev) return;
+ prev = current;
+ }
+}
+
+function typeIntoFallback(doc: Y.Doc, xmlFragment: Y.XmlFragment, char: string): void {
+ const json = fragmentJson(xmlFragment);
+ const fallback = findFallback(json);
+ if (!fallback) throw new Error('expected a rawMdxFallback node in the fragment');
+ const current = (fallback.content ?? []).map((child) => child.text ?? '').join('');
+ fallback.content = [{ type: 'text', text: current + char }];
+ writeFragment(doc, xmlFragment, json);
+}
+
+function appendToLastParagraph(doc: Y.Doc, xmlFragment: Y.XmlFragment, suffix: string): void {
+ const json = fragmentJson(xmlFragment);
+ const last = json.content?.[json.content.length - 1];
+ const textNode = last?.content?.[0];
+ if (!textNode?.text) throw new Error('expected trailing paragraph text');
+ textNode.text += suffix;
+ writeFragment(doc, xmlFragment, json);
+}
+
+describe('divergent rawMdxFallback must not become authoritative source', () => {
+ test('S4: typing twice into a divergent fallback preserves the source bytes it stands for', async () => {
+ const { doc, xmlFragment, ytext, cleanup } = loadDivergentDoc(guardEmptyShape);
+
+ typeIntoFallback(doc, xmlFragment, 'x');
+ await quiesce(xmlFragment, ytext);
+ typeIntoFallback(doc, xmlFragment, 'y');
+ await quiesce(xmlFragment, ytext);
+
+ const after = ytext.toString();
+ expect(after).toContain(BROKEN_BLOCK);
+ expect(after.split(BROKEN_BLOCK).length - 1).toBe(1);
+ expect(after).toContain('Para one.');
+ expect(after).toContain('Para two.');
+
+ cleanup();
+ });
+
+ test('protective re-derive dispatches a-then-b within the same drain', () => {
+ const dispatches: ObserverDispatchKind[] = [];
+ const { doc, xmlFragment, ytext, cleanup } = loadDivergentDoc(guardEmptyShape, (kind) =>
+ dispatches.push(kind),
+ );
+
+ dispatches.length = 0;
+ resetMetrics();
+ __resetBridgeWatchdogForTests();
+ typeIntoFallback(doc, xmlFragment, 'x');
+
+ const userDispatches = dispatches.filter((kind) => kind !== 'none');
+ expect(userDispatches).toEqual(['a', 'b']);
+ expect(ytext.toString()).toContain(BROKEN_BLOCK);
+ expect(getMetrics().bridgeSplitBrainRederives).toBe(1);
+
+ cleanup();
+ });
+
+ test('S4 sentinel producer: «unknown:type»-shaped divergence is equally protected', async () => {
+ const { doc, xmlFragment, ytext, cleanup } = loadDivergentDoc(sentinelShape);
+
+ typeIntoFallback(doc, xmlFragment, 'x');
+ await quiesce(xmlFragment, ytext);
+ typeIntoFallback(doc, xmlFragment, 'y');
+ await quiesce(xmlFragment, ytext);
+
+ const after = ytext.toString();
+ expect(after).toContain(BROKEN_BLOCK);
+ expect(after).not.toContain('«unknown:someFutureType»');
+
+ cleanup();
+ });
+
+ test('S6: one fallback keystroke then an ordinary edit elsewhere preserves the source bytes', async () => {
+ const { doc, xmlFragment, ytext, cleanup } = loadDivergentDoc(guardEmptyShape);
+
+ typeIntoFallback(doc, xmlFragment, 'x');
+ await quiesce(xmlFragment, ytext);
+ appendToLastParagraph(doc, xmlFragment, ' EDITED');
+ await quiesce(xmlFragment, ytext);
+
+ const after = ytext.toString();
+ expect(after).toContain(BROKEN_BLOCK);
+ expect(after.split(BROKEN_BLOCK).length - 1).toBe(1);
+ expect(after).toContain('EDITED');
+
+ cleanup();
+ });
+
+ test('S5: blur-upgrade on an empty divergent fallback keeps Y.Text intact and keeps the broken-block chrome', async () => {
+ const { doc, xmlFragment, ytext, cleanup } = loadDivergentDoc(guardEmptyShape);
+
+ const upgraded = mdManager.parseWithFallback('') as PmJson;
+ const json = fragmentJson(xmlFragment);
+ const fallback = findFallback(json);
+ if (!fallback) throw new Error('expected a rawMdxFallback node in the fragment');
+ const index = json.content?.indexOf(fallback) ?? -1;
+ if (index < 0 || !json.content || !upgraded.content) {
+ throw new Error('expected top-level fallback and upgrade content');
+ }
+ json.content.splice(index, 1, ...upgraded.content);
+ writeFragment(doc, xmlFragment, json);
+ await quiesce(xmlFragment, ytext);
+
+ expect(ytext.toString()).toBe(SOURCE);
+ expect(findFallback(fragmentJson(xmlFragment))).not.toBeNull();
+
+ cleanup();
+ });
+
+ test('S3 bound: a divergent fallback at rest is safe — an edit elsewhere alone merges cleanly', async () => {
+ const { doc, xmlFragment, ytext, cleanup } = loadDivergentDoc(guardEmptyShape);
+
+ appendToLastParagraph(doc, xmlFragment, ' EDITED');
+ await quiesce(xmlFragment, ytext);
+
+ expect(ytext.toString()).toBe(SOURCE.replace('Para two.', 'Para two. EDITED'));
+
+ cleanup();
+ });
+
+ test('error-recovery: a serialize throw during a fallback drain must not let the baseline reset destroy the source bytes', async () => {
+ const fault = makeSerializeFault();
+ const { doc, xmlFragment, ytext, cleanup } = loadDivergentDoc(
+ guardEmptyShape,
+ undefined,
+ fault,
+ );
+
+ fault.arm();
+ typeIntoFallback(doc, xmlFragment, 'x');
+ await quiesce(xmlFragment, ytext);
+ expect(fault.fired).toBe(true);
+
+ appendToLastParagraph(doc, xmlFragment, ' EDITED');
+ await quiesce(xmlFragment, ytext);
+
+ const after = ytext.toString();
+ expect(after).toContain(BROKEN_BLOCK);
+ expect(after.split(BROKEN_BLOCK).length - 1).toBe(1);
+ expect(after).toContain('Para one.');
+ expect(after).toContain('EDITED');
+
+ cleanup();
+ });
+
+ test('error-recovery double failure: when the recovery serialize also throws, the next drain still preserves the source bytes', async () => {
+ const fault = makeSerializeFault();
+ const { doc, xmlFragment, ytext, cleanup } = loadDivergentDoc(
+ guardEmptyShape,
+ undefined,
+ fault,
+ );
+
+ fault.arm(2);
+ typeIntoFallback(doc, xmlFragment, 'x');
+ await quiesce(xmlFragment, ytext);
+ expect(fault.fired).toBe(true);
+
+ appendToLastParagraph(doc, xmlFragment, ' EDITED');
+ await quiesce(xmlFragment, ytext);
+
+ const after = ytext.toString();
+ expect(after).toContain(BROKEN_BLOCK);
+ expect(after.split(BROKEN_BLOCK).length - 1).toBe(1);
+ expect(after).toContain('Para one.');
+ expect(after).toContain('EDITED');
+
+ cleanup();
+ });
+
+ test('identity-gate dispatch pin: blur-upgrade fires a same-drain a-then-b re-derive with one counted emission', async () => {
+ const dispatches: ObserverDispatchKind[] = [];
+ const { doc, xmlFragment, ytext, cleanup } = loadDivergentDoc(guardEmptyShape, (kind) =>
+ dispatches.push(kind),
+ );
+
+ const upgraded = mdManager.parseWithFallback('') as PmJson;
+ const json = fragmentJson(xmlFragment);
+ const fallback = findFallback(json);
+ if (!fallback) throw new Error('expected a rawMdxFallback node in the fragment');
+ const index = json.content?.indexOf(fallback) ?? -1;
+ if (index < 0 || !json.content || !upgraded.content) {
+ throw new Error('expected top-level fallback and upgrade content');
+ }
+ json.content.splice(index, 1, ...upgraded.content);
+
+ dispatches.length = 0;
+ resetMetrics();
+ __resetBridgeWatchdogForTests();
+ writeFragment(doc, xmlFragment, json);
+
+ const userDispatches = dispatches.filter((kind) => kind !== 'none');
+ expect(userDispatches).toEqual(['a', 'b']);
+ expect(getMetrics().bridgeSplitBrainRederives).toBe(1);
+ expect(findFallback(fragmentJson(xmlFragment))).not.toBeNull();
+ expect(ytext.toString()).toBe(SOURCE);
+
+ cleanup();
+ });
+});
diff --git a/packages/server/src/server-observers.ts b/packages/server/src/server-observers.ts
index c73757165..0876728aa 100644
--- a/packages/server/src/server-observers.ts
+++ b/packages/server/src/server-observers.ts
@@ -14,11 +14,17 @@ import type { Schema } from '@tiptap/pm/model';
import { updateYFragment, yXmlFragmentToProseMirrorRootNode } from '@tiptap/y-tiptap';
import type * as Y from 'yjs';
import { attachQuiescenceTracker } from './bridge-quiescence.ts';
-import { assertBridgeInvariant, emitObserverAPathBFired } from './bridge-watchdog.ts';
+import {
+ assertBridgeInvariant,
+ type BridgeSplitBrainSite,
+ emitBridgeSplitBrainRederive,
+ emitObserverAPathBFired,
+} from './bridge-watchdog.ts';
import { recordFrontmatterEditSurface } from './frontmatter-telemetry.ts';
import {
incrementBridgeMergeCheckpointCreated,
incrementBridgeMergeContentLoss,
+ incrementBridgeSplitBrainRederives,
incrementObserverAPathBFires,
incrementServerObserverError,
incrementServerObserverFire,
@@ -72,6 +78,10 @@ export interface SetupServerObserversOpts {
onDispatch?: ObserverDispatchHook;
}
+function settlesSplitBrain(settledText: string, md: string): boolean {
+ return settledText !== md && normalizeBridge(settledText) !== normalizeBridge(md);
+}
+
export function setupServerObservers(opts: SetupServerObserversOpts): () => void {
const { doc, xmlFragment, ytext, mdManager, schema } = opts;
@@ -126,6 +136,23 @@ export function setupServerObservers(opts: SetupServerObserversOpts): () => void
});
};
+ const recordSplitBrainRederive = (site: BridgeSplitBrainSite): void => {
+ try {
+ if (emitBridgeSplitBrainRederive(site, opts.docName)) {
+ incrementBridgeSplitBrainRederives();
+ console.warn(
+ JSON.stringify({
+ event: 'bridge-split-brain-rederive',
+ 'doc.name': opts.docName ?? null,
+ site,
+ }),
+ );
+ }
+ } catch (telErr) {
+ console.warn('[Server Observer A] Split-brain telemetry failed:', telErr);
+ }
+ };
+
let lastSyncedXmlMd = '';
let xmlDirty = false;
let textDirty = false;
@@ -153,7 +180,13 @@ export function setupServerObservers(opts: SetupServerObserversOpts): () => void
const frontmatter = readCurrentFm();
const md = prependFrontmatter(frontmatter, body);
- if (lastSyncedXmlMd === md) return;
+ if (lastSyncedXmlMd === md) {
+ if (settlesSplitBrain(ytext.toString(), md)) {
+ textDirty = true;
+ recordSplitBrainRederive('identity-gate');
+ }
+ return;
+ }
const currentText = ytext.toString();
@@ -198,14 +231,38 @@ export function setupServerObservers(opts: SetupServerObserversOpts): () => void
}
incrementServerObserverFire('a');
- lastSyncedXmlMd = ytext.toString();
+ const settledText = ytext.toString();
+ if (settlesSplitBrain(settledText, md)) {
+ lastSyncedXmlMd = md;
+ textDirty = true;
+ recordSplitBrainRederive('post-merge');
+ } else {
+ lastSyncedXmlMd = settledText;
+ }
} catch (err) {
incrementServerObserverError('a');
console.error('[Server Observer A] Failed to sync tree→text:', err);
try {
- lastSyncedXmlMd = ytext.toString();
+ const recoveryJson = yXmlFragmentToProseMirrorRootNode(xmlFragment, schema).toJSON();
+ const recoveryBody = mdManager.serialize(recoveryJson);
+ const recoveryMd = prependFrontmatter(readCurrentFm(), recoveryBody);
+ if (settlesSplitBrain(ytext.toString(), recoveryMd)) {
+ lastSyncedXmlMd = recoveryMd;
+ textDirty = true;
+ recordSplitBrainRederive('error-recovery');
+ } else {
+ lastSyncedXmlMd = ytext.toString();
+ }
} catch (innerErr) {
- console.warn('[Server Observer A] Baseline recovery also failed:', innerErr);
+ console.warn(
+ '[Server Observer A] Baseline recovery also failed',
+ JSON.stringify({
+ 'doc.name': opts.docName ?? null,
+ originalError: err instanceof Error ? err.message : String(err),
+ recoveryError: innerErr instanceof Error ? innerErr.message : String(innerErr),
+ }),
+ );
+ lastSyncedXmlMd = '';
}
}
};
@@ -382,12 +439,12 @@ export function setupServerObservers(opts: SetupServerObserversOpts): () => void
}
const ranA = xmlDirty;
- const ranB = textDirty;
if (xmlDirty) {
xmlDirty = false;
opts.onDispatch?.('a');
runObserverASync();
}
+ const ranB = textDirty;
if (textDirty) {
textDirty = false;
opts.onDispatch?.('b');