Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

fix(open-knowledge): merge-granularity marker-split content loss (#2745)

Nick Gomez16df5b18a7b4
diff --git a/.changeset/line-aligned-bridge-materialization.md b/.changeset/line-aligned-bridge-materialization.md
new file mode 100644
index 000000000..0da5371c2
--- /dev/null
+++ b/.changeset/line-aligned-bridge-materialization.md
@@ -0,0 +1,5 @@
+---
+"@inkeep/open-knowledge": patch
+---
+
+Fixed a rare content-corruption bug where text written to a document while a collaborator's connection was catching up could split freshly-applied content in two. When a file changed on disk (or an agent rewrote a document) at the same time as an offline-typed edit from another client arrived, part of the new content could end up interleaved with the concurrent edit — observed as a line split in the middle with the other client's text in between. The sync bridge now applies changed content as whole lines in single contiguous runs, which makes this interleaving structurally impossible; unchanged lines still keep their edit history and attribution.
diff --git a/packages/app/tests/integration/external-change-stale-anchor-interleave.test.ts b/packages/app/tests/integration/external-change-stale-anchor-interleave.test.ts
new file mode 100644
index 000000000..47cf47134
--- /dev/null
+++ b/packages/app/tests/integration/external-change-stale-anchor-interleave.test.ts
@@ -0,0 +1,138 @@
+/**
+ * Stale-anchor interleave over an external-change wholesale replace.
+ *
+ * A sync-paused client (inbound paused, outbound flowing — the
+ * ControllableWebSocket contract) types a source-mode insert anchored to
+ * pre-replace Y.Text items while the file watcher applies a wholesale disk
+ * replace. The bridge materializer must land the replaced content so that no
+ * concurrent stale-anchored op can integrate INSIDE it: the replaced marker
+ * line must survive contiguously on every converged peer, for either relative
+ * clientID ordering (Yjs orders concurrent same-position inserts by clientID,
+ * so a materializer that stitches new content out of reused old items splits
+ * it only when the concurrent writer's clientID sorts first).
+ *
+ * Deterministic distillation of fuzz seeds 1784396512201 / 1784413394675 /
+ * 1784413395177 (failClass content-preservation): forced clientIDs replace the
+ * seeds' random-ordering coin flip.
+ *
+ * Per-test docName isolation; client lifecycle in try/finally.
+ */
+
+import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
+import { writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { setTimeout as wait } from 'node:timers/promises';
+import { applyExternalChange } from '@inkeep/open-knowledge-server';
+import { HARNESS_BOOT_TIMEOUT_MS } from './harness-boot-timeout';
+import {
+ agentWriteMd,
+ assertBridgeInvariant,
+ createTestClients,
+ createTestServer,
+ pollUntil,
+ type TestServer,
+} from './test-harness';
+
+let server: TestServer;
+
+beforeAll(async () => {
+ server = await createTestServer();
+}, HARNESS_BOOT_TIMEOUT_MS);
+
+afterAll(async () => {
+ await server.cleanup();
+});
+
+const SEED_CONTENT = 'M3-echo foxtrot\n';
+const REPLACED_CONTENT = 'M6-hotel golf\n';
+const CONCURRENT_LINE = 'M8-golf alpha alpha';
+
+/**
+ * Drive the interleave with the paused client's Y.Doc forced to `clientId`.
+ * Returns the converged text (asserted byte-identical across peers first).
+ */
+async function runInterleave(clientId: number): Promise<string> {
+ const docName = `stale-anchor-${clientId}-${crypto.randomUUID()}`;
+ const clients = await createTestClients(server.port, {
+ count: 2,
+ docName,
+ perClientOptions: { skipInvariantWatcher: true, syncControl: true },
+ });
+ const [live, paused] = clients;
+ try {
+ // Force the paused client's clientID BEFORE it authors any Y items, so
+ // the concurrent-insert ordering is deterministic instead of the fuzz
+ // harness's per-run coin flip. The server side allocates a random
+ // uint32; 1 sorts below it (and 0xfffffffe above it) with probability
+ // ~1 - 2^-32.
+ paused.doc.clientID = clientId;
+
+ await agentWriteMd(server.port, SEED_CONTENT, { docName, position: 'replace' });
+ for (const c of clients) {
+ await pollUntil(() => c.ytext.toString().includes('M3-'), 5000);
+ }
+ await wait(300);
+
+ // Divergence window: the paused client stops RECEIVING; its own edits
+ // still flow out (inbound-only pause).
+ paused.pauseSync();
+
+ // Server-side growth the paused client never sees (mirrors the seeds'
+ // mid-window wysiwyg + jsx agent writes). A char-granular diff of the
+ // subsequent replace would retain the doc-start 'M' and a trailing '\n'
+ // from DIFFERENT generations with a large tombstone range between them —
+ // the seam the stale-anchored insert integrates into.
+ await agentWriteMd(
+ server.port,
+ '\n\nM4-echo delta golf\n\n<Steps>\n\n<Step>\n\nM5-jsx-bravo step body.\n\n</Step>\n\n</Steps>\n',
+ { docName, position: 'append' },
+ );
+ await pollUntil(() => live.ytext.toString().includes('M5-'), 5000);
+
+ // External wholesale replace lands on the server and reaches the live
+ // client only.
+ writeFileSync(join(server.contentDir, `${docName}.md`), REPLACED_CONTENT, 'utf-8');
+ applyExternalChange(server.instance.hocuspocus, docName, REPLACED_CONTENT);
+ await pollUntil(() => live.ytext.toString().includes('M6-'), 5000);
+
+ // Concurrent stale-anchored insert: the paused client still sees the
+ // pre-replace bytes, so this append anchors on pre-replace items.
+ paused.doc.transact(() => {
+ paused.ytext.insert(paused.ytext.length, `\n\n${CONCURRENT_LINE}\n`);
+ });
+ await wait(300);
+
+ paused.resumeSync();
+ await pollUntil(
+ () =>
+ live.ytext.toString() === paused.ytext.toString() &&
+ live.ytext.toString().includes(CONCURRENT_LINE),
+ 10_000,
+ );
+ await wait(600);
+
+ const texts = clients.map((c) => c.ytext.toString());
+ expect(texts[1]).toBe(texts[0]);
+ for (const c of clients) assertBridgeInvariant(c.ytext, c.fragment);
+ return texts[0];
+ } finally {
+ for (const c of clients) await c.cleanup();
+ }
+}
+
+describe('external-change wholesale replace vs stale-anchored concurrent insert', () => {
+ test('replaced content survives contiguously when the concurrent writer clientID sorts first', async () => {
+ const converged = await runInterleave(1);
+ // The replaced marker line must not be split by the concurrent insert
+ // integrating inside a reused-item seam.
+ expect(converged).toContain(REPLACED_CONTENT.trimEnd());
+ // The concurrent insert itself must also survive intact.
+ expect(converged).toContain(CONCURRENT_LINE);
+ }, 30_000);
+
+ test('replaced content survives contiguously when the concurrent writer clientID sorts last', async () => {
+ const converged = await runInterleave(0xfffffffe);
+ expect(converged).toContain(REPLACED_CONTENT.trimEnd());
+ expect(converged).toContain(CONCURRENT_LINE);
+ }, 30_000);
+});
diff --git a/packages/core/src/bridge/apply-diff.ts b/packages/core/src/bridge/apply-diff.ts
index 514d41f88..73293e380 100644
--- a/packages/core/src/bridge/apply-diff.ts
+++ b/packages/core/src/bridge/apply-diff.ts
@@ -1,10 +1,6 @@
-import DiffMatchPatch from 'diff-match-patch';
import type * as Y from 'yjs';
import { diffLinesFast } from './diff-lines.ts';
-const dmpDiff = new DiffMatchPatch();
-dmpDiff.Diff_Timeout = 0.25;
-
const APPLY_FAST_DIFF_MAX_BYTES = 256 * 1024;
export function applyIncrementalDiff(ytext: Y.Text, currentText: string, newText: string): void {
@@ -46,17 +42,16 @@ export function applyFastDiff(ytext: Y.Text, currentText: string, newText: strin
applyByPrefixSuffixMiddleReplace(ytext, currentText, newText);
return;
}
- const diffs = dmpDiff.diff_main(currentText, newText);
- dmpDiff.diff_cleanupSemantic(diffs);
+ const changes = diffLinesFast(currentText, newText);
let offset = 0;
- for (const [type, text] of diffs) {
- if (type === 0) {
- offset += text.length;
- } else if (type === -1) {
- ytext.delete(offset, text.length);
- } else if (type === 1) {
- ytext.insert(offset, text);
- offset += text.length;
+ for (const change of changes) {
+ if (change.removed) {
+ ytext.delete(offset, change.value.length);
+ } else if (change.added) {
+ ytext.insert(offset, change.value);
+ offset += change.value.length;
+ } else {
+ offset += change.value.length;
}
}
}
@@ -74,6 +69,9 @@ function applyByPrefixSuffixMiddleReplace(
) {
prefixLen++;
}
+ if (prefixLen > 0) {
+ prefixLen = currentText.lastIndexOf('\n', prefixLen - 1) + 1;
+ }
let suffixLen = 0;
while (
@@ -83,6 +81,16 @@ function applyByPrefixSuffixMiddleReplace(
) {
suffixLen++;
}
+ if (suffixLen > 0) {
+ const curStart = currentText.length - suffixLen;
+ const newStart = newText.length - suffixLen;
+ const curAligned = curStart === 0 || currentText.charCodeAt(curStart - 1) === 10;
+ const newAligned = newStart === 0 || newText.charCodeAt(newStart - 1) === 10;
+ if (!(curAligned && newAligned)) {
+ const firstNewline = currentText.indexOf('\n', curStart);
+ suffixLen = firstNewline === -1 ? 0 : currentText.length - (firstNewline + 1);
+ }
+ }
const deleteLen = currentText.length - prefixLen - suffixLen;
const insertStr = newText.slice(prefixLen, newText.length - suffixLen);
diff --git a/packages/core/src/bridge/diff-lines.ts b/packages/core/src/bridge/diff-lines.ts
index 3ff09de2d..4dc384fdc 100644
--- a/packages/core/src/bridge/diff-lines.ts
+++ b/packages/core/src/bridge/diff-lines.ts
@@ -14,13 +14,20 @@ export function diffLinesFast(oldStr: string, newStr: string): DiffChange[] {
const { chars1, chars2, lineArray } = dmp.diff_linesToChars_(oldStr, newStr);
const diffs = dmp.diff_main(chars1, chars2, false);
dmp.diff_charsToLines_(diffs, lineArray);
- dmp.diff_cleanupSemantic(diffs);
const result: DiffChange[] = [];
for (const [op, text] of diffs) {
- if (op === DiffMatchPatch.DIFF_DELETE) {
+ if (text.length === 0) continue;
+ const prev = result[result.length - 1];
+ const removed = op === DiffMatchPatch.DIFF_DELETE;
+ const added = op === DiffMatchPatch.DIFF_INSERT;
+ if (prev && Boolean(prev.removed) === removed && Boolean(prev.added) === added) {
+ prev.value += text;
+ continue;
+ }
+ if (removed) {
result.push({ value: text, removed: true });
- } else if (op === DiffMatchPatch.DIFF_INSERT) {
+ } else if (added) {
result.push({ value: text, added: true });
} else {
result.push({ value: text });
diff --git a/packages/server/src/agent-sessions.test.ts b/packages/server/src/agent-sessions.test.ts
index 1a188c274..6eea4ed45 100644
--- a/packages/server/src/agent-sessions.test.ts
+++ b/packages/server/src/agent-sessions.test.ts
@@ -891,17 +891,20 @@ describe('applyAgentMarkdownWrite — position: "replace" atomic-overwrite contr
const session = await manager.getSession('doc-patch-incremental.md', 'agent-patch-inc');
const ytext = session.dc.document.getText('source');
- const seed = 'aaaa bbbb cccc\n';
+ const seed = 'head line\n\naaaa bbbb cccc\n\ntail line\n';
session.dc.document.transact(() => {
applyAgentMarkdownWrite(session.dc.document, seed, 'replace');
}, session.origin);
expect(ytext.toString()).toBe(seed);
- // A `patch` whose recomposed body shares prefix/suffix with the prior
- // content must emit a MINIMAL DMP delta (the differing middle only) — the
+ // A `patch` whose recomposed body shares lines with the prior content
+ // must emit a minimal LINE-ALIGNED delta (the changed line only) — the
// opposite of the atomic `replace` pin. This is the contract
- // edit_document depends on: a surgical find/replace stays item-preserving
- // instead of churning the whole doc through replaceRawBody.
+ // edit_document depends on: a surgical find/replace preserves the
+ // untouched lines' Items instead of churning the whole doc through
+ // replaceRawBody. (Granularity is line-level, not character-level:
+ // sub-line Item reuse inside changed content creates stale-anchor
+ // interleave seams — see applyFastDiff.)
let insertCharCount = 0;
let deleteCharCount = 0;
const observer = (event: Y.YTextEvent): void => {
@@ -913,16 +916,19 @@ describe('applyAgentMarkdownWrite — position: "replace" atomic-overwrite contr
}
};
ytext.observe(observer);
- const patchedBody = 'aaaa XXXX cccc\n';
+ const patchedBody = 'head line\n\naaaa XXXX cccc\n\ntail line\n';
session.dc.document.transact(() => {
applyAgentMarkdownWrite(session.dc.document, patchedBody, 'patch');
}, session.origin);
ytext.unobserve(observer);
expect(ytext.toString()).toBe(patchedBody);
- // Minimal delta: only the differing middle is touched, not the whole doc.
+ // Minimal line-aligned delta: only the changed line is touched, not the
+ // whole doc.
expect(deleteCharCount).toBeLessThan(seed.length);
expect(insertCharCount).toBeLessThan(patchedBody.length);
+ expect(deleteCharCount).toBeLessThanOrEqual('aaaa bbbb cccc\n'.length);
+ expect(insertCharCount).toBeLessThanOrEqual('aaaa XXXX cccc\n'.length);
// An empty-body `patch` CLEARS the doc — it is deliberately excluded from
// the append/prepend empty-payload no-op guard, so it behaves like an
diff --git a/packages/server/src/api-agent-patch.test.ts b/packages/server/src/api-agent-patch.test.ts
index 09cc103f6..4db821900 100644
--- a/packages/server/src/api-agent-patch.test.ts
+++ b/packages/server/src/api-agent-patch.test.ts
@@ -252,14 +252,18 @@ describe('POST /api/agent-patch — surgical/incremental write shape', () => {
// primitive. Under the atomic primitive (replaceRawBody) the patch is
// delete(0,len)+insert(0,whole) — deleteCharCount === wholeDocLen and
// insertCharCount === newDocLen, both RED here. Under the incremental
- // primitive (composeAndWriteRawBody → applyFastDiff) only the matched
- // span is touched.
+ // primitive (composeAndWriteRawBody → applyFastDiff) only the changed
+ // LINE is touched (line-aligned granularity: sub-line Item reuse inside
+ // changed content creates stale-anchor interleave seams — see
+ // applyFastDiff).
expect(deleteCharCount).toBeLessThan(wholeDocLen);
expect(insertCharCount).toBeLessThan(newDocLen);
- // Sanity: the delta is on the order of the edited token, not the
- // document. Generous ceiling absorbs diff_cleanupSemantic boundary jitter.
- expect(deleteCharCount).toBeLessThan(64);
- expect(insertCharCount).toBeLessThan(64);
+ // Sanity: the delta is on the order of the edited line, not the
+ // document.
+ const changedLineOld = 'Beta paragraph two with the TARGET token buried in the middle.\n';
+ const changedLineNew = changedLineOld.replace('TARGET', 'REPLACED-TOKEN');
+ expect(deleteCharCount).toBeLessThanOrEqual(changedLineOld.length);
+ expect(insertCharCount).toBeLessThanOrEqual(changedLineNew.length);
} finally {
await sessionManager.closeAll();
rmSync(projectDir, { recursive: true, force: true });
diff --git a/packages/server/src/bridge-intake.ts b/packages/server/src/bridge-intake.ts
index 3245b111b..83f457c2e 100644
--- a/packages/server/src/bridge-intake.ts
+++ b/packages/server/src/bridge-intake.ts
@@ -4,13 +4,14 @@
* name is the contract.
*
* - `composeAndWriteRawBody` — file-watcher + agent-write: parse → ytext-
- * first `applyFastDiff` → fragment derive. Character-level DMP preserves
- * unrelated Y.Text Items + their origins.
+ * first `applyFastDiff` → fragment derive. Line-aligned diff preserves
+ * unrelated whole-line Y.Text Items + their origins; changed lines land
+ * as fresh contiguous runs (stale-anchor interleave safety).
* - `replaceRawBody` — rollback: parse → ytext-first FULL OVERWRITE
* (delete(0, len) + insert(0, raw)) → fragment derive. The non-
* incremental replacement is the load-bearing signal to Y.UndoManager
- * that "this is a rollback, not an edit"; DMP-based diff would over-
- * preserve Items the user explicitly rolled back.
+ * that "this is a rollback, not an edit"; diff-based application would
+ * over-preserve Items the user explicitly rolled back.
* - `deriveFragmentFromYtext` — agent-undo: `Y.UndoManager.undo()` has
* already mutated ytext to the post-undo state; this primitive ONLY
* derives the fragment from `parse(ytext.toString())`. Writes zero
@@ -93,7 +94,7 @@ function buildParseOpts(embedResolver: EmbedResolverArg):
}
/**
- * Apply raw composed bytes to Y.Text via incremental DMP diff and derive
+ * Apply raw composed bytes to Y.Text via an incremental line-aligned diff and derive
* XmlFragment via parse.
*
* MUST be called inside an outer `doc.transact(..., origin)` block
@@ -102,7 +103,8 @@ function buildParseOpts(embedResolver: EmbedResolverArg):
*
* Bytes flow:
* - `ytext` receives `rawContent` verbatim via `applyFastDiff`
- * (character-level DMP, item-preserving) — NO canonicalization. Run
+ * (line-aligned diff, item-preserving for unchanged lines) — NO
+ * canonicalization. Run
* FIRST per the file-level write-order rationale.
* - `xmlFragment` receives `parse(body-without-FM)` via
* `updateYFragment` (item-preservation aware structural diff,
@@ -184,9 +186,9 @@ export function composeAndWriteRawBody(
// bytes, silently reverting the write.
//
// Both writes happen inside the caller's outer transact for atomicity and
- // to share one origin object. applyFastDiff is character-level DMP that
- // preserves unrelated Y.Text Items + their origins; updateYFragment is
- // item-preservation aware (precedent #11(a)).
+ // to share one origin object. applyFastDiff is a line-aligned diff that
+ // preserves unrelated whole-line Y.Text Items + their origins; updateYFragment
+ // is item-preservation aware (precedent #11(a)).
if (currentYText !== rawContent) {
applyFastDiff(ytext, currentYText, rawContent);
}
@@ -201,11 +203,11 @@ export function composeAndWriteRawBody(
* Replace the entire Y.Text contents with `rawContent` via delete/insert
* and derive XmlFragment via parse — the atomic-overwrite semantics.
*
- * The full overwrite (vs `applyFastDiff`'s incremental DMP) is the load-
- * bearing signal to `Y.UndoManager` that this is a non-incremental
- * replacement: the caller discards the user's recent edits, so DMP-based
- * Item preservation would defeat the contract by re-using Items the
- * writer explicitly overwrote.
+ * The full overwrite (vs `applyFastDiff`'s incremental line-aligned diff)
+ * is the load-bearing signal to `Y.UndoManager` that this is a
+ * non-incremental replacement: the caller discards the user's recent
+ * edits, so diff-based Item preservation would defeat the contract by
+ * re-using Items the writer explicitly overwrote.
*
* Callers:
* - `handleRollback` under `ROLLBACK_ORIGIN` — restore a historical version.