Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

fix(ok): preserve doc-edge blank runs authored in the WYSIWYG (#3132)

Nick Gomez4ed1036c6190
diff --git a/.changeset/quiet-pans-repeat.md b/.changeset/quiet-pans-repeat.md
new file mode 100644
index 000000000..57c48f314
--- /dev/null
+++ b/.changeset/quiet-pans-repeat.md
@@ -0,0 +1,5 @@
+---
+'@inkeep/open-knowledge': patch
+---
+
+Blank lines you add at the very top or the very bottom of a document now survive. Press Enter twice or more at either end and the blank lines reach the file on disk and every other open window, instead of disappearing the next time anything else touches the document. The threshold is two: a single blank line at either end is still not preserved, because one blank line there cannot be told apart from spacing nothing typed. At the bottom that is the empty line the editor keeps below the last block so you can carry on typing, and at the top it is the blank line that separates frontmatter from the body. The threshold is the same for every document, including ones where neither of those is present.
diff --git a/packages/app/src/editor/block-spans.ts b/packages/app/src/editor/block-spans.ts
index 150a0d27c..94af1f354 100644
--- a/packages/app/src/editor/block-spans.ts
+++ b/packages/app/src/editor/block-spans.ts
@@ -22,7 +22,11 @@
* decoration plugin's module graph.
*/
-import { type MarkdownManager, stripFrontmatter } from '@inkeep/open-knowledge-core';
+import {
+ type MarkdownManager,
+ MIN_CARRIED_EDGE_EMPTIES,
+ stripFrontmatter,
+} from '@inkeep/open-knowledge-core';
import type { Node as PmNode } from '@tiptap/pm/model';
/** 1-based inclusive line spans of top-level body blocks, in full-source coordinates. */
@@ -176,18 +180,31 @@ export function blockIndexForLine(spans: SourceBlockSpans['spans'], line: number
}
/**
- * Top-level child count comparable against body block spans. A doc whose last
- * block isn't a paragraph renders with a trailing empty paragraph (the
- * type-here affordance below a final heading/list) that has NO source
- * counterpart — parse of the source never yields an empty paragraph. Without
- * this allowance the span↔doc count comparison fails PERMANENTLY on
- * heading-final docs, silently disabling decorations and navigation.
+ * Top-level child count comparable against body block spans.
+ *
+ * Two different things can put an empty paragraph at the end of the PM doc,
+ * and only one of them has a source counterpart. A doc whose last block isn't
+ * a paragraph renders with the type-here affordance below that final
+ * heading/list, which the source does not spell. A doc-edge blank run the user
+ * authored does reach the source, and parse gives it back as the same empty
+ * paragraphs — but only from `MIN_CARRIED_EDGE_EMPTIES` up; below the floor a
+ * run stays on the boundary snapshot and yields no block, precisely because at
+ * one paragraph it is indistinguishable from the affordance.
+ *
+ * So the floor is also the discriminator: a trailing empty run shorter than it
+ * has no source blocks behind it and must come off the count, while a run at or
+ * above it is matched paragraph-for-paragraph and must not. Getting this wrong
+ * fails silently — the span↔doc comparison would fail PERMANENTLY on every
+ * affected doc, disabling decorations and navigation with no error.
*/
export function comparableChildCount(doc: PmNode): number {
- const last = doc.childCount > 0 ? doc.child(doc.childCount - 1) : null;
- const trailingEmptyParagraph =
- last !== null && last.type.name === 'paragraph' && last.content.size === 0;
- return trailingEmptyParagraph ? doc.childCount - 1 : doc.childCount;
+ let trailingEmpty = 0;
+ for (let i = doc.childCount - 1; i >= 0; i--) {
+ const child = doc.child(i);
+ if (child.type.name !== 'paragraph' || child.content.size !== 0) break;
+ trailingEmpty++;
+ }
+ return trailingEmpty < MIN_CARRIED_EDGE_EMPTIES ? doc.childCount - trailingEmpty : doc.childCount;
}
/**
diff --git a/packages/app/src/editor/mode-switch-position-resolver.test.ts b/packages/app/src/editor/mode-switch-position-resolver.test.ts
index ad9cffadb..84c3769b5 100644
--- a/packages/app/src/editor/mode-switch-position-resolver.test.ts
+++ b/packages/app/src/editor/mode-switch-position-resolver.test.ts
@@ -197,6 +197,44 @@ describe('count tripwire', () => {
);
expect(affordanceDoc.childCount === mdastCount).toBe(false);
});
+
+ test('a source-carried trailing blank run stays in the comparable count', () => {
+ // The affordance allowance and a doc-edge blank run put the same shape at
+ // the end of the doc, so subtracting unconditionally would under-count
+ // every document the user ended with two or more blank lines — and the
+ // tripwire failure is silent, so nothing would report it.
+ const carried = '# A\n\nbody\n\n\n';
+ const doc = snap(carried).doc;
+ const blocks = computeSourceBlocks(carried, md).blocks;
+
+ expect(blocks.length).toBe(4);
+ expect(comparableChildCount(doc)).toBe(blocks.length);
+
+ // The affordance twin, for contrast: same trailing shape, one paragraph
+ // shorter, no source counterpart, so it does come off the count.
+ const affordanceOnly = '## H\n';
+ const headingJson = md.parse(affordanceOnly) as { type: string; content: unknown[] };
+ const affordanceDoc = schema.nodeFromJSON({
+ ...headingJson,
+ content: [...headingJson.content, { type: 'paragraph' }],
+ });
+ expect(affordanceDoc.childCount).toBe(2);
+ expect(comparableChildCount(affordanceDoc)).toBe(
+ computeSourceBlocks(affordanceOnly, md).blocks.length,
+ );
+ });
+
+ test('a leading blank run is never subtracted, at or below the floor', () => {
+ // Only the TAIL carries an artifact the source does not spell. Every
+ // leading empty paragraph came from source bytes, so reasoning by symmetry
+ // and subtracting a sub-floor head run would under-count and disable
+ // decorations permanently on documents that open with blank lines.
+ for (const source of ['\n\n# A\n\nbody\n', '\n\n\n# A\n\nbody\n']) {
+ const doc = snap(source).doc;
+ expect(comparableChildCount(doc)).toBe(computeSourceBlocks(source, md).blocks.length);
+ expect(comparableChildCount(doc)).toBe(doc.childCount);
+ }
+ });
});
describe('content-equality gating', () => {
diff --git a/packages/app/tests/integration/doc-edge-blank-runs.test.ts b/packages/app/tests/integration/doc-edge-blank-runs.test.ts
new file mode 100644
index 000000000..461806354
--- /dev/null
+++ b/packages/app/tests/integration/doc-edge-blank-runs.test.ts
@@ -0,0 +1,332 @@
+/**
+ * Blank runs the user authors at the LEADING or TRAILING edge of a document.
+ *
+ * The interior case has its own suite. This one covers the two ends, where the
+ * gesture is identical from the user's side — hold Enter at the top or the
+ * bottom of the page — but the bytes take a different route to `Y.Text` and to
+ * disk, and never arrive.
+ *
+ * Multi-client on purpose: the server is the sole fragment writer, so a run
+ * that is only ever held in the authoring client's own fragment looks correct
+ * to that client for as long as nothing re-derives. Two real WebSocket peers
+ * make "reached the source bytes" and "survived a re-derive" separable, which
+ * single-client coverage cannot do.
+ *
+ * Every assertion is raw-byte. The bridge comparator collapses three-or-more
+ * newlines on both sides, so a `toContain` or a `startsWith` would sit inside
+ * the tolerance and pass on a document that lost the run it is about.
+ */
+
+import { setTimeout as wait } from 'node:timers/promises';
+import { afterAll, beforeAll, describe, expect, test } from 'vitest';
+import * as Y from 'yjs';
+import { HARNESS_BOOT_TIMEOUT_MS } from './harness-boot-timeout';
+import {
+ agentWriteMd,
+ createTestClients,
+ createTestServer,
+ pollUntil,
+ readTestDoc,
+ serializeFragment,
+ type TestClient,
+ type TestServer,
+} from './test-harness';
+
+let server: TestServer;
+
+beforeAll(async () => {
+ server = await createTestServer();
+}, HARNESS_BOOT_TIMEOUT_MS);
+
+afterAll(async () => {
+ await server.cleanup();
+});
+
+/** What pressing Enter mints in the WYSIWYG: a top-level empty paragraph. */
+const para = () => new Y.XmlElement('paragraph');
+
+/** Top-level children of a fragment that hold no text — the blank lines. */
+function countBlankLineNodes(fragment: Y.XmlFragment): number {
+ let count = 0;
+ for (let i = 0; i < fragment.length; i++) {
+ if (String(fragment.get(i)) === '<paragraph></paragraph>') count += 1;
+ }
+ return count;
+}
+
+/**
+ * Wait for the settled state, returning quietly if it never arrives.
+ *
+ * The `expect` that follows is the oracle. A throwing poll would report a
+ * document that never converged as a harness error and hide which bytes are
+ * actually wrong, so the budget is spent here and the diagnosis is left to the
+ * byte comparison.
+ */
+async function settle(predicate: () => boolean, timeoutMs: number): Promise<void> {
+ const deadline = Date.now() + timeoutMs;
+ while (Date.now() < deadline) {
+ if (predicate()) return;
+ await wait(25);
+ }
+}
+
+/**
+ * A converged two-peer document holding exactly `raw`, seeded through the
+ * agent byte-writer. Seeds carry no edge run of their own, so this poll
+ * settles identically before and after the fix.
+ */
+async function seedDocument(raw: string, fragmentBody = raw): Promise<TestClient[]> {
+ const docName = `doc-edge-${crypto.randomUUID()}`;
+ const clients = await createTestClients(server.port, {
+ count: 2,
+ docName,
+ // These tests drive the exact divergence the watcher reports, and its
+ // throw would pre-empt the byte assertions that say what diverged.
+ perClientOptions: { skipInvariantWatcher: true },
+ });
+ await agentWriteMd(server.port, raw, { docName, position: 'replace' });
+ await pollUntil(
+ () =>
+ clients.every(
+ (c) => c.ytext.toString() === raw && serializeFragment(c.fragment) === fragmentBody,
+ ),
+ 10_000,
+ );
+ return clients;
+}
+
+/** The three surfaces that have to agree, checked byte for byte on every peer. */
+async function expectEverywhereExactly(
+ clients: TestClient[],
+ expected: string,
+ blankNodes: number,
+ diskTimeoutMs: number,
+ expectedFragment = expected,
+): Promise<void> {
+ for (const c of clients) {
+ expect(c.ytext.toString()).toBe(expected);
+ expect(serializeFragment(c.fragment)).toBe(expectedFragment);
+ expect(countBlankLineNodes(c.fragment)).toBe(blankNodes);
+ }
+ // Persistence is debounced, so the bytes land after convergence.
+ const docName = clients[0].docName;
+ await settle(() => readTestDoc(server.contentDir, docName) === expected, diskTimeoutMs);
+ expect(readTestDoc(server.contentDir, docName)).toBe(expected);
+}
+
+describe('doc-edge blank runs on the CRDT path', () => {
+ test('a trailing blank run authored in the WYSIWYG reaches the source bytes', async () => {
+ const clients = await seedDocument('Above.\n\nBelow.\n');
+ try {
+ const a = clients[0];
+ // Enter pressed twice at the very end of the document.
+ a.doc.transact(() => {
+ a.fragment.insert(a.fragment.length, [para(), para()]);
+ });
+
+ const expected = 'Above.\n\nBelow.\n\n\n';
+ await settle(() => clients.every((c) => c.ytext.toString() === expected), 6000);
+ await expectEverywhereExactly(clients, expected, 2, 10_000);
+ } finally {
+ for (const c of clients) await c.cleanup();
+ }
+ });
+
+ test('a trailing blank run on a frontmatter document reaches the source bytes', async () => {
+ // The other branch of the merge byte-space: an FM-bearing document keeps a
+ // single newline in the boundary slot rather than stripping it, and the
+ // body the carrier sees starts one newline in. Every other case here is
+ // FM-less, so without this one the composition that every real document
+ // has is only covered at unit level.
+ const clients = await seedDocument(
+ '---\ntitle: Edge\n---\n\nAbove.\n\nBelow.\n',
+ 'Above.\n\nBelow.\n',
+ );
+ try {
+ const a = clients[0];
+ a.doc.transact(() => {
+ a.fragment.insert(a.fragment.length, [para(), para()]);
+ });
+
+ const expected = '---\ntitle: Edge\n---\n\nAbove.\n\nBelow.\n\n\n';
+ await settle(() => clients.every((c) => c.ytext.toString() === expected), 6000);
+ await expectEverywhereExactly(clients, expected, 2, 10_000, 'Above.\n\nBelow.\n\n\n');
+ } finally {
+ for (const c of clients) await c.cleanup();
+ }
+ });
+
+ test('a leading blank run authored in the WYSIWYG reaches the source bytes', async () => {
+ const clients = await seedDocument('Above.\n\nBelow.\n');
+ try {
+ const a = clients[0];
+ // Enter pressed twice at the very start of the document.
+ a.doc.transact(() => {
+ a.fragment.insert(0, [para(), para()]);
+ });
+
+ // The head carries no preceding block, so two empty paragraphs are two
+ // newlines here where the same two at the tail would be three.
+ const expected = '\n\nAbove.\n\nBelow.\n';
+ await settle(() => clients.every((c) => c.ytext.toString() === expected), 6000);
+ await expectEverywhereExactly(clients, expected, 2, 10_000);
+ } finally {
+ for (const c of clients) await c.cleanup();
+ }
+ });
+
+ test('a trailing blank run on a single-block document reaches the source bytes', async () => {
+ // One block means the trailing run is also the document's only gap, so
+ // there is no interior run for a positional comparison to fall back on.
+ const clients = await seedDocument('Hello.\n');
+ try {
+ const a = clients[0];
+ a.doc.transact(() => {
+ a.fragment.insert(a.fragment.length, [para(), para(), para()]);
+ });
+
+ const expected = 'Hello.\n\n\n\n';
+ await settle(() => clients.every((c) => c.ytext.toString() === expected), 6000);
+ await expectEverywhereExactly(clients, expected, 3, 10_000);
+ } finally {
+ for (const c of clients) await c.cleanup();
+ }
+ });
+
+ test('a source-mode edit beside a trailing blank run does not destroy it', async () => {
+ const clients = await seedDocument('Above.\n\nBelow.\n');
+ try {
+ const a = clients[0];
+ const b = clients[1];
+ a.doc.transact(() => {
+ a.fragment.insert(a.fragment.length, [para(), para()]);
+ });
+ await settle(() => a.ytext.toString() === 'Above.\n\nBelow.\n\n\n', 4000);
+
+ // The other peer in source mode: a direct Y.Text write, which is exactly
+ // what CodeMirror does on a keystroke.
+ b.doc.transact(() => {
+ b.ytext.insert(0, 'X');
+ });
+
+ const expected = 'XAbove.\n\nBelow.\n\n\n';
+ await settle(() => clients.every((c) => c.ytext.toString() === expected), 6000);
+ await expectEverywhereExactly(clients, expected, 2, 8000);
+ } finally {
+ for (const c of clients) await c.cleanup();
+ }
+ });
+
+ test('a fragment-only blank run survives the three-way merge seam at either position', async () => {
+ // The seam the cases above never reach. Both surfaces have to diverge
+ // inside ONE settlement window for the merge to run at all, and the merge
+ // projects its three inputs into a byte space that strips the doc-start
+ // run from frontmatter-less documents and re-attaches it from the Y.Text
+ // side. That projection was written while the fragment structurally could
+ // not hold a doc-start run, so a run only the fragment has is precisely
+ // what it was never asked to carry, and it was dropped here.
+ //
+ // The concurrent Y.Text keystroke is scaffolding to force that window, and
+ // it does not reach the fragment under this gesture. That is a property of
+ // the gesture, not of the blank run: the interior arm is the identical
+ // transaction one index over, and it strands the keystroke exactly the same
+ // way while its own run survives. Both arms assert every surface byte-exact
+ // so a change to either behaviour fails loudly instead of passing over a
+ // lost run.
+ async function mergeSeam(insertAt: number, ytext: string, fragment: string): Promise<void> {
+ const clients = await seedDocument('Above.\n\nBelow.\n');
+ try {
+ const a = clients[0];
+ a.doc.transact(() => {
+ a.fragment.insert(insertAt, [para(), para()]);
+ a.ytext.insert(a.ytext.toString().length - 1, '!');
+ });
+
+ await settle(
+ () =>
+ clients.every(
+ (c) => c.ytext.toString() === ytext && serializeFragment(c.fragment) === fragment,
+ ),
+ 8000,
+ );
+ for (const c of clients) {
+ expect(c.ytext.toString()).toBe(ytext);
+ expect(serializeFragment(c.fragment)).toBe(fragment);
+ expect(countBlankLineNodes(c.fragment)).toBe(2);
+ }
+ const docName = clients[0].docName;
+ await settle(() => readTestDoc(server.contentDir, docName) === ytext, 8000);
+ expect(readTestDoc(server.contentDir, docName)).toBe(ytext);
+ } finally {
+ for (const c of clients) await c.cleanup();
+ }
+ }
+
+ await mergeSeam(0, '\n\nAbove.\n\nBelow.!\n', '\n\nAbove.\n\nBelow.\n');
+ await mergeSeam(1, 'Above.\n\n\n\nBelow.!\n', 'Above.\n\n\n\nBelow.\n');
+ });
+
+ test('an external write that carries a trailing run lands it in every fragment', async () => {
+ // The reverse direction. The agent byte-writer puts the run into the
+ // source bytes verbatim, so the question is whether the fragment every
+ // peer renders from the following derive still has it — a run the
+ // fragment cannot carry retracts here and takes the disk copy with it on
+ // the next persistence pass.
+ const clients = await seedDocument('Alpha.\n\nOmega.\n');
+ try {
+ const a = clients[0];
+ a.doc.transact(() => {
+ a.fragment.insert(a.fragment.length, [para(), para()]);
+ });
+ await settle(() => a.ytext.toString() === 'Alpha.\n\nOmega.\n\n\n', 4000);
+
+ const expected = 'Alpha edited.\n\nOmega.\n\n\n';
+ await agentWriteMd(server.port, expected, { docName: a.docName, position: 'replace' });
+
+ await settle(() => clients.every((c) => c.ytext.toString() === expected), 6000);
+ await expectEverywhereExactly(clients, expected, 2, 8000);
+ } finally {
+ for (const c of clients) await c.cleanup();
+ }
+ });
+
+ test('a text edit and a trailing run in one transaction both reach the source bytes', async () => {
+ // Both halves land in a single transaction, so the two sides cannot
+ // normalize-compare equal and no already-in-sync shortcut is reachable.
+ // Whatever carries the text change to Y.Text has to carry the run with it.
+ const clients = await seedDocument('Above.\n\nBelow.\n');
+ try {
+ const a = clients[0];
+ a.doc.transact(() => {
+ const first = a.fragment.get(0) as { get(i: number): unknown };
+ (first.get(0) as { insert(i: number, s: string): void }).insert(0, 'Z');
+ a.fragment.insert(a.fragment.length, [para(), para()]);
+ });
+
+ const expected = 'ZAbove.\n\nBelow.\n\n\n';
+ await settle(() => clients.every((c) => c.ytext.toString() === expected), 6000);
+ await expectEverywhereExactly(clients, expected, 2, 8000);
+ } finally {
+ for (const c of clients) await c.cleanup();
+ }
+ });
+
+ test('CONTROL: an interior blank run still reaches the source bytes unchanged', async () => {
+ // Colocated deliberately. The interior path already works, and the edge
+ // rule is a change to the same carrier, so a regression there would
+ // otherwise only surface in a different file and a different run.
+ const clients = await seedDocument('Above.\n\nBelow.\n');
+ try {
+ const a = clients[0];
+ a.doc.transact(() => {
+ a.fragment.insert(1, [para(), para()]);
+ });
+
+ const expected = 'Above.\n\n\n\nBelow.\n';
+ await settle(() => clients.every((c) => c.ytext.toString() === expected), 6000);
+ await expectEverywhereExactly(clients, expected, 2, 10_000);
+ } finally {
+ for (const c of clients) await c.cleanup();
+ }
+ });
+});
diff --git a/packages/app/tests/integration/interior-blank-runs.test.ts b/packages/app/tests/integration/interior-blank-runs.test.ts
index dd634c343..9bbaf68b4 100644
--- a/packages/app/tests/integration/interior-blank-runs.test.ts
+++ b/packages/app/tests/integration/interior-blank-runs.test.ts
@@ -153,10 +153,12 @@ describe('interior blank runs on the CRDT path', () => {
});
test('a trailing blank run in the file does not hide a new interior one', async () => {
- // The fragment cannot carry doc-edge runs, so the source and the fragment
- // disagree about them permanently. A comparison that counted them would
- // read the two sides as incomparable on this document and never see the
- // interior addition at all.
+ // A doc-edge run and an interior one are separate dimensions: folding them
+ // into a single positional comparison would read the two sides as
+ // incomparable on this document and never see the interior addition at
+ // all. The trailing run in the fixture is not scenery — it has to still be
+ // there afterwards, which is why every assertion below is raw-byte and the
+ // node count covers both ends.
const docName = `blank-run-edge-${crypto.randomUUID()}`;
const raw = 'Above.\n\nBelow.\n\n\n';
const clients = await createTestClients(server.port, {
@@ -174,19 +176,22 @@ describe('interior blank runs on the CRDT path', () => {
a.fragment.insert(1, [new Y.XmlElement('paragraph'), new Y.XmlElement('paragraph')]);
});
- await pollUntil(
- () => clients.every((c) => c.ytext.toString().startsWith('Above.\n\n\n\nBelow.')),
- 5000,
+ // Two empty paragraphs for the interior gap, plus the two the trailing
+ // run of three newlines is carried by.
+ const expected = 'Above.\n\n\n\nBelow.\n\n\n';
+ await pollUntil(() => clients.every((c) => c.ytext.toString() === expected), 5000).catch(
+ () => {},
);
await wait(500);
for (const c of clients) {
- expect(c.ytext.toString()).toContain('Above.\n\n\n\nBelow.');
- expect(countBlankLineNodes(c.fragment)).toBe(2);
+ expect(c.ytext.toString()).toBe(expected);
+ expect(serializeFragment(c.fragment)).toBe(expected);
+ expect(countBlankLineNodes(c.fragment)).toBe(4);
}
- await pollUntil(
- () => readTestDoc(server.contentDir, docName).startsWith('Above.\n\n\n\nBelow.'),
- 10_000,
+ await pollUntil(() => readTestDoc(server.contentDir, docName) === expected, 10_000).catch(
+ () => {},
);
+ expect(readTestDoc(server.contentDir, docName)).toBe(expected);
} finally {
for (const c of clients) await c.cleanup();
}
diff --git a/packages/core/src/bridge/doc-boundary-space.ts b/packages/core/src/bridge/doc-boundary-space.ts
index 8890f6739..a1dff381b 100644
--- a/packages/core/src/bridge/doc-boundary-space.ts
+++ b/packages/core/src/bridge/doc-boundary-space.ts
@@ -1,13 +1,35 @@
import { FRONTMATTER_RE, stripFrontmatter } from '../extensions/frontmatter.ts';
+import { carriedEdgeEmpties } from '../markdown/doc-edge-blank-runs.ts';
const LEADING_BOUNDARY_RE = /^(?:\r?\n)+/;
+function fragmentCarriesDocStartRun(fragmentBody: string): boolean {
+ return carriedEdgeEmpties(fragmentBody).leading > 0;
+}
+
+export interface MergeBoundarySpace {
+ project(text: string): string;
+ unproject(merged: string, raw: string): string;
+}
+
+export function createMergeBoundarySpace(fragmentBody: string): MergeBoundarySpace {
+ const carriesDocStartRun = fragmentCarriesDocStartRun(fragmentBody);
+ return {
+ project: (text) => projectMergeBoundarySpace(text, carriesDocStartRun),
+ unproject: (merged, raw) => unprojectMergeBoundarySpace(merged, raw, carriesDocStartRun),
+ };
+}
+
export interface DocBoundarySplit {
boundary: string;
text: string;
}
-export function splitLeadingDocBoundary(text: string): DocBoundarySplit {
+export function splitLeadingDocBoundary(
+ text: string,
+ carriesDocStartRun: boolean,
+): DocBoundarySplit {
+ if (carriesDocStartRun) return { boundary: '', text };
const { frontmatter, body } = stripFrontmatter(text);
const match = body.match(LEADING_BOUNDARY_RE);
if (!match) return { boundary: '', text };
@@ -24,8 +46,21 @@ export function reattachLeadingDocBoundary(text: string, boundary: string): stri
return frontmatter + boundary + body;
}
-export function projectMergeBoundarySpace(text: string): string {
- const stripped = splitLeadingDocBoundary(text).text;
+export function projectMergeBoundarySpace(text: string, carriesDocStartRun: boolean): string {
+ if (carriesDocStartRun) return text;
+ const stripped = splitLeadingDocBoundary(text, carriesDocStartRun).text;
if (stripFrontmatter(stripped).frontmatter === '') return stripped;
return reattachLeadingDocBoundary(stripped, '\n');
}
+
+export function unprojectMergeBoundarySpace(
+ merged: string,
+ raw: string,
+ carriesDocStartRun: boolean,
+): string {
+ if (carriesDocStartRun) return merged;
+ return reattachLeadingDocBoundary(
+ splitLeadingDocBoundary(merged, carriesDocStartRun).text,
+ splitLeadingDocBoundary(raw, carriesDocStartRun).boundary,
+ );
+}
diff --git a/packages/core/src/bridge/index.ts b/packages/core/src/bridge/index.ts
index fe65c0547..84b7b4604 100644
--- a/packages/core/src/bridge/index.ts
+++ b/packages/core/src/bridge/index.ts
@@ -31,10 +31,8 @@ export {
} from './compose.ts';
export { type DiffChange, diffLinesFast } from './diff-lines.ts';
export {
- type DocBoundarySplit,
- projectMergeBoundarySpace,
- reattachLeadingDocBoundary,
- splitLeadingDocBoundary,
+ createMergeBoundarySpace,
+ type MergeBoundarySpace,
} from './doc-boundary-space.ts';
export {
applyPatchToFm,
diff --git a/packages/core/src/bridge/normalize.ts b/packages/core/src/bridge/normalize.ts
index da41900cd..650461462 100644
--- a/packages/core/src/bridge/normalize.ts
+++ b/packages/core/src/bridge/normalize.ts
@@ -1,3 +1,5 @@
+import { carriedEdgeEmpties } from '../markdown/doc-edge-blank-runs.ts';
+
const COMMONMARK_ESCAPE_RE = /\\([!"#$%&'()*+,\-./:;<=>?@[\]^_`{|}~])/g;
const TABLE_ALIGN_ROW_RE = /^\s*\|?\s*:?-+:?\s*(\|\s*:?-+:?\s*)+\|?\s*$/;
@@ -228,6 +230,10 @@ function interiorBlankRunLengths(s: string): number[] {
}
export function addsBlankLines(baseline: string, candidate: string): boolean {
+ const beforeEdges = carriedEdgeEmpties(baseline);
+ const afterEdges = carriedEdgeEmpties(candidate);
+ if (afterEdges.leading > beforeEdges.leading) return true;
+ if (afterEdges.trailing > beforeEdges.trailing) return true;
const before = interiorBlankRunLengths(baseline);
const after = interiorBlankRunLengths(candidate);
if (before.length !== after.length) return false;
diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts
index f9998e9ab..7afa43d2e 100644
--- a/packages/core/src/index.ts
+++ b/packages/core/src/index.ts
@@ -534,6 +534,7 @@ export {
withSkillPointer,
} from './handoff/index.ts';
export { selectFenceChar, widenFenceLength } from './markdown/code-fence.ts';
+export { MIN_CARRIED_EDGE_EMPTIES } from './markdown/doc-edge-blank-runs.ts';
export {
HTML_MAX_BYTES,
HtmlPayloadTooLargeError,
@@ -1392,9 +1393,9 @@ export {
compareRoundTripStructural,
composeWithDerivedBody,
composeWithDerivedFrontmatter,
+ createMergeBoundarySpace,
createStructuralFreshnessChecker,
type DiffChange,
- type DocBoundarySplit,
DUPLICATION_GATE_MIN_LINE_LENGTH,
defaultScheduler,
detectAppliedToleranceClasses,
@@ -1423,6 +1424,7 @@ export {
type InvariantViolation,
isParseEquivalentBridge,
MAX_FM_REGION_BYTES,
+ type MergeBoundarySpace,
mergeThreeWay,
normalizeBridge,
overMultipliedBodyLines,
@@ -1432,11 +1434,9 @@ export {
parseFencedFmRegion,
parseFmRegion,
pendingContentLines,
- projectMergeBoundarySpace,
readFmKeys,
readFmMap,
readFmRegionWithError,
- reattachLeadingDocBoundary,
type Scheduler,
type StructuralDegradeLabel,
type StructuralDivergenceReason,
@@ -1444,7 +1444,6 @@ export {
type StructuralFreshnessChecker,
type StructuralFreshnessCheckerOptions,
setToleranceTelemetryHook,
- splitLeadingDocBoundary,
structuralDivergence,
type ToleranceClassSeverity,
type ToleranceFireRecord,
diff --git a/packages/core/src/markdown/doc-edge-blank-runs.ts b/packages/core/src/markdown/doc-edge-blank-runs.ts
new file mode 100644
index 000000000..22b26e8b4
--- /dev/null
+++ b/packages/core/src/markdown/doc-edge-blank-runs.ts
@@ -0,0 +1,72 @@
+import type { Root as MdastRoot, Paragraph } from 'mdast';
+import type { SourceDocBoundary } from './mdast-augmentation.ts';
+
+export const MIN_CARRIED_EDGE_EMPTIES = 2;
+
+export interface DocEdgeEmpties {
+ leading: number;
+ trailing: number;
+}
+
+function applyFloor(empties: number): number {
+ return empties >= MIN_CARRIED_EDGE_EMPTIES ? empties : 0;
+}
+
+export function carriedEdgeEmpties(source: string): DocEdgeEmpties {
+ const afterBom = source.charCodeAt(0) === 0xfeff ? source.slice(1) : source;
+ const afterHead = afterBom.replace(/^\n+/, '');
+ if (afterHead.replace(/\n+$/, '').length === 0) return { leading: 0, trailing: 0 };
+ return {
+ leading: applyFloor(afterBom.length - afterHead.length),
+ trailing: applyFloor(afterHead.length - afterHead.replace(/\n+$/, '').length - 1),
+ };
+}
+
+function blankLineParagraph(line: number, offset: number): Paragraph {
+ const point = { line, column: 1, offset };
+ return { type: 'paragraph', children: [], position: { start: point, end: { ...point } } };
+}
+
+export function materializeDocEdgeBlankRuns(
+ root: MdastRoot,
+ boundary: SourceDocBoundary,
+): SourceDocBoundary | undefined {
+ const children = root.children;
+ const head =
+ children.length === 0 || boundary.leading === undefined
+ ? 0
+ : applyFloor(boundary.leading.length);
+ const tail =
+ children.length === 0 || boundary.trailing === undefined
+ ? 0
+ : applyFloor(boundary.trailing.length - 1);
+ if (head === 0 && tail === 0) return boundary;
+
+ const lastEnd = children[children.length - 1]?.position?.end;
+ const headParagraphs: Paragraph[] = [];
+ const tailParagraphs: Paragraph[] = [];
+
+ if (head > 0) {
+ for (let i = 0; i < head; i++) headParagraphs.push(blankLineParagraph(1 + i, i));
+ }
+ if (tail > 0 && typeof lastEnd?.offset === 'number' && typeof lastEnd.line === 'number') {
+ for (let i = 0; i < tail; i++) {
+ tailParagraphs.push(blankLineParagraph(lastEnd.line + 1 + i, lastEnd.offset + 1 + i));
+ }
+ }
+ if (headParagraphs.length === 0 && tailParagraphs.length === 0) return boundary;
+
+ root.children = [...headParagraphs, ...children, ...tailParagraphs];
+ if (headParagraphs.length > 0) delete boundary.leading;
+ if (tailParagraphs.length > 0) delete boundary.trailing;
+
+ if (
+ boundary.bom !== undefined ||
+ boundary.leading !== undefined ||
+ boundary.trailing !== undefined
+ ) {
+ return boundary;
+ }
+ if (root.data !== undefined) delete root.data.sourceDocBoundary;
+ return undefined;
+}
diff --git a/packages/core/src/markdown/parse-with-fallback.ts b/packages/core/src/markdown/parse-with-fallback.ts
index 9864b3b60..d988f4b8b 100644
--- a/packages/core/src/markdown/parse-with-fallback.ts
+++ b/packages/core/src/markdown/parse-with-fallback.ts
@@ -121,9 +121,9 @@ export function parseRecursive(
try {
const region = findFallbackRegion(source, offset);
- const beforeSrc = source.slice(0, region.start);
const brokenSrc = source.slice(region.start, region.end);
- const afterSrc = source.slice(region.end);
+ const beforeSrc = source.slice(0, region.start).replace(/\n+$/, '');
+ const afterSrc = source.slice(region.end).replace(/^\n+/, '');
const beforeDoc = beforeSrc.trim()
? parseRecursive(beforeSrc, parse, depth + 1, budget)
diff --git a/packages/core/src/markdown/pipeline.ts b/packages/core/src/markdown/pipeline.ts
index 4426ab46e..449e6f12f 100644
--- a/packages/core/src/markdown/pipeline.ts
+++ b/packages/core/src/markdown/pipeline.ts
@@ -23,6 +23,7 @@ import { commentPromoterPlugin } from './comment-promoter.ts';
import { dedentBlockJsxClose } from './dedent-block-jsx-close.ts';
import { detailsAccordionPromoterPlugin } from './details-accordion-promoter.ts';
import { divAlignPromoterPlugin } from './div-align-promoter.ts';
+import { materializeDocEdgeBlankRuns } from './doc-edge-blank-runs.ts';
import { emptyTaskItemUnmintPlugin, mintEmptyTaskItemContent } from './empty-task-item.ts';
import { encodeEntityRefs, restoreEntityRefsPlugin } from './entity-ref-guard.ts';
import { highlightPromoterPlugin } from './highlight-promoter.ts';
@@ -218,7 +219,8 @@ export function parseMd(rawSource: string, processor: Processor): PmNode {
file.value = source;
const transformed = processor.runSync(tree, file) as MdastRoot;
insertInteriorBlankRunParagraphs(transformed, source);
- const boundary = captureDocBoundary(transformed, source, hadBom);
+ const captured = captureDocBoundary(transformed, source, hadBom);
+ const boundary = captured ? materializeDocEdgeBlankRuns(transformed, captured) : undefined;
const doc = (processor as unknown as { stringify(tree: unknown): PmNode }).stringify(transformed);
if (!boundary) return doc;
return doc.type.create({ ...doc.attrs, sourceDocBoundary: boundary }, doc.content, doc.marks);
@@ -245,7 +247,8 @@ function parseToMdast(
file.value = source;
const transformed = processor.runSync(tree, file) as MdastRoot;
if (materializeBlankRuns) insertInteriorBlankRunParagraphs(transformed, source);
- captureDocBoundary(transformed, source, hadBom);
+ const captured = captureDocBoundary(transformed, source, hadBom);
+ if (materializeBlankRuns && captured) materializeDocEdgeBlankRuns(transformed, captured);
return transformed;
}
diff --git a/packages/server/src/server-observers-paired-write-baseline.test.ts b/packages/server/src/server-observers-paired-write-baseline.test.ts
index aff7aa8f9..766875262 100644
--- a/packages/server/src/server-observers-paired-write-baseline.test.ts
+++ b/packages/server/src/server-observers-paired-write-baseline.test.ts
@@ -2,8 +2,11 @@
* Regression: Observer A's paired-write callback must refresh the raw Y.Text
* witness from `ytext.toString()` (raw bytes), NOT `serialize(fragment)`
* (canonical bytes). Under the Y.Text-is-truth contract the two diverge on any
- * input where parse→serialize normalizes (e.g., a leading "\n\n" delimiter that
- * mdast drops). If the witness is set from `serialize(fragment)`, the next
+ * input where parse→serialize normalizes — here a trailing blank line, which
+ * is worth one empty paragraph and so stays below `MIN_CARRIED_EDGE_EMPTIES`:
+ * it rides the doc-node boundary attribute, which a Y.XmlFragment has no
+ * surface to hold, so the fragment serialization drops it. If the witness is
+ * set from `serialize(fragment)`, the next
* non-paired XmlFragment mutation (a real WYSIWYG keystroke) fails Observer A's
* strict-equality Path A gate (`currentText === lastSyncedYTextBytes`) and forces
* Path B's mergeThreeWay to run on every keystroke. Under stress (large content
@@ -54,20 +57,23 @@ describe('Observer A paired-write baseline — raw ytext, not canonical fragment
// composeBody for 'append' on empty ytext: '' + '\n\n' + payload, so ytext
// ends up with leading \n\n ('\n\n' delimiter is
// unconditional). This mirrors the exact path agent-write-md takes for
- // the first turn.
- const fixturePayload = '## Section 1\n\nLorem ipsum dolor sit amet.\n';
+ // the first turn. The payload closes with a blank line, which is what puts
+ // raw and canonical bytes apart: the leading run is two empty paragraphs
+ // the fragment carries, the trailing one is a single empty paragraph it
+ // cannot, so the fragment serialization comes back a blank line shorter.
+ const fixturePayload = '## Section 1\n\nLorem ipsum dolor sit amet.\n\n';
const composedAppend = `\n\n${fixturePayload}`;
doc.transact(() => {
composeAndWriteRawBody(doc, composedAppend, 'agent');
}, AGENT_WRITE_ORIGIN);
// Confirm the divergence shape that motivates this regression test:
- // ytext keeps the leading \n\n; serialize(fragment) does not.
+ // ytext keeps the trailing blank line; serialize(fragment) does not.
const ytextAfterAgent = ytext.toString();
const fragmentJson = yXmlFragmentToProseMirrorRootNode(xmlFragment, schema).toJSON();
const fragmentSerialized = mdManager.serialize(fragmentJson);
- expect(ytextAfterAgent.startsWith('\n\n')).toBe(true);
- expect(fragmentSerialized.startsWith('\n\n')).toBe(false);
+ expect(ytextAfterAgent.endsWith('amet.\n\n')).toBe(true);
+ expect(fragmentSerialized.endsWith('amet.\n\n')).toBe(false);
expect(ytextAfterAgent === fragmentSerialized).toBe(false);
const pathBFiresBefore = getMetrics().observerAPathBFires;
diff --git a/packages/server/src/server-observers.path-b-doc-boundary.test.ts b/packages/server/src/server-observers.path-b-doc-boundary.test.ts
index bdcccf097..766d8df9b 100644
--- a/packages/server/src/server-observers.path-b-doc-boundary.test.ts
+++ b/packages/server/src/server-observers.path-b-doc-boundary.test.ts
@@ -33,11 +33,10 @@
import {
BridgeMergeContentLossError,
+ createMergeBoundarySpace,
MarkdownManager,
prependFrontmatter,
- reattachLeadingDocBoundary,
sharedExtensions,
- splitLeadingDocBoundary,
stripFrontmatter,
} from '@inkeep/open-knowledge-core';
import { getSchema } from '@tiptap/core';
@@ -282,9 +281,12 @@ describe('in-sync residual merge (sibling router branch) — byte-preservation g
* counterpart of the divergent-fallback harness's `mdManager` proxy), with a
* crafted `info.result` in MERGE byte-space (boundary normalized to one `\n`),
* exactly what `mergeThreeWayImpl` returns before the call site re-attaches the
- * boundary. The fixture's doc boundary is an NG-class multi-blank run, so the
- * merge-space single-`\n` form and the re-attached form differ observably — a
- * regression that applied raw `info.result` would collapse the run.
+ * boundary. The fixture's doc boundary is an NG-class EMPTY slot — no blank
+ * line at all between the close fence and the body — so the merge-space
+ * single-`\n` form and the re-attached form differ observably; a regression
+ * that applied raw `info.result` would leave a blank line the user never
+ * authored. The sibling case pins the other branch, where the run is
+ * node-carried and the two byte-spaces coincide.
*/
describe('Path B doc-boundary alignment: content-loss recovery arm re-projects the boundary', () => {
const ENV_KEYS = ['NODE_ENV', 'OK_RETHROW_BRIDGE_LOSS'] as const;
@@ -307,11 +309,17 @@ describe('Path B doc-boundary alignment: content-loss recovery arm re-projects t
});
const FM2 = '---\ntitle: Boundary recovery\n---\n';
- // NG-class doc boundary: a multi-blank run ('\n\n') between the FM close
- // fence and the body. The merge space normalizes the run to one '\n', so the
- // boundary re-attached from the current Y.Text differs observably from the
- // merge-space as-computed bytes.
- const RAW_NG = `${FM2}\n\nFirst paragraph body.\n\nSecond paragraph stays.\n`;
+ // NG-class doc boundary: the body starts on the line after the close fence
+ // with no blank separator at all. The merge space always spells that slot as
+ // one '\n', so the boundary re-attached from the current Y.Text (nothing)
+ // differs observably from the merge-space as-computed bytes. A multi-blank
+ // run cannot serve as the vehicle: from MIN_CARRIED_EDGE_EMPTIES up the run
+ // is node-carried, both byte-spaces spell it identically, and the
+ // re-projection is the identity — the sibling case below pins that branch.
+ const RAW_NG = `${FM2}First paragraph body.\n\nSecond paragraph stays.\n`;
+ // What the fragment serializes to for these arms: no doc-start run, so the
+ // boundary slot is an asymmetry the projection owns.
+ const FRAGMENT_WITHOUT_RUN = 'First paragraph body.\n\nSecond paragraph stays.\n';
test('applies the boundary-reattached as-computed bytes, not raw info.result', () => {
__resetBridgeWatchdogForTests();
@@ -367,19 +375,76 @@ describe('Path B doc-boundary alignment: content-loss recovery arm re-projects t
expect(getMetrics().bridgeMergeContentLoss).toBe(1);
const finalText = ytext.toString();
- // The current Y.Text's multi-blank boundary is re-attached verbatim — the
+ // The current Y.Text's empty boundary slot is restored verbatim — the
// recovery applied projectMerged(info.result), not the merge-space bytes.
- expect(finalText).toContain(`${FM2}\n\n`);
+ expect(finalText).toContain(`${FM2}Z`);
// Exactly the as-computed bytes with the boundary re-projected from the
// current Y.Text. A regression applying raw `info.result` would land the
// merge-space single-'\n' boundary here instead.
- const boundary = splitLeadingDocBoundary(RAW_NG).boundary;
- const expected = reattachLeadingDocBoundary(
- splitLeadingDocBoundary(asComputedMergeSpace).text,
- boundary,
+ // Expected through the production entry point, so this asserts the
+ // recovery matches what the seam itself computes rather than re-deriving
+ // the projection with a hand-supplied carrying answer.
+ const expected = createMergeBoundarySpace(FRAGMENT_WITHOUT_RUN).unproject(
+ asComputedMergeSpace,
+ RAW_NG,
);
expect(finalText).toBe(expected);
- expect(finalText).toBe(`${FM2}\n\nZFirst paragraph body. \n\nSecond paragraph stays.\n`);
+ expect(finalText).toBe(`${FM2}ZFirst paragraph body. \n\nSecond paragraph stays.\n`);
+
+ cleanup();
+ });
+
+ // The other branch of the same re-projection. At MIN_CARRIED_EDGE_EMPTIES the
+ // doc-start run is node-carried, so both merge inputs already spell it and
+ // merge space IS raw space at the boundary — the crafted `info.result` below
+ // therefore carries the run, as a real merge over these inputs would. The
+ // re-projection must leave it alone: stripping it would delete bytes the
+ // merge kept, and re-attaching over it would invent a blank line.
+ test('a node-carried doc-start run survives the recovery arm verbatim', () => {
+ __resetBridgeWatchdogForTests();
+ resetMetrics();
+
+ const rawCarried = `${FM2}\n\nFirst paragraph body.\n\nSecond paragraph stays.\n`;
+ const asComputedMergeSpace = `${FM2}\n\nZFirst paragraph body. \n\nSecond paragraph stays.\n`;
+ const throwingMerge = (): never => {
+ throw new BridgeMergeContentLossError({
+ baseline: rawCarried,
+ userText: asComputedMergeSpace,
+ agentText: rawCarried,
+ result: asComputedMergeSpace,
+ lostSubstrings: ['a-dropped-line'],
+ which: 'substring',
+ side: 'user',
+ });
+ };
+
+ const doc = new Y.Doc();
+ const xmlFragment = doc.getXmlFragment('default');
+ const ytext = doc.getText('source');
+ doc.transact(() => {
+ composeAndWriteRawBody(doc, rawCarried, 'file-watcher');
+ }, FILE_WATCHER_ORIGIN);
+ const cleanup = setupServerObservers({
+ doc,
+ xmlFragment,
+ ytext,
+ mdManager,
+ schema,
+ docName: 'pathb-boundary-recovery-carried',
+ mergeThreeWay: throwingMerge,
+ });
+ expect(ytext.toString()).toBe(rawCarried);
+
+ typeIntoSource(
+ doc,
+ ytext,
+ rawCarried.indexOf('First paragraph body.') + 'First paragraph body.'.length,
+ ' ',
+ );
+ typeIntoParagraph(doc, xmlFragment, 'First paragraph');
+
+ expect(getMetrics().bridgeMergeContentLoss).toBe(1);
+ expect(ytext.toString()).toBe(asComputedMergeSpace);
cleanup();
});
diff --git a/packages/server/src/server-observers.ts b/packages/server/src/server-observers.ts
index bad55753c..7ec07e7ea 100644
--- a/packages/server/src/server-observers.ts
+++ b/packages/server/src/server-observers.ts
@@ -36,6 +36,7 @@ import {
BridgeMergeContentLossError,
comparePmStructural,
composeWithDerivedBody,
+ createMergeBoundarySpace,
DUPLICATION_GATE_MIN_LINE_LENGTH,
fnv1aDigest,
fragmentHoldsPendingContent,
@@ -45,9 +46,6 @@ import {
overMultipliedBodyLines,
pendingContentLines,
prependFrontmatter,
- projectMergeBoundarySpace,
- reattachLeadingDocBoundary,
- splitLeadingDocBoundary,
stripFrontmatter,
} from '@inkeep/open-knowledge-core';
import type { Schema } from '@tiptap/pm/model';
@@ -1903,7 +1901,17 @@ export function setupServerObservers(opts: SetupServerObserversOpts): () => void
// tolerated, since that is the container-interior case the fragment
// cannot represent.
if (normCurrent === normMd && !addsBlankLines(currentText, md)) {
- setActiveSpanAttributes({ 'observer.a.path': 'gated-in-sync' });
+ // Why the gate certified, as a two-value enum. Both outcomes stamp the
+ // same path, so a settlement over bytes the tolerance set swallowed is
+ // indistinguishable in traces from one where the sides genuinely
+ // agreed — and the doc-edge class was exactly the second kind reported
+ // as the first. The tolerance is comparison-only and cannot narrow, so
+ // there is no watchdog for a divergence resting inside it; this
+ // attribute is the only runtime signal such a class can have.
+ setActiveSpanAttributes({
+ 'observer.a.path': 'gated-in-sync',
+ 'observer.a.gate_reason': currentText === md ? 'bytes-identical' : 'tolerance-equivalent',
+ });
recordSettledBaselines(md);
return;
}
@@ -2033,18 +2041,21 @@ export function setupServerObservers(opts: SetupServerObserversOpts): () => void
// and fabricate content. Project all three inputs into one merge
// byte-space, merge, and re-attach the current Y.Text's boundary
// bytes verbatim — Y.Text is the only surface that can author them
- // (precedent #38). Both apply paths project through the same closure
- // so they cannot drift.
- const { boundary } = splitLeadingDocBoundary(currentText);
+ // (precedent #38). The space is built from `body` — the fragment's
+ // own serialization, before the frontmatter region is composed onto
+ // it, so the separator newline is not read as part of a carried run
+ // — and every input and the way back out go through that one
+ // answer, so they cannot drift.
+ const boundarySpace = createMergeBoundarySpace(body);
const projectMerged = (merged: string): string =>
- reattachLeadingDocBoundary(splitLeadingDocBoundary(merged).text, boundary);
+ boundarySpace.unproject(merged, currentText);
const mergeThreeWayFn = opts.mergeThreeWay ?? mergeThreeWay;
try {
const mergedText = projectMerged(
mergeThreeWayFn(
- projectMergeBoundarySpace(mergeBase),
- projectMergeBoundarySpace(md),
- projectMergeBoundarySpace(currentText),
+ boundarySpace.project(mergeBase),
+ boundarySpace.project(md),
+ boundarySpace.project(currentText),
),
);
applyFastDiff(ytext, currentText, mergedText);