Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

Preserve blank lines typed in the WYSIWYG (#3004)

Nick Gomez06176758bc95
diff --git a/.changeset/blank-line-preservation.md b/.changeset/blank-line-preservation.md
new file mode 100644
index 000000000..378d74412
--- /dev/null
+++ b/.changeset/blank-line-preservation.md
@@ -0,0 +1,14 @@
+---
+'@inkeep/open-knowledge': minor
+---
+
+Blank lines you type in the visual editor now stay where you put them. Previously the editor showed the space while you were typing and then quietly took it back — switching to markdown source and returning, reloading the page, or a teammate opening the same document all collapsed a run of blank lines down to one.
+
+| Before | After |
+| --- | --- |
+| Press Enter three times, reload — one blank line | Press Enter three times, reload — three blank lines |
+| The file on disk kept your blank lines, the editor did not show them | The editor shows what the file actually contains |
+
+This covers blank lines between top-level blocks, which is where they come from when you press Enter. Two places still collapse them, both deliberately: runs at the very top or very bottom of a document, and runs inside a list, a quote, or a table cell — where CommonMark treats the blank line as meaningful structure rather than space. Nothing new is written into your files: no `&nbsp;`, no `<br />`, just the newlines that were already there.
+
+Worth knowing: blank lines still render as nothing when a document is published to GitHub or any other CommonMark renderer. This makes the editor honest about your file, it does not add vertical space to published output.
diff --git a/packages/app/package.json b/packages/app/package.json
index 47718cdd4..2e541d7ed 100644
--- a/packages/app/package.json
+++ b/packages/app/package.json
@@ -26,7 +26,7 @@
"measure:fuzz": "bash scripts/measure-fuzz.sh",
"measure:stress": "bash scripts/measure-stress.sh",
"perf:compare": "bash scripts/perf-compare.sh",
- "test:e2e": "playwright test tests/stress/ux-interactions.e2e.ts tests/stress/file-tree-create.e2e.ts tests/stress/file-tree-drag-to-root.e2e.ts tests/stress/file-tree-deselect-to-root.e2e.ts tests/stress/create-then-rename-editable.e2e.ts tests/stress/find-replace.e2e.ts tests/stress/editor-tabs.e2e.ts tests/stress/crdt-stress.e2e.ts tests/stress/slash-command.e2e.ts tests/stress/paste-fidelity.e2e.ts tests/stress/fr-7a-disconnect-source-mode.e2e.ts tests/stress/docs-open.e2e.ts tests/stress/frozen-table-headers.e2e.ts tests/stress/asset-embed.e2e.ts tests/stress/asset-embed-advanced.e2e.ts tests/stress/asset-click-dispatch.e2e.ts tests/stress/handoff.e2e.ts tests/stress/multi-agent-presence.e2e.ts tests/stress/editor-mode-persistence.e2e.ts tests/stress/lint-config-toggle.e2e.ts tests/stress/schema-edit-fields-view.e2e.ts tests/stress/sidebar-search-pill.e2e.ts tests/stress/navigation-history.e2e.ts tests/stress/command-palette-semantic.e2e.ts tests/stress/agent-activity-panel.e2e.ts tests/stress/drop-pipeline-auto-open.e2e.ts tests/stress/command-palette-flicker.e2e.ts tests/stress/cm6-list-hanging-indent.e2e.ts tests/stress/ng7-rapid-nav-coherence.e2e.ts tests/stress/reveal-on-activate.e2e.ts tests/stress/selection-indicator.e2e.ts tests/stress/new-file-cross-doc-bleed.e2e.ts tests/stress/editor-mode-flip-cross-doc-bleed.e2e.ts tests/stress/editor-area-viewport-resize.e2e.ts tests/stress/qa-sidebar-responsive.e2e.ts tests/stress/prd-6955-reassertion-repro.e2e.ts tests/stress/prd-6955-reassertion-wedge.e2e.ts tests/stress/prd-6914-repro.e2e.ts tests/stress/showall-lazy-tree.e2e.ts tests/stress/tabs-component-strip.e2e.ts tests/stress/source-find-scroll.e2e.ts tests/stress/harness-app-warmth.e2e.ts tests/stress/qa-canary-authoring-both-modes.e2e.ts tests/stress/qa-canary-live-typing.e2e.ts tests/stress/keystroke-cadence-danger-space.e2e.ts tests/stress/jsx-unregistered-backspace-delete.e2e.ts tests/stress/jsx-unregistered-ime-concurrent.e2e.ts tests/stress/jsx-backspace-delete.e2e.ts tests/stress/link-authoring-bytes.e2e.ts tests/stress/link-authoring-apex.e2e.ts tests/stress/command-palette-parity.e2e.ts tests/stress/settings-search.e2e.ts tests/stress/chip-popover-positioning.e2e.ts tests/stress/docs-open-scroll-restore.e2e.ts tests/stress/file-tree-decoration-chip-strip-repro.e2e.ts tests/stress/grip-click-nodeselect.e2e.ts tests/stress/jsx-halo-semantics.e2e.ts tests/stress/jsx-image-body-click.e2e.ts tests/stress/jsx-prop-panel-placeholder.e2e.ts tests/stress/jsx-range-encompass-halo.e2e.ts tests/stress/jsx-wildcard-convert.e2e.ts tests/stress/markdown-lint.e2e.ts tests/stress/unified-problems.e2e.ts tests/stress/mid-type-recovery.e2e.ts tests/stress/observer-a-multi-client.e2e.ts tests/stress/outline-toolbar-occlusion.e2e.ts tests/stress/quiet-tree-hidden-doc.e2e.ts tests/stress/raw-mdx-fallback-onblur-upgrade.e2e.ts tests/stress/rename-noext-probe.e2e.ts tests/stress/show-ok-folders.e2e.ts tests/stress/source-polish.e2e.ts tests/stress/warm-skeleton-scroll-restore.e2e.ts tests/stress/props-toggle-scroll-preserve.e2e.ts tests/stress/yjs-no-dual-import.e2e.ts tests/stress/slash-command-auto-open.e2e.ts tests/stress/graph-panel-surfaces.e2e.ts tests/stress/clipboard-relative-url-source-fallback.e2e.ts tests/stress/asset-embed-real-fidelity.e2e.ts tests/stress/prop-upload.e2e.ts tests/stress/rename-content-preservation.e2e.ts tests/stress/rename-consolidation.e2e.ts tests/stress/outline-navigation.e2e.ts tests/stress/prd-7627-default-swatch.e2e.ts tests/stress/clipboard-dollar-math-source-fallback.e2e.ts tests/stress/source-mode-undo-keymap.e2e.ts tests/stress/timeline-recovered-restore.e2e.ts",
+ "test:e2e": "playwright test tests/stress/ux-interactions.e2e.ts tests/stress/file-tree-create.e2e.ts tests/stress/file-tree-drag-to-root.e2e.ts tests/stress/file-tree-deselect-to-root.e2e.ts tests/stress/create-then-rename-editable.e2e.ts tests/stress/find-replace.e2e.ts tests/stress/editor-tabs.e2e.ts tests/stress/crdt-stress.e2e.ts tests/stress/slash-command.e2e.ts tests/stress/paste-fidelity.e2e.ts tests/stress/fr-7a-disconnect-source-mode.e2e.ts tests/stress/docs-open.e2e.ts tests/stress/frozen-table-headers.e2e.ts tests/stress/asset-embed.e2e.ts tests/stress/asset-embed-advanced.e2e.ts tests/stress/asset-click-dispatch.e2e.ts tests/stress/handoff.e2e.ts tests/stress/multi-agent-presence.e2e.ts tests/stress/editor-mode-persistence.e2e.ts tests/stress/lint-config-toggle.e2e.ts tests/stress/schema-edit-fields-view.e2e.ts tests/stress/sidebar-search-pill.e2e.ts tests/stress/navigation-history.e2e.ts tests/stress/command-palette-semantic.e2e.ts tests/stress/agent-activity-panel.e2e.ts tests/stress/drop-pipeline-auto-open.e2e.ts tests/stress/command-palette-flicker.e2e.ts tests/stress/cm6-list-hanging-indent.e2e.ts tests/stress/ng7-rapid-nav-coherence.e2e.ts tests/stress/reveal-on-activate.e2e.ts tests/stress/selection-indicator.e2e.ts tests/stress/new-file-cross-doc-bleed.e2e.ts tests/stress/editor-mode-flip-cross-doc-bleed.e2e.ts tests/stress/editor-area-viewport-resize.e2e.ts tests/stress/qa-sidebar-responsive.e2e.ts tests/stress/prd-6955-reassertion-repro.e2e.ts tests/stress/prd-6955-reassertion-wedge.e2e.ts tests/stress/prd-6914-repro.e2e.ts tests/stress/showall-lazy-tree.e2e.ts tests/stress/tabs-component-strip.e2e.ts tests/stress/source-find-scroll.e2e.ts tests/stress/harness-app-warmth.e2e.ts tests/stress/qa-canary-authoring-both-modes.e2e.ts tests/stress/qa-canary-live-typing.e2e.ts tests/stress/keystroke-cadence-danger-space.e2e.ts tests/stress/jsx-unregistered-backspace-delete.e2e.ts tests/stress/jsx-unregistered-ime-concurrent.e2e.ts tests/stress/jsx-backspace-delete.e2e.ts tests/stress/link-authoring-bytes.e2e.ts tests/stress/link-authoring-apex.e2e.ts tests/stress/command-palette-parity.e2e.ts tests/stress/settings-search.e2e.ts tests/stress/chip-popover-positioning.e2e.ts tests/stress/docs-open-scroll-restore.e2e.ts tests/stress/file-tree-decoration-chip-strip-repro.e2e.ts tests/stress/grip-click-nodeselect.e2e.ts tests/stress/jsx-halo-semantics.e2e.ts tests/stress/jsx-image-body-click.e2e.ts tests/stress/jsx-prop-panel-placeholder.e2e.ts tests/stress/jsx-range-encompass-halo.e2e.ts tests/stress/jsx-wildcard-convert.e2e.ts tests/stress/markdown-lint.e2e.ts tests/stress/unified-problems.e2e.ts tests/stress/mid-type-recovery.e2e.ts tests/stress/observer-a-multi-client.e2e.ts tests/stress/outline-toolbar-occlusion.e2e.ts tests/stress/quiet-tree-hidden-doc.e2e.ts tests/stress/raw-mdx-fallback-onblur-upgrade.e2e.ts tests/stress/rename-noext-probe.e2e.ts tests/stress/show-ok-folders.e2e.ts tests/stress/source-polish.e2e.ts tests/stress/warm-skeleton-scroll-restore.e2e.ts tests/stress/props-toggle-scroll-preserve.e2e.ts tests/stress/yjs-no-dual-import.e2e.ts tests/stress/slash-command-auto-open.e2e.ts tests/stress/graph-panel-surfaces.e2e.ts tests/stress/clipboard-relative-url-source-fallback.e2e.ts tests/stress/asset-embed-real-fidelity.e2e.ts tests/stress/prop-upload.e2e.ts tests/stress/rename-content-preservation.e2e.ts tests/stress/rename-consolidation.e2e.ts tests/stress/outline-navigation.e2e.ts tests/stress/prd-7627-default-swatch.e2e.ts tests/stress/clipboard-dollar-math-source-fallback.e2e.ts tests/stress/source-mode-undo-keymap.e2e.ts tests/stress/timeline-recovered-restore.e2e.ts tests/stress/blank-line-preservation.e2e.ts",
"test:e2e:install-browsers": "playwright install chromium webkit firefox",
"test:visual": "playwright test --config playwright.visual.config.ts",
"test:visual:update": "playwright test --config playwright.visual.config.ts --update-snapshots",
diff --git a/packages/app/src/editor/extensions/markdown-lint-decorations.test.ts b/packages/app/src/editor/extensions/markdown-lint-decorations.test.ts
index 4b615c3fd..0ece09a69 100644
--- a/packages/app/src/editor/extensions/markdown-lint-decorations.test.ts
+++ b/packages/app/src/editor/extensions/markdown-lint-decorations.test.ts
@@ -140,6 +140,49 @@ describe('computeSourceBlockSpans', () => {
test('empty body → no spans', () => {
expect(computeSourceBlockSpans('', md).spans).toHaveLength(0);
});
+
+ // Decorations and Problems-panel navigation are gated on spans matching the
+ // PM doc's top-level children one for one. A preserved blank run adds
+ // paragraphs the source parse alone does not see, so without a synthesized
+ // span per blank line the whole feature silently switches off for any
+ // document containing one.
+ test('a preserved blank run keeps span/child parity', () => {
+ const source = 'a\n\n\n\nb\n';
+ expect(computeSourceBlockSpans(source, md).spans).toHaveLength(
+ md.parse(source).content?.length ?? 0,
+ );
+ });
+
+ test('each blank line of a run gets its own single-line span', () => {
+ expect(computeSourceBlockSpans('a\n\n\n\nb\n', md).spans).toEqual([
+ { start: 1, end: 1 },
+ { start: 3, end: 3 },
+ { start: 4, end: 4 },
+ { start: 5, end: 5 },
+ ]);
+ });
+
+ test('synthesized spans shift with frontmatter like every other span', () => {
+ const { spans, fmLineCount } = computeSourceBlockSpans('---\nt: X\n---\na\n\n\nb\n', md);
+ expect(fmLineCount).toBe(3);
+ expect(spans).toEqual([
+ { start: 4, end: 4 },
+ { start: 6, end: 6 },
+ { start: 7, end: 7 },
+ ]);
+ });
+});
+
+describe('diagnostics on a preserved blank run', () => {
+ test('anchor to the blank line they are about, not the next block', () => {
+ const byBlock = mapDiagnosticsToBlocks('a\n\n\n\nb\n', [diag(3)], md);
+ expect([...byBlock.keys()]).toEqual([1]);
+ });
+
+ test('the canonical single blank line still anchors to the next block', () => {
+ const byBlock = mapDiagnosticsToBlocks('a\n\nb\n', [diag(2)], md);
+ expect([...byBlock.keys()]).toEqual([1]);
+ });
});
describe('blockIndexForLine', () => {
diff --git a/packages/app/src/editor/extensions/markdown-lint-decorations.ts b/packages/app/src/editor/extensions/markdown-lint-decorations.ts
index c47dd8ebc..8df2dff93 100644
--- a/packages/app/src/editor/extensions/markdown-lint-decorations.ts
+++ b/packages/app/src/editor/extensions/markdown-lint-decorations.ts
@@ -78,7 +78,10 @@ export interface SourceBlockSpans {
export function computeSourceBlockSpans(source: string, md: MarkdownManager): SourceBlockSpans {
const { frontmatter, body } = stripFrontmatter(source);
const fmLineCount = frontmatter === '' ? 0 : frontmatter.split('\n').length - 1;
- const spans = md.parseToMdast(body).children.map((child) => ({
+ // The editor view, not the CommonMark one: a preserved blank line is a
+ // paragraph in the PM doc, and this array is index-aligned with those
+ // children. Losing the alignment silently disables every decoration.
+ const spans = md.parseToEditorMdast(body).children.map((child) => ({
start: (child.position?.start.line ?? Number.POSITIVE_INFINITY) + fmLineCount,
end: (child.position?.end.line ?? Number.NEGATIVE_INFINITY) + fmLineCount,
}));
diff --git a/packages/app/src/editor/provider-pool-replay-diverged.test.ts b/packages/app/src/editor/provider-pool-replay-diverged.test.ts
index 06dd2778a..064194761 100644
--- a/packages/app/src/editor/provider-pool-replay-diverged.test.ts
+++ b/packages/app/src/editor/provider-pool-replay-diverged.test.ts
@@ -158,3 +158,30 @@ describe('content-level replay against content the server has moved past', () =>
expect(emittedEvents(warn)).not.toContain('ok-buffer-replay-diverged');
});
});
+
+describe('content-level replay of an edit the comparator cannot see', () => {
+ /** The buffered edit is blank lines and nothing else. */
+ const BLANK_RUN_MD = '# Notes\n\n\n\nSettled paragraph.\n';
+
+ it('recovers buffered blank lines the server state lacks', async () => {
+ // The surface comparison collapses blank runs on both sides, so this edit
+ // reads as "already there" and the recycle would drop it with no event and
+ // no checkpoint. It is only visible to the direction check.
+ const docName = `pp-blank-${randomUUID()}`;
+ const { delta, fullState } = buildTwoSurfaceState(BASE_MD, BLANK_RUN_MD);
+ pool = new ProviderPool(3, DUMMY_WS);
+ const entry = pool.open(docName);
+ if (!entry) throw new Error('expected entry');
+ entry.observerCleanup = () => {};
+ const ytext = entry.provider.document.getText('source');
+ ytext.insert(0, BASE_MD);
+ pool.__test_seedBufferedUpdate(docName, delta, { fullState, durable: false });
+ entry.provider.emit('synced', { state: true });
+
+ await vi.waitFor(() => {
+ expect(emittedEvents(warn)).toContain('ok-buffer-replay-content-applied');
+ });
+ expect(ytext.toString()).toContain('# Notes\n\n\n\nSettled paragraph.');
+ expect(emittedEvents(warn)).not.toContain('ok-buffer-replay-diverged');
+ });
+});
diff --git a/packages/app/src/editor/provider-pool.ts b/packages/app/src/editor/provider-pool.ts
index a900c9a14..28e527a9d 100644
--- a/packages/app/src/editor/provider-pool.ts
+++ b/packages/app/src/editor/provider-pool.ts
@@ -1,5 +1,6 @@
import { HocuspocusProvider } from '@hocuspocus/provider';
import {
+ addsBlankLines,
LINEAGE_EPOCH_KEY,
MarkdownManager,
normalizeBridge,
@@ -2820,8 +2821,14 @@ export class ProviderPool {
const theirs = provider.document.getText('source').toString();
const { body: theirsBody } = stripFrontmatter(theirs);
const theirsNorm = normalizeBridge(theirsBody);
- const ytextClean = normalizeBridge(oursYtextBody) === theirsNorm;
- const fragClean = normalizeBridge(oursFragBody) === theirsNorm;
+ // `addsBlankLines` breaks the tie the normalized compare cannot: it
+ // collapses blank runs on both sides, so buffered blank lines the
+ // server's rebuilt state lacks would read as "nothing to restore" and
+ // the recycle would discard them.
+ const ytextClean =
+ normalizeBridge(oursYtextBody) === theirsNorm && !addsBlankLines(theirsBody, oursYtextBody);
+ const fragClean =
+ normalizeBridge(oursFragBody) === theirsNorm && !addsBlankLines(theirsBody, oursFragBody);
if (ytextClean && fragClean) return true;
let ours: string;
if (ytextClean) {
diff --git a/packages/app/tests/integration/_fixtures/init-load-byte-stable-corpus.ts b/packages/app/tests/integration/_fixtures/init-load-byte-stable-corpus.ts
index 7813f922b..c8fa8cc0d 100644
--- a/packages/app/tests/integration/_fixtures/init-load-byte-stable-corpus.ts
+++ b/packages/app/tests/integration/_fixtures/init-load-byte-stable-corpus.ts
@@ -68,7 +68,7 @@ export const CORPUS: CorpusEntry[] = [
{
filename: 'ng1-multiblank.md',
ng: ['blank-line-count-normalization'],
- description: 'Three blank lines between paragraphs (remark collapses to 2).',
+ description: 'Three blank lines between paragraphs (preserved as blank-line paragraphs).',
body:
'First paragraph.' +
NL +
diff --git a/packages/app/tests/integration/interior-blank-runs.test.ts b/packages/app/tests/integration/interior-blank-runs.test.ts
new file mode 100644
index 000000000..dd634c343
--- /dev/null
+++ b/packages/app/tests/integration/interior-blank-runs.test.ts
@@ -0,0 +1,217 @@
+/**
+ * Interior blank runs across real peers and across a disk load.
+ *
+ * The retraction this feature removes only ever showed itself on a FULL
+ * re-derive of the fragment from source bytes — mode toggle, reload, remote
+ * peer. Single-client coverage cannot see it: the client that typed the blank
+ * lines still holds them in its own fragment. So the contract is pinned here,
+ * over WebSocket peers, where the server is the sole fragment writer and every
+ * client's view arrives from the same derive.
+ *
+ * Both assertions are raw-byte, not normalized. The comparator collapses
+ * three-or-more newlines on both sides, so a wrong blank count rests inside
+ * the bridge tolerance and no watchdog would ever report it.
+ */
+
+import { writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { setTimeout as wait } from 'node:timers/promises';
+import { afterAll, beforeAll, describe, expect, test } from 'vitest';
+import * as Y from 'yjs';
+import { HARNESS_BOOT_TIMEOUT_MS } from './harness-boot-timeout';
+import {
+ agentWriteMd,
+ createTestClients,
+ createTestServer,
+ pollUntil,
+ readTestDoc,
+ serializeFragment,
+ type TestServer,
+} from './test-harness';
+
+let server: TestServer;
+
+beforeAll(async () => {
+ server = await createTestServer();
+}, HARNESS_BOOT_TIMEOUT_MS);
+
+afterAll(async () => {
+ await server.cleanup();
+});
+
+/** Top-level children of a fragment that hold no text — the blank lines. */
+function countBlankLineNodes(fragment: Y.XmlFragment): number {
+ let count = 0;
+ for (let i = 0; i < fragment.length; i++) {
+ const child = fragment.get(i);
+ if (
+ child instanceof Object &&
+ 'toString' in child &&
+ child.toString() === '<paragraph></paragraph>'
+ ) {
+ count += 1;
+ }
+ }
+ return count;
+}
+
+describe('interior blank runs on the CRDT path', () => {
+ test('a blank run written by one peer reaches the other as blank lines, byte for byte', async () => {
+ const docName = `blank-run-peers-${crypto.randomUUID()}`;
+ const raw = 'First block.\n\n\n\n\nSecond block.\n';
+ const clients = await createTestClients(server.port, {
+ count: 2,
+ docName,
+ perClientOptions: { skipInvariantWatcher: true },
+ });
+ try {
+ await agentWriteMd(server.port, raw, { docName, position: 'replace' });
+ await pollUntil(() => clients.every((c) => c.ytext.toString() === raw), 5000);
+ await wait(500);
+
+ for (const c of clients) {
+ // The remote peer's WYSIWYG holds the blank lines as real nodes, and
+ // serializing them reproduces the exact bytes rather than a
+ // normalize-equal approximation.
+ expect(countBlankLineNodes(c.fragment)).toBe(3);
+ expect(serializeFragment(c.fragment)).toBe(raw);
+ }
+ expect(clients[0].ytext.toString()).toBe(clients[1].ytext.toString());
+ } finally {
+ for (const c of clients) await c.cleanup();
+ }
+ });
+
+ test('a WYSIWYG edit beside a blank run converges on both peers without disturbing it', async () => {
+ const docName = `blank-run-edit-${crypto.randomUUID()}`;
+ const raw = 'Alpha.\n\n\n\nOmega.\n';
+ const clients = await createTestClients(server.port, {
+ count: 2,
+ docName,
+ perClientOptions: { skipInvariantWatcher: true },
+ });
+ try {
+ await agentWriteMd(server.port, raw, { docName, position: 'replace' });
+ await pollUntil(() => clients.every((c) => c.ytext.toString() === raw), 5000);
+ await wait(500);
+
+ const a = clients[0];
+ a.doc.transact(() => {
+ const first = a.fragment.get(0);
+ const text = (first as { get(i: number): unknown }).get(0);
+ (text as { insert(i: number, s: string): void }).insert(0, 'Z');
+ });
+ await pollUntil(() => clients.every((c) => c.ytext.toString().includes('ZAlpha')), 5000);
+ await wait(500);
+ await pollUntil(
+ () => clients.every((c) => serializeFragment(c.fragment).includes('ZAlpha')),
+ 5000,
+ );
+
+ const expected = 'ZAlpha.\n\n\n\nOmega.\n';
+ for (const c of clients) {
+ expect(c.ytext.toString()).toBe(expected);
+ expect(serializeFragment(c.fragment)).toBe(expected);
+ expect(countBlankLineNodes(c.fragment)).toBe(2);
+ }
+ } finally {
+ for (const c of clients) await c.cleanup();
+ }
+ });
+
+ test('blank lines created in the WYSIWYG reach the source bytes', async () => {
+ const docName = `blank-run-authored-${crypto.randomUUID()}`;
+ const raw = 'Above.\n\nBelow.\n';
+ const clients = await createTestClients(server.port, {
+ count: 2,
+ docName,
+ perClientOptions: { skipInvariantWatcher: true },
+ });
+ try {
+ await agentWriteMd(server.port, raw, { docName, position: 'replace' });
+ await pollUntil(() => clients.every((c) => c.ytext.toString() === raw), 5000);
+ await wait(500);
+
+ // What pressing Enter twice at the end of the first paragraph produces.
+ const a = clients[0];
+ a.doc.transact(() => {
+ a.fragment.insert(1, [new Y.XmlElement('paragraph'), new Y.XmlElement('paragraph')]);
+ });
+
+ const expected = 'Above.\n\n\n\nBelow.\n';
+ await pollUntil(() => clients.every((c) => c.ytext.toString() === expected), 5000);
+ await wait(500);
+ for (const c of clients) {
+ expect(c.ytext.toString()).toBe(expected);
+ expect(countBlankLineNodes(c.fragment)).toBe(2);
+ }
+ // Persistence is debounced, so the bytes land after convergence.
+ await pollUntil(() => readTestDoc(server.contentDir, docName) === expected, 10_000);
+ } finally {
+ for (const c of clients) await c.cleanup();
+ }
+ });
+
+ test('a trailing blank run in the file does not hide a new interior one', async () => {
+ // The fragment cannot carry doc-edge runs, so the source and the fragment
+ // disagree about them permanently. A comparison that counted them would
+ // read the two sides as incomparable on this document and never see the
+ // interior addition at all.
+ const docName = `blank-run-edge-${crypto.randomUUID()}`;
+ const raw = 'Above.\n\nBelow.\n\n\n';
+ const clients = await createTestClients(server.port, {
+ count: 2,
+ docName,
+ perClientOptions: { skipInvariantWatcher: true },
+ });
+ try {
+ await agentWriteMd(server.port, raw, { docName, position: 'replace' });
+ await pollUntil(() => clients.every((c) => c.ytext.toString() === raw), 5000);
+ await wait(500);
+
+ const a = clients[0];
+ a.doc.transact(() => {
+ a.fragment.insert(1, [new Y.XmlElement('paragraph'), new Y.XmlElement('paragraph')]);
+ });
+
+ await pollUntil(
+ () => clients.every((c) => c.ytext.toString().startsWith('Above.\n\n\n\nBelow.')),
+ 5000,
+ );
+ await wait(500);
+ for (const c of clients) {
+ expect(c.ytext.toString()).toContain('Above.\n\n\n\nBelow.');
+ expect(countBlankLineNodes(c.fragment)).toBe(2);
+ }
+ await pollUntil(
+ () => readTestDoc(server.contentDir, docName).startsWith('Above.\n\n\n\nBelow.'),
+ 10_000,
+ );
+ } finally {
+ for (const c of clients) await c.cleanup();
+ }
+ });
+
+ test('a document already on disk gains its blank lines on first open, with no migration step', async () => {
+ const docName = `blank-run-disk-${crypto.randomUUID()}`;
+ const raw = 'On disk before the upgrade.\n\n\n\n\n\nStill here.\n';
+ // Written straight to disk, so the document exists with no CRDT state at
+ // all — the shape every pre-existing document has after an upgrade.
+ writeFileSync(join(server.contentDir, `${docName}.md`), raw, 'utf8');
+
+ const clients = await createTestClients(server.port, {
+ count: 1,
+ docName,
+ perClientOptions: { skipInvariantWatcher: true },
+ });
+ try {
+ await pollUntil(() => clients[0].ytext.toString() === raw, 10_000);
+ await wait(500);
+ expect(countBlankLineNodes(clients[0].fragment)).toBe(4);
+ expect(serializeFragment(clients[0].fragment)).toBe(raw);
+ expect(readTestDoc(server.contentDir, docName)).toBe(raw);
+ } finally {
+ for (const c of clients) await c.cleanup();
+ }
+ });
+});
diff --git a/packages/app/tests/stress/blank-line-preservation.e2e.ts b/packages/app/tests/stress/blank-line-preservation.e2e.ts
new file mode 100644
index 000000000..4d0c30f7f
--- /dev/null
+++ b/packages/app/tests/stress/blank-line-preservation.e2e.ts
@@ -0,0 +1,79 @@
+/**
+ * The user-facing shape of blank-line preservation: press Enter a few times,
+ * leave, come back, and the blank lines are still there.
+ *
+ * Every path that used to lose them is a full re-derive of the WYSIWYG
+ * fragment from the source bytes, so the two round trips that matter are the
+ * source-mode toggle and a reload. Both are exercised here against the real
+ * browser keymap, which is the one surface the jsdom and integration tiers
+ * cannot reach.
+ */
+import { randomUUID } from 'node:crypto';
+import type { Page } from '@playwright/test';
+import { expect, test, waitForActiveProviderSynced as waitForProvider } from './_helpers';
+
+const sourceToggle = (page: Page) => page.getByRole('radio', { name: 'Markdown source' });
+const visualToggle = (page: Page) => page.getByRole('radio', { name: 'Visual editor' });
+
+function readSource(page: Page): Promise<string> {
+ return page.evaluate(() => window.__activeProvider?.document.getText('source').toString() ?? '');
+}
+
+/** Top-level paragraphs with no text — what a preserved blank line renders as. */
+function countBlankParagraphs(page: Page): Promise<number> {
+ return page.evaluate(() => {
+ const editor = document.querySelector('.ProseMirror:not(.composer-prosemirror)');
+ if (!editor) return -1;
+ return [...editor.children].filter(
+ (child) => child.tagName === 'P' && (child.textContent ?? '') === '',
+ ).length;
+ });
+}
+
+test.describe('blank lines typed in the visual editor', () => {
+ let docName: string;
+
+ test.beforeEach(async ({ page, api }) => {
+ docName = `test-blank-lines-${randomUUID().slice(0, 8)}`;
+ await api.createPage(`${docName}.md`);
+ await api.replaceDoc(docName, 'Above.\n\nBelow.\n');
+ await page.goto(`/#/${docName}`);
+ await waitForProvider(page);
+ await page.waitForSelector('.ProseMirror:not(.composer-prosemirror)');
+ await page.waitForFunction(
+ () =>
+ document
+ .querySelector('.ProseMirror:not(.composer-prosemirror)')
+ ?.textContent?.includes('Above'),
+ null,
+ { timeout: 10_000 },
+ );
+ });
+
+ test('survive a source-mode round trip and a reload', async ({ page }) => {
+ // Three Enters at the end of the first paragraph: three empty paragraphs,
+ // i.e. three blank lines the user means to keep.
+ await page.locator('.ProseMirror:not(.composer-prosemirror) p').first().click();
+ await page.keyboard.press('End');
+ await page.keyboard.press('Enter');
+ await page.keyboard.press('Enter');
+ await page.keyboard.press('Enter');
+
+ await expect.poll(() => readSource(page), { timeout: 10_000 }).toBe('Above.\n\n\n\n\nBelow.\n');
+
+ await sourceToggle(page).click();
+ await page.waitForSelector('.cm-content', { timeout: 10_000 });
+ await visualToggle(page).click();
+ await page.waitForSelector('.ProseMirror:not(.composer-prosemirror)');
+
+ await expect.poll(() => countBlankParagraphs(page), { timeout: 10_000 }).toBe(3);
+ expect(await readSource(page)).toBe('Above.\n\n\n\n\nBelow.\n');
+
+ await page.reload();
+ await waitForProvider(page);
+ await page.waitForSelector('.ProseMirror:not(.composer-prosemirror)');
+
+ await expect.poll(() => countBlankParagraphs(page), { timeout: 10_000 }).toBe(3);
+ expect(await readSource(page)).toBe('Above.\n\n\n\n\nBelow.\n');
+ });
+});
diff --git a/packages/core/src/bridge/index.ts b/packages/core/src/bridge/index.ts
index 4669ca409..287afac69 100644
--- a/packages/core/src/bridge/index.ts
+++ b/packages/core/src/bridge/index.ts
@@ -68,6 +68,7 @@ export {
tryLineLevelCombine,
} from './merge-three-way.ts';
export {
+ addsBlankLines,
BRIDGE_TOLERANCE_CLASSES,
type BridgeToleranceClass,
detectAppliedToleranceClasses,
diff --git a/packages/core/src/bridge/normalize.ts b/packages/core/src/bridge/normalize.ts
index 4b8890476..da41900cd 100644
--- a/packages/core/src/bridge/normalize.ts
+++ b/packages/core/src/bridge/normalize.ts
@@ -222,6 +222,18 @@ export const BRIDGE_TOLERANCE_CLASSES = [
export type BridgeToleranceClass = (typeof BRIDGE_TOLERANCE_CLASSES)[number];
+function interiorBlankRunLengths(s: string): number[] {
+ const interior = s.replace(/^\n+/, '').replace(/\n+$/, '');
+ return [...interior.matchAll(/\n{2,}/g)].map((m) => m[0].length);
+}
+
+export function addsBlankLines(baseline: string, candidate: string): boolean {
+ const before = interiorBlankRunLengths(baseline);
+ const after = interiorBlankRunLengths(candidate);
+ if (before.length !== after.length) return false;
+ return after.some((length, i) => length > (before[i] ?? 0));
+}
+
export function detectAppliedToleranceClasses(left: string, right: string): BridgeToleranceClass[] {
const classes: BridgeToleranceClass[] = [];
diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts
index 0aa2f4d6f..734a01da8 100644
--- a/packages/core/src/index.ts
+++ b/packages/core/src/index.ts
@@ -1238,6 +1238,7 @@ export {
// Bridge — observer/CRDT-bridge shared utilities (precedent #14)
export {
+ addsBlankLines,
applyFastDiff,
applyIncrementalDiff,
applyPatchToFm,
diff --git a/packages/core/src/markdown/index.ts b/packages/core/src/markdown/index.ts
index 8b36807fd..a64ecf306 100644
--- a/packages/core/src/markdown/index.ts
+++ b/packages/core/src/markdown/index.ts
@@ -68,6 +68,7 @@ import {
createParseProcessor,
createSerializeProcessor,
parseMd,
+ parseMdToEditorMdast,
parseMdToMdast,
serializeMd,
} from './pipeline.ts';
@@ -167,6 +168,13 @@ export class MarkdownManager {
return parseMdToMdast(markdown, this.parseProcessor);
}
+ parseToEditorMdast(markdown: string): MdastRoot {
+ if (!markdown.trim()) {
+ return { type: 'root', children: [] };
+ }
+ return parseMdToEditorMdast(markdown, this.parseProcessor);
+ }
+
parseWithFallback(markdown: string, opts?: ParseContext): JSONContent {
if (!markdown.trim()) {
return { type: 'doc', content: [{ type: 'paragraph', content: [] }] };
diff --git a/packages/core/src/markdown/interior-blank-runs.ts b/packages/core/src/markdown/interior-blank-runs.ts
new file mode 100644
index 000000000..f6d95ac98
--- /dev/null
+++ b/packages/core/src/markdown/interior-blank-runs.ts
@@ -0,0 +1,64 @@
+import type { Root as MdastRoot, Paragraph } from 'mdast';
+
+const MIN_PRESERVED_GAP_NEWLINES = 3;
+
+const PRESERVED_GAP_SHAPE = /^\n+$/;
+
+function blankLineParagraphCount(gap: string): number {
+ if (gap.length < MIN_PRESERVED_GAP_NEWLINES) return 0;
+ if (!PRESERVED_GAP_SHAPE.test(gap)) return 0;
+ return gap.length - 2;
+}
+
+interface InteriorBlankRun {
+ beforeChildIndex: number;
+ lines: number[];
+ offsets: number[];
+}
+
+function findInteriorBlankRuns(root: MdastRoot, source: string): InteriorBlankRun[] {
+ const runs: InteriorBlankRun[] = [];
+ const children = root.children;
+ for (let i = 1; i < children.length; i++) {
+ const prev = children[i - 1];
+ const next = children[i];
+ const prevEnd = prev?.position?.end?.offset;
+ const prevEndLine = prev?.position?.end?.line;
+ const nextStart = next?.position?.start?.offset;
+ if (typeof prevEnd !== 'number' || typeof nextStart !== 'number') continue;
+ if (typeof prevEndLine !== 'number' || nextStart < prevEnd) continue;
+ const count = blankLineParagraphCount(source.slice(prevEnd, nextStart));
+ if (count === 0) continue;
+ const lines: number[] = [];
+ const offsets: number[] = [];
+ for (let j = 0; j < count; j++) {
+ lines.push(prevEndLine + 2 + j);
+ offsets.push(prevEnd + 2 + j);
+ }
+ runs.push({ beforeChildIndex: i, lines, offsets });
+ }
+ return runs;
+}
+
+export function insertInteriorBlankRunParagraphs(root: MdastRoot, source: string): void {
+ const runs = findInteriorBlankRuns(root, source);
+ if (runs.length === 0) return;
+ const out: MdastRoot['children'] = [];
+ let runIndex = 0;
+ for (let i = 0; i < root.children.length; i++) {
+ const run = runs[runIndex];
+ if (run && run.beforeChildIndex === i) {
+ for (let j = 0; j < run.lines.length; j++) {
+ out.push(blankLineParagraph(run.lines[j] as number, run.offsets[j] as number));
+ }
+ runIndex += 1;
+ }
+ out.push(root.children[i] as MdastRoot['children'][number]);
+ }
+ root.children = out;
+}
+
+function blankLineParagraph(line: number, offset: number): Paragraph {
+ const point = { line, column: 1, offset };
+ return { type: 'paragraph', children: [], position: { start: point, end: { ...point } } };
+}
diff --git a/packages/core/src/markdown/mdast-augmentation.ts b/packages/core/src/markdown/mdast-augmentation.ts
index a686bd9dd..b1d866d2b 100644
--- a/packages/core/src/markdown/mdast-augmentation.ts
+++ b/packages/core/src/markdown/mdast-augmentation.ts
@@ -104,13 +104,9 @@ export interface SourceDocBoundary {
bom?: true;
leading?: string;
trailing?: string;
- gapBlankLines?: ReadonlyArray<number | null>;
}
declare module 'mdast' {
- interface Data {
- sourcePrecedingBlankLines?: number;
- }
interface RootData {
sourceDocBoundary?: SourceDocBoundary;
}
diff --git a/packages/core/src/markdown/pipeline.ts b/packages/core/src/markdown/pipeline.ts
index 21bc33473..8bcbe2692 100644
--- a/packages/core/src/markdown/pipeline.ts
+++ b/packages/core/src/markdown/pipeline.ts
@@ -27,6 +27,7 @@ import { encodeEntityRefs, restoreEntityRefsPlugin } from './entity-ref-guard.ts
import { highlightPromoterPlugin } from './highlight-promoter.ts';
import { imagePromoterPlugin } from './image-promoter.ts';
import { indentedCodePromoterPlugin } from './indented-code-promoter.ts';
+import { insertInteriorBlankRunParagraphs } from './interior-blank-runs.ts';
import { mathPromoterPlugin } from './math-promoter.ts';
import type { SourceDocBoundary } from './mdast-augmentation.ts';
import { mergedPostParseWalkerPlugin } from './merged-walker.ts';
@@ -38,6 +39,7 @@ import { stripTrailingEdge } from './strip-trailing-edge.ts';
import { remarkTags } from './tag-to-markdown.ts';
import { voidBrPromoterPlugin } from './void-br-promoter.ts';
import { remarkWikiLink } from './wiki-link-micromark.ts';
+import { pruneZeroEmissionBlocks } from './zero-emission-blocks.ts';
interface PipelineOptions {
schema: Schema;
@@ -148,8 +150,6 @@ function splitDocumentHeadBom(source: string): { source: string; hadBom: boolean
: { source, hadBom: false };
}
-const GAP_CAPTURE_MIN_NEWLINES = 3;
-
const LEADING_BOUNDARY_SHAPE = /^\n+$/;
const TRAILING_BOUNDARY_SHAPE = /^\n{2,}$/;
@@ -160,7 +160,6 @@ function captureDocBoundary(
): SourceDocBoundary | undefined {
let leading: string | undefined;
let trailing: string | undefined;
- let gapBlankLines: Array<number | null> | undefined;
const children = root.children;
if (children.length > 0) {
@@ -174,32 +173,15 @@ function captureDocBoundary(
const gap = source.slice(lastEnd);
if (TRAILING_BOUNDARY_SHAPE.test(gap)) trailing = gap;
}
- const gaps: Array<number | null> = [];
- let anyGap = false;
- for (let i = 1; i < children.length; i++) {
- const prevEnd = children[i - 1]?.position?.end?.offset;
- const nextStart = children[i]?.position?.start?.offset;
- let blanks: number | null = null;
- if (typeof prevEnd === 'number' && typeof nextStart === 'number' && nextStart >= prevEnd) {
- const gap = source.slice(prevEnd, nextStart);
- if (new RegExp(`^\\n{${GAP_CAPTURE_MIN_NEWLINES},}$`).test(gap)) {
- blanks = gap.length - 1;
- anyGap = true;
- }
- }
- gaps.push(blanks);
- }
- if (anyGap) gapBlankLines = gaps;
}
- if (!hadBom && leading === undefined && trailing === undefined && !gapBlankLines) {
+ if (!hadBom && leading === undefined && trailing === undefined) {
return undefined;
}
const boundary: SourceDocBoundary = {
...(hadBom ? { bom: true as const } : {}),
...(leading !== undefined ? { leading } : {}),
...(trailing !== undefined ? { trailing } : {}),
- ...(gapBlankLines ? { gapBlankLines } : {}),
};
root.data ??= {};
root.data.sourceDocBoundary = boundary;
@@ -209,12 +191,7 @@ function captureDocBoundary(
function readDocBoundary(value: unknown): SourceDocBoundary | undefined {
if (typeof value !== 'object' || value === null) return undefined;
const v = value as Record<string, unknown>;
- const out: {
- bom?: true;
- leading?: string;
- trailing?: string;
- gapBlankLines?: Array<number | null>;
- } = {};
+ const out: { bom?: true; leading?: string; trailing?: string } = {};
if (v.bom === true) out.bom = true;
if (typeof v.leading === 'string' && LEADING_BOUNDARY_SHAPE.test(v.leading)) {
out.leading = v.leading;
@@ -222,17 +199,7 @@ function readDocBoundary(value: unknown): SourceDocBoundary | undefined {
if (typeof v.trailing === 'string' && TRAILING_BOUNDARY_SHAPE.test(v.trailing)) {
out.trailing = v.trailing;
}
- if (
- Array.isArray(v.gapBlankLines) &&
- v.gapBlankLines.every(
- (g) =>
- g === null ||
- (typeof g === 'number' && Number.isInteger(g) && g >= GAP_CAPTURE_MIN_NEWLINES - 1),
- )
- ) {
- out.gapBlankLines = v.gapBlankLines as Array<number | null>;
- }
- return out.bom || out.leading || out.trailing || out.gapBlankLines ? out : undefined;
+ return out.bom || out.leading || out.trailing ? out : undefined;
}
export function parseMd(rawSource: string, processor: Processor): PmNode {
@@ -246,6 +213,7 @@ export function parseMd(rawSource: string, processor: Processor): PmNode {
const tree = processor.parse(file);
file.value = source;
const transformed = processor.runSync(tree, file) as MdastRoot;
+ insertInteriorBlankRunParagraphs(transformed, source);
const boundary = captureDocBoundary(transformed, source, hadBom);
const doc = (processor as unknown as { stringify(tree: unknown): PmNode }).stringify(transformed);
if (!boundary) return doc;
@@ -253,6 +221,18 @@ export function parseMd(rawSource: string, processor: Processor): PmNode {
}
export function parseMdToMdast(rawSource: string, processor: Processor): MdastRoot {
+ return parseToMdast(rawSource, processor, false);
+}
+
+export function parseMdToEditorMdast(rawSource: string, processor: Processor): MdastRoot {
+ return parseToMdast(rawSource, processor, true);
+}
+
+function parseToMdast(
+ rawSource: string,
+ processor: Processor,
+ materializeBlankRuns: boolean,
+): MdastRoot {
const { source: rawAfterBom, hadBom } = splitDocumentHeadBom(rawSource);
const source = dedentBlockJsxClose(rawAfterBom);
const protected_ = encodeEntityRefs(protectFromMdx(encodeBackslashEscapes(source)));
@@ -260,6 +240,7 @@ export function parseMdToMdast(rawSource: string, processor: Processor): MdastRo
const tree = processor.parse(file);
file.value = source;
const transformed = processor.runSync(tree, file) as MdastRoot;
+ if (materializeBlankRuns) insertInteriorBlankRunParagraphs(transformed, source);
captureDocBoundary(transformed, source, hadBom);
return transformed;
}
@@ -273,18 +254,9 @@ export function serializeMd(doc: PmNode, processor: Processor, opts: SerializeMd
stripTrailingEdge(mdast);
+ pruneZeroEmissionBlocks(mdast);
+
const boundary = readDocBoundary(doc.attrs?.sourceDocBoundary);
- const gaps = boundary?.gapBlankLines;
- if (gaps && gaps.length === mdast.children.length - 1) {
- for (let i = 1; i < mdast.children.length; i++) {
- const blanks = gaps[i - 1];
- if (typeof blanks === 'number') {
- const child = mdast.children[i];
- child.data ??= {};
- child.data.sourcePrecedingBlankLines = blanks;
- }
- }
- }
let out = String(processor.stringify(mdast));
if (boundary?.leading) out = boundary.leading + out;
diff --git a/packages/core/src/markdown/position-aware-join.ts b/packages/core/src/markdown/position-aware-join.ts
index 5a5e4c42a..91cdf658f 100644
--- a/packages/core/src/markdown/position-aware-join.ts
+++ b/packages/core/src/markdown/position-aware-join.ts
@@ -1,10 +1,11 @@
import type { Join } from 'mdast-util-to-markdown';
import type { Position } from 'unist';
+import { isBlankLineParagraph } from './zero-emission-blocks.ts';
type MaybePositioned = { position?: Position };
type MaybeTyped = {
type?: string;
- data?: { sourceContiguousNext?: boolean; sourcePrecedingBlankLines?: number };
+ data?: { sourceContiguousNext?: boolean };
};
type FlowNode = Parameters<Join>[0];
@@ -15,9 +16,8 @@ function isContiguousHeadingWithParagraph(left: FlowNode, right: FlowNode): bool
}
export const positionAwareBlankLineJoin: Join = (left, right) => {
+ if (isBlankLineParagraph(left)) return 0;
if (isContiguousHeadingWithParagraph(left, right)) return 0;
- const dataGap = (right as MaybeTyped).data?.sourcePrecedingBlankLines;
- if (typeof dataGap === 'number' && dataGap >= 2) return dataGap;
const lp = (left as MaybePositioned).position;
const rp = (right as MaybePositioned).position;
if (!lp || !rp) return undefined;
diff --git a/packages/core/src/markdown/zero-emission-blocks.ts b/packages/core/src/markdown/zero-emission-blocks.ts
new file mode 100644
index 000000000..78fc0100c
--- /dev/null
+++ b/packages/core/src/markdown/zero-emission-blocks.ts
@@ -0,0 +1,80 @@
+import type { Root as MdastRoot, Paragraph, PhrasingContent, RootContent } from 'mdast';
+
+type WithData = { data?: Record<string, unknown> };
+type WithValue = { value?: unknown };
+type WithChildren = { children?: unknown[] };
+
+function sourceRawOf(node: unknown): unknown {
+ return (node as WithData).data?.sourceRaw;
+}
+
+function emitsNoInlineBytes(node: PhrasingContent): boolean {
+ if (node.type === 'text') return (node as WithValue).value === '';
+ if (node.type === 'mdxJsxTextElement') return sourceRawOf(node) === '';
+ return false;
+}
+
+export function isBlankLineParagraph(node: unknown): boolean {
+ if ((node as { type?: string }).type !== 'paragraph') return false;
+ const children = (node as Paragraph).children ?? [];
+ return children.every(emitsNoInlineBytes);
+}
+
+function tableWidth(node: RootContent): number {
+ const rows = ((node as WithChildren).children ?? []) as WithChildren[];
+ const headerWidth = rows[0]?.children?.length ?? 0;
+ let bodyWidth = 0;
+ for (let i = 1; i < rows.length; i++) {
+ bodyWidth = Math.max(bodyWidth, rows[i]?.children?.length ?? 0);
+ }
+ const align = (node as { align?: unknown[] }).align ?? [];
+ return Math.max(headerWidth, bodyWidth, align.length);
+}
+
+function emitsNoBlockBytes(node: RootContent): boolean {
+ switch (node.type) {
+ case 'table':
+ return tableWidth(node) === 0;
+ case 'code':
+ return (
+ (node as WithData).data?.sourceStyle === 'indented' &&
+ /^\n*$/.test(((node as WithValue).value as string | undefined) ?? '')
+ );
+ case 'thematicBreak':
+ return sourceRawOf(node) === '';
+ case 'html':
+ return (((node as WithValue).value as string | undefined) ?? '') === '';
+ case 'mdxJsxFlowElement':
+ return (
+ sourceRawOf(node) === undefined &&
+ (node as WithData).data?.htmlBoundary === undefined &&
+ ((node as WithChildren).children ?? []).length === 0 &&
+ !(node as { name?: unknown }).name
+ );
+ case 'list':
+ return ((node as WithChildren).children ?? []).length === 0;
+ default:
+ return false;
+ }
+}
+
+export function pruneZeroEmissionBlocks(root: MdastRoot): void {
+ const kept: MdastRoot['children'] = [];
+ for (const child of root.children) {
+ if (child.type === 'table' && tableWidth(child) === 0) {
+ console.warn(
+ JSON.stringify({
+ event: 'table-serialize-dropped-empty',
+ rows: ((child as WithChildren).children ?? []).length,
+ }),
+ );
+ continue;
+ }
+ if (emitsNoBlockBytes(child)) continue;
+ if (child.type === 'paragraph' && child.children.length > 0 && isBlankLineParagraph(child)) {
+ child.children = [];
+ }
+ kept.push(child);
+ }
+ root.children = kept;
+}
diff --git a/packages/server/src/map-driven-observer-a.test.ts b/packages/server/src/map-driven-observer-a.test.ts
index c55fd7a18..1635f9a0f 100644
--- a/packages/server/src/map-driven-observer-a.test.ts
+++ b/packages/server/src/map-driven-observer-a.test.ts
@@ -354,7 +354,7 @@ describe('map-driven Observer A — default Path A behavior', () => {
test('a parse/serialize throw inside the splice reports parse-error instead of vanishing', () => {
const throwingManager = {
- parseToMdast: () => {
+ parseToEditorMdast: () => {
throw new Error('synthetic parser regression');
},
serialize: () => '',
@@ -377,12 +377,12 @@ describe('map-driven Observer A — default Path A behavior', () => {
test('a sustained parse-error fallback warns once with the error message, then stays counter-only', () => {
const raw = '# Heading\n\nFirst.\n\nSecond.\n';
const { doc, xmlFragment, ytext } = createTestDoc();
- // Real manager except parseToMdast — only the splice computation uses
- // parseToMdast, so the drain itself (serialize + the incremental-diff
+ // Real manager except parseToEditorMdast — only the splice computation
+ // uses that view, so the drain itself (serialize + the incremental-diff
// fallback) still completes while every splice attempt throws.
const throwingManager = new Proxy(mdManager, {
get(target, prop) {
- if (prop === 'parseToMdast') {
+ if (prop === 'parseToEditorMdast') {
return () => {
throw new Error('synthetic parser regression');
};
diff --git a/packages/server/src/map-driven-splice.ts b/packages/server/src/map-driven-splice.ts
index 054915ed3..4084df8ef 100644
--- a/packages/server/src/map-driven-splice.ts
+++ b/packages/server/src/map-driven-splice.ts
@@ -56,9 +56,14 @@ export function computeMapDrivenBodySplice(
let newBody: string;
let newChildren: readonly RootContent[];
try {
- oldChildren = mdManager.parseToMdast(oldBody).children;
+ // The editor view on both sides: a blank-line run between two blocks is
+ // a paragraph in the PM document but pure gap bytes to a CommonMark
+ // parse, so the sibling walk below would see two identical block lists
+ // and splice a byte range that excludes the very change it is meant to
+ // carry.
+ oldChildren = mdManager.parseToEditorMdast(oldBody).children;
newBody = mdManager.serialize(newPmJson);
- newChildren = mdManager.parseToMdast(newBody).children;
+ newChildren = mdManager.parseToEditorMdast(newBody).children;
} catch (err) {
// Swallowing is the contract (caller falls back to the whole-body diff
// path), but the swallow must not be silent: a systemic parse/serialize
diff --git a/packages/server/src/persistence-staleness-watchdog.test.ts b/packages/server/src/persistence-staleness-watchdog.test.ts
index a5b7a41d4..f0657afad 100644
--- a/packages/server/src/persistence-staleness-watchdog.test.ts
+++ b/packages/server/src/persistence-staleness-watchdog.test.ts
@@ -1,3 +1,4 @@
+import { normalizeBridge } from '@inkeep/open-knowledge-core';
import { afterEach, beforeEach, describe, expect, test } from 'vitest';
import * as Y from 'yjs';
import { isPersistenceExcludedDoc } from './cc1-broadcast.ts';
@@ -185,6 +186,41 @@ describe('staleness detection and forced store', () => {
expect(rig.forceCalls).toEqual(['fm-doc']);
});
+ test('rescues a blank run the normalized comparator cannot see', async () => {
+ // The comparator collapses blank runs on both sides, so a widened interior
+ // run reads as converged and the backstop would leave the edit in memory
+ // forever. The base and the memory below are deliberately normalize-equal.
+ const base = 'Above.\n\nBelow.\n';
+ const widened = 'Above.\n\n\n\nBelow.\n';
+ expect(normalizeBridge(base)).toBe(normalizeBridge(widened));
+
+ const doc = makeDoc(widened);
+ rig.docs.set('blank-run-doc', doc);
+ rig.bases.set('blank-run-doc', base);
+ rig.disk.set('blank-run-doc', base);
+ rig.txAges.set('blank-run-doc', GRACE_MS * 10);
+
+ await rig.watchdog.sweep();
+ expect(rig.forceCalls).toEqual(['blank-run-doc']);
+
+ // The store advanced the base, so the next sweep sees convergence.
+ await rig.watchdog.sweep();
+ expect(rig.forceCalls).toEqual(['blank-run-doc']);
+ });
+
+ test('leaves a doc alone when the base carries the wider run', async () => {
+ // The reverse direction is the collapse class the comparator exists to
+ // tolerate — forcing a store here would write the narrower form to disk.
+ const doc = makeDoc('- a\n\n- b\n');
+ rig.docs.set('collapsed-doc', doc);
+ rig.bases.set('collapsed-doc', '- a\n\n\n- b\n');
+ rig.disk.set('collapsed-doc', '- a\n\n\n- b\n');
+ rig.txAges.set('collapsed-doc', GRACE_MS * 10);
+
+ await rig.watchdog.sweep();
+ expect(rig.forceCalls).toEqual([]);
+ });
+
test('materializes a never-persisted doc with content and no disk file', async () => {
const doc = makeDoc('# brand new\n');
rig.docs.set('new-doc', doc);
diff --git a/packages/server/src/persistence-staleness-watchdog.ts b/packages/server/src/persistence-staleness-watchdog.ts
index 9ffdadd30..20dfbc8f9 100644
--- a/packages/server/src/persistence-staleness-watchdog.ts
+++ b/packages/server/src/persistence-staleness-watchdog.ts
@@ -44,7 +44,7 @@
* tripwire, writeTracker registration) applies unchanged.
*/
-import { normalizeBridge } from '@inkeep/open-knowledge-core';
+import { addsBlankLines, normalizeBridge } from '@inkeep/open-knowledge-core';
import type * as Y from 'yjs';
import { getMsSinceLastUserTx } from './bridge-quiescence.ts';
import { isPersistenceExcludedDoc } from './cc1-broadcast.ts';
@@ -182,8 +182,23 @@ export function createPersistenceStalenessWatchdog(
return base === undefined ? undefined : normalizeBridge(base);
}
- function isDivergent(candidate: string, normalizedBase: string | undefined): boolean {
- return normalizedBase === undefined ? candidate !== '' : candidate !== normalizedBase;
+ /**
+ * `rawYText` / `base` carry the unnormalized bytes because the normalized
+ * comparison collapses blank runs on both sides: a run the user just added
+ * reads as no divergence, and the durability backstop would never fire for
+ * an edit that slipped the debounced store. The reverse direction stays
+ * non-divergent — fewer blank lines than the base is the collapse class the
+ * comparator exists to tolerate.
+ */
+ function isDivergent(
+ candidate: string,
+ normalizedBase: string | undefined,
+ rawYText: string,
+ base: string | undefined,
+ ): boolean {
+ if (normalizedBase === undefined) return candidate !== '';
+ if (candidate !== normalizedBase) return true;
+ return base !== undefined && addsBlankLines(base, rawYText);
}
async function sweepOnce(): Promise<void> {
@@ -226,7 +241,7 @@ export function createPersistenceStalenessWatchdog(
}
const candidate = candidateFromRaw(rawYText);
const normalizedBase = normalizedBaseFor(base);
- if (!isDivergent(candidate, normalizedBase)) {
+ if (!isDivergent(candidate, normalizedBase, rawYText, base)) {
attempts.delete(documentName);
continue;
}
@@ -354,9 +369,14 @@ export function createPersistenceStalenessWatchdog(
// Re-read both sides: a store that completed yet left the same bytes
// divergent chose not to write them (no-op class) — mark declined so
// we don't loop on it. Convergence (or content that moved on) re-arms.
- const afterCandidate = candidateFromRaw(document.getText('source').toString());
- const afterNormalizedBase = normalizedBaseFor(getBase(documentName));
- if (isDivergent(afterCandidate, afterNormalizedBase) && afterCandidate === candidate) {
+ const afterRawYText = document.getText('source').toString();
+ const afterBase = getBase(documentName);
+ const afterCandidate = candidateFromRaw(afterRawYText);
+ const afterNormalizedBase = normalizedBaseFor(afterBase);
+ if (
+ isDivergent(afterCandidate, afterNormalizedBase, afterRawYText, afterBase) &&
+ afterCandidate === candidate
+ ) {
attempts.set(documentName, {
fingerprint: candidate,
atMs: nowMs,
diff --git a/packages/server/src/persistence-tripwire-paste.test.ts b/packages/server/src/persistence-tripwire-paste.test.ts
index 6ed2396cb..5c0f9acc8 100644
--- a/packages/server/src/persistence-tripwire-paste.test.ts
+++ b/packages/server/src/persistence-tripwire-paste.test.ts
@@ -49,7 +49,10 @@ const FIXTURE_DIR = resolve(import.meta.dirname, 'persistence-tripwire.fixtures'
* doubled blank line — that is what makes this 51 bytes rather than the 43
* the same five paragraphs occupy with single blank lines.
*/
-const USER_DOC = 'hello\n\n\n\nkjnekandkjawnkjd\n\n\n\nwkajnd\n\n\n\nwk\n\n\n\nwwjwj\n';
+// One blank-line paragraph per gap serializes to three newlines: the canonical
+// block separator plus the blank line itself. Four newlines would be the shape
+// that re-parses to TWO paragraphs per gap, doubling the run on every trip.
+const USER_DOC = 'hello\n\n\nkjnekandkjawnkjd\n\n\nwkajnd\n\n\nwk\n\n\nwwjwj\n';
const USER_DOC_LINE = 'kjnekandkjawnkjd';
const BROWSER_ORIGIN = {
@@ -180,7 +183,7 @@ describe('persistence tripwire vs a whole-document paste', () => {
// the shape genuinely IS an integer doubling, and the epoch-recovery guard
// still needs that verdict. Pinning it here keeps a future "fix" from
// weakening the classifier instead of the action it drives.
- expect(USER_DOC.length).toBe(51);
+ expect(USER_DOC.length).toBe(47);
expect(classifyDuplication(`${USER_DOC}\n${USER_DOC}`, USER_DOC)).toEqual({
kind: 'block',
reason: 'structural-duplication',
diff --git a/packages/server/src/persistence.ts b/packages/server/src/persistence.ts
index bbcf1f4f7..6643e0fb0 100644
--- a/packages/server/src/persistence.ts
+++ b/packages/server/src/persistence.ts
@@ -22,6 +22,7 @@ import { dirname, relative, resolve } from 'node:path';
import type { Extension } from '@hocuspocus/server';
import type { MarkdownManager } from '@inkeep/open-knowledge-core';
import {
+ addsBlankLines,
BridgeInvariantViolationError,
type ConfigValidationError,
DOCUMENT_OPEN_BYTE_LIMIT,
@@ -1675,8 +1676,16 @@ export function createPersistenceExtension(options?: PersistenceOptions): Persis
// Routed through the shared helper so this compare and the
// staleness watchdog's divergence predicate stay one derivation.
const normalizedMarkdown = normalizedSourceForm(ytextSnapshot);
+ // The blank-line tolerance is symmetric, so blank lines a user just
+ // added normalize away and this compare would call the store a no-op —
+ // the edit would live in the CRDT and never reach the file. The
+ // opposite direction stays a no-op, since a candidate with FEWER blank
+ // lines than the base is the mount artifact / container-collapse class
+ // this gate exists to suppress.
let markdownSemanticallyUnchanged =
- currentBase !== undefined && normalizedMarkdown === normalizeBridge(currentBase);
+ currentBase !== undefined &&
+ normalizedMarkdown === normalizeBridge(currentBase) &&
+ !addsBlankLines(currentBase, markdown);
// G8 (ephemeral single-file mode only): also treat the candidate as a
// no-op when it equals the CANONICAL serialization of the on-disk base.
// A round-trip-unstable file (`## H\nP`) load-canonicalizes on open —
diff --git a/packages/server/src/server-observers.ts b/packages/server/src/server-observers.ts
index c9edcd892..4dc02579f 100644
--- a/packages/server/src/server-observers.ts
+++ b/packages/server/src/server-observers.ts
@@ -29,6 +29,7 @@ import type {
StructuralDivergenceReason,
} from '@inkeep/open-knowledge-core';
import {
+ addsBlankLines,
applyFastDiff,
applyIncrementalDiff,
BridgeInvariantViolationError,
@@ -1887,7 +1888,13 @@ export function setupServerObservers(opts: SetupServerObserversOpts): () => void
// classification below went lazy).
const normCurrent = normalizeBridge(currentText);
const normMd = normalizeBridge(md);
- if (normCurrent === normMd) {
+ // The blank-line tolerance is symmetric, so a run the user just added in
+ // the WYSIWYG normalizes away and this gate would certify a settlement
+ // that drops it. The fragment is the authority for its own blank lines;
+ // the reverse direction (source richer than the fragment) stays
+ // tolerated, since that is the container-interior case the fragment
+ // cannot represent.
+ if (normCurrent === normMd && !addsBlankLines(currentText, md)) {
setActiveSpanAttributes({ 'observer.a.path': 'gated-in-sync' });
recordSettledBaselines(md);
return;
@@ -2153,6 +2160,13 @@ export function setupServerObservers(opts: SetupServerObserversOpts): () => void
// residual doc on every WYSIWYG edit (regressing the residual-merge
// steady state); the bytes are already safe either way.
if (settlesSplitBrainChecked(appliedYText, md, normMd, normApplied)) {
+ // When the drain left Y.Text untouched, B's raw-witness comparand
+ // still equals the live bytes and it would early-exit, leaving the
+ // divergence to rest until some later fragment change happens to
+ // re-trigger the identity gate. Move to the diverged-attach witness
+ // shape in exactly that case so B re-derives now. A drain that DID
+ // move Y.Text keeps the residual-merge steady state.
+ if (appliedYText === preMergeBaseline) recordDivergedAttachBaselines(md);
textDirty = true;
recordSplitBrainRederive('post-merge');
}