Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

fix(open-knowledge): boundary-whitespace serialization fidelity (#2111)

Nick Gomezf349f38093c9
diff --git a/.changeset/serialize-boundary-whitespace-fidelity.md b/.changeset/serialize-boundary-whitespace-fidelity.md
new file mode 100644
index 000000000..97e50448e
--- /dev/null
+++ b/.changeset/serialize-boundary-whitespace-fidelity.md
@@ -0,0 +1,5 @@
+---
+"@inkeep/open-knowledge-core": patch
+---
+
+Fix markdown serializer boundary-whitespace defects on the WYSIWYG-to-source path. An insignificant trailing space or tab at a block edge now serializes to a literal character instead of a visible ` ` / `	` character reference, so source mode no longer shows the escape where a space was typed (a leading tab, or a leading run of four or more spaces, still encodes, since it would otherwise trigger an indented code block on re-parse and turn the paragraph into a code block). Strikethrough (`~~`) and highlight (`==`) marks authored with boundary whitespace now char-ref-encode that whitespace so the mark survives re-parse instead of silently dropping. Emphasis and strong already behaved correctly and are unchanged.
diff --git a/packages/app/tests/integration/bug3-source-mode-writeback.test.ts b/packages/app/tests/integration/bug3-source-mode-writeback.test.ts
new file mode 100644
index 000000000..b08b3ec2d
--- /dev/null
+++ b/packages/app/tests/integration/bug3-source-mode-writeback.test.ts
@@ -0,0 +1,113 @@
+import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
+import { setTimeout as wait } from 'node:timers/promises';
+import { updateYFragment } from '@tiptap/y-tiptap';
+import { HARNESS_BOOT_TIMEOUT_MS } from './harness-boot-timeout';
+import {
+ agentWriteMd,
+ awaitDocQuiescence,
+ createTestClient,
+ createTestServer,
+ getServerState,
+ mdManager,
+ schema,
+ type TestClient,
+ type TestServer,
+} from './test-harness';
+
+const STEPS = [
+ '<Steps>',
+ '',
+ '<Step>',
+ '',
+ 'Content one.',
+ '',
+ '</Step>',
+ '',
+ '<Step>',
+ '',
+ 'Content two.',
+ '',
+ '</Step>',
+ '',
+ '</Steps>',
+ '',
+].join('\n');
+
+let server: TestServer;
+
+beforeAll(async () => {
+ server = await createTestServer();
+}, HARNESS_BOOT_TIMEOUT_MS);
+
+afterAll(async () => {
+ await server.cleanup();
+});
+
+/** A genuine WYSIWYG-side fragment commit (null origin => server Observer A sees
+ * a real WYSIWYG mutation, xmlDirty=true) — the same channel the hidden-but-
+ * mounted TipTap binding republishes through. Non-vacuous: it really changes
+ * the fragment, so Observer A runs and its serialize-vs-ytext diff is exercised. */
+function applyWysiwygEdit(client: TestClient, markdownAfterEdit: string): void {
+ const pmNode = schema.nodeFromJSON(mdManager.parse(markdownAfterEdit));
+ client.doc.transact(() => {
+ updateYFragment(client.doc, client.fragment, pmNode, {
+ mapping: new Map(),
+ isOMark: new Map(),
+ });
+ });
+}
+
+const INDENTED_STEP = /\n[ \t]+<\/?Step\b/; // a <Step>/</Step> tag gaining leading indentation
+const INDENTED_STEPS = /\n[ \t]+<\/?Steps\b/;
+
+describe('bug #3 — source-mode write-back guard (re-indent facet closed by #1991)', () => {
+ test('the faithful <Steps> parses to a jsxComponent and is a serialize fixed point', () => {
+ const tree = mdManager.parse(STEPS) as { content?: Array<{ type?: string }> };
+ const topTypes = (tree.content ?? []).map((n) => n.type);
+ expect(topTypes).toContain('jsxComponent');
+ expect(mdManager.serialize(mdManager.parse(STEPS))).toBe(
+ '<Steps>\n\n<Step>\n\nContent one.\n\n</Step>\n\n<Step>\n\nContent two.\n\n</Step>\n\n</Steps>\n',
+ );
+ });
+
+ test('V1 baseline: an isolated source keystroke stays byte-verbatim (no Observer-A write-back)', async () => {
+ const docName = `bug3-v1-${crypto.randomUUID()}`;
+ await agentWriteMd(server.port, STEPS, { docName, position: 'replace' });
+ await wait(300);
+ const client = await createTestClient(server.port, docName);
+ try {
+ const ytext = client.doc.getText('source');
+ await awaitDocQuiescence(client.doc);
+ expect(ytext.toString()).toBe(STEPS); // seed landed verbatim
+ const at = ytext.toString().indexOf('Content one.') + 'Content one'.length;
+ client.doc.transact(() => ytext.insert(at, 'X'));
+ const expected = ytext.toString();
+ await awaitDocQuiescence(client.doc);
+ expect(getServerState(server, docName)?.ytext.toString()).toBe(expected);
+ } finally {
+ await client.cleanup();
+ }
+ });
+
+ test('a concurrent WYSIWYG fragment commit does NOT re-indent the <Steps> in Y.Text', async () => {
+ const docName = `bug3-writeback-${crypto.randomUUID()}`;
+ await agentWriteMd(server.port, STEPS, { docName, position: 'replace' });
+ await wait(300);
+ const client = await createTestClient(server.port, docName);
+ try {
+ const ytext = client.doc.getText('source');
+ await awaitDocQuiescence(client.doc);
+ expect(ytext.toString()).toBe(STEPS);
+
+ applyWysiwygEdit(client, STEPS.replace('Content two.', 'Content two, edited.'));
+ await awaitDocQuiescence(client.doc);
+
+ const after = getServerState(server, docName)?.ytext.toString() ?? '';
+ expect(after).toContain('Content two, edited.'); // the edit landed (non-vacuous)
+ expect(after).not.toMatch(INDENTED_STEP);
+ expect(after).not.toMatch(INDENTED_STEPS);
+ } finally {
+ await client.cleanup();
+ }
+ });
+});
diff --git a/packages/core/src/markdown/serialize-fidelity.test-helper.ts b/packages/core/src/markdown/serialize-fidelity.test-helper.ts
new file mode 100644
index 000000000..8f9fdb08e
--- /dev/null
+++ b/packages/core/src/markdown/serialize-fidelity.test-helper.ts
@@ -0,0 +1,39 @@
+import { expect } from 'bun:test';
+import type { JSONContent } from '@tiptap/core';
+import type { Nodes } from 'mdast';
+import { sharedExtensions } from '../extensions/shared.ts';
+import { MarkdownManager } from './index.ts';
+import { isInlineWhitespaceNumericCharRef } from './whitespace-char-ref.ts';
+
+const md = new MarkdownManager({ extensions: sharedExtensions });
+
+const NUMERIC_CHAR_REF_SCAN = /&#(?:x[0-9A-Fa-f]+|X[0-9A-Fa-f]+|[0-9]+);/g;
+
+export function assertFragmentSerializesTo(fragment: JSONContent, expectedBytes: string): void {
+ expect(md.serialize(fragment)).toBe(expectedBytes);
+}
+
+export function assertNoBoundaryLeak(fragment: JSONContent): void {
+ const out = md.serialize(fragment);
+ const leaked = (out.match(NUMERIC_CHAR_REF_SCAN) ?? []).filter(isInlineWhitespaceNumericCharRef);
+ expect(leaked).toEqual([]);
+}
+
+export function assertMarkSurvives(fragment: JSONContent, markType: string): void {
+ const out = md.serialize(fragment);
+ expect(mdastNodeTypes(out)).toContain(markType);
+}
+
+function mdastNodeTypes(markdown: string): string[] {
+ const out: string[] = [];
+ const walk = (node: Nodes): void => {
+ out.push(node.type);
+ if ('children' in node) {
+ for (const child of node.children) {
+ walk(child);
+ }
+ }
+ };
+ walk(md.parseToMdast(markdown));
+ return out;
+}
diff --git a/packages/core/src/markdown/to-markdown-handlers.ts b/packages/core/src/markdown/to-markdown-handlers.ts
index c04277c04..f622bdd21 100644
--- a/packages/core/src/markdown/to-markdown-handlers.ts
+++ b/packages/core/src/markdown/to-markdown-handlers.ts
@@ -567,11 +567,15 @@ export const toMarkdownHandlers = {
const tracker = state.createTracker(info);
const exit = state.enter('mark');
let value = tracker.move('==');
- value += state.containerPhrasing(node as Parents, {
- before: value,
- after: '==',
- ...tracker.current(),
- });
+ value += encodeAttentionBoundaries(
+ state.containerPhrasing(node as Parents, {
+ before: value,
+ after: '==',
+ ...tracker.current(),
+ }),
+ info,
+ '==',
+ );
value += tracker.move('==');
exit();
return value;
@@ -786,6 +790,12 @@ function safeText(state: State, value: string, info: Info): string {
return false;
}
if (u.character === '=' && u.atBreak === true) return false;
+ if (u.character === ' ' && u.inConstruct === 'phrasing' && u.after === '[\\r\\n]') {
+ return false;
+ }
+ if (u.character === '\t' && u.inConstruct === 'phrasing' && u.after === '[\\r\\n]') {
+ return false;
+ }
return true;
});
let result: string;
@@ -873,14 +883,16 @@ function escapeActiveDelimiterRuns(value: string, info: Info): string {
}
function encodeAttentionBoundaries(between: string, info: Info, delim: string): string {
- const marker = delim.startsWith('_') ? '_' : '*';
+ const marker: '*' | '_' = delim.startsWith('_') ? '_' : '*';
+ const before = info.before ?? '';
+ const after = info.after ?? '';
let result = between;
const head = result.charCodeAt(0);
- if (shouldEncodeAttentionBoundary(info.before.charCodeAt(info.before.length - 1), head, marker)) {
+ if (shouldEncodeAttentionBoundary(before.charCodeAt(before.length - 1), head, marker)) {
result = encodeCharacterReference(head) + result.slice(1);
}
const tail = result.charCodeAt(result.length - 1);
- if (shouldEncodeAttentionBoundary(info.after.charCodeAt(0), tail, marker)) {
+ if (shouldEncodeAttentionBoundary(after.charCodeAt(0), tail, marker)) {
result = result.slice(0, -1) + encodeCharacterReference(tail);
}
return result;
@@ -950,11 +962,15 @@ function serializeDelete(
const tracker = state.createTracker(info);
const exit = state.enter('strikethrough');
let value = tracker.move(delim);
- value += state.containerPhrasing(node, {
- ...tracker.current(),
- before: value,
- after: '~',
- });
+ value += encodeAttentionBoundaries(
+ state.containerPhrasing(node, {
+ ...tracker.current(),
+ before: value,
+ after: '~',
+ }),
+ info,
+ delim,
+ );
value += tracker.move(delim);
exit();
return value;
diff --git a/turbo.json b/turbo.json
index 24d04f431..e99d47c58 100644
--- a/turbo.json
+++ b/turbo.json
@@ -30,7 +30,7 @@
"test": {
"dependsOn": ["^build"],
"cache": true,
- "inputs": ["src/**/*.ts", "src/**/*.tsx", "tests/contract/**/*.ts"]
+ "inputs": ["src/**/*.ts", "src/**/*.tsx", "src/**/*.json", "tests/contract/**/*.ts"]
},
"test:dom": {
"dependsOn": ["^build"],