Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
fix(open-knowledge): boundary-whitespace serialization fidelity (#2111)
Nick Gomezf349f38093c9
diff --git a/.changeset/serialize-boundary-whitespace-fidelity.md b/.changeset/serialize-boundary-whitespace-fidelity.mdnew file mode 100644index 000000000..97e50448e--- /dev/null+++ b/.changeset/serialize-boundary-whitespace-fidelity.md@@ -0,0 +1,5 @@+---+"@inkeep/open-knowledge-core": patch+---++Fix markdown serializer boundary-whitespace defects on the WYSIWYG-to-source path. An insignificant trailing space or tab at a block edge now serializes to a literal character instead of a visible ` ` / `	` character reference, so source mode no longer shows the escape where a space was typed (a leading tab, or a leading run of four or more spaces, still encodes, since it would otherwise trigger an indented code block on re-parse and turn the paragraph into a code block). Strikethrough (`~~`) and highlight (`==`) marks authored with boundary whitespace now char-ref-encode that whitespace so the mark survives re-parse instead of silently dropping. Emphasis and strong already behaved correctly and are unchanged.diff --git a/packages/app/tests/integration/bug3-source-mode-writeback.test.ts b/packages/app/tests/integration/bug3-source-mode-writeback.test.tsnew file mode 100644index 000000000..b08b3ec2d--- /dev/null+++ b/packages/app/tests/integration/bug3-source-mode-writeback.test.ts@@ -0,0 +1,113 @@+import { afterAll, beforeAll, describe, expect, test } from 'bun:test';+import { setTimeout as wait } from 'node:timers/promises';+import { updateYFragment } from '@tiptap/y-tiptap';+import { HARNESS_BOOT_TIMEOUT_MS } from './harness-boot-timeout';+import {+ agentWriteMd,+ awaitDocQuiescence,+ createTestClient,+ createTestServer,+ getServerState,+ mdManager,+ schema,+ type TestClient,+ type TestServer,+} from './test-harness';++const STEPS = [+ '<Steps>',+ '',+ '<Step>',+ '',+ 'Content one.',+ '',+ '</Step>',+ '',+ '<Step>',+ '',+ 'Content two.',+ '',+ '</Step>',+ '',+ '</Steps>',+ '',+].join('\n');++let server: TestServer;++beforeAll(async () => {+ server = await createTestServer();+}, HARNESS_BOOT_TIMEOUT_MS);++afterAll(async () => {+ await server.cleanup();+});++/** A genuine WYSIWYG-side fragment commit (null origin => server Observer A sees+ * a real WYSIWYG mutation, xmlDirty=true) — the same channel the hidden-but-+ * mounted TipTap binding republishes through. Non-vacuous: it really changes+ * the fragment, so Observer A runs and its serialize-vs-ytext diff is exercised. */+function applyWysiwygEdit(client: TestClient, markdownAfterEdit: string): void {+ const pmNode = schema.nodeFromJSON(mdManager.parse(markdownAfterEdit));+ client.doc.transact(() => {+ updateYFragment(client.doc, client.fragment, pmNode, {+ mapping: new Map(),+ isOMark: new Map(),+ });+ });+}++const INDENTED_STEP = /\n[ \t]+<\/?Step\b/; // a <Step>/</Step> tag gaining leading indentation+const INDENTED_STEPS = /\n[ \t]+<\/?Steps\b/;++describe('bug #3 — source-mode write-back guard (re-indent facet closed by #1991)', () => {+ test('the faithful <Steps> parses to a jsxComponent and is a serialize fixed point', () => {+ const tree = mdManager.parse(STEPS) as { content?: Array<{ type?: string }> };+ const topTypes = (tree.content ?? []).map((n) => n.type);+ expect(topTypes).toContain('jsxComponent');+ expect(mdManager.serialize(mdManager.parse(STEPS))).toBe(+ '<Steps>\n\n<Step>\n\nContent one.\n\n</Step>\n\n<Step>\n\nContent two.\n\n</Step>\n\n</Steps>\n',+ );+ });++ test('V1 baseline: an isolated source keystroke stays byte-verbatim (no Observer-A write-back)', async () => {+ const docName = `bug3-v1-${crypto.randomUUID()}`;+ await agentWriteMd(server.port, STEPS, { docName, position: 'replace' });+ await wait(300);+ const client = await createTestClient(server.port, docName);+ try {+ const ytext = client.doc.getText('source');+ await awaitDocQuiescence(client.doc);+ expect(ytext.toString()).toBe(STEPS); // seed landed verbatim+ const at = ytext.toString().indexOf('Content one.') + 'Content one'.length;+ client.doc.transact(() => ytext.insert(at, 'X'));+ const expected = ytext.toString();+ await awaitDocQuiescence(client.doc);+ expect(getServerState(server, docName)?.ytext.toString()).toBe(expected);+ } finally {+ await client.cleanup();+ }+ });++ test('a concurrent WYSIWYG fragment commit does NOT re-indent the <Steps> in Y.Text', async () => {+ const docName = `bug3-writeback-${crypto.randomUUID()}`;+ await agentWriteMd(server.port, STEPS, { docName, position: 'replace' });+ await wait(300);+ const client = await createTestClient(server.port, docName);+ try {+ const ytext = client.doc.getText('source');+ await awaitDocQuiescence(client.doc);+ expect(ytext.toString()).toBe(STEPS);++ applyWysiwygEdit(client, STEPS.replace('Content two.', 'Content two, edited.'));+ await awaitDocQuiescence(client.doc);++ const after = getServerState(server, docName)?.ytext.toString() ?? '';+ expect(after).toContain('Content two, edited.'); // the edit landed (non-vacuous)+ expect(after).not.toMatch(INDENTED_STEP);+ expect(after).not.toMatch(INDENTED_STEPS);+ } finally {+ await client.cleanup();+ }+ });+});diff --git a/packages/core/src/markdown/serialize-fidelity.test-helper.ts b/packages/core/src/markdown/serialize-fidelity.test-helper.tsnew file mode 100644index 000000000..8f9fdb08e--- /dev/null+++ b/packages/core/src/markdown/serialize-fidelity.test-helper.ts@@ -0,0 +1,39 @@+import { expect } from 'bun:test';+import type { JSONContent } from '@tiptap/core';+import type { Nodes } from 'mdast';+import { sharedExtensions } from '../extensions/shared.ts';+import { MarkdownManager } from './index.ts';+import { isInlineWhitespaceNumericCharRef } from './whitespace-char-ref.ts';++const md = new MarkdownManager({ extensions: sharedExtensions });++const NUMERIC_CHAR_REF_SCAN = /&#(?:x[0-9A-Fa-f]+|X[0-9A-Fa-f]+|[0-9]+);/g;++export function assertFragmentSerializesTo(fragment: JSONContent, expectedBytes: string): void {+ expect(md.serialize(fragment)).toBe(expectedBytes);+}++export function assertNoBoundaryLeak(fragment: JSONContent): void {+ const out = md.serialize(fragment);+ const leaked = (out.match(NUMERIC_CHAR_REF_SCAN) ?? []).filter(isInlineWhitespaceNumericCharRef);+ expect(leaked).toEqual([]);+}++export function assertMarkSurvives(fragment: JSONContent, markType: string): void {+ const out = md.serialize(fragment);+ expect(mdastNodeTypes(out)).toContain(markType);+}++function mdastNodeTypes(markdown: string): string[] {+ const out: string[] = [];+ const walk = (node: Nodes): void => {+ out.push(node.type);+ if ('children' in node) {+ for (const child of node.children) {+ walk(child);+ }+ }+ };+ walk(md.parseToMdast(markdown));+ return out;+}diff --git a/packages/core/src/markdown/to-markdown-handlers.ts b/packages/core/src/markdown/to-markdown-handlers.tsindex c04277c04..f622bdd21 100644--- a/packages/core/src/markdown/to-markdown-handlers.ts+++ b/packages/core/src/markdown/to-markdown-handlers.ts@@ -567,11 +567,15 @@ export const toMarkdownHandlers = {const tracker = state.createTracker(info);const exit = state.enter('mark');let value = tracker.move('==');- value += state.containerPhrasing(node as Parents, {- before: value,- after: '==',- ...tracker.current(),- });+ value += encodeAttentionBoundaries(+ state.containerPhrasing(node as Parents, {+ before: value,+ after: '==',+ ...tracker.current(),+ }),+ info,+ '==',+ );value += tracker.move('==');exit();return value;@@ -786,6 +790,12 @@ function safeText(state: State, value: string, info: Info): string {return false;}if (u.character === '=' && u.atBreak === true) return false;+ if (u.character === ' ' && u.inConstruct === 'phrasing' && u.after === '[\\r\\n]') {+ return false;+ }+ if (u.character === '\t' && u.inConstruct === 'phrasing' && u.after === '[\\r\\n]') {+ return false;+ }return true;});let result: string;@@ -873,14 +883,16 @@ function escapeActiveDelimiterRuns(value: string, info: Info): string {}function encodeAttentionBoundaries(between: string, info: Info, delim: string): string {- const marker = delim.startsWith('_') ? '_' : '*';+ const marker: '*' | '_' = delim.startsWith('_') ? '_' : '*';+ const before = info.before ?? '';+ const after = info.after ?? '';let result = between;const head = result.charCodeAt(0);- if (shouldEncodeAttentionBoundary(info.before.charCodeAt(info.before.length - 1), head, marker)) {+ if (shouldEncodeAttentionBoundary(before.charCodeAt(before.length - 1), head, marker)) {result = encodeCharacterReference(head) + result.slice(1);}const tail = result.charCodeAt(result.length - 1);- if (shouldEncodeAttentionBoundary(info.after.charCodeAt(0), tail, marker)) {+ if (shouldEncodeAttentionBoundary(after.charCodeAt(0), tail, marker)) {result = result.slice(0, -1) + encodeCharacterReference(tail);}return result;@@ -950,11 +962,15 @@ function serializeDelete(const tracker = state.createTracker(info);const exit = state.enter('strikethrough');let value = tracker.move(delim);- value += state.containerPhrasing(node, {- ...tracker.current(),- before: value,- after: '~',- });+ value += encodeAttentionBoundaries(+ state.containerPhrasing(node, {+ ...tracker.current(),+ before: value,+ after: '~',+ }),+ info,+ delim,+ );value += tracker.move(delim);exit();return value;diff --git a/turbo.json b/turbo.jsonindex 24d04f431..e99d47c58 100644--- a/turbo.json+++ b/turbo.json@@ -30,7 +30,7 @@"test": {"dependsOn": ["^build"],"cache": true,- "inputs": ["src/**/*.ts", "src/**/*.tsx", "tests/contract/**/*.ts"]+ "inputs": ["src/**/*.ts", "src/**/*.tsx", "src/**/*.json", "tests/contract/**/*.ts"]},"test:dom": {"dependsOn": ["^build"],