Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
fix(open-knowledge): don't over-escape boundary-whitespace char-refs (PRD-6767) (#1627)
Nick Gomez523f1d8adf43
diff --git a/.changeset/fix-boundary-space-overescape.md b/.changeset/fix-boundary-space-overescape.mdnew file mode 100644index 000000000..603f7ac45--- /dev/null+++ b/.changeset/fix-boundary-space-overescape.md@@ -0,0 +1,7 @@+---+"@inkeep/open-knowledge-core": patch+---++fix(markdown): stop over-escaping phrasing-boundary whitespace character references++A space or tab at a phrasing boundary (the edge of a paragraph, list item, or blockquote's inline content) was serializing to a backslash-escaped character reference (`\ ` / `\	`) instead of a bare ` ` / `	`. Per CommonMark the escaped form renders as the literal visible text ` ` in GitHub, the docs site, and Open Knowledge's own re-parsed editor. `escapeEntityAmpersands` now leaves character references that `mdast-util-to-markdown` synthesizes for boundary-whitespace preservation un-escaped, while continuing to escape user-authored entities.diff --git a/packages/core/src/markdown/roundtrip-boundary-space-overescape.test.ts b/packages/core/src/markdown/roundtrip-boundary-space-overescape.test.tsnew file mode 100644index 000000000..55ba420b3--- /dev/null+++ b/packages/core/src/markdown/roundtrip-boundary-space-overescape.test.ts@@ -0,0 +1,149 @@+import { describe, expect, test } from 'bun:test';+import type { JSONContent } from '@tiptap/core';+import { sharedExtensions } from '../extensions/shared.ts';+import { MarkdownManager } from './index.ts';++const md = new MarkdownManager({ extensions: sharedExtensions });++const NBSP = '\u00A0';+const SPACE = ' ';+const TAB = '\t';++const strong = (t: string): JSONContent => ({+ type: 'text',+ marks: [{ type: 'strong', attrs: { sourceDelimiter: '**' } }],+ text: t,+});+const text = (t: string): JSONContent => ({ type: 'text', text: t });+const paragraph = (...inline: JSONContent[]): JSONContent => ({+ type: 'doc',+ content: [{ type: 'paragraph', content: inline }],+});+const listItem = (...inline: JSONContent[]): JSONContent => ({+ type: 'doc',+ content: [+ {+ type: 'list',+ attrs: {+ ordered: false,+ start: 1,+ spread: false,+ bulletMarker: '-',+ listMarkerDelimiter: null,+ },+ content: [+ {+ type: 'listItem',+ attrs: { checked: null, spread: false },+ content: [{ type: 'paragraph', content: inline }],+ },+ ],+ },+ ],+});++const expectNoBoundaryOverEscape = (out: string): void => {+ expect(out).not.toContain('\\ ');+ expect(out).not.toContain('\\	');+ expect(out).not.toMatch(/\\&#x?[0-9A-Fa-f]/);+};++describe('boundary whitespace must not over-escape — source NBSP round-trip (path i)', () => {+ const trailing: ReadonlyArray<readonly [string, string]> = [+ ['paragraph', `abc${NBSP}\n`],+ ['emphasis', `*a*${NBSP}\n`],+ ['strong', `**a**${NBSP}\n`],+ ['inline code', `\`a\`${NBSP}\n`],+ ['link', `[a](b)${NBSP}\n`],+ ['strikethrough', `~~a~~${NBSP}\n`],+ ['wiki-link', `[[a]]${NBSP}\n`],+ ['blockquote', `> a${NBSP}\n`],+ ['nested list item', `- x\n - y${NBSP}\n`],+ ];+ for (const [label, src] of trailing) {+ test(`trailing NBSP after ${label}`, () => {+ expectNoBoundaryOverEscape(md.serialize(md.parse(src)));+ });+ }++ const leading: ReadonlyArray<readonly [string, string]> = [+ ['paragraph', `${NBSP}abc\n`],+ ['emphasis', `${NBSP}*a*\n`],+ ['strong', `${NBSP}**a**\n`],+ ['inline code', `${NBSP}\`a\`\n`],+ ['link', `${NBSP}[a](b)\n`],+ ['strikethrough', `${NBSP}~~a~~\n`],+ ['wiki-link', `${NBSP}[[a]]\n`],+ ['blockquote', `> ${NBSP}a\n`],+ ];+ for (const [label, src] of leading) {+ test(`leading NBSP before ${label}`, () => {+ expectNoBoundaryOverEscape(md.serialize(md.parse(src)));+ });+ }+});++describe('boundary whitespace must not over-escape — PM-doc text node (path ii)', () => {+ test('trailing regular space after strong', () => {+ expectNoBoundaryOverEscape(md.serialize(paragraph(strong('work.'), text(SPACE))));+ });+ test('leading regular space before strong', () => {+ expectNoBoundaryOverEscape(md.serialize(paragraph(text(SPACE), strong('bold'))));+ });+ test('trailing regular space after plain text', () => {+ expectNoBoundaryOverEscape(md.serialize(paragraph(text('abc'), text(SPACE))));+ });+ test('trailing regular space in list item', () => {+ expectNoBoundaryOverEscape(md.serialize(listItem(strong('x'), text(SPACE))));+ });++ test('trailing tab after strong', () => {+ expectNoBoundaryOverEscape(md.serialize(paragraph(strong('work.'), text(TAB))));+ });+ test('leading tab before strong', () => {+ expectNoBoundaryOverEscape(md.serialize(paragraph(text(TAB), strong('bold'))));+ });+ test('trailing tab in list item', () => {+ expectNoBoundaryOverEscape(md.serialize(listItem(strong('x'), text(TAB))));+ });+});++describe('guard: user-authored entities round-trip byte-identically (no over-correction)', () => {+ test('bare numeric entity', () => {+ expect(md.serialize(md.parse('aAb\n'))).toBe('aAb\n');+ });+ test('bare whitespace numeric entity (hex space)', () => {+ expect(md.serialize(md.parse('a b\n'))).toBe('a b\n');+ });+ test('bare whitespace numeric entity (decimal space)', () => {+ expect(md.serialize(md.parse('a b\n'))).toBe('a b\n');+ });+ test('bare whitespace numeric entity (hex tab)', () => {+ expect(md.serialize(md.parse('a	b\n'))).toBe('a	b\n');+ });+ test('bare named entity', () => {+ expect(md.serialize(md.parse('a&b\n'))).toBe('a&b\n');+ });+ test('escaped named entity', () => {+ expect(md.serialize(md.parse('a\\öb\n'))).toBe('a\\öb\n');+ });+ test('escaped non-whitespace numeric entity (hex)', () => {+ expect(md.serialize(md.parse('a\\Ab\n'))).toBe('a\\Ab\n');+ });+ test('escaped non-whitespace numeric entity (decimal)', () => {+ expect(md.serialize(md.parse('a\\Ab\n'))).toBe('a\\Ab\n');+ });+ test('user-authored escaped numeric entity (byte-identical to the corruption)', () => {+ expect(md.serialize(md.parse('Entity \\  stays.\n'))).toBe('Entity \\  stays.\n');+ });+});++describe('guard: heading / table asymmetry — a fix must not start encoding them', () => {+ test('ATX heading with trailing NBSP', () => {+ expect(md.serialize(md.parse(`# h${NBSP}\n`))).not.toContain('\\ ');+ });+ test('table cell with trailing NBSP', () => {+ const tbl = `| a${NBSP} | b |\n| --- | --- |\n| c | d |\n`;+ expect(md.serialize(md.parse(tbl))).not.toContain('\\ ');+ });+});diff --git a/packages/core/src/markdown/to-markdown-handlers.ts b/packages/core/src/markdown/to-markdown-handlers.tsindex cd0c935f6..62542c87d 100644--- a/packages/core/src/markdown/to-markdown-handlers.ts+++ b/packages/core/src/markdown/to-markdown-handlers.ts@@ -619,7 +619,26 @@ export function formatLinkUrl(url: string): string {}function escapeEntityAmpersands(s: string): string {- return s.replace(/(?<!\\)&(?=[A-Za-z][A-Za-z0-9]*;|#[0-9]+;|#[xX][0-9A-Fa-f]+;)/g, '\\&');+ return s.replace(+ /(?<!\\)&(?=(#[0-9]+|#[xX][0-9A-Fa-f]+|[A-Za-z][A-Za-z0-9]*);)/g,+ (match, body: string) => (isWhitespaceNumericCharRef(body) ? match : `\\${match}`),+ );+}++function isWhitespaceNumericCharRef(body: string): boolean {+ if (body.charCodeAt(0) !== 0x23 /* '#' */) return false;+ const code =+ body[1] === 'x' || body[1] === 'X'+ ? Number.parseInt(body.slice(2), 16)+ : Number.parseInt(body.slice(1), 10);+ return (+ code === 0x09 ||+ code === 0x0a ||+ code === 0x0b ||+ code === 0x0c ||+ code === 0x0d ||+ code === 0x20+ );}function safeText(state: State, value: string, info: Info): string {