Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

fix(open-knowledge): decode boundary-whitespace char-refs for display (#1853)

Nick Gomezca6b7ff476e9
diff --git a/.changeset/wysiwyg-x20-boundary-whitespace-display.md b/.changeset/wysiwyg-x20-boundary-whitespace-display.md
new file mode 100644
index 000000000..8106c31e5
--- /dev/null
+++ b/.changeset/wysiwyg-x20-boundary-whitespace-display.md
@@ -0,0 +1,5 @@
+---
+"@inkeep/open-knowledge": patch
+---
+
+Fix literal ` ` characters appearing in the editor after closing and reopening a document. A space at the edge of a paragraph, bold/italic run, inline code, or list item is stored by the byte-fidelity engine as the numeric character reference ` ` (or `	` for a tab); on a cold reopen that reference was restored as the visible literal text ` ` instead of a real space. Whitespace numeric character references now display as the decoded space/tab in the editor while still serializing to the exact same bytes on disk, so source byte-fidelity is unchanged. User-authored entity references (`&`, `A`) and escaped `\ ` are untouched, and the sourceLiteral security gate continues to reject any non-whitespace divergence between displayed text and persisted bytes.
diff --git a/packages/app/tests/integration/wysiwyg-x20-cold-reopen.test.ts b/packages/app/tests/integration/wysiwyg-x20-cold-reopen.test.ts
new file mode 100644
index 000000000..5735b7ef8
--- /dev/null
+++ b/packages/app/tests/integration/wysiwyg-x20-cold-reopen.test.ts
@@ -0,0 +1,145 @@
+import { afterEach, describe, expect, test } from 'bun:test';
+import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+import { yXmlFragmentToProseMirrorRootNode } from '@tiptap/y-tiptap';
+import type * as Y from 'yjs';
+import {
+ agentWriteMd,
+ createRestartableServer,
+ createTestClient,
+ createTestServer,
+ pollDiskContentStable,
+ pollUntil,
+ schema,
+ type TestServer,
+} from './test-harness';
+
+type AnyServer = { instance: TestServer['instance'] };
+
+function serverFragment(server: AnyServer, docName: string): Y.XmlFragment | undefined {
+ return server.instance.hocuspocus.documents.get(docName)?.getXmlFragment('default');
+}
+
+function fragmentVisibleText(fragment: Y.XmlFragment): string {
+ return yXmlFragmentToProseMirrorRootNode(fragment, schema).textContent;
+}
+
+describe('WYSIWYG   literal after close + cold reopen (the reported bug)', () => {
+ test('agent write of a boundary space, cold restart, reopen: shows a space, bytes stable', async () => {
+ let server = await createRestartableServer({ debounce: 100, maxDebounce: 400 });
+ const docName = `x20-restart-${crypto.randomUUID()}`;
+ const docFile = join(server.contentDir, `${docName}.md`);
+ try {
+ await agentWriteMd(server.port, 'before ', {
+ docName,
+ position: 'replace',
+ });
+ const preDisk = await pollDiskContentStable(docFile, (c) => c.includes('before '), {
+ timeoutMs: 8000,
+ settleMs: 300,
+ });
+ expect(preDisk).toContain('before ');
+
+ server = await server.killAndRestartOnSamePort({ downtimeMs: 300 });
+
+ const client = await createTestClient(server.port, docName);
+ try {
+ await pollUntil(() => serverFragment(server, docName) !== undefined, 10_000, 50);
+ const fragment = serverFragment(server, docName);
+ if (!fragment) throw new Error('doc not loaded on server after cold restart');
+
+ const shown = fragmentVisibleText(fragment);
+ expect(shown).not.toContain(' ');
+ expect(shown).toContain('before ');
+
+ const postDisk = await pollDiskContentStable(docFile, (c) => c.includes('before '), {
+ timeoutMs: 5000,
+ settleMs: 300,
+ });
+ expect(postDisk).toContain('before ');
+ } finally {
+ await client.cleanup();
+ }
+ } finally {
+ await server.shutdown();
+ }
+ }, 30_000);
+});
+
+describe('WYSIWYG   literal — pure cold load of a doc already stored with  ', () => {
+ let server: TestServer | undefined;
+ let dir: string | undefined;
+
+ afterEach(async () => {
+ await server?.cleanup();
+ server = undefined;
+ if (dir) {
+ rmSync(dir, { recursive: true, force: true });
+ dir = undefined;
+ }
+ });
+
+ test('cold load shows a real space, not literal  , and bytes are byte-stable', async () => {
+ dir = mkdtempSync(join(tmpdir(), 'ok-x20-coldload-'));
+ const docName = `x20-${crypto.randomUUID()}`;
+ writeFileSync(join(dir, `${docName}.md`), 'alpha \n', 'utf-8');
+
+ server = await createTestServer({ contentDir: dir, keepContentDir: true });
+ const client = await createTestClient(server.port, docName);
+ try {
+ await pollUntil(
+ () => serverFragment(server as TestServer, docName) !== undefined,
+ 10_000,
+ 50,
+ );
+ const fragment = serverFragment(server, docName);
+ if (!fragment) throw new Error('doc not loaded on server');
+
+ const shown = fragmentVisibleText(fragment);
+ expect(shown).not.toContain(' '); // RED before fix
+ expect(shown).toContain('alpha ');
+
+ const disk = await pollDiskContentStable(
+ join(dir, `${docName}.md`),
+ (c) => c.includes('alpha '),
+ { timeoutMs: 5000, settleMs: 300 },
+ );
+ expect(disk).toContain('alpha ');
+ } finally {
+ await client.cleanup();
+ }
+ }, 30_000);
+
+ test('cold load of ADJACENT refs shows multiple spaces and keeps bytes byte-stable', async () => {
+ dir = mkdtempSync(join(tmpdir(), 'ok-x20-adjacent-'));
+ const docName = `x20adj-${crypto.randomUUID()}`;
+ writeFileSync(join(dir, `${docName}.md`), 'gamma  delta\n', 'utf-8');
+
+ server = await createTestServer({ contentDir: dir, keepContentDir: true });
+ const client = await createTestClient(server.port, docName);
+ try {
+ await pollUntil(
+ () => serverFragment(server as TestServer, docName) !== undefined,
+ 10_000,
+ 50,
+ );
+ const fragment = serverFragment(server, docName);
+ if (!fragment) throw new Error('doc not loaded on server');
+
+ const shown = fragmentVisibleText(fragment);
+ expect(shown).not.toContain(' ');
+ expect(shown).toContain('gamma delta'); // two real spaces
+
+ const disk = await pollDiskContentStable(
+ join(dir, `${docName}.md`),
+ (c) => c.includes('gamma  delta'),
+ { timeoutMs: 5000, settleMs: 300 },
+ );
+ expect(disk).toContain('gamma  delta');
+ } finally {
+ await client.cleanup();
+ }
+ }, 30_000);
+});
diff --git a/packages/core/src/extensions/source-literal-mark.test.ts b/packages/core/src/extensions/source-literal-mark.test.ts
index 6d7380700..d986d9545 100644
--- a/packages/core/src/extensions/source-literal-mark.test.ts
+++ b/packages/core/src/extensions/source-literal-mark.test.ts
@@ -81,3 +81,48 @@ describe('isValidSourceLiteralRaw — rejects hidden injections', () => {
expect(isValidSourceLiteralRaw(raw, visible)).toBe(false);
});
});
+
+describe('isValidSourceLiteralRaw — inline-whitespace numeric char-ref divergence class', () => {
+ test('accepts a whitespace numeric ref whose decoded char is the visible text', () => {
+ expect(isValidSourceLiteralRaw(' ', ' ')).toBe(true); // hex space
+ expect(isValidSourceLiteralRaw(' ', ' ')).toBe(true); // capital X
+ expect(isValidSourceLiteralRaw(' ', ' ')).toBe(true); // decimal space
+ expect(isValidSourceLiteralRaw('	', '\t')).toBe(true); // hex tab
+ expect(isValidSourceLiteralRaw('	', '\t')).toBe(true); // decimal tab
+ });
+
+ test('accepts a RUN of whitespace refs decoding to the visible run (coalesced segment)', () => {
+ expect(isValidSourceLiteralRaw('  ', ' ')).toBe(true);
+ expect(isValidSourceLiteralRaw('		', '\t\t')).toBe(true);
+ expect(isValidSourceLiteralRaw(' 	', ' \t')).toBe(true);
+ });
+
+ test('rejects a run whose decoded length does NOT match the visible run', () => {
+ expect(isValidSourceLiteralRaw('  ', ' ')).toBe(false); // 2 refs, 1 space
+ expect(isValidSourceLiteralRaw(' ', ' ')).toBe(false); // 1 ref, 2 spaces
+ expect(isValidSourceLiteralRaw(' 
', ' ')).toBe(false); // newline member
+ });
+
+ test('rejects a VERTICAL-whitespace ref displayed as the control char (structure smuggle)', () => {
+ expect(isValidSourceLiteralRaw('
', '\n')).toBe(false);
+ expect(isValidSourceLiteralRaw('
', '\n')).toBe(false);
+ expect(isValidSourceLiteralRaw('
', '\r')).toBe(false);
+ expect(isValidSourceLiteralRaw('', '\f')).toBe(false);
+ });
+
+ test('rejects a non-whitespace numeric ref displayed as its decoded char', () => {
+ expect(isValidSourceLiteralRaw('A', 'A')).toBe(false);
+ expect(isValidSourceLiteralRaw('&', '&')).toBe(false);
+ });
+
+ test('rejects a whitespace ref whose decoded char is NOT the visible text', () => {
+ expect(isValidSourceLiteralRaw(' ', 'x')).toBe(false);
+ expect(isValidSourceLiteralRaw(' ', 'attacker')).toBe(false);
+ expect(isValidSourceLiteralRaw('	', 'safe')).toBe(false);
+ });
+
+ test('rejects a ref with a trailing payload (not a bare numeric ref)', () => {
+ expect(isValidSourceLiteralRaw('&#x20;<script>', ' ')).toBe(false);
+ expect(isValidSourceLiteralRaw('&#x20;extra', ' ')).toBe(false);
+ });
+});
diff --git a/packages/core/src/extensions/source-literal-mark.ts b/packages/core/src/extensions/source-literal-mark.ts
index b53df9a21..61f83f5d5 100644
--- a/packages/core/src/extensions/source-literal-mark.ts
+++ b/packages/core/src/extensions/source-literal-mark.ts
@@ -1,4 +1,5 @@
import { Mark } from '@tiptap/core';
+import { decodeInlineWhitespaceNumericCharRefRun } from '../markdown/whitespace-char-ref.ts';
export const SourceLiteralMark = Mark.create({
name: 'sourceLiteral',
@@ -28,7 +29,8 @@ export function isValidSourceLiteralRaw(sourceRaw: unknown, visibleText: unknown
const normalizedRaw = sourceRaw.replaceAll(' ', ' ');
const normalizedVisible = visibleText.replaceAll(' ', ' ');
if (normalizedRaw === normalizedVisible) return true;
- return stripMarkdownBackslashEscapes(normalizedRaw) === normalizedVisible;
+ if (stripMarkdownBackslashEscapes(normalizedRaw) === normalizedVisible) return true;
+ return decodeInlineWhitespaceNumericCharRefRun(normalizedRaw) === normalizedVisible;
}
function stripMarkdownBackslashEscapes(s: string): string {
diff --git a/packages/core/src/markdown/boundary-whitespace-pm-display.test.ts b/packages/core/src/markdown/boundary-whitespace-pm-display.test.ts
new file mode 100644
index 000000000..c6a9e40bf
--- /dev/null
+++ b/packages/core/src/markdown/boundary-whitespace-pm-display.test.ts
@@ -0,0 +1,157 @@
+import { describe, expect, test } from 'bun:test';
+import type { JSONContent } from '@tiptap/core';
+import { sharedExtensions } from '../extensions/shared.ts';
+import { MarkdownManager } from './index.ts';
+
+const md = new MarkdownManager({ extensions: sharedExtensions });
+
+type PmJSON = JSONContent;
+
+function toJSON(node: unknown): PmJSON {
+ const n = node as { toJSON?: () => PmJSON };
+ return n && typeof n.toJSON === 'function' ? n.toJSON() : (node as PmJSON);
+}
+
+function pmText(node: unknown, out: string[] = []): string {
+ const n = toJSON(node);
+ if (!n || typeof n !== 'object') return out.join('');
+ if (typeof n.text === 'string') out.push(n.text);
+ if (Array.isArray(n.content)) for (const c of n.content) pmText(c, out);
+ return out.join('');
+}
+
+function pmSourceLiteralSegments(
+ node: unknown,
+ out: Array<{ text: string; sourceRaw: string }> = [],
+): Array<{ text: string; sourceRaw: string }> {
+ const n = toJSON(node);
+ if (!n || typeof n !== 'object') return out;
+ if (typeof n.text === 'string' && Array.isArray(n.marks)) {
+ const mark = n.marks.find((mk) => (mk as { type?: string }).type === 'sourceLiteral') as
+ | { attrs?: { sourceRaw?: string } }
+ | undefined;
+ if (mark) out.push({ text: n.text, sourceRaw: mark.attrs?.sourceRaw ?? '' });
+ }
+ if (Array.isArray(n.content)) for (const c of n.content) pmSourceLiteralSegments(c, out);
+ return out;
+}
+
+describe('boundary-whitespace char-ref displays decoded in the PM doc (RED — the reported bug)', () => {
+ test('interior boundary space displays as a real space, not literal &#x20;', () => {
+ const shown = pmText(md.parse('a&#x20;b\n'));
+ expect(shown).not.toContain('&#x20;');
+ expect(shown).toBe('a b');
+ });
+
+ test('trailing boundary space displays as a real space (trailing-trim risk)', () => {
+ const shown = pmText(md.parse('abc&#x20;\n'));
+ expect(shown).not.toContain('&#x20;');
+ expect(shown).toBe('abc ');
+ });
+
+ test('leading boundary space displays as a real space', () => {
+ const shown = pmText(md.parse('&#x20;abc\n'));
+ expect(shown).not.toContain('&#x20;');
+ expect(shown).toBe(' abc');
+ });
+
+ test('decimal space form (&#32;) decodes for display', () => {
+ expect(pmText(md.parse('a&#32;b\n'))).toBe('a b');
+ });
+
+ test('hex tab form (&#x9;) decodes for display', () => {
+ expect(pmText(md.parse('a&#x9;b\n'))).toBe('a\tb');
+ });
+
+ test('decoded boundary space carries sourceLiteral.sourceRaw for byte-fidelity', () => {
+ expect(pmSourceLiteralSegments(md.parse('a&#x20;b\n'))).toEqual([
+ { text: ' ', sourceRaw: '&#x20;' },
+ ]);
+ });
+
+ test('decoded boundary tab carries sourceLiteral.sourceRaw', () => {
+ expect(pmSourceLiteralSegments(md.parse('a&#x9;b\n'))).toEqual([
+ { text: '\t', sourceRaw: '&#x9;' },
+ ]);
+ });
+
+ test('adjacent identical refs display as multiple spaces in ONE coalesced segment', () => {
+ expect(pmText(md.parse('a&#x20;&#x20;b\n'))).toBe('a b');
+ expect(pmSourceLiteralSegments(md.parse('a&#x20;&#x20;b\n'))).toEqual([
+ { text: ' ', sourceRaw: '&#x20;&#x20;' },
+ ]);
+ });
+
+ test('adjacent identical tab refs coalesce', () => {
+ expect(pmText(md.parse('a&#x9;&#x9;b\n'))).toBe('a\t\tb');
+ expect(pmSourceLiteralSegments(md.parse('a&#x9;&#x9;b\n'))).toEqual([
+ { text: '\t\t', sourceRaw: '&#x9;&#x9;' },
+ ]);
+ });
+
+ test('mixed adjacent whitespace refs coalesce into one segment', () => {
+ expect(pmText(md.parse('a&#x20;&#x9;b\n'))).toBe('a \tb');
+ expect(pmSourceLiteralSegments(md.parse('a&#x20;&#x9;b\n'))).toEqual([
+ { text: ' \t', sourceRaw: '&#x20;&#x9;' },
+ ]);
+ });
+
+ test('a long run (n>=4) coalesces into ONE segment, not two equal-mark pairs', () => {
+ expect(pmText(md.parse('a&#x20;&#x20;&#x20;&#x20;b\n'))).toBe('a b');
+ expect(pmSourceLiteralSegments(md.parse('a&#x20;&#x20;&#x20;&#x20;b\n'))).toEqual([
+ { text: ' ', sourceRaw: '&#x20;&#x20;&#x20;&#x20;' },
+ ]);
+ });
+
+ test('whitespace refs do NOT coalesce across an interleaved non-whitespace entity', () => {
+ expect(pmText(md.parse('a&#x20;&amp;&#x20;b\n'))).toBe('a &amp; b');
+ expect(pmSourceLiteralSegments(md.parse('a&#x20;&amp;&#x20;b\n'))).toEqual([
+ { text: ' ', sourceRaw: '&#x20;' },
+ { text: '&amp;', sourceRaw: '&amp;' },
+ { text: ' ', sourceRaw: '&#x20;' },
+ ]);
+ });
+});
+
+describe('guard: byte-fidelity is unchanged by the display decode', () => {
+ const byteStable = [
+ 'a&#x20;b\n',
+ 'abc&#x20;\n',
+ '&#x20;abc\n',
+ 'a&#32;b\n',
+ 'a&#x9;b\n',
+ 'a&#x41;b\n',
+ 'a&amp;b\n',
+ 'a\\&#x20;b\n',
+ 'a&#x20;&#x20;b\n',
+ 'a&#x9;&#x9;b\n',
+ 'a&#x20;&#x20;&#x20;b\n',
+ 'a&#x20;&#x9;b\n',
+ 'a&#x20;&#x20;\n',
+ '&#x20;&#x20;a\n',
+ 'a&#x20;&amp;&#x20;b\n',
+ '**a&#x20;&#x20;b**\n',
+ 'a&#x20;&#x20;&#x20;&#x20;b\n',
+ 'a&#x20;&#x20;&#x20;&#x20;&#x20;b\n',
+ 'a&#x20;&#x20;&#x20;&#x20;&#x20;&#x20;b\n',
+ 'a&#x9;&#x9;&#x9;&#x9;b\n',
+ 'a&#x20;&#x9;&#x20;&#x9;&#x20;b\n',
+ ] as const;
+ for (const src of byteStable) {
+ test(`round-trips byte-identically: ${JSON.stringify(src)}`, () => {
+ expect(md.serialize(md.parse(src))).toBe(src);
+ });
+ }
+});
+
+describe('guard: scope is strictly whitespace numeric refs (locked non-goal)', () => {
+ test('non-whitespace numeric ref (&#x41; = "A") stays literal in display', () => {
+ expect(pmText(md.parse('a&#x41;b\n'))).toBe('a&#x41;b');
+ });
+ test('named ref (&amp;) stays literal in display', () => {
+ expect(pmText(md.parse('a&amp;b\n'))).toBe('a&amp;b');
+ });
+ test('a user-escaped \\&#x20; stays literal (escape = literal intent)', () => {
+ expect(pmText(md.parse('x\\&#x20;y\n'))).toContain('&#x20;');
+ });
+});
diff --git a/packages/core/src/markdown/index.ts b/packages/core/src/markdown/index.ts
index 1644f78cd..1b13478a8 100644
--- a/packages/core/src/markdown/index.ts
+++ b/packages/core/src/markdown/index.ts
@@ -68,6 +68,10 @@ import {
} from './pipeline.ts';
import { normalizeDocRelativeAssetUrl } from './resolve-image-url.ts';
import { toMarkdownHandlers } from './to-markdown-handlers.ts';
+import {
+ decodeInlineWhitespaceNumericCharRefRun,
+ isInlineWhitespaceNumericCharRef,
+} from './whitespace-char-ref.ts';
interface MdastToPmState {
all: (node: MdastNodes) => PmNode[];
@@ -452,9 +456,29 @@ function buildMdastToPmHandlers(
),
];
markers.sort((a, b) => a.offset - b.offset);
+ const coalesced: Marker[] = [];
+ for (const marker of markers) {
+ const prev = coalesced[coalesced.length - 1];
+ if (
+ marker.kind === 'entity' &&
+ prev?.kind === 'entity' &&
+ prev.offset + prev.length === marker.offset &&
+ decodeInlineWhitespaceNumericCharRefRun(prev.raw) !== null &&
+ isInlineWhitespaceNumericCharRef(marker.raw)
+ ) {
+ coalesced[coalesced.length - 1] = {
+ kind: 'entity',
+ offset: prev.offset,
+ length: prev.length + marker.length,
+ raw: prev.raw + marker.raw,
+ };
+ } else {
+ coalesced.push(marker);
+ }
+ }
const fragments: PmNode[] = [];
let lastIdx = 0;
- for (const marker of markers) {
+ for (const marker of coalesced) {
if (marker.offset > lastIdx) {
const segment = value.slice(lastIdx, marker.offset).replaceAll('\u00A0', ' ');
if (segment) fragments.push(schema.text(segment));
@@ -467,8 +491,11 @@ function buildMdastToPmHandlers(
if (marker.kind === 'escape') {
fragments.push(schema.text(segmentText, [m.escapeMark.create()]));
} else if (m.sourceLiteral) {
+ const decodedWhitespace = decodeInlineWhitespaceNumericCharRefRun(marker.raw);
fragments.push(
- schema.text(segmentText, [m.sourceLiteral.create({ sourceRaw: marker.raw })]),
+ schema.text(decodedWhitespace ?? segmentText, [
+ m.sourceLiteral.create({ sourceRaw: marker.raw }),
+ ]),
);
} else {
fragments.push(schema.text(segmentText));
diff --git a/packages/core/src/markdown/whitespace-char-ref.test.ts b/packages/core/src/markdown/whitespace-char-ref.test.ts
new file mode 100644
index 000000000..ba4c3cda0
--- /dev/null
+++ b/packages/core/src/markdown/whitespace-char-ref.test.ts
@@ -0,0 +1,76 @@
+import { describe, expect, test } from 'bun:test';
+import {
+ decodeInlineWhitespaceNumericCharRef,
+ decodeInlineWhitespaceNumericCharRefRun,
+ isInlineWhitespaceNumericCharRef,
+} from './whitespace-char-ref.ts';
+
+describe('isInlineWhitespaceNumericCharRef', () => {
+ test('true for space + tab numeric refs (hex + decimal, X case)', () => {
+ for (const ref of ['&#x20;', '&#X20;', '&#32;', '&#x9;', '&#X9;', '&#9;', '&#x09;']) {
+ expect(isInlineWhitespaceNumericCharRef(ref)).toBe(true);
+ }
+ });
+
+ test('false for vertical whitespace (newline / CR / VT / FF)', () => {
+ for (const ref of ['&#xA;', '&#10;', '&#xD;', '&#13;', '&#xB;', '&#xC;']) {
+ expect(isInlineWhitespaceNumericCharRef(ref)).toBe(false);
+ }
+ });
+
+ test('false for non-whitespace numeric refs', () => {
+ for (const ref of ['&#x41;', '&#65;', '&#x26;', '&#x2003;' /* em space */]) {
+ expect(isInlineWhitespaceNumericCharRef(ref)).toBe(false);
+ }
+ });
+
+ test('false for named refs and malformed input', () => {
+ for (const ref of ['&amp;', '&nbsp;', '&#x20', '#x20;', '&#;', '&#xZZ;', '&#x20;x', '', ' ']) {
+ expect(isInlineWhitespaceNumericCharRef(ref)).toBe(false);
+ }
+ });
+});
+
+describe('decodeInlineWhitespaceNumericCharRef', () => {
+ test('decodes space + tab forms to the real character', () => {
+ expect(decodeInlineWhitespaceNumericCharRef('&#x20;')).toBe(' ');
+ expect(decodeInlineWhitespaceNumericCharRef('&#32;')).toBe(' ');
+ expect(decodeInlineWhitespaceNumericCharRef('&#x9;')).toBe('\t');
+ expect(decodeInlineWhitespaceNumericCharRef('&#9;')).toBe('\t');
+ });
+
+ test('returns null for everything that is not an inline-whitespace numeric ref', () => {
+ for (const ref of ['&#xA;', '&#x41;', '&amp;', '&#x20', 'plain', '']) {
+ expect(decodeInlineWhitespaceNumericCharRef(ref)).toBeNull();
+ }
+ });
+});
+
+describe('decodeInlineWhitespaceNumericCharRefRun', () => {
+ test('decodes a single ref (run of one)', () => {
+ expect(decodeInlineWhitespaceNumericCharRefRun('&#x20;')).toBe(' ');
+ expect(decodeInlineWhitespaceNumericCharRefRun('&#x9;')).toBe('\t');
+ });
+
+ test('decodes a run of back-to-back whitespace refs', () => {
+ expect(decodeInlineWhitespaceNumericCharRefRun('&#x20;&#x20;')).toBe(' ');
+ expect(decodeInlineWhitespaceNumericCharRefRun('&#x9;&#x9;')).toBe('\t\t');
+ expect(decodeInlineWhitespaceNumericCharRefRun('&#x20;&#x9;&#x20;')).toBe(' \t ');
+ expect(decodeInlineWhitespaceNumericCharRefRun('&#x20;&#32;')).toBe(' '); // hex + decimal
+ });
+
+ test('returns null when ANY member is not inline whitespace', () => {
+ expect(decodeInlineWhitespaceNumericCharRefRun('&#x20;&#xA;')).toBeNull(); // newline member
+ expect(decodeInlineWhitespaceNumericCharRefRun('&#x20;&#x41;')).toBeNull(); // letter member
+ });
+
+ test('returns null for any gap, embedded, or trailing non-ref bytes', () => {
+ expect(decodeInlineWhitespaceNumericCharRefRun('&#x20; &#x20;')).toBeNull(); // literal space gap
+ expect(decodeInlineWhitespaceNumericCharRefRun('&#x20;x&#x20;')).toBeNull();
+ expect(decodeInlineWhitespaceNumericCharRefRun('&#x20;<script>')).toBeNull();
+ expect(decodeInlineWhitespaceNumericCharRefRun('x&#x20;')).toBeNull();
+ expect(decodeInlineWhitespaceNumericCharRefRun('&#x20;extra')).toBeNull();
+ expect(decodeInlineWhitespaceNumericCharRefRun('')).toBeNull();
+ expect(decodeInlineWhitespaceNumericCharRefRun(' ')).toBeNull();
+ });
+});
diff --git a/packages/core/src/markdown/whitespace-char-ref.ts b/packages/core/src/markdown/whitespace-char-ref.ts
new file mode 100644
index 000000000..a100cad76
--- /dev/null
+++ b/packages/core/src/markdown/whitespace-char-ref.ts
@@ -0,0 +1,38 @@
+const INLINE_WHITESPACE_BY_CODE: ReadonlyMap<number, string> = new Map([
+ [0x20, ' '],
+ [0x09, '\t'],
+]);
+
+function numericCharRefCodePoint(ref: string): number | null {
+ const match = /^&#(x[0-9A-Fa-f]+|X[0-9A-Fa-f]+|[0-9]+);$/.exec(ref);
+ if (!match) return null;
+ const body = match[1];
+ const hex = body[0] === 'x' || body[0] === 'X';
+ const code = Number.parseInt(hex ? body.slice(1) : body, hex ? 16 : 10);
+ return Number.isNaN(code) ? null : code;
+}
+
+export function isInlineWhitespaceNumericCharRef(ref: string): boolean {
+ const code = numericCharRefCodePoint(ref);
+ return code !== null && INLINE_WHITESPACE_BY_CODE.has(code);
+}
+
+export function decodeInlineWhitespaceNumericCharRef(ref: string): string | null {
+ const code = numericCharRefCodePoint(ref);
+ return code === null ? null : (INLINE_WHITESPACE_BY_CODE.get(code) ?? null);
+}
+
+const NUMERIC_CHAR_REF_TOKEN = /&#(?:x[0-9A-Fa-f]+|X[0-9A-Fa-f]+|[0-9]+);/g;
+
+export function decodeInlineWhitespaceNumericCharRefRun(refRun: string): string | null {
+ NUMERIC_CHAR_REF_TOKEN.lastIndex = 0;
+ const tokens = refRun.match(NUMERIC_CHAR_REF_TOKEN);
+ if (tokens === null || tokens.join('') !== refRun) return null;
+ let decoded = '';
+ for (const token of tokens) {
+ const char = decodeInlineWhitespaceNumericCharRef(token);
+ if (char === null) return null;
+ decoded += char;
+ }
+ return decoded;
+}