Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

feat(Markdown): export the character reference decoder, and decode references in RichText with it (#7013)

Cindy Zhangcb30e207e0b5
diff --git a/.changeset/decode-markdown-character-references.md b/.changeset/decode-markdown-character-references.md
new file mode 100644
index 0000000000..0e32a787b2
--- /dev/null
+++ b/.changeset/decode-markdown-character-references.md
@@ -0,0 +1,8 @@
+---
+'@astryxdesign/core': patch
+---
+
+[feat] Export `decodeMarkdownCharacterReferences` from `@astryxdesign/core/Markdown/parser` and `@astryxdesign/core/Markdown`
+@cixzhang
+
+It decodes character references the way `Markdown` renders them — `©`, `©`, and `©` become `©`; unknown names and references without their semicolon stay as written — using the same table `Markdown` uses. It works on plain text, so leave code and backslash-escaped references out. The parser subpath has no use-client boundary, so server code can call it.
diff --git a/packages/core/src/Markdown/Markdown.doc.mjs b/packages/core/src/Markdown/Markdown.doc.mjs
index d8901d2526..b3b41adad6 100644
--- a/packages/core/src/Markdown/Markdown.doc.mjs
+++ b/packages/core/src/Markdown/Markdown.doc.mjs
@@ -277,6 +277,11 @@ export const docs = {
description:
'Import parseMarkdownAst or parseInlineAst from @astryxdesign/core/Markdown/parser when server or React Server Component code needs to run plugins against the canonical readonly tree. The parser and plugin subpaths have no use-client boundary. The Markdown component remains client-owned, so function-bearing plugin entries must not be passed across an RSC serialization boundary.',
},
+ {
+ guidance: true,
+ description:
+ 'Use decodeMarkdownCharacterReferences (from @astryxdesign/core/Markdown/parser or @astryxdesign/core/Markdown) to show text the way Markdown renders its character references: ©, ©, and © become ©, while unknown names and references without their semicolon stay as written. It decodes plain text and knows nothing of Markdown, so skip code and backslash-escaped references yourself. It uses the same table Markdown renders with.',
+ },
{
guidance: true,
description:
diff --git a/packages/core/src/Markdown/Markdown.public.test.ts b/packages/core/src/Markdown/Markdown.public.test.ts
index 306cf8ae9f..da9a5c154c 100644
--- a/packages/core/src/Markdown/Markdown.public.test.ts
+++ b/packages/core/src/Markdown/Markdown.public.test.ts
@@ -10,11 +10,13 @@
import {describe, expectTypeOf, it} from 'vitest';
import {
createIncrementalState,
+ decodeMarkdownCharacterReferences,
parseInline,
parseMarkdown,
parseMarkdownIncremental,
visitMarkdownNodes,
} from './index';
+import {decodeMarkdownCharacterReferences as decodeFromParser} from './parser/index';
import {
createMarkdownFenceTransform,
createMarkdownHeadingLinks,
@@ -92,6 +94,15 @@ function legacyBlockText(node: BlockNode): string {
}
}
+describe('decodeMarkdownCharacterReferences entry points (spec:AST-061 DEC-5)', () => {
+ it('is the same function from the Markdown entry and the parser subpath', () => {
+ expectTypeOf(decodeMarkdownCharacterReferences).toEqualTypeOf<
+ (text: string) => string
+ >();
+ expectTypeOf(decodeFromParser).toEqualTypeOf<(text: string) => string>();
+ });
+});
+
describe('Markdown public parser types', () => {
it('keeps default and legacy parser calls on the legacy unions', () => {
expectTypeOf(parseInline('plain')).toEqualTypeOf<InlineNode[]>();
diff --git a/packages/core/src/Markdown/characterReferences.test.ts b/packages/core/src/Markdown/characterReferences.test.ts
new file mode 100644
index 0000000000..c8ad8d5d2f
--- /dev/null
+++ b/packages/core/src/Markdown/characterReferences.test.ts
@@ -0,0 +1,47 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+import {describe, expect, it} from 'vitest';
+import {decodeMarkdownCharacterReferences} from './characterReferences';
+import {parseInline} from './parser';
+
+/** Character reference cases every surface decodes the same way (spec:AST-061 DEC-5). */
+const CHARACTER_REFERENCE_CASES: ReadonlyArray<readonly [string, string]> = [
+ ['&copy;', '©'],
+ ['&#169;', '©'],
+ ['&#xA9;', '©'],
+ ['&#XA9;', '©'],
+ ['&amp;copy;', '&copy;'],
+ ['Fish &amp; chips', 'Fish & chips'],
+ ['&NotEqualTilde;', '\u2242\u0338'],
+ ['&unknown;', '&unknown;'],
+ ['&copy', '&copy'],
+ ['& copy;', '& copy;'],
+ ['&#0;', '\uFFFD'],
+ ['&#xD800;', '\uFFFD'],
+ ['&#x110000;', '\uFFFD'],
+ ['&#12345678;', '&#12345678;'],
+ ['a&nbsp;b', 'a\u00A0b'],
+ ['no references', 'no references'],
+];
+
+describe('decodeMarkdownCharacterReferences (spec:AST-061 DEC-5)', () => {
+ it('decodes valid references and leaves everything else as written', () => {
+ for (const [source, decoded] of CHARACTER_REFERENCE_CASES) {
+ expect(decodeMarkdownCharacterReferences(source), source).toBe(decoded);
+ }
+ });
+
+ it('decodes exactly as Markdown renders text', () => {
+ for (const [source] of CHARACTER_REFERENCE_CASES) {
+ const rendered = parseInline(source)
+ .map(node => ('content' in node ? node.content : ''))
+ .join('');
+ expect(decodeMarkdownCharacterReferences(source), source).toBe(rendered);
+ }
+ });
+
+ it('returns the same string when there is nothing to decode', () => {
+ const text = 'plain & simple';
+ expect(decodeMarkdownCharacterReferences(text)).toBe(text);
+ });
+});
diff --git a/packages/core/src/Markdown/characterReferences.ts b/packages/core/src/Markdown/characterReferences.ts
index e1a6edf5e6..a1cab412d1 100644
--- a/packages/core/src/Markdown/characterReferences.ts
+++ b/packages/core/src/Markdown/characterReferences.ts
@@ -7,9 +7,13 @@
* `&#xH;` reference names, and where the reference ends. decodeLiteralText:
* text with backslash escapes and character references resolved, for text
* that is not parsed as Markdown, such as an image's alt text.
+ * decodeMarkdownCharacterReferences: the public decoder, exported from
+ * `@astryxdesign/core/Markdown/parser` and `@astryxdesign/core/Markdown`.
* @position Used by parser.ts while it parses inline text, so references
* render as the characters they name (spec:AST-061 FR7, DEC-3). Code spans
- * and code blocks never reach it and keep references literal.
+ * and code blocks never reach it and keep references literal. The RichText
+ * surfaces decode with the same function, so every surface reads references
+ * from this one table (spec:AST-061 DEC-5).
*/
/**
@@ -116,3 +120,37 @@ export function decodeLiteralText(text: string): string {
}
return output;
}
+
+/**
+ * Returns `text` with every valid named or numeric character reference
+ * replaced by the characters it names, as Markdown renders it: `&copy;`,
+ * `&#169;`, and `&#xA9;` all become `©`, and a numeric reference to NUL, a
+ * surrogate, or a code point past U+10FFFF becomes U+FFFD. An unknown name, a
+ * reference without its `;`, and all other text stay as written.
+ *
+ * It works on plain text and knows nothing of Markdown, so the caller decides
+ * where it applies: keep backslash-escaped references and code literal by not
+ * passing them in (spec:AST-061 DEC-5).
+ *
+ * @example
+ * ```
+ * decodeMarkdownCharacterReferences('Fish &amp; chips &copy; 2026'); // 'Fish & chips © 2026'
+ * decodeMarkdownCharacterReferences('&unknown; &copy'); // '&unknown; &copy'
+ * ```
+ */
+export function decodeMarkdownCharacterReferences(text: string): string {
+ let output = '';
+ let index = 0;
+ let ampersand = text.indexOf('&');
+ while (ampersand !== -1) {
+ const reference = matchCharacterReference(text, ampersand);
+ if (reference == null) {
+ ampersand = text.indexOf('&', ampersand + 1);
+ continue;
+ }
+ output += text.slice(index, ampersand) + reference.value;
+ index = reference.end;
+ ampersand = text.indexOf('&', index);
+ }
+ return index === 0 ? text : output + text.slice(index);
+}
diff --git a/packages/core/src/Markdown/index.ts b/packages/core/src/Markdown/index.ts
index 38aae88ec4..0be59d24b3 100644
--- a/packages/core/src/Markdown/index.ts
+++ b/packages/core/src/Markdown/index.ts
@@ -4,7 +4,8 @@
/**
* @file index.ts
- * @output Exports Markdown component, canonical and compatibility parser functions, and types
+ * @output Exports Markdown component, canonical and compatibility parser functions,
+ * the character reference decoder, and types
* @position Component entry point; re-exported by /packages/core/src/index.ts
*/
@@ -56,6 +57,7 @@ export {
parseInline,
parseInlineAst,
} from './parser';
+export {decodeMarkdownCharacterReferences} from './characterReferences';
export type {
BlockNode,
BlockNodeWithMath,
diff --git a/packages/core/src/Markdown/parser.public.test.ts b/packages/core/src/Markdown/parser.public.test.ts
index 5c914774a7..c6ed04dd08 100644
--- a/packages/core/src/Markdown/parser.public.test.ts
+++ b/packages/core/src/Markdown/parser.public.test.ts
@@ -16,6 +16,7 @@ import {
} from '@astryxdesign/core/Markdown/plugins';
import {
createIncrementalState,
+ decodeMarkdownCharacterReferences,
parseInline,
parseInlineAst,
parseMarkdown,
@@ -30,6 +31,15 @@ import type {
} from '@astryxdesign/core/Markdown/parser';
describe('@astryxdesign/core/Markdown/parser', () => {
+ it('exports the character reference decoder Markdown renders with (spec:AST-061 DEC-5)', () => {
+ expectTypeOf(decodeMarkdownCharacterReferences).toEqualTypeOf<
+ (text: string) => string
+ >();
+ expect(decodeMarkdownCharacterReferences('&copy; &unknown;')).toBe(
+ '© &unknown;',
+ );
+ });
+
it('is a generated public package subpath', () => {
const packageJson = JSON.parse(
readFileSync(join(process.cwd(), 'packages/core/package.json'), 'utf8'),
diff --git a/packages/core/src/Markdown/parser.ts b/packages/core/src/Markdown/parser.ts
index fc3f43943a..0ad6ffd222 100644
--- a/packages/core/src/Markdown/parser.ts
+++ b/packages/core/src/Markdown/parser.ts
@@ -43,6 +43,10 @@ import {
isAsciiPunctuation,
matchCharacterReference,
} from './characterReferences';
+
+// The decoder Markdown renders with, public from the parser subpath
+// (spec:AST-061 DEC-5).
+export {decodeMarkdownCharacterReferences} from './characterReferences';
import {isSafeMarkdownParserUrl} from './url';
// ---------------------------------------------------------------------------
diff --git a/packages/core/src/Markdown/parser/index.ts b/packages/core/src/Markdown/parser/index.ts
index 630cf2c21d..d8739b5222 100644
--- a/packages/core/src/Markdown/parser/index.ts
+++ b/packages/core/src/Markdown/parser/index.ts
@@ -3,12 +3,14 @@
/**
* @file index.ts
* @input Markdown source and public parser options
- * @output Server-safe legacy and canonical Markdown parser entry points and types
+ * @output Server-safe legacy and canonical Markdown parser entry points and types,
+ * and the character reference decoder Markdown renders with
* @position Public subpath entry point: `@astryxdesign/core/Markdown/parser`
*/
export {
createIncrementalState,
+ decodeMarkdownCharacterReferences,
parseInline,
parseInlineAst,
parseMarkdown,
diff --git a/packages/richtext/src/markdownCharacterReferences.test.ts b/packages/richtext/src/markdownCharacterReferences.test.ts
new file mode 100644
index 0000000000..5514e96c02
--- /dev/null
+++ b/packages/richtext/src/markdownCharacterReferences.test.ts
@@ -0,0 +1,202 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+import {describe, it, expect} from 'vitest';
+import {createHeadlessEditor} from '@lexical/headless';
+import {$getRoot} from 'lexical';
+import {parseInline, parseMarkdown} from '@astryxdesign/core/Markdown/parser';
+import {DEFAULT_NODES} from './editorNodes';
+import {
+ editorStateJSONToMarkdown,
+ markdownToEditorStateJSON,
+} from './markdownSerializers';
+
+interface SerializedNode {
+ readonly type: string;
+ readonly text?: string;
+ readonly children?: ReadonlyArray<SerializedNode>;
+}
+
+/** The text RichText imports for a one-paragraph document. */
+function richTextText(markdown: string): string {
+ const {root} = JSON.parse(markdownToEditorStateJSON(markdown)) as {
+ root: SerializedNode;
+ };
+ const texts: Array<string> = [];
+ const visit = (node: SerializedNode) => {
+ if (node.text != null) {
+ texts.push(node.text);
+ }
+ node.children?.forEach(visit);
+ };
+ visit(root);
+ return texts.join('');
+}
+
+/** The text core Markdown renders for the same paragraph. */
+function markdownText(markdown: string): string {
+ return parseInline(markdown)
+ .map(node =>
+ 'content' in node
+ ? String(node.content)
+ : 'children' in node && Array.isArray(node.children)
+ ? node.children
+ .map(child => ('content' in child ? String(child.content) : ''))
+ .join('')
+ : '',
+ )
+ .join('');
+}
+
+// One table for both surfaces (spec:AST-061 DEC-5).
+const CASES = [
+ '&copy;',
+ '&#169;',
+ '&#xA9; and &#XA9;',
+ 'Fish &amp; chips',
+ '&amp;copy;',
+ '&NotEqualTilde;',
+ '&unknown;',
+ '&copy without a semicolon',
+ '& copy;',
+ '&#0; &#xD800; &#x110000;',
+ '&#12345678;',
+ 'a&nbsp;b',
+ '\\&copy; is escaped',
+ // Backslashes inside a code span are literal and close nothing early.
+ 'Paths like `C:\\` &amp; `D:\\` work',
+ 'Path `C:\\` then &#12345678; and &copy;',
+ // An escaped opener, and spans of one, two, and three backticks.
+ '\\`&amp;` stays text',
+ '``a ` &amp; b`` and &amp;',
+ 'Then ```a `` &amp; b``` and &copy;',
+ // An unclosed run is literal.
+ '`unclosed &amp; text',
+ '\\&#169; is escaped too',
+ '\\\\&copy; follows an escaped backslash',
+ 'code `&copy;` stays',
+];
+
+describe('character references (spec:AST-061 FR7, DEC-5)', () => {
+ it('import exactly as core Markdown renders them', () => {
+ for (const source of CASES) {
+ expect(richTextText(`${source}\n`), source).toBe(markdownText(source));
+ }
+ });
+
+ it('decode the same way in headings, lists, quotes, links, and table cells', () => {
+ const {root} = JSON.parse(
+ markdownToEditorStateJSON(
+ [
+ '# Caf&eacute; &#x2014; menu',
+ '',
+ '- Fish &amp; chips',
+ '> &copy; 2026',
+ '',
+ '[Terms &amp; conditions](https://example.com/?a=1&amp;b=2 "T&amp;C")',
+ '',
+ '| Item &amp; size | Price |',
+ '| --- | --- |',
+ '| Tea \\| &frac12; `&copy;` **&#169;** | &#12345678; \\&copy; |',
+ '',
+ ].join('\n'),
+ ),
+ ) as {root: SerializedNode & {children: Array<SerializedNode>}};
+ const texts: Array<string> = [];
+ const links: Array<{url?: unknown; title?: unknown}> = [];
+ const visit = (node: SerializedNode & {url?: unknown; title?: unknown}) => {
+ if (node.text != null) {
+ texts.push(node.text);
+ }
+ if (node.type === 'link') {
+ links.push({url: node.url, title: node.title});
+ }
+ (node.children as Array<typeof node> | undefined)?.forEach(visit);
+ };
+ visit(root);
+ const all = texts.join('|');
+ expect(all).toContain('Café — menu');
+ expect(all).toContain('Fish & chips');
+ expect(all).toContain('© 2026');
+ expect(all).toContain('Terms & conditions');
+ expect(all).toContain('Item & size');
+ expect(all).toContain('½');
+ // Inline code keeps the reference; strong text decodes it.
+ expect(texts).toContain('&copy;');
+ expect(texts).toContain('©');
+ // Not a reference, and an escaped one: literal.
+ expect(all).toContain('&#12345678;');
+ expect(all).toContain('&copy;');
+ expect(links).toEqual([
+ {url: 'https://example.com/?a=1&b=2', title: 'T&C'},
+ ]);
+ });
+
+ it('decode thousands of references in one pass, without extra nodes', () => {
+ for (const count of [2000, 6000, 10000]) {
+ const unit = '&copy; &#169; &#12345678; \\&amp; &nope; ';
+ const markdown = `${unit.repeat(count / 5)}\n`;
+ const started = performance.now();
+ const {root} = JSON.parse(markdownToEditorStateJSON(markdown)) as {
+ root: SerializedNode;
+ };
+ const elapsed = performance.now() - started;
+ const paragraph = root.children?.[0];
+ // One paragraph, one text node: nothing split per reference.
+ expect(paragraph?.children, String(count)).toHaveLength(1);
+ const expected = '© © &#12345678; &amp; &nope; '.repeat(count / 5);
+ // Compared as a flag, so a mismatch does not print the whole text.
+ expect(paragraph?.children?.[0].text === expected, String(count)).toBe(
+ true,
+ );
+ expect(elapsed, String(count)).toBeLessThan(5000);
+ }
+ });
+
+ it('find code spans the CommonMark way: only a run of the same length closes one', () => {
+ // A double run inside a single-backtick span neither closes it nor opens
+ // one, so the whole is code and the reference stays as written.
+ expect(richTextText('`one`` &amp; two`\n')).toBe('one`` &amp; two');
+ // Fenced code and CRLF: the reference after the fence decodes.
+ expect(richTextText('```\r\n&copy;\r\n```\r\n\r\n&copy;\r\n')).toBe(
+ '&copy;©',
+ );
+ });
+
+ it('stay literal in fenced code, as core Markdown keeps them', () => {
+ const markdown = '```\n&copy; &#169;\n```\n';
+ expect(richTextText(markdown)).toBe('&copy; &#169;');
+ expect(parseMarkdown(markdown)[0]).toMatchObject({
+ content: '&copy; &#169;',
+ });
+ });
+
+ it('keep an untouched paragraph exact and write decoded text back so it reads the same (spec:AST-062)', () => {
+ const markdown = 'A &copy; B &amp;copy; C &#169;\n';
+ expect(editorStateJSONToMarkdown(markdownToEditorStateJSON(markdown))).toBe(
+ markdown,
+ );
+ const editor = createHeadlessEditor({
+ namespace: 'astryx-character-reference-test',
+ nodes: [...DEFAULT_NODES],
+ onError(error: Error) {
+ throw error;
+ },
+ });
+ editor.setEditorState(
+ editor.parseEditorState(markdownToEditorStateJSON(markdown)),
+ );
+ editor.update(
+ () => {
+ const text = $getRoot().getAllTextNodes()[0];
+ text?.setTextContent(`${text.getTextContent()}!`);
+ },
+ {discrete: true},
+ );
+ const exported = editorStateJSONToMarkdown(
+ JSON.stringify(editor.getEditorState().toJSON()),
+ );
+ // The decoded `&copy;` text is escaped, so it does not decode again.
+ expect(exported).toBe('A © B \\&copy; C ©!\n');
+ expect(richTextText(exported)).toBe('A © B &copy; C ©!');
+ });
+});
diff --git a/packages/richtext/src/markdownCharacterReferences.ts b/packages/richtext/src/markdownCharacterReferences.ts
new file mode 100644
index 0000000000..2ff4f18815
--- /dev/null
+++ b/packages/richtext/src/markdownCharacterReferences.ts
@@ -0,0 +1,339 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file markdownCharacterReferences.ts
+ * @input Uses lexical, @lexical/link, @lexical/code, and core Markdown's
+ * decodeMarkdownCharacterReferences.
+ * @output Exports protectCharacterReferences, which swaps every character
+ * reference in Markdown source for a private-use stand-in before Lexical
+ * imports it, and $restoreCharacterReferences, which puts the decoded text
+ * in place of the stand-ins afterwards.
+ * @position Used by importMarkdownKeepingSource (markdownSource.ts), so every
+ * block — paragraphs, headings, lists, quotes, table cells, link text and
+ * destinations — decodes references with the one decoder core Markdown
+ * renders with (spec:AST-061 FR7, DEC-5), in one pass over the source and
+ * one over the imported text. The stand-ins mean nothing to Lexical's
+ * Markdown import, so a decoded `*` never starts emphasis, and no `&#digits;`
+ * reaches Lexical's own decoding, which ignores escapes and throws on a
+ * number past Unicode. Code keeps references as written.
+ */
+
+import {$isCodeNode} from '@lexical/code';
+import {$isLinkNode} from '@lexical/link';
+import {
+ $isElementNode,
+ $isTextNode,
+ type ElementNode,
+ type LexicalNode,
+} from 'lexical';
+import {decodeMarkdownCharacterReferences} from '@astryxdesign/core/Markdown/parser';
+
+/** What a stand-in replaced: a reference as written, or an escaped `&`. */
+type StandIn =
+ | {readonly kind: 'reference'; readonly source: string}
+ | {readonly kind: 'ampersand'};
+
+export interface ProtectedMarkdown {
+ /** The source with stand-ins in place of references. */
+ readonly markdown: string;
+ /** What each stand-in replaced. */
+ readonly standIns: ReadonlyMap<string, StandIn>;
+}
+
+// Something that looks like a reference; the decoder decides if it is one.
+const REFERENCE_LIKE = /&(?:#[xX][0-9a-fA-F]+|#[0-9]+|[A-Za-z][A-Za-z0-9]*);/y;
+const FENCE_OPEN = /^ {0,3}(`{3,}|~{3,})/;
+
+const PRIVATE_USE_RANGES: ReadonlyArray<readonly [number, number]> = [
+ [0xe000, 0xf8ff],
+ [0xf0000, 0xffffd],
+ [0x100000, 0x10fffd],
+];
+
+/** Private-use characters that do not occur in `text`, in order. */
+function* absentCharacters(text: string): Generator<string> {
+ const present = new Set(text);
+ for (const [first, last] of PRIVATE_USE_RANGES) {
+ for (let codePoint = first; codePoint <= last; codePoint++) {
+ const character = String.fromCodePoint(codePoint);
+ if (!present.has(character)) {
+ yield character;
+ }
+ }
+ }
+}
+
+/**
+ * The ranges of `text` that are code: fenced code blocks, and code spans,
+ * where a backtick run opens a span only if a later run of the same length
+ * closes it (CommonMark 0.31). One scan collects the runs; runs of each length
+ * are searched by position, so the whole is O(n log n).
+ */
+function codeRanges(text: string): Array<readonly [number, number]> {
+ const ranges: Array<readonly [number, number]> = [];
+ // Fenced code blocks, line by line.
+ let lineStart = 0;
+ let fence: {readonly marker: string; readonly start: number} | null = null;
+ while (lineStart <= text.length) {
+ const lineEnd = text.indexOf('\n', lineStart);
+ const end = lineEnd === -1 ? text.length : lineEnd;
+ const line = text.slice(lineStart, end);
+ if (fence == null) {
+ const open = FENCE_OPEN.exec(line);
+ if (open != null) {
+ fence = {marker: open[1], start: lineStart};
+ }
+ } else if (
+ new RegExp(
+ `^ {0,3}\\${fence.marker[0]}{${fence.marker.length},}[ \\t]*$`,
+ ).test(line)
+ ) {
+ ranges.push([fence.start, end]);
+ fence = null;
+ }
+ if (lineEnd === -1) {
+ break;
+ }
+ lineStart = lineEnd + 1;
+ }
+ if (fence != null) {
+ ranges.push([fence.start, text.length]);
+ }
+ // Code spans, outside fenced code. A backtick run is a string of backticks
+ // with no backtick on either side. Backslashes are literal inside a code
+ // span, so they never hide a closing run; an odd number of backslashes
+ // before a run only makes its first backtick literal, so the rest of the
+ // run (if any) may still open a span.
+ const runs: Array<{readonly start: number; readonly length: number}> = [];
+ let index = 0;
+ let fenceIndex = 0;
+ while (index < text.length) {
+ const current = ranges[fenceIndex];
+ if (current != null && index >= current[0]) {
+ index = current[1];
+ fenceIndex++;
+ continue;
+ }
+ if (text[index] === '`') {
+ let end = index;
+ while (text[end] === '`') {
+ end++;
+ }
+ runs.push({start: index, length: end - index});
+ index = end;
+ continue;
+ }
+ index++;
+ }
+ // Each run's position in the list of runs of its length, for finding the
+ // next run of a given length after a point by binary search.
+ const byLength = new Map<number, Array<number>>();
+ for (const run of runs) {
+ const starts = byLength.get(run.length) ?? [];
+ starts.push(run.start);
+ byLength.set(run.length, starts);
+ }
+ const nextRunAfter = (length: number, after: number): number | null => {
+ const starts = byLength.get(length);
+ if (starts == null) {
+ return null;
+ }
+ let low = 0;
+ let high = starts.length;
+ while (low < high) {
+ const middle = (low + high) >> 1;
+ if (starts[middle] < after) {
+ low = middle + 1;
+ } else {
+ high = middle;
+ }
+ }
+ return starts[low] ?? null;
+ };
+ let resumeAt = 0;
+ for (const run of runs) {
+ if (run.start < resumeAt) {
+ continue;
+ }
+ let backslashes = 0;
+ while (text[run.start - 1 - backslashes] === '\\') {
+ backslashes++;
+ }
+ const isEscaped = backslashes % 2 === 1;
+ const openStart = isEscaped ? run.start + 1 : run.start;
+ const openLength = isEscaped ? run.length - 1 : run.length;
+ if (openLength === 0) {
+ continue;
+ }
+ const closer = nextRunAfter(openLength, run.start + run.length);
+ if (closer == null) {
+ continue;
+ }
+ ranges.push([openStart, closer + openLength]);
+ resumeAt = closer + openLength;
+ }
+ return ranges.sort((a, b) => a[0] - b[0]);
+}
+
+/**
+ * Returns `markdown` with every character reference outside code replaced by
+ * a private-use stand-in that occurs nowhere else in it, and the escaped `&`
+ * of an escaped reference replaced by one too. One pass over the source.
+ */
+export function protectCharacterReferences(
+ markdown: string,
+): ProtectedMarkdown {
+ const standIns = new Map<string, StandIn>();
+ if (!markdown.includes('&')) {
+ return {markdown, standIns};
+ }
+ const available = absentCharacters(markdown);
+ const byReference = new Map<string, string>();
+ let ampersand: string | null = null;
+ const standInFor = (reference: string | null): string | null => {
+ const existing = reference == null ? ampersand : byReference.get(reference);
+ if (existing != null) {
+ return existing;
+ }
+ const next = available.next();
+ if (next.done === true) {
+ // Every private-use character is in the text or already used: leave
+ // the rest as written.
+ return null;
+ }
+ if (reference == null) {
+ ampersand = next.value;
+ standIns.set(next.value, {kind: 'ampersand'});
+ } else {
+ byReference.set(reference, next.value);
+ standIns.set(next.value, {kind: 'reference', source: reference});
+ }
+ return next.value;
+ };
+ const code = codeRanges(markdown);
+ let output = '';
+ let copied = 0;
+ let codeIndex = 0;
+ let index = markdown.indexOf('&');
+ while (index !== -1) {
+ while (code[codeIndex] != null && code[codeIndex][1] <= index) {
+ codeIndex++;
+ }
+ const range = code[codeIndex];
+ if (range != null && range[0] <= index) {
+ index = markdown.indexOf('&', range[1]);
+ continue;
+ }
+ REFERENCE_LIKE.lastIndex = index;
+ const match = REFERENCE_LIKE.exec(markdown);
+ if (match == null) {
+ index = markdown.indexOf('&', index + 1);
+ continue;
+ }
+ let backslashes = 0;
+ while (markdown[index - 1 - backslashes] === '\\') {
+ backslashes++;
+ }
+ const reference = match[0];
+ const end = index + reference.length;
+ if (backslashes % 2 === 1) {
+ // An escaped `&`: the reference stays literal, without its backslash.
+ const standIn = standInFor(null);
+ if (standIn != null) {
+ output += markdown.slice(copied, index - 1) + standIn;
+ copied = index + 1;
+ }
+ } else if (decodeMarkdownCharacterReferences(reference) !== reference) {
+ const standIn = standInFor(reference);
+ if (standIn != null) {
+ output += markdown.slice(copied, index) + standIn;
+ copied = end;
+ }
+ } else if (reference.startsWith('&#')) {
+ // Not a reference (too many digits): literal, but kept from Lexical's
+ // own numeric decoding.
+ const standIn = standInFor(null);
+ if (standIn != null) {
+ output += markdown.slice(copied, index) + standIn;
+ copied = index + 1;
+ }
+ }
+ index = markdown.indexOf('&', end);
+ }
+ return {markdown: output + markdown.slice(copied), standIns};
+}
+
+/** `text` with each stand-in replaced as `replace` says. */
+function restored(
+ text: string,
+ standIns: ReadonlyMap<string, StandIn>,
+ replace: (standIn: StandIn) => string,
+): string {
+ let output = '';
+ let changed = false;
+ for (const character of text) {
+ const standIn = standIns.get(character);
+ if (standIn == null) {
+ output += character;
+ } else {
+ output += replace(standIn);
+ changed = true;
+ }
+ }
+ return changed ? output : text;
+}
+
+const decoded = (standIn: StandIn): string =>
+ standIn.kind === 'ampersand'
+ ? '&'
+ : decodeMarkdownCharacterReferences(standIn.source);
+const asWritten = (standIn: StandIn): string =>
+ standIn.kind === 'ampersand' ? '&' : standIn.source;
+
+/**
+ * Replaces the stand-ins under `node` with the characters their references
+ * name — or, in code Lexical read as code, with the references as written —
+ * in text, link destinations, and link titles. One pass over the text.
+ */
+export function $restoreCharacterReferences(
+ node: ElementNode,
+ standIns: ReadonlyMap<string, StandIn>,
+): void {
+ if (standIns.size === 0) {
+ return;
+ }
+ const visit = (current: LexicalNode, inCode: boolean) => {
+ if ($isTextNode(current)) {
+ const text = current.getTextContent();
+ const next = restored(
+ text,
+ standIns,
+ inCode || current.hasFormat('code') ? asWritten : decoded,
+ );
+ if (next !== text) {
+ current.setTextContent(next);
+ }
+ return;
+ }
+ if ($isLinkNode(current)) {
+ const url = restored(current.getURL(), standIns, decoded);
+ if (url !== current.getURL()) {
+ current.setURL(url);
+ }
+ const title = current.getTitle();
+ if (title != null) {
+ const nextTitle = restored(title, standIns, decoded);
+ if (nextTitle !== title) {
+ current.setTitle(nextTitle);
+ }
+ }
+ }
+ if ($isElementNode(current)) {
+ const isCode = inCode || $isCodeNode(current);
+ for (const child of current.getChildren()) {
+ visit(child, isCode);
+ }
+ }
+ };
+ visit(node, false);
+}
diff --git a/packages/richtext/src/markdownSource.test.tsx b/packages/richtext/src/markdownSource.test.tsx
index 218a7509e1..80a1dbc64c 100644
--- a/packages/richtext/src/markdownSource.test.tsx
+++ b/packages/richtext/src/markdownSource.test.tsx
@@ -24,6 +24,10 @@ import {
} from 'lexical';
import {DEFAULT_NODES} from './editorNodes';
import {normalizeListIndentation} from './markdownListIndentation';
+import {
+ $restoreCharacterReferences,
+ protectCharacterReferences,
+} from './markdownCharacterReferences';
import {DEFAULT_TRANSFORMERS} from './markdownTable';
import {
editorStateJSONToMarkdown,
@@ -354,12 +358,16 @@ describe('Markdown source preservation (spec:AST-062)', () => {
});
editor.update(
() => {
- $convertFromMarkdownString(
- normalizeListIndentation(
- markdown.replace(/^\uFEFF/, '').replace(/\r\n/g, '\n'),
- ),
- [...DEFAULT_TRANSFORMERS],
- );
+ const {markdown: protectedMarkdown, standIns} =
+ protectCharacterReferences(
+ normalizeListIndentation(
+ markdown.replace(/^\uFEFF/, '').replace(/\r\n/g, '\n'),
+ ),
+ );
+ $convertFromMarkdownString(protectedMarkdown, [
+ ...DEFAULT_TRANSFORMERS,
+ ]);
+ $restoreCharacterReferences($getRoot(), standIns);
$nestFollowingLists($getRoot());
$joinSoftLineBreaks($getRoot());
},
diff --git a/packages/richtext/src/markdownSource.ts b/packages/richtext/src/markdownSource.ts
index 37215303cb..eb361c3662 100644
--- a/packages/richtext/src/markdownSource.ts
+++ b/packages/richtext/src/markdownSource.ts
@@ -49,6 +49,10 @@ import {
} from 'lexical';
import {isMarkedHardLineBreak} from './markdownHardLineBreak';
import {normalizeListIndentation} from './markdownListIndentation';
+import {
+ $restoreCharacterReferences,
+ protectCharacterReferences,
+} from './markdownCharacterReferences';
/** The whitespace and content one chunk of Markdown source was split into. */
export interface MarkdownChunk {
@@ -309,11 +313,12 @@ export function importMarkdownKeepingSource(
const holder = $createParagraphNode();
root.append(holder);
// Lexical imports LF lines; the record keeps the authored endings.
- $convertFromMarkdownString(
+ // Character references go through as stand-ins and come back decoded.
+ const {markdown: chunkMarkdown, standIns} = protectCharacterReferences(
withoutCarriageReturns(importChunks[index]?.content ?? chunk.content),
- transformers,
- holder,
);
+ $convertFromMarkdownString(chunkMarkdown, transformers, holder);
+ $restoreCharacterReferences(holder, standIns);
$joinSoftLineBreaks(holder);
for (const node of holder.getChildren()) {
$setState(node, groupState, {group: `${importId}:${index}`});