An open source design system that's fully customizable and agent ready
TypeScript
feat(Markdown): export the character reference decoder, and decode references in RichText with it (#7013)
Cindy Zhangcb30e207e0b5
diff --git a/.changeset/decode-markdown-character-references.md b/.changeset/decode-markdown-character-references.mdnew file mode 100644index 0000000000..0e32a787b2--- /dev/null+++ b/.changeset/decode-markdown-character-references.md@@ -0,0 +1,8 @@+---+'@astryxdesign/core': patch+---++[feat] Export `decodeMarkdownCharacterReferences` from `@astryxdesign/core/Markdown/parser` and `@astryxdesign/core/Markdown`+@cixzhang++It decodes character references the way `Markdown` renders them — `©`, `©`, and `©` become `©`; unknown names and references without their semicolon stay as written — using the same table `Markdown` uses. It works on plain text, so leave code and backslash-escaped references out. The parser subpath has no use-client boundary, so server code can call it.diff --git a/packages/core/src/Markdown/Markdown.doc.mjs b/packages/core/src/Markdown/Markdown.doc.mjsindex d8901d2526..b3b41adad6 100644--- a/packages/core/src/Markdown/Markdown.doc.mjs+++ b/packages/core/src/Markdown/Markdown.doc.mjs@@ -277,6 +277,11 @@ export const docs = {description:'Import parseMarkdownAst or parseInlineAst from @astryxdesign/core/Markdown/parser when server or React Server Component code needs to run plugins against the canonical readonly tree. The parser and plugin subpaths have no use-client boundary. The Markdown component remains client-owned, so function-bearing plugin entries must not be passed across an RSC serialization boundary.',},+ {+ guidance: true,+ description:+ 'Use decodeMarkdownCharacterReferences (from @astryxdesign/core/Markdown/parser or @astryxdesign/core/Markdown) to show text the way Markdown renders its character references: ©, ©, and © become ©, while unknown names and references without their semicolon stay as written. It decodes plain text and knows nothing of Markdown, so skip code and backslash-escaped references yourself. It uses the same table Markdown renders with.',+ },{guidance: true,description:diff --git a/packages/core/src/Markdown/Markdown.public.test.ts b/packages/core/src/Markdown/Markdown.public.test.tsindex 306cf8ae9f..da9a5c154c 100644--- a/packages/core/src/Markdown/Markdown.public.test.ts+++ b/packages/core/src/Markdown/Markdown.public.test.ts@@ -10,11 +10,13 @@import {describe, expectTypeOf, it} from 'vitest';import {createIncrementalState,+ decodeMarkdownCharacterReferences,parseInline,parseMarkdown,parseMarkdownIncremental,visitMarkdownNodes,} from './index';+import {decodeMarkdownCharacterReferences as decodeFromParser} from './parser/index';import {createMarkdownFenceTransform,createMarkdownHeadingLinks,@@ -92,6 +94,15 @@ function legacyBlockText(node: BlockNode): string {}}+describe('decodeMarkdownCharacterReferences entry points (spec:AST-061 DEC-5)', () => {+ it('is the same function from the Markdown entry and the parser subpath', () => {+ expectTypeOf(decodeMarkdownCharacterReferences).toEqualTypeOf<+ (text: string) => string+ >();+ expectTypeOf(decodeFromParser).toEqualTypeOf<(text: string) => string>();+ });+});+describe('Markdown public parser types', () => {it('keeps default and legacy parser calls on the legacy unions', () => {expectTypeOf(parseInline('plain')).toEqualTypeOf<InlineNode[]>();diff --git a/packages/core/src/Markdown/characterReferences.test.ts b/packages/core/src/Markdown/characterReferences.test.tsnew file mode 100644index 0000000000..c8ad8d5d2f--- /dev/null+++ b/packages/core/src/Markdown/characterReferences.test.ts@@ -0,0 +1,47 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++import {describe, expect, it} from 'vitest';+import {decodeMarkdownCharacterReferences} from './characterReferences';+import {parseInline} from './parser';++/** Character reference cases every surface decodes the same way (spec:AST-061 DEC-5). */+const CHARACTER_REFERENCE_CASES: ReadonlyArray<readonly [string, string]> = [+ ['©', '©'],+ ['©', '©'],+ ['©', '©'],+ ['©', '©'],+ ['&copy;', '©'],+ ['Fish & chips', 'Fish & chips'],+ ['≂̸', '\u2242\u0338'],+ ['&unknown;', '&unknown;'],+ ['©', '©'],+ ['& copy;', '& copy;'],+ ['�', '\uFFFD'],+ ['�', '\uFFFD'],+ ['�', '\uFFFD'],+ ['�', '�'],+ ['a b', 'a\u00A0b'],+ ['no references', 'no references'],+];++describe('decodeMarkdownCharacterReferences (spec:AST-061 DEC-5)', () => {+ it('decodes valid references and leaves everything else as written', () => {+ for (const [source, decoded] of CHARACTER_REFERENCE_CASES) {+ expect(decodeMarkdownCharacterReferences(source), source).toBe(decoded);+ }+ });++ it('decodes exactly as Markdown renders text', () => {+ for (const [source] of CHARACTER_REFERENCE_CASES) {+ const rendered = parseInline(source)+ .map(node => ('content' in node ? node.content : ''))+ .join('');+ expect(decodeMarkdownCharacterReferences(source), source).toBe(rendered);+ }+ });++ it('returns the same string when there is nothing to decode', () => {+ const text = 'plain & simple';+ expect(decodeMarkdownCharacterReferences(text)).toBe(text);+ });+});diff --git a/packages/core/src/Markdown/characterReferences.ts b/packages/core/src/Markdown/characterReferences.tsindex e1a6edf5e6..a1cab412d1 100644--- a/packages/core/src/Markdown/characterReferences.ts+++ b/packages/core/src/Markdown/characterReferences.ts@@ -7,9 +7,13 @@* `&#xH;` reference names, and where the reference ends. decodeLiteralText:* text with backslash escapes and character references resolved, for text* that is not parsed as Markdown, such as an image's alt text.+ * decodeMarkdownCharacterReferences: the public decoder, exported from+ * `@astryxdesign/core/Markdown/parser` and `@astryxdesign/core/Markdown`.* @position Used by parser.ts while it parses inline text, so references* render as the characters they name (spec:AST-061 FR7, DEC-3). Code spans- * and code blocks never reach it and keep references literal.+ * and code blocks never reach it and keep references literal. The RichText+ * surfaces decode with the same function, so every surface reads references+ * from this one table (spec:AST-061 DEC-5).*//**@@ -116,3 +120,37 @@ export function decodeLiteralText(text: string): string {}return output;}++/**+ * Returns `text` with every valid named or numeric character reference+ * replaced by the characters it names, as Markdown renders it: `©`,+ * `©`, and `©` all become `©`, and a numeric reference to NUL, a+ * surrogate, or a code point past U+10FFFF becomes U+FFFD. An unknown name, a+ * reference without its `;`, and all other text stay as written.+ *+ * It works on plain text and knows nothing of Markdown, so the caller decides+ * where it applies: keep backslash-escaped references and code literal by not+ * passing them in (spec:AST-061 DEC-5).+ *+ * @example+ * ```+ * decodeMarkdownCharacterReferences('Fish & chips © 2026'); // 'Fish & chips © 2026'+ * decodeMarkdownCharacterReferences('&unknown; ©'); // '&unknown; ©'+ * ```+ */+export function decodeMarkdownCharacterReferences(text: string): string {+ let output = '';+ let index = 0;+ let ampersand = text.indexOf('&');+ while (ampersand !== -1) {+ const reference = matchCharacterReference(text, ampersand);+ if (reference == null) {+ ampersand = text.indexOf('&', ampersand + 1);+ continue;+ }+ output += text.slice(index, ampersand) + reference.value;+ index = reference.end;+ ampersand = text.indexOf('&', index);+ }+ return index === 0 ? text : output + text.slice(index);+}diff --git a/packages/core/src/Markdown/index.ts b/packages/core/src/Markdown/index.tsindex 38aae88ec4..0be59d24b3 100644--- a/packages/core/src/Markdown/index.ts+++ b/packages/core/src/Markdown/index.ts@@ -4,7 +4,8 @@/*** @file index.ts- * @output Exports Markdown component, canonical and compatibility parser functions, and types+ * @output Exports Markdown component, canonical and compatibility parser functions,+ * the character reference decoder, and types* @position Component entry point; re-exported by /packages/core/src/index.ts*/@@ -56,6 +57,7 @@ export {parseInline,parseInlineAst,} from './parser';+export {decodeMarkdownCharacterReferences} from './characterReferences';export type {BlockNode,BlockNodeWithMath,diff --git a/packages/core/src/Markdown/parser.public.test.ts b/packages/core/src/Markdown/parser.public.test.tsindex 5c914774a7..c6ed04dd08 100644--- a/packages/core/src/Markdown/parser.public.test.ts+++ b/packages/core/src/Markdown/parser.public.test.ts@@ -16,6 +16,7 @@ import {} from '@astryxdesign/core/Markdown/plugins';import {createIncrementalState,+ decodeMarkdownCharacterReferences,parseInline,parseInlineAst,parseMarkdown,@@ -30,6 +31,15 @@ import type {} from '@astryxdesign/core/Markdown/parser';describe('@astryxdesign/core/Markdown/parser', () => {+ it('exports the character reference decoder Markdown renders with (spec:AST-061 DEC-5)', () => {+ expectTypeOf(decodeMarkdownCharacterReferences).toEqualTypeOf<+ (text: string) => string+ >();+ expect(decodeMarkdownCharacterReferences('© &unknown;')).toBe(+ '© &unknown;',+ );+ });+it('is a generated public package subpath', () => {const packageJson = JSON.parse(readFileSync(join(process.cwd(), 'packages/core/package.json'), 'utf8'),diff --git a/packages/core/src/Markdown/parser.ts b/packages/core/src/Markdown/parser.tsindex fc3f43943a..0ad6ffd222 100644--- a/packages/core/src/Markdown/parser.ts+++ b/packages/core/src/Markdown/parser.ts@@ -43,6 +43,10 @@ import {isAsciiPunctuation,matchCharacterReference,} from './characterReferences';++// The decoder Markdown renders with, public from the parser subpath+// (spec:AST-061 DEC-5).+export {decodeMarkdownCharacterReferences} from './characterReferences';import {isSafeMarkdownParserUrl} from './url';// ---------------------------------------------------------------------------diff --git a/packages/core/src/Markdown/parser/index.ts b/packages/core/src/Markdown/parser/index.tsindex 630cf2c21d..d8739b5222 100644--- a/packages/core/src/Markdown/parser/index.ts+++ b/packages/core/src/Markdown/parser/index.ts@@ -3,12 +3,14 @@/*** @file index.ts* @input Markdown source and public parser options- * @output Server-safe legacy and canonical Markdown parser entry points and types+ * @output Server-safe legacy and canonical Markdown parser entry points and types,+ * and the character reference decoder Markdown renders with* @position Public subpath entry point: `@astryxdesign/core/Markdown/parser`*/export {createIncrementalState,+ decodeMarkdownCharacterReferences,parseInline,parseInlineAst,parseMarkdown,diff --git a/packages/richtext/src/markdownCharacterReferences.test.ts b/packages/richtext/src/markdownCharacterReferences.test.tsnew file mode 100644index 0000000000..5514e96c02--- /dev/null+++ b/packages/richtext/src/markdownCharacterReferences.test.ts@@ -0,0 +1,202 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++import {describe, it, expect} from 'vitest';+import {createHeadlessEditor} from '@lexical/headless';+import {$getRoot} from 'lexical';+import {parseInline, parseMarkdown} from '@astryxdesign/core/Markdown/parser';+import {DEFAULT_NODES} from './editorNodes';+import {+ editorStateJSONToMarkdown,+ markdownToEditorStateJSON,+} from './markdownSerializers';++interface SerializedNode {+ readonly type: string;+ readonly text?: string;+ readonly children?: ReadonlyArray<SerializedNode>;+}++/** The text RichText imports for a one-paragraph document. */+function richTextText(markdown: string): string {+ const {root} = JSON.parse(markdownToEditorStateJSON(markdown)) as {+ root: SerializedNode;+ };+ const texts: Array<string> = [];+ const visit = (node: SerializedNode) => {+ if (node.text != null) {+ texts.push(node.text);+ }+ node.children?.forEach(visit);+ };+ visit(root);+ return texts.join('');+}++/** The text core Markdown renders for the same paragraph. */+function markdownText(markdown: string): string {+ return parseInline(markdown)+ .map(node =>+ 'content' in node+ ? String(node.content)+ : 'children' in node && Array.isArray(node.children)+ ? node.children+ .map(child => ('content' in child ? String(child.content) : ''))+ .join('')+ : '',+ )+ .join('');+}++// One table for both surfaces (spec:AST-061 DEC-5).+const CASES = [+ '©',+ '©',+ '© and ©',+ 'Fish & chips',+ '&copy;',+ '≂̸',+ '&unknown;',+ '© without a semicolon',+ '& copy;',+ '� � �',+ '�',+ 'a b',+ '\\© is escaped',+ // Backslashes inside a code span are literal and close nothing early.+ 'Paths like `C:\\` & `D:\\` work',+ 'Path `C:\\` then � and ©',+ // An escaped opener, and spans of one, two, and three backticks.+ '\\`&` stays text',+ '``a ` & b`` and &',+ 'Then ```a `` & b``` and ©',+ // An unclosed run is literal.+ '`unclosed & text',+ '\\© is escaped too',+ '\\\\© follows an escaped backslash',+ 'code `©` stays',+];++describe('character references (spec:AST-061 FR7, DEC-5)', () => {+ it('import exactly as core Markdown renders them', () => {+ for (const source of CASES) {+ expect(richTextText(`${source}\n`), source).toBe(markdownText(source));+ }+ });++ it('decode the same way in headings, lists, quotes, links, and table cells', () => {+ const {root} = JSON.parse(+ markdownToEditorStateJSON(+ [+ '# Café — menu',+ '',+ '- Fish & chips',+ '> © 2026',+ '',+ '[Terms & conditions](https://example.com/?a=1&b=2 "T&C")',+ '',+ '| Item & size | Price |',+ '| --- | --- |',+ '| Tea \\| ½ `©` **©** | � \\© |',+ '',+ ].join('\n'),+ ),+ ) as {root: SerializedNode & {children: Array<SerializedNode>}};+ const texts: Array<string> = [];+ const links: Array<{url?: unknown; title?: unknown}> = [];+ const visit = (node: SerializedNode & {url?: unknown; title?: unknown}) => {+ if (node.text != null) {+ texts.push(node.text);+ }+ if (node.type === 'link') {+ links.push({url: node.url, title: node.title});+ }+ (node.children as Array<typeof node> | undefined)?.forEach(visit);+ };+ visit(root);+ const all = texts.join('|');+ expect(all).toContain('Café — menu');+ expect(all).toContain('Fish & chips');+ expect(all).toContain('© 2026');+ expect(all).toContain('Terms & conditions');+ expect(all).toContain('Item & size');+ expect(all).toContain('½');+ // Inline code keeps the reference; strong text decodes it.+ expect(texts).toContain('©');+ expect(texts).toContain('©');+ // Not a reference, and an escaped one: literal.+ expect(all).toContain('�');+ expect(all).toContain('©');+ expect(links).toEqual([+ {url: 'https://example.com/?a=1&b=2', title: 'T&C'},+ ]);+ });++ it('decode thousands of references in one pass, without extra nodes', () => {+ for (const count of [2000, 6000, 10000]) {+ const unit = '© © � \\& &nope; ';+ const markdown = `${unit.repeat(count / 5)}\n`;+ const started = performance.now();+ const {root} = JSON.parse(markdownToEditorStateJSON(markdown)) as {+ root: SerializedNode;+ };+ const elapsed = performance.now() - started;+ const paragraph = root.children?.[0];+ // One paragraph, one text node: nothing split per reference.+ expect(paragraph?.children, String(count)).toHaveLength(1);+ const expected = '© © � & &nope; '.repeat(count / 5);+ // Compared as a flag, so a mismatch does not print the whole text.+ expect(paragraph?.children?.[0].text === expected, String(count)).toBe(+ true,+ );+ expect(elapsed, String(count)).toBeLessThan(5000);+ }+ });++ it('find code spans the CommonMark way: only a run of the same length closes one', () => {+ // A double run inside a single-backtick span neither closes it nor opens+ // one, so the whole is code and the reference stays as written.+ expect(richTextText('`one`` & two`\n')).toBe('one`` & two');+ // Fenced code and CRLF: the reference after the fence decodes.+ expect(richTextText('```\r\n©\r\n```\r\n\r\n©\r\n')).toBe(+ '©©',+ );+ });++ it('stay literal in fenced code, as core Markdown keeps them', () => {+ const markdown = '```\n© ©\n```\n';+ expect(richTextText(markdown)).toBe('© ©');+ expect(parseMarkdown(markdown)[0]).toMatchObject({+ content: '© ©',+ });+ });++ it('keep an untouched paragraph exact and write decoded text back so it reads the same (spec:AST-062)', () => {+ const markdown = 'A © B &copy; C ©\n';+ expect(editorStateJSONToMarkdown(markdownToEditorStateJSON(markdown))).toBe(+ markdown,+ );+ const editor = createHeadlessEditor({+ namespace: 'astryx-character-reference-test',+ nodes: [...DEFAULT_NODES],+ onError(error: Error) {+ throw error;+ },+ });+ editor.setEditorState(+ editor.parseEditorState(markdownToEditorStateJSON(markdown)),+ );+ editor.update(+ () => {+ const text = $getRoot().getAllTextNodes()[0];+ text?.setTextContent(`${text.getTextContent()}!`);+ },+ {discrete: true},+ );+ const exported = editorStateJSONToMarkdown(+ JSON.stringify(editor.getEditorState().toJSON()),+ );+ // The decoded `©` text is escaped, so it does not decode again.+ expect(exported).toBe('A © B \\© C ©!\n');+ expect(richTextText(exported)).toBe('A © B © C ©!');+ });+});diff --git a/packages/richtext/src/markdownCharacterReferences.ts b/packages/richtext/src/markdownCharacterReferences.tsnew file mode 100644index 0000000000..2ff4f18815--- /dev/null+++ b/packages/richtext/src/markdownCharacterReferences.ts@@ -0,0 +1,339 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file markdownCharacterReferences.ts+ * @input Uses lexical, @lexical/link, @lexical/code, and core Markdown's+ * decodeMarkdownCharacterReferences.+ * @output Exports protectCharacterReferences, which swaps every character+ * reference in Markdown source for a private-use stand-in before Lexical+ * imports it, and $restoreCharacterReferences, which puts the decoded text+ * in place of the stand-ins afterwards.+ * @position Used by importMarkdownKeepingSource (markdownSource.ts), so every+ * block — paragraphs, headings, lists, quotes, table cells, link text and+ * destinations — decodes references with the one decoder core Markdown+ * renders with (spec:AST-061 FR7, DEC-5), in one pass over the source and+ * one over the imported text. The stand-ins mean nothing to Lexical's+ * Markdown import, so a decoded `*` never starts emphasis, and no `&#digits;`+ * reaches Lexical's own decoding, which ignores escapes and throws on a+ * number past Unicode. Code keeps references as written.+ */++import {$isCodeNode} from '@lexical/code';+import {$isLinkNode} from '@lexical/link';+import {+ $isElementNode,+ $isTextNode,+ type ElementNode,+ type LexicalNode,+} from 'lexical';+import {decodeMarkdownCharacterReferences} from '@astryxdesign/core/Markdown/parser';++/** What a stand-in replaced: a reference as written, or an escaped `&`. */+type StandIn =+ | {readonly kind: 'reference'; readonly source: string}+ | {readonly kind: 'ampersand'};++export interface ProtectedMarkdown {+ /** The source with stand-ins in place of references. */+ readonly markdown: string;+ /** What each stand-in replaced. */+ readonly standIns: ReadonlyMap<string, StandIn>;+}++// Something that looks like a reference; the decoder decides if it is one.+const REFERENCE_LIKE = /&(?:#[xX][0-9a-fA-F]+|#[0-9]+|[A-Za-z][A-Za-z0-9]*);/y;+const FENCE_OPEN = /^ {0,3}(`{3,}|~{3,})/;++const PRIVATE_USE_RANGES: ReadonlyArray<readonly [number, number]> = [+ [0xe000, 0xf8ff],+ [0xf0000, 0xffffd],+ [0x100000, 0x10fffd],+];++/** Private-use characters that do not occur in `text`, in order. */+function* absentCharacters(text: string): Generator<string> {+ const present = new Set(text);+ for (const [first, last] of PRIVATE_USE_RANGES) {+ for (let codePoint = first; codePoint <= last; codePoint++) {+ const character = String.fromCodePoint(codePoint);+ if (!present.has(character)) {+ yield character;+ }+ }+ }+}++/**+ * The ranges of `text` that are code: fenced code blocks, and code spans,+ * where a backtick run opens a span only if a later run of the same length+ * closes it (CommonMark 0.31). One scan collects the runs; runs of each length+ * are searched by position, so the whole is O(n log n).+ */+function codeRanges(text: string): Array<readonly [number, number]> {+ const ranges: Array<readonly [number, number]> = [];+ // Fenced code blocks, line by line.+ let lineStart = 0;+ let fence: {readonly marker: string; readonly start: number} | null = null;+ while (lineStart <= text.length) {+ const lineEnd = text.indexOf('\n', lineStart);+ const end = lineEnd === -1 ? text.length : lineEnd;+ const line = text.slice(lineStart, end);+ if (fence == null) {+ const open = FENCE_OPEN.exec(line);+ if (open != null) {+ fence = {marker: open[1], start: lineStart};+ }+ } else if (+ new RegExp(+ `^ {0,3}\\${fence.marker[0]}{${fence.marker.length},}[ \\t]*$`,+ ).test(line)+ ) {+ ranges.push([fence.start, end]);+ fence = null;+ }+ if (lineEnd === -1) {+ break;+ }+ lineStart = lineEnd + 1;+ }+ if (fence != null) {+ ranges.push([fence.start, text.length]);+ }+ // Code spans, outside fenced code. A backtick run is a string of backticks+ // with no backtick on either side. Backslashes are literal inside a code+ // span, so they never hide a closing run; an odd number of backslashes+ // before a run only makes its first backtick literal, so the rest of the+ // run (if any) may still open a span.+ const runs: Array<{readonly start: number; readonly length: number}> = [];+ let index = 0;+ let fenceIndex = 0;+ while (index < text.length) {+ const current = ranges[fenceIndex];+ if (current != null && index >= current[0]) {+ index = current[1];+ fenceIndex++;+ continue;+ }+ if (text[index] === '`') {+ let end = index;+ while (text[end] === '`') {+ end++;+ }+ runs.push({start: index, length: end - index});+ index = end;+ continue;+ }+ index++;+ }+ // Each run's position in the list of runs of its length, for finding the+ // next run of a given length after a point by binary search.+ const byLength = new Map<number, Array<number>>();+ for (const run of runs) {+ const starts = byLength.get(run.length) ?? [];+ starts.push(run.start);+ byLength.set(run.length, starts);+ }+ const nextRunAfter = (length: number, after: number): number | null => {+ const starts = byLength.get(length);+ if (starts == null) {+ return null;+ }+ let low = 0;+ let high = starts.length;+ while (low < high) {+ const middle = (low + high) >> 1;+ if (starts[middle] < after) {+ low = middle + 1;+ } else {+ high = middle;+ }+ }+ return starts[low] ?? null;+ };+ let resumeAt = 0;+ for (const run of runs) {+ if (run.start < resumeAt) {+ continue;+ }+ let backslashes = 0;+ while (text[run.start - 1 - backslashes] === '\\') {+ backslashes++;+ }+ const isEscaped = backslashes % 2 === 1;+ const openStart = isEscaped ? run.start + 1 : run.start;+ const openLength = isEscaped ? run.length - 1 : run.length;+ if (openLength === 0) {+ continue;+ }+ const closer = nextRunAfter(openLength, run.start + run.length);+ if (closer == null) {+ continue;+ }+ ranges.push([openStart, closer + openLength]);+ resumeAt = closer + openLength;+ }+ return ranges.sort((a, b) => a[0] - b[0]);+}++/**+ * Returns `markdown` with every character reference outside code replaced by+ * a private-use stand-in that occurs nowhere else in it, and the escaped `&`+ * of an escaped reference replaced by one too. One pass over the source.+ */+export function protectCharacterReferences(+ markdown: string,+): ProtectedMarkdown {+ const standIns = new Map<string, StandIn>();+ if (!markdown.includes('&')) {+ return {markdown, standIns};+ }+ const available = absentCharacters(markdown);+ const byReference = new Map<string, string>();+ let ampersand: string | null = null;+ const standInFor = (reference: string | null): string | null => {+ const existing = reference == null ? ampersand : byReference.get(reference);+ if (existing != null) {+ return existing;+ }+ const next = available.next();+ if (next.done === true) {+ // Every private-use character is in the text or already used: leave+ // the rest as written.+ return null;+ }+ if (reference == null) {+ ampersand = next.value;+ standIns.set(next.value, {kind: 'ampersand'});+ } else {+ byReference.set(reference, next.value);+ standIns.set(next.value, {kind: 'reference', source: reference});+ }+ return next.value;+ };+ const code = codeRanges(markdown);+ let output = '';+ let copied = 0;+ let codeIndex = 0;+ let index = markdown.indexOf('&');+ while (index !== -1) {+ while (code[codeIndex] != null && code[codeIndex][1] <= index) {+ codeIndex++;+ }+ const range = code[codeIndex];+ if (range != null && range[0] <= index) {+ index = markdown.indexOf('&', range[1]);+ continue;+ }+ REFERENCE_LIKE.lastIndex = index;+ const match = REFERENCE_LIKE.exec(markdown);+ if (match == null) {+ index = markdown.indexOf('&', index + 1);+ continue;+ }+ let backslashes = 0;+ while (markdown[index - 1 - backslashes] === '\\') {+ backslashes++;+ }+ const reference = match[0];+ const end = index + reference.length;+ if (backslashes % 2 === 1) {+ // An escaped `&`: the reference stays literal, without its backslash.+ const standIn = standInFor(null);+ if (standIn != null) {+ output += markdown.slice(copied, index - 1) + standIn;+ copied = index + 1;+ }+ } else if (decodeMarkdownCharacterReferences(reference) !== reference) {+ const standIn = standInFor(reference);+ if (standIn != null) {+ output += markdown.slice(copied, index) + standIn;+ copied = end;+ }+ } else if (reference.startsWith('&#')) {+ // Not a reference (too many digits): literal, but kept from Lexical's+ // own numeric decoding.+ const standIn = standInFor(null);+ if (standIn != null) {+ output += markdown.slice(copied, index) + standIn;+ copied = index + 1;+ }+ }+ index = markdown.indexOf('&', end);+ }+ return {markdown: output + markdown.slice(copied), standIns};+}++/** `text` with each stand-in replaced as `replace` says. */+function restored(+ text: string,+ standIns: ReadonlyMap<string, StandIn>,+ replace: (standIn: StandIn) => string,+): string {+ let output = '';+ let changed = false;+ for (const character of text) {+ const standIn = standIns.get(character);+ if (standIn == null) {+ output += character;+ } else {+ output += replace(standIn);+ changed = true;+ }+ }+ return changed ? output : text;+}++const decoded = (standIn: StandIn): string =>+ standIn.kind === 'ampersand'+ ? '&'+ : decodeMarkdownCharacterReferences(standIn.source);+const asWritten = (standIn: StandIn): string =>+ standIn.kind === 'ampersand' ? '&' : standIn.source;++/**+ * Replaces the stand-ins under `node` with the characters their references+ * name — or, in code Lexical read as code, with the references as written —+ * in text, link destinations, and link titles. One pass over the text.+ */+export function $restoreCharacterReferences(+ node: ElementNode,+ standIns: ReadonlyMap<string, StandIn>,+): void {+ if (standIns.size === 0) {+ return;+ }+ const visit = (current: LexicalNode, inCode: boolean) => {+ if ($isTextNode(current)) {+ const text = current.getTextContent();+ const next = restored(+ text,+ standIns,+ inCode || current.hasFormat('code') ? asWritten : decoded,+ );+ if (next !== text) {+ current.setTextContent(next);+ }+ return;+ }+ if ($isLinkNode(current)) {+ const url = restored(current.getURL(), standIns, decoded);+ if (url !== current.getURL()) {+ current.setURL(url);+ }+ const title = current.getTitle();+ if (title != null) {+ const nextTitle = restored(title, standIns, decoded);+ if (nextTitle !== title) {+ current.setTitle(nextTitle);+ }+ }+ }+ if ($isElementNode(current)) {+ const isCode = inCode || $isCodeNode(current);+ for (const child of current.getChildren()) {+ visit(child, isCode);+ }+ }+ };+ visit(node, false);+}diff --git a/packages/richtext/src/markdownSource.test.tsx b/packages/richtext/src/markdownSource.test.tsxindex 218a7509e1..80a1dbc64c 100644--- a/packages/richtext/src/markdownSource.test.tsx+++ b/packages/richtext/src/markdownSource.test.tsx@@ -24,6 +24,10 @@ import {} from 'lexical';import {DEFAULT_NODES} from './editorNodes';import {normalizeListIndentation} from './markdownListIndentation';+import {+ $restoreCharacterReferences,+ protectCharacterReferences,+} from './markdownCharacterReferences';import {DEFAULT_TRANSFORMERS} from './markdownTable';import {editorStateJSONToMarkdown,@@ -354,12 +358,16 @@ describe('Markdown source preservation (spec:AST-062)', () => {});editor.update(() => {- $convertFromMarkdownString(- normalizeListIndentation(- markdown.replace(/^\uFEFF/, '').replace(/\r\n/g, '\n'),- ),- [...DEFAULT_TRANSFORMERS],- );+ const {markdown: protectedMarkdown, standIns} =+ protectCharacterReferences(+ normalizeListIndentation(+ markdown.replace(/^\uFEFF/, '').replace(/\r\n/g, '\n'),+ ),+ );+ $convertFromMarkdownString(protectedMarkdown, [+ ...DEFAULT_TRANSFORMERS,+ ]);+ $restoreCharacterReferences($getRoot(), standIns);$nestFollowingLists($getRoot());$joinSoftLineBreaks($getRoot());},diff --git a/packages/richtext/src/markdownSource.ts b/packages/richtext/src/markdownSource.tsindex 37215303cb..eb361c3662 100644--- a/packages/richtext/src/markdownSource.ts+++ b/packages/richtext/src/markdownSource.ts@@ -49,6 +49,10 @@ import {} from 'lexical';import {isMarkedHardLineBreak} from './markdownHardLineBreak';import {normalizeListIndentation} from './markdownListIndentation';+import {+ $restoreCharacterReferences,+ protectCharacterReferences,+} from './markdownCharacterReferences';/** The whitespace and content one chunk of Markdown source was split into. */export interface MarkdownChunk {@@ -309,11 +313,12 @@ export function importMarkdownKeepingSource(const holder = $createParagraphNode();root.append(holder);// Lexical imports LF lines; the record keeps the authored endings.- $convertFromMarkdownString(+ // Character references go through as stand-ins and come back decoded.+ const {markdown: chunkMarkdown, standIns} = protectCharacterReferences(withoutCarriageReturns(importChunks[index]?.content ?? chunk.content),- transformers,- holder,);+ $convertFromMarkdownString(chunkMarkdown, transformers, holder);+ $restoreCharacterReferences(holder, standIns);$joinSoftLineBreaks(holder);for (const node of holder.getChildren()) {$setState(node, groupState, {group: `${importId}:${index}`});