An open source design system that's fully customizable and agent ready
TypeScript
fix(richtext): export text with long runs of spaces in linear time (#7085)
Cindy Zhang74a62ce8ee35
diff --git a/packages/richtext/src/markdownCharacterReferences.ts b/packages/richtext/src/markdownCharacterReferences.tsindex 2fc59d37fa..062631619c 100644--- a/packages/richtext/src/markdownCharacterReferences.ts+++ b/packages/richtext/src/markdownCharacterReferences.ts@@ -71,7 +71,7 @@ const PRIVATE_USE_RANGES: ReadonlyArray<readonly [number, number]> = [];/** Private-use characters that do not occur in `text`, in order. */-function* absentCharacters(text: string): Generator<string> {+export function* absentCharacters(text: string): Generator<string> {const present = new Set(text);for (const [first, last] of PRIVATE_USE_RANGES) {for (let codePoint = first; codePoint <= last; codePoint++) {diff --git a/packages/richtext/src/markdownSource.ts b/packages/richtext/src/markdownSource.tsindex 6ac218c7c7..3f9d102a9e 100644--- a/packages/richtext/src/markdownSource.ts+++ b/packages/richtext/src/markdownSource.ts@@ -27,11 +27,7 @@* written instead.*/-import {- $convertFromMarkdownString,- $convertToMarkdownString,- type Transformer,-} from '@lexical/markdown';+import {$convertFromMarkdownString, type Transformer} from '@lexical/markdown';import {$isCodeNode} from '@lexical/code';import {$isListItemNode, $isListNode} from '@lexical/list';import {@@ -69,6 +65,7 @@ import {withExtensionExport,} from './markdownExtensions';import {$isRichTextExtensionNode} from './markdownExtensionNode';+import {$convertToMarkdownKeepingTimeLinear} from './markdownSpaceRuns';import type {MarkdownPluginEntry} from '@astryxdesign/core/Markdown/plugins';/** The whitespace and content one chunk of Markdown source was split into. */@@ -459,7 +456,15 @@ export function $joinSoftLineBreaks(element: ElementNode): void {// The spaces around a soft break are part of it.const previous = child.getPreviousSibling();if ($isTextNode(previous) && !previous.hasFormat('code')) {- previous.setTextContent(previous.getTextContent().replace(/[ \t]+$/, ''));+ // Counted from the end: a pattern such as /[ \t]+$/ retries from every+ // space in a long run that does not end the text, so its time grows+ // with the square of the run.+ const text = previous.getTextContent();+ let end = text.length;+ while (end > 0 && (text[end - 1] === ' ' || text[end - 1] === '\t')) {+ end--;+ }+ previous.setTextContent(text.slice(0, end));}const next = child.getNextSibling();if ($isTextNode(next) && !next.hasFormat('code')) {@@ -632,7 +637,7 @@ function $canonicalMarkdown(descendants,descendants.map(node => node.getTextContent()).join(''),);- let markdown = $convertToMarkdownString(+ let markdown = $convertToMarkdownKeepingTimeLinear(withExtensionExport(transformers),childrenOf(placeholders.size === 0@@ -808,7 +813,7 @@ function $regeneratedMarkdown(descendantsOf(nodes),text + token,);- let marked = $convertToMarkdownString(+ let marked = $convertToMarkdownKeepingTimeLinear(withExtensionExport(transformers),childrenOf(nodes.map(node => markedView(node, token, placeholders))),)diff --git a/packages/richtext/src/markdownSpaceRuns.test.ts b/packages/richtext/src/markdownSpaceRuns.test.tsnew file mode 100644index 0000000000..33ba9b0bd2--- /dev/null+++ b/packages/richtext/src/markdownSpaceRuns.test.ts@@ -0,0 +1,138 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++import {describe, expect, it} from 'vitest';+import {createHeadlessEditor} from '@lexical/headless';+import {$getRoot, $isTextNode} from 'lexical';+import {markdownToEditorStateJSON} from './markdownSerializers';+import {DEFAULT_NODES} from './editorNodes';+import {DEFAULT_TRANSFORMERS} from './markdownTable';+import {+ $exportMarkdownKeepingSource,+ importMarkdownKeepingSource,+} from './markdownSource';++/** Imports `markdown`, appends `x` to its last text, and exports it. */+function editAndExport(markdown: string): string {+ const editor = createHeadlessEditor({+ nodes: [...DEFAULT_NODES],+ onError(error) {+ throw error;+ },+ });+ importMarkdownKeepingSource(editor, markdown, [...DEFAULT_TRANSFORMERS]);+ editor.update(+ () => {+ const last = $getRoot().getLastDescendant();+ if (!$isTextNode(last)) {+ throw new Error('No text to edit');+ }+ last.setTextContent(`${last.getTextContent()}x`);+ },+ {discrete: true},+ );+ return editor+ .getEditorState()+ .read(() => $exportMarkdownKeepingSource([...DEFAULT_TRANSFORMERS]));+}++/**+ * The least CPU time of five runs of `run`, in milliseconds. CPU time, not+ * elapsed time: on a loaded test machine, other work stretches elapsed time+ * but not the time the editor spends computing.+ */+function leastCpuTime(run: () => void): number {+ run();+ let least = Number.POSITIVE_INFINITY;+ for (let round = 0; round < 5; round++) {+ const started = process.cpuUsage();+ run();+ const used = process.cpuUsage(started);+ least = Math.min(least, (used.user + used.system) / 1000);+ }+ return least;+}++const RUN = ' '.repeat(40_000);++describe('runs of spaces inside text', () => {+ it.each([+ ['plain text', 'a b\n', 'a bx\n'],+ ['bold text', '**a b** c\n', '**a b** cx\n'],+ ['link text', '[a b](u) c\n', '[a b](u) cx\n'],+ ['tabs and spaces', 'a\t\t b\n', 'a\t\t bx\n'],+ ['no-break spaces', 'a\u00a0\u00a0b\n', 'a\u00a0\u00a0bx\n'],+ [+ 'a table cell',+ '| h |\n| - |\n| a b |\n',+ '| h |\n| --- |\n| a bx |\n',+ ],+ ])('keeps each space through an edit of %s', (_, markdown, expected) => {+ expect(editAndExport(markdown)).toBe(expected);+ });++ it.each([+ ['plain text', `a${RUN}b\nc\n`],+ ['bold text', `**a${RUN}b**\nc\n`],+ ['link text', `[a${RUN}b](u)\nc\n`],+ ['a table cell', `| h |\n| - |\n| a${RUN}b |\n`],+ ])(+ 'imports, edits, and exports %s with a 40 KB run within 100 ms',+ (_, markdown) => {+ expect(leastCpuTime(() => editAndExport(markdown))).toBeLessThan(100);+ },+ );+});++/** Every link's address and title, in order. */+function links(markdown: string): Array<[string, string | null]> {+ const found: Array<[string, string | null]> = [];+ const visit = (node: {+ type: string;+ url?: string;+ title?: string | null;+ children?: unknown[];+ }) => {+ if (node.type === 'link') {+ found.push([node.url ?? '', node.title ?? null]);+ }+ (node.children as Array<typeof node> | undefined)?.forEach(visit);+ };+ visit(+ (JSON.parse(markdownToEditorStateJSON(markdown)) as {root: {type: string}})+ .root,+ );+ return found;+}++describe('stand-ins never touch what the export writes besides text', () => {+ it.each([+ ['an address', 'a b [x](https://e.com/\uE000) c\n'],+ ['a title', 'a b [x](https://e.com "t\uE000") c\n'],+ [+ 'a table cell link address',+ '| h |\n| - |\n| a b [x](https://e.com/\uE000) |\n',+ ],+ ])('keeps a private-use character in %s through an edit', (_, markdown) => {+ const before = links(markdown);+ expect(before.length).toBe(1);+ expect(links(editAndExport(markdown))).toEqual(before);+ });++ it('keeps every address and title exact when they hold private-use characters', () => {+ let state = 7085;+ const random = () => {+ state = (state * 1103515245 + 12345) % 2147483648;+ return state / 2147483648;+ };+ const privateUse = () =>+ String.fromCodePoint(0xe000 + Math.floor(random() * 6));+ for (let round = 0; round < 60; round++) {+ const url = `https://e.com/${privateUse()}${privateUse()}`;+ const title = `t${privateUse()}`;+ const markdown = `a${' '.repeat(2 + Math.floor(random() * 4))}b [x](${url} "${title}") c${' '.repeat(3)}d\n`;+ const before = links(markdown);+ expect(before).toEqual([[url, title]]);+ expect(links(editAndExport(markdown)), markdown).toEqual(before);+ }+ });+});diff --git a/packages/richtext/src/markdownSpaceRuns.ts b/packages/richtext/src/markdownSpaceRuns.tsnew file mode 100644index 0000000000..72afc59be9--- /dev/null+++ b/packages/richtext/src/markdownSpaceRuns.ts@@ -0,0 +1,154 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file markdownSpaceRuns.ts+ * @input Lexical nodes to export as Markdown, and the transformers to use+ * @output $convertToMarkdownKeepingTimeLinear: Lexical's Markdown export, in+ * time linear in the text even when a text holds long runs of spaces+ * @position Every Markdown export in this package goes through it+ * (markdownSource.ts, markdownTable.ts). Lexical's text-format export splits+ * each text into leading spaces, content, and trailing spaces with+ * `/^(\s*)(.*?)(\s*)$/s`, which retries from every character of a long run+ * of spaces inside the text, so its time grows with the square of the run.+ * Each such run is exported as private-use stand-ins, which that pattern+ * passes in one step each, and put back afterwards. Leading and trailing+ * spaces, which decide where marks attach, are left as they are.+ */++import type {ElementNode, LexicalNode} from 'lexical';+import type {Transformer} from '@lexical/markdown';+import {$isCodeNode} from '@lexical/code';+import {$convertToMarkdownString} from '@lexical/markdown';+import {$isElementNode, $isTextNode} from 'lexical';+import {absentCharacters} from './markdownCharacterReferences';++/** Two or more spaces with other characters on both sides. */+const INNER_SPACE_RUN = /(?<=\S)\s{2,}(?=\S)/g;++/** Whether Lexical exports the text node's text as written, untouched. */+function isLiteralText(node: LexicalNode): boolean {+ return (+ $isTextNode(node) &&+ (node.hasFormat('code') || $isCodeNode(node.getParent()))+ );+}++/**+ * Lexical's `$convertToMarkdownString(transformers, root)`, with every run of+ * two or more spaces inside a text exported as stand-ins and put back, so the+ * export takes time linear in the text. The tree is read through views, never+ * changed.+ */+export function $convertToMarkdownKeepingTimeLinear(+ transformers: Array<Transformer>,+ root: ElementNode,+): string {+ // Every string the export can write — texts, and the properties nodes+ // serialize, such as a link's address and title — so a stand-in occurs+ // nowhere else in the output.+ const written: string[] = [];+ const spaces = new Set<string>();+ const visit = (node: LexicalNode) => {+ written.push(JSON.stringify(node.exportJSON()));+ if ($isTextNode(node)) {+ const text = node.getTextContent();+ written.push(text);+ if (!isLiteralText(node)) {+ for (const run of text.match(INNER_SPACE_RUN) ?? []) {+ for (const space of run) {+ spaces.add(space);+ }+ }+ }+ } else if ($isElementNode(node)) {+ node.getChildren().forEach(visit);+ }+ };+ root.getChildren().forEach(visit);+ if (spaces.size === 0) {+ return $convertToMarkdownString(transformers, root);+ }+ const free = absentCharacters(written.join(''));+ const standIns = new Map<string, string>();+ for (const space of spaces) {+ const standIn = free.next().value;+ if (standIn == null) {+ return $convertToMarkdownString(transformers, root);+ }+ standIns.set(space, standIn);+ }+ const placed = {count: 0};+ const children = viewsOf(root.getChildren(), standIns, placed);+ const view = Object.create(root) as typeof root;+ view.getChildren = <T extends LexicalNode>() => children as Array<T>;+ let markdown = $convertToMarkdownString(transformers, view);+ // Every stand-in in the output must be one this export placed; otherwise+ // the plain export, slower but exact, is the answer.+ let found = 0;+ for (const standIn of standIns.values()) {+ found += markdown.split(standIn).length - 1;+ }+ if (found !== placed.count) {+ return $convertToMarkdownString(transformers, root);+ }+ for (const [space, standIn] of standIns) {+ markdown = markdown.split(standIn).join(space);+ }+ return markdown;+}++/**+ * Views of `nodes` whose texts hold stand-ins for their inner runs of+ * spaces; a node with no such run is itself. Each view's siblings are the+ * views beside it, as Lexical's exporter reads neighbors to place marks.+ */+function viewsOf(+ nodes: ReadonlyArray<LexicalNode>,+ standIns: ReadonlyMap<string, string>,+ placed: {count: number},+): LexicalNode[] {+ const views = nodes.map(node => viewOf(node, standIns, placed));+ views.forEach((view, index) => {+ if (view !== nodes[index]) {+ view.getPreviousSibling = <T extends LexicalNode>() =>+ (views[index - 1] ?? null) as T | null;+ view.getNextSibling = <T extends LexicalNode>() =>+ (views[index + 1] ?? null) as T | null;+ }+ });+ return views;+}++function viewOf(+ node: LexicalNode,+ standIns: ReadonlyMap<string, string>,+ placed: {count: number},+): LexicalNode {+ if ($isTextNode(node)) {+ if (isLiteralText(node)) {+ return node;+ }+ const text = node.getTextContent();+ const stood = text.replace(INNER_SPACE_RUN, run => {+ placed.count += run.length;+ return Array.from(run, space => standIns.get(space) ?? space).join('');+ });+ if (stood === text) {+ return node;+ }+ const view = Object.create(node) as typeof node;+ view.getTextContent = () => stood;+ return view;+ }+ if ($isElementNode(node) && !$isCodeNode(node)) {+ const original = node.getChildren();+ const children = viewsOf(original, standIns, placed);+ if (children.every((child, index) => child === original[index])) {+ return node;+ }+ const view = Object.create(node) as typeof node;+ view.getChildren = <T extends LexicalNode>() => children as Array<T>;+ return view;+ }+ return node;+}diff --git a/packages/richtext/src/markdownTable.ts b/packages/richtext/src/markdownTable.tsindex 9e233a3791..84b5ebeab9 100644--- a/packages/richtext/src/markdownTable.ts+++ b/packages/richtext/src/markdownTable.ts@@ -17,7 +17,6 @@import {$convertFromMarkdownString,- $convertToMarkdownString,CODE,TEXT_FORMAT_TRANSFORMERS,TEXT_MATCH_TRANSFORMERS,@@ -47,6 +46,7 @@ import {TASK_LIST} from './markdownTaskList';import {ORDERED_LIST_KEEPING_START} from './markdownOrderedList';import {LINK_KEEPING_DESTINATIONS} from './markdownLink';import {BACKTICK_CODE, TILDE_CODE} from './markdownCodeFence';+import {$convertToMarkdownKeepingTimeLinear} from './markdownSpaceRuns';/*** Cells hold inline Markdown only, so they are imported and exported with the@@ -169,12 +169,34 @@ function $createCell(function cellMarkdown(cell: TableCellNode): string {// A cell is one line of Markdown, so paragraph breaks inside it become// spaces, and literal pipes are escaped to stay inside the cell.- return $convertToMarkdownString(CELL_TRANSFORMERS, cell)- .replace(/\s*\n+\s*/g, ' ')+ return oneLine($convertToMarkdownKeepingTimeLinear(CELL_TRANSFORMERS, cell)).replace(/\|/g, '\\|').trim();}+/**+ * `markdown` with each run of spaces that holds a line break made one space.+ * Read line by line: a pattern matching optional spaces around line breaks+ * retries from every space in a long run without a line break, so its time+ * grows with the square of the run.+ */+function oneLine(markdown: string): string {+ if (!markdown.includes('\n')) {+ return markdown;+ }+ const lines = markdown.split('\n');+ let joined = lines[0].trimEnd();+ for (let index = 1; index < lines.length; index++) {+ const isLast = index === lines.length - 1;+ const line = isLast ? lines[index].trimStart() : lines[index].trim();+ // A blank line is part of the run of spaces around it.+ if (line !== '' || isLast) {+ joined += ` ${line}`;+ }+ }+ return joined;+}+/*** GFM pipe tables. The header row is the first row and renders as header* cells; delimiter-row colons set each column's alignment; rows shorter than