An open source design system that's fully customizable and agent ready
TypeScript
fix(richtext): nest list items the way CommonMark does (#7002)
Cindy Zhang643fe130e7cc
diff --git a/packages/richtext/src/RichTextEditor.doc.mjs b/packages/richtext/src/RichTextEditor.doc.mjsindex 5d9088507a..afc959b8f9 100644--- a/packages/richtext/src/RichTextEditor.doc.mjs+++ b/packages/richtext/src/RichTextEditor.doc.mjs@@ -116,7 +116,7 @@ export const docs = {name: 'transformers',type: 'ReadonlyArray<Transformer>',description:- 'Markdown transformers: the single source of truth for markdown behaviour. Defaults to thematic breaks, the standard @lexical/markdown TRANSFORMERS, hard line breaks (a line break typed with Shift+Enter exports as a backslash before the line ending, so it reads back as a line break), and GFM tables; a custom array replaces the default, thematic breaks and tables included. In Lexical the same array drives all three markdown operations (shortcut typing, markdown->state import, state->markdown export); this prop wires shortcut typing today and is the intended input for the serialization APIs added in later phases. Pass a custom array to support additional node types (e.g. transformers layered in via the nodes extension point) consistently across all three. Shortcut typing is only applied when hasMarkdownShortcuts is true.',+ 'Markdown transformers: the single source of truth for markdown behaviour. Defaults to thematic breaks, the standard @lexical/markdown TRANSFORMERS (with each nested list written at the content column of the item above it, so it reads back nested), hard line breaks (a line break typed with Shift+Enter exports as a backslash before the line ending, so it reads back as a line break), and GFM tables; a custom array replaces the default, thematic breaks and tables included. In Lexical the same array drives all three markdown operations (shortcut typing, markdown->state import, state->markdown export); this prop wires shortcut typing today and is the intended input for the serialization APIs added in later phases. Pass a custom array to support additional node types (e.g. transformers layered in via the nodes extension point) consistently across all three. Shortcut typing is only applied when hasMarkdownShortcuts is true.',default: 'TRANSFORMERS',},{diff --git a/packages/richtext/src/markdownListExport.ts b/packages/richtext/src/markdownListExport.tsnew file mode 100644index 0000000000..48aa318b3a--- /dev/null+++ b/packages/richtext/src/markdownListExport.ts@@ -0,0 +1,77 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file markdownListExport.ts+ * @input Uses @lexical/list (ListNode, ListItemNode) and @lexical/markdown+ * (ElementTransformer).+ * @output Exports LIST_EXPORT, which writes lists the way Lexical's list+ * transformers do except for nesting: a nested list is indented to its+ * parent item's content column instead of four spaces per level.+ * @position Part of DEFAULT_TRANSFORMERS (markdownTable.ts), ahead of+ * Lexical's list transformers, which still import. Import reads nesting the+ * CommonMark way (markdownListIndentation.ts), so a child written four+ * spaces under `100. parent` (content column five) would read back at the+ * parent's level; indenting to the content column keeps an edited list's+ * structure when it is read again (spec:AST-062 FR3).+ */++import {+ $isListItemNode,+ $isListNode,+ ListItemNode,+ ListNode,+} from '@lexical/list';+import type {ElementTransformer} from '@lexical/markdown';+import type {ElementNode} from 'lexical';++/** The bullet Lexical recorded for a list on import (`-`, `*`, or `+`). */+function bulletOf(list: ListNode): string {+ const serialized = list.exportJSON() as {$?: {mdListMarker?: unknown}};+ const marker = serialized.$?.mdListMarker;+ return marker === '*' || marker === '+' ? marker : '-';+}++function exportList(+ list: ListNode,+ exportChildren: (node: ElementNode) => string,+ indent: string,+): string {+ const lines: Array<string> = [];+ const type = list.getListType();+ let index = 0;+ // Where a nested list goes: the content column of the item before it.+ let childIndent = `${indent} `;+ for (const item of list.getChildren()) {+ if (!$isListItemNode(item)) {+ continue;+ }+ const first = item.getFirstChild();+ if (item.getChildrenSize() === 1 && $isListNode(first)) {+ lines.push(exportList(first, exportChildren, childIndent));+ continue;+ }+ const marker =+ type === 'number' ? `${list.getStart() + index}. ` : `${bulletOf(list)} `;+ const checkbox =+ type === 'check' ? `[${item.getChecked() === true ? 'x' : ' '}] ` : '';+ let text = exportChildren(item);+ if (type !== 'number') {+ // As Lexical does: keep `1. ` at the start of a bullet item literal.+ text = text.replace(/^(\s{0,3}\d+)(\.\s)/, '$1\\$2');+ }+ lines.push(indent + marker + checkbox + text);+ childIndent = indent + ' '.repeat(marker.length);+ index++;+ }+ return lines.join('\n');+}++export const LIST_EXPORT: ElementTransformer = {+ dependencies: [ListNode, ListItemNode],+ export: (node, exportChildren) =>+ $isListNode(node) ? exportList(node, exportChildren, '') : null,+ // Export only: Lexical's list transformers import.+ regExp: /(?!)/,+ replace: () => false,+ type: 'element',+};diff --git a/packages/richtext/src/markdownListIndentation.test.ts b/packages/richtext/src/markdownListIndentation.test.tsnew file mode 100644index 0000000000..c1d34e7b03--- /dev/null+++ b/packages/richtext/src/markdownListIndentation.test.ts@@ -0,0 +1,235 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++import {describe, it, expect} from 'vitest';+import {createHeadlessEditor} from '@lexical/headless';+import {$createListItemNode, $isListItemNode, $isListNode} from '@lexical/list';+import {$createTextNode, $getRoot, type LexicalNode} from 'lexical';+import {DEFAULT_NODES} from './editorNodes';+import {normalizeListIndentation} from './markdownListIndentation';+import {+ editorStateJSONToMarkdown,+ markdownToEditorStateJSON,+} from './markdownSerializers';++interface SerializedNode {+ readonly type: string;+ readonly text?: string;+ readonly children?: ReadonlyArray<SerializedNode>;+}++/** Each list item's depth and text, in document order. */+function listItems(markdown: string): Array<string> {+ const {root} = JSON.parse(markdownToEditorStateJSON(markdown)) as {+ root: SerializedNode;+ };+ const items: Array<string> = [];+ const visit = (node: SerializedNode, depth: number) => {+ if (node.type === 'listitem') {+ const text = (node.children ?? [])+ .filter(child => child.type === 'text')+ .map(child => child.text)+ .join('');+ if (text !== '') {+ items.push(`${depth}:${text}`);+ }+ }+ const nextDepth = node.type === 'list' ? depth + 1 : depth;+ for (const child of node.children ?? []) {+ visit(child, nextDepth);+ }+ };+ visit(root, -1);+ return items;+}++/** Edits the imported document and exports it again. */+function editAndExport(markdown: string, edit: () => void): string {+ const editor = createHeadlessEditor({+ namespace: 'astryx-list-nesting-test',+ nodes: [...DEFAULT_NODES],+ onError(error: Error) {+ throw error;+ },+ });+ editor.setEditorState(+ editor.parseEditorState(markdownToEditorStateJSON(markdown)),+ );+ editor.update(edit, {discrete: true});+ return editorStateJSONToMarkdown(+ JSON.stringify(editor.getEditorState().toJSON()),+ );+}++/** Every list item node, in document order. */+function $listItems(): Array<LexicalNode> {+ const items: Array<LexicalNode> = [];+ const visit = (node: LexicalNode) => {+ if ($isListItemNode(node)) {+ items.push(node);+ }+ if ($isListNode(node) || $isListItemNode(node)) {+ node.getChildren().forEach(visit);+ }+ };+ $getRoot().getChildren().forEach(visit);+ return items;+}++describe('list nesting (spec:AST-061 FR5)', () => {+ it('nests an item under the item whose content it is indented to', () => {+ expect(listItems('- a\n - b\n - c\n- d\n')).toEqual([+ '0:a',+ '1:b',+ '2:c',+ '0:d',+ ]);+ expect(listItems('1. a\n 1. b\n 2. c\n2. d\n')).toEqual([+ '0:a',+ '1:b',+ '1:c',+ '0:d',+ ]);+ expect(listItems('- a\n - four spaces\n')).toEqual([+ '0:a',+ '1:four spaces',+ ]);+ expect(listItems('- a\n\t- tab\n')).toEqual(['0:a', '1:tab']);+ });++ it('keeps an item that does not reach its parent content at the parent level', () => {+ // One space is not enough to reach the content of `- a`.+ expect(listItems('- a\n - b\n')).toEqual(['0:a', '0:b']);+ // `10. ` puts its content at column four, so a three-space item starts+ // a list of its own, and a four-space item does not reach that one's.+ expect(listItems('10. a\n - b\n - c\n')).toEqual([+ '0:a',+ '0:b',+ '0:c',+ ]);+ });++ it('keeps nesting across blank lines in a loose list', () => {+ expect(listItems('- a\n\n - b\n\n- c\n')).toEqual(['0:a', '1:b', '0:c']);+ });++ it('leaves non-list lines, fenced code, and the line count alone', () => {+ const markdown =+ '- a\n continued\n\n```\n - not a list\n```\n\nText\n - b\n';+ const normalized = normalizeListIndentation(markdown);+ expect(normalized.split('\n')).toHaveLength(markdown.split('\n').length);+ expect(normalized).toContain('```\n - not a list\n```');+ expect(normalized).toContain('- a\n continued');+ });++ it('measures the content column across a tab after the marker', () => {+ expect(listItems('-\tparent\n\t-\tchild\n\t\t-\tgrandchild\n')).toEqual([+ '0:parent',+ '1:child',+ '2:grandchild',+ ]);+ expect(listItems('1.\ta\n\t1.\tb\n')).toEqual(['0:a', '1:b']);+ expect(listItems('- a\n -\tb\n - c\n')).toEqual([+ '0:a',+ '1:b',+ '2:c',+ ]);+ expect(listItems('- a\r\n - b\r\n - c\r\n')).toEqual([+ '0:a',+ '1:b',+ '2:c',+ ]);+ });++ it('writes an edited nested list so it reads back at the same depths (spec:AST-062 FR3)', () => {+ for (const parent of ['9.', '10.', '100.']) {+ const child = ' '.repeat(parent.length + 1);+ const markdown = `${parent} a\n${child}- one\n${child}- two\n`;+ expect(listItems(markdown), parent).toEqual(['0:a', '1:one', '1:two']);+ // Edit a child.+ const edited = editAndExport(markdown, () => {+ const one = $getRoot()+ .getAllTextNodes()+ .find(node => node.getTextContent() === 'one');+ one?.setTextContent('one!');+ });+ expect(listItems(edited), `${parent} edited`).toEqual([+ '0:a',+ '1:one!',+ '1:two',+ ]);+ // Insert a child after the last one.+ const inserted = editAndExport(markdown, () => {+ const two = $listItems().find(node => node.getTextContent() === 'two');+ two?.insertAfter(+ $createListItemNode().append($createTextNode('three')),+ );+ });+ expect(listItems(inserted), `${parent} inserted`).toEqual([+ '0:a',+ '1:one',+ '1:two',+ '1:three',+ ]);+ // Move the second child before the first.+ const moved = editAndExport(markdown, () => {+ const items = $listItems();+ const one = items.find(node => node.getTextContent() === 'one');+ const two = items.find(node => node.getTextContent() === 'two');+ if (one != null && two != null) {+ one.insertBefore(two);+ }+ });+ expect(listItems(moved), `${parent} moved`).toEqual([+ '0:a',+ '1:two',+ '1:one',+ ]);+ }+ // A list of another type nested under the item above, as CommonMark+ // reads it, through the same edits.+ for (const parent of ['9.', '100.']) {+ const child = ' '.repeat(parent.length + 1);+ const markdown = `${parent} a\n${child}- one\n${child}- two\n${parent.replace(/\d+/, n => String(Number(n) + 1))} b\n`;+ expect(listItems(markdown), parent).toEqual([+ '0:a',+ '1:one',+ '1:two',+ '0:b',+ ]);+ const edited = editAndExport(markdown, () => {+ const one = $getRoot()+ .getAllTextNodes()+ .find(node => node.getTextContent() === 'one');+ one?.setTextContent('one!');+ });+ expect(listItems(edited), `${parent} mixed edited`).toEqual([+ '0:a',+ '1:one!',+ '1:two',+ '0:b',+ ]);+ }+ // Three levels, and CRLF line endings.+ for (const ending of ['\n', '\r\n']) {+ const markdown = ['- a', ' - b', ' - c', ''].join(ending);+ const edited = editAndExport(markdown, () => {+ const c = $getRoot()+ .getAllTextNodes()+ .find(node => node.getTextContent() === 'c');+ c?.setTextContent('c!');+ });+ expect(listItems(edited), JSON.stringify(ending)).toEqual([+ '0:a',+ '1:b',+ '2:c!',+ ]);+ }+ });++ it('keeps the authored indentation of an untouched list (spec:AST-062)', () => {+ const markdown = '- a\n - b\n\n1. a\n 1. b\n';+ expect(editorStateJSONToMarkdown(markdownToEditorStateJSON(markdown))).toBe(+ markdown,+ );+ });+});diff --git a/packages/richtext/src/markdownListIndentation.ts b/packages/richtext/src/markdownListIndentation.tsnew file mode 100644index 0000000000..52cde72da1--- /dev/null+++ b/packages/richtext/src/markdownListIndentation.ts@@ -0,0 +1,89 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file markdownListIndentation.ts+ * @input Markdown source text.+ * @output Exports normalizeListIndentation, which rewrites the indentation of+ * every list item line so its nesting reads the same to Lexical as it does+ * to CommonMark.+ * @position Used by markdownSource.ts on the text it imports (never on the+ * bytes it records). Lexical nests a list item one level per four spaces or+ * one tab; CommonMark nests an item under the item whose content it is+ * indented to, so `- a` / ` - b` (two spaces) and `1. a` / ` 1. b`+ * (three) are nested lists that Lexical would flatten (spec:AST-061 FR5).+ */++/** A list item line: indentation, marker, and the spaces after it. */+const LIST_ITEM = /^([ \t]*)([-+*]|\d{1,9}[.)])( {1,4}|\t|$)/;+const FENCE_OPEN = /^[ \t]*(`{3,}|~{3,})/;+const BLANK_LINE = /^[ \t]*\r?$/;++/** Columns of leading whitespace, with tabs stopping every four columns. */+function columnsOf(whitespace: string): number {+ let columns = 0;+ for (const character of whitespace) {+ columns = character === '\t' ? columns + 4 - (columns % 4) : columns + 1;+ }+ return columns;+}++/**+ * Rewrites each list item line's indentation as four spaces per CommonMark+ * nesting level, leaving every other line, and every line inside fenced code,+ * as it is. The line count and every line's content after its indentation+ * are unchanged.+ */+export function normalizeListIndentation(markdown: string): string {+ const lines = markdown.split('\n');+ // The content column of each open list item, outermost first.+ const open: Array<number> = [];+ let fence: string | null = null;+ let afterBlankLine = false;+ for (let index = 0; index < lines.length; index++) {+ const line = lines[index];+ if (fence != null) {+ if (+ new RegExp(`^[ \\t]*${fence[0]}{${fence.length},}[ \\t]*\\r?$`).test(+ line,+ )+ ) {+ fence = null;+ }+ continue;+ }+ if (BLANK_LINE.test(line)) {+ afterBlankLine = true;+ continue;+ }+ const item = LIST_ITEM.exec(line);+ const indent = columnsOf(/^[ \t]*/.exec(line)?.[0] ?? '');+ if (item == null) {+ const opening = FENCE_OPEN.exec(line);+ if (opening != null) {+ fence = opening[1];+ }+ // A line after a blank line belongs to the items it is indented into;+ // without a blank line it continues the last item's paragraph lazily.+ if (afterBlankLine) {+ while (open.length > 0 && indent < open[open.length - 1]) {+ open.pop();+ }+ }+ afterBlankLine = false;+ continue;+ }+ afterBlankLine = false;+ // An item nests under every open item whose content it reaches.+ while (open.length > 0 && indent < open[open.length - 1]) {+ open.pop();+ }+ const depth = open.length;+ // The content column, counted across the whole prefix so a tab after+ // the marker stops at the right column; a marker that ends the line has+ // its content one column after it.+ const markerEnd = columnsOf(item[0]) + (item[3] === '' ? 1 : 0);+ open.push(markerEnd);+ lines[index] = ' '.repeat(depth * 4) + line.slice(item[1].length);+ }+ return lines.join('\n');+}diff --git a/packages/richtext/src/markdownSource.test.tsx b/packages/richtext/src/markdownSource.test.tsxindex 4a2cedc345..218a7509e1 100644--- a/packages/richtext/src/markdownSource.test.tsx+++ b/packages/richtext/src/markdownSource.test.tsx@@ -23,6 +23,7 @@ import {type SerializedLexicalNode,} from 'lexical';import {DEFAULT_NODES} from './editorNodes';+import {normalizeListIndentation} from './markdownListIndentation';import {DEFAULT_TRANSFORMERS} from './markdownTable';import {editorStateJSONToMarkdown,@@ -30,6 +31,7 @@ import {} from './markdownSerializers';import {$joinSoftLineBreaks,+ $nestFollowingLists,absentToken,splitMarkdownChunks,} from './markdownSource';@@ -335,8 +337,9 @@ describe('Markdown source preservation (spec:AST-062)', () => {it('imports the same structure as importing the whole document at once', () => {// Node state aside, chunked import must build exactly the tree Lexical's- // own import builds, soft breaks joined: lazy continuation lines, loose- // lists, and line breaks all land in the same blocks.+ // own import builds from the list-normalized document, soft breaks+ // joined: lazy continuation lines, loose lists, and line breaks all land+ // in the same blocks.const structureOf = (json: string): unknown =>JSON.parse(json, (key, value: unknown) =>key === '$' ? undefined : value,@@ -352,9 +355,12 @@ describe('Markdown source preservation (spec:AST-062)', () => {editor.update(() => {$convertFromMarkdownString(- markdown.replace(/^\uFEFF/, '').replace(/\r\n/g, '\n'),+ normalizeListIndentation(+ markdown.replace(/^\uFEFF/, '').replace(/\r\n/g, '\n'),+ ),[...DEFAULT_TRANSFORMERS],);+ $nestFollowingLists($getRoot());$joinSoftLineBreaks($getRoot());},{discrete: true},diff --git a/packages/richtext/src/markdownSource.ts b/packages/richtext/src/markdownSource.tsindex d8ccc53d47..37215303cb 100644--- a/packages/richtext/src/markdownSource.ts+++ b/packages/richtext/src/markdownSource.ts@@ -32,6 +32,7 @@ import {type Transformer,} from '@lexical/markdown';import {$isCodeNode} from '@lexical/code';+import {$isListItemNode, $isListNode} from '@lexical/list';import {$createParagraphNode,$createTextNode,@@ -47,6 +48,7 @@ import {type LexicalNode,} from 'lexical';import {isMarkedHardLineBreak} from './markdownHardLineBreak';+import {normalizeListIndentation} from './markdownListIndentation';/** The whitespace and content one chunk of Markdown source was split into. */export interface MarkdownChunk {@@ -290,6 +292,10 @@ export function importMarkdownKeepingSource(const byteOrderMark = markdown.startsWith('\uFEFF');const body = byteOrderMark ? markdown.slice(1) : markdown;const chunks = splitMarkdownChunks(body);+ // Lexical reads the same chunks with list nesting spelled its way; the+ // records keep the authored bytes. Only list item indentation changes, so+ // the chunks line up one for one.+ const importChunks = splitMarkdownChunks(normalizeListIndentation(body));const lineEnding = lineEndingOf(body, '\n');editor.update(() => {@@ -304,7 +310,7 @@ export function importMarkdownKeepingSource(root.append(holder);// Lexical imports LF lines; the record keeps the authored endings.$convertFromMarkdownString(- withoutCarriageReturns(chunk.content),+ withoutCarriageReturns(importChunks[index]?.content ?? chunk.content),transformers,holder,);@@ -317,6 +323,7 @@ export function importMarkdownKeepingSource(}holder.remove();});+ $nestFollowingLists(root);if (root.getChildrenSize() === 0) {// Nothing to import: keep one empty paragraph, as Lexical does.root.append($createParagraphNode());@@ -375,6 +382,35 @@ export function $joinSoftLineBreaks(element: ElementNode): void {}}+/**+ * Moves a list that begins nested right after another list into that list's+ * last item. Lexical starts a new top-level list for an item of a different+ * type (a bullet under `1. item`) and nests it there, while CommonMark nests+ * it inside the item above; without this, writing it back would lose the+ * nesting (spec:AST-061 FR5, spec:AST-062 FR3).+ */+export function $nestFollowingLists(root: ElementNode): void {+ for (const node of root.getChildren()) {+ const previous = node.getPreviousSibling();+ if (!$isListNode(node) || !$isListNode(previous)) {+ continue;+ }+ // Lexical keeps a nested list in an item of its own after its parent.+ let first = node.getFirstChild();+ while (+ $isListItemNode(first) &&+ first.getChildrenSize() === 1 &&+ $isListNode(first.getFirstChild())+ ) {+ previous.append(first);+ first = node.getFirstChild();+ }+ if (node.getChildrenSize() === 0) {+ node.remove();+ }+ }+}+/*** Turns the first pass's chunk stamps into groups. Consecutive nodes stamped* with one chunk form a group, and the group also covers every later chunk updiff --git a/packages/richtext/src/markdownTable.ts b/packages/richtext/src/markdownTable.tsindex f22b0b65bb..1cc303c9e8 100644--- a/packages/richtext/src/markdownTable.ts+++ b/packages/richtext/src/markdownTable.ts@@ -39,6 +39,7 @@ import {import type {ElementFormatType} from 'lexical';import {HARD_LINE_BREAK} from './markdownHardLineBreak';import {THEMATIC_BREAK} from './markdownThematicBreak';+import {LIST_EXPORT} from './markdownListExport';/*** Cells hold inline Markdown only, so they are imported and exported with the@@ -236,13 +237,14 @@ export const TABLE: MultilineElementTransformer = {};/**- * The editor's default Markdown transformers: thematic breaks, Lexical's- * standard set, hard line breaks for breaks typed in the editor, and GFM- * tables. Thematic breaks come first so a line such as `* * *` is a rule- * rather than a list item.+ * The editor's default Markdown transformers: thematic breaks, list export+ * that nests to each parent's content column, Lexical's standard set, hard+ * line breaks for breaks typed in the editor, and GFM tables. Thematic breaks+ * come first so a line such as `* * *` is a rule rather than a list item.*/export const DEFAULT_TRANSFORMERS: ReadonlyArray<Transformer> = [THEMATIC_BREAK,+ LIST_EXPORT,...TRANSFORMERS,HARD_LINE_BREAK,TABLE,