Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

fix(richtext): nest list items the way CommonMark does (#7002)

Cindy Zhang643fe130e7cc
diff --git a/packages/richtext/src/RichTextEditor.doc.mjs b/packages/richtext/src/RichTextEditor.doc.mjs
index 5d9088507a..afc959b8f9 100644
--- a/packages/richtext/src/RichTextEditor.doc.mjs
+++ b/packages/richtext/src/RichTextEditor.doc.mjs
@@ -116,7 +116,7 @@ export const docs = {
name: 'transformers',
type: 'ReadonlyArray<Transformer>',
description:
- 'Markdown transformers: the single source of truth for markdown behaviour. Defaults to thematic breaks, the standard @lexical/markdown TRANSFORMERS, hard line breaks (a line break typed with Shift+Enter exports as a backslash before the line ending, so it reads back as a line break), and GFM tables; a custom array replaces the default, thematic breaks and tables included. In Lexical the same array drives all three markdown operations (shortcut typing, markdown->state import, state->markdown export); this prop wires shortcut typing today and is the intended input for the serialization APIs added in later phases. Pass a custom array to support additional node types (e.g. transformers layered in via the nodes extension point) consistently across all three. Shortcut typing is only applied when hasMarkdownShortcuts is true.',
+ 'Markdown transformers: the single source of truth for markdown behaviour. Defaults to thematic breaks, the standard @lexical/markdown TRANSFORMERS (with each nested list written at the content column of the item above it, so it reads back nested), hard line breaks (a line break typed with Shift+Enter exports as a backslash before the line ending, so it reads back as a line break), and GFM tables; a custom array replaces the default, thematic breaks and tables included. In Lexical the same array drives all three markdown operations (shortcut typing, markdown->state import, state->markdown export); this prop wires shortcut typing today and is the intended input for the serialization APIs added in later phases. Pass a custom array to support additional node types (e.g. transformers layered in via the nodes extension point) consistently across all three. Shortcut typing is only applied when hasMarkdownShortcuts is true.',
default: 'TRANSFORMERS',
},
{
diff --git a/packages/richtext/src/markdownListExport.ts b/packages/richtext/src/markdownListExport.ts
new file mode 100644
index 0000000000..48aa318b3a
--- /dev/null
+++ b/packages/richtext/src/markdownListExport.ts
@@ -0,0 +1,77 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file markdownListExport.ts
+ * @input Uses @lexical/list (ListNode, ListItemNode) and @lexical/markdown
+ * (ElementTransformer).
+ * @output Exports LIST_EXPORT, which writes lists the way Lexical's list
+ * transformers do except for nesting: a nested list is indented to its
+ * parent item's content column instead of four spaces per level.
+ * @position Part of DEFAULT_TRANSFORMERS (markdownTable.ts), ahead of
+ * Lexical's list transformers, which still import. Import reads nesting the
+ * CommonMark way (markdownListIndentation.ts), so a child written four
+ * spaces under `100. parent` (content column five) would read back at the
+ * parent's level; indenting to the content column keeps an edited list's
+ * structure when it is read again (spec:AST-062 FR3).
+ */
+
+import {
+ $isListItemNode,
+ $isListNode,
+ ListItemNode,
+ ListNode,
+} from '@lexical/list';
+import type {ElementTransformer} from '@lexical/markdown';
+import type {ElementNode} from 'lexical';
+
+/** The bullet Lexical recorded for a list on import (`-`, `*`, or `+`). */
+function bulletOf(list: ListNode): string {
+ const serialized = list.exportJSON() as {$?: {mdListMarker?: unknown}};
+ const marker = serialized.$?.mdListMarker;
+ return marker === '*' || marker === '+' ? marker : '-';
+}
+
+function exportList(
+ list: ListNode,
+ exportChildren: (node: ElementNode) => string,
+ indent: string,
+): string {
+ const lines: Array<string> = [];
+ const type = list.getListType();
+ let index = 0;
+ // Where a nested list goes: the content column of the item before it.
+ let childIndent = `${indent} `;
+ for (const item of list.getChildren()) {
+ if (!$isListItemNode(item)) {
+ continue;
+ }
+ const first = item.getFirstChild();
+ if (item.getChildrenSize() === 1 && $isListNode(first)) {
+ lines.push(exportList(first, exportChildren, childIndent));
+ continue;
+ }
+ const marker =
+ type === 'number' ? `${list.getStart() + index}. ` : `${bulletOf(list)} `;
+ const checkbox =
+ type === 'check' ? `[${item.getChecked() === true ? 'x' : ' '}] ` : '';
+ let text = exportChildren(item);
+ if (type !== 'number') {
+ // As Lexical does: keep `1. ` at the start of a bullet item literal.
+ text = text.replace(/^(\s{0,3}\d+)(\.\s)/, '$1\\$2');
+ }
+ lines.push(indent + marker + checkbox + text);
+ childIndent = indent + ' '.repeat(marker.length);
+ index++;
+ }
+ return lines.join('\n');
+}
+
+export const LIST_EXPORT: ElementTransformer = {
+ dependencies: [ListNode, ListItemNode],
+ export: (node, exportChildren) =>
+ $isListNode(node) ? exportList(node, exportChildren, '') : null,
+ // Export only: Lexical's list transformers import.
+ regExp: /(?!)/,
+ replace: () => false,
+ type: 'element',
+};
diff --git a/packages/richtext/src/markdownListIndentation.test.ts b/packages/richtext/src/markdownListIndentation.test.ts
new file mode 100644
index 0000000000..c1d34e7b03
--- /dev/null
+++ b/packages/richtext/src/markdownListIndentation.test.ts
@@ -0,0 +1,235 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+import {describe, it, expect} from 'vitest';
+import {createHeadlessEditor} from '@lexical/headless';
+import {$createListItemNode, $isListItemNode, $isListNode} from '@lexical/list';
+import {$createTextNode, $getRoot, type LexicalNode} from 'lexical';
+import {DEFAULT_NODES} from './editorNodes';
+import {normalizeListIndentation} from './markdownListIndentation';
+import {
+ editorStateJSONToMarkdown,
+ markdownToEditorStateJSON,
+} from './markdownSerializers';
+
+interface SerializedNode {
+ readonly type: string;
+ readonly text?: string;
+ readonly children?: ReadonlyArray<SerializedNode>;
+}
+
+/** Each list item's depth and text, in document order. */
+function listItems(markdown: string): Array<string> {
+ const {root} = JSON.parse(markdownToEditorStateJSON(markdown)) as {
+ root: SerializedNode;
+ };
+ const items: Array<string> = [];
+ const visit = (node: SerializedNode, depth: number) => {
+ if (node.type === 'listitem') {
+ const text = (node.children ?? [])
+ .filter(child => child.type === 'text')
+ .map(child => child.text)
+ .join('');
+ if (text !== '') {
+ items.push(`${depth}:${text}`);
+ }
+ }
+ const nextDepth = node.type === 'list' ? depth + 1 : depth;
+ for (const child of node.children ?? []) {
+ visit(child, nextDepth);
+ }
+ };
+ visit(root, -1);
+ return items;
+}
+
+/** Edits the imported document and exports it again. */
+function editAndExport(markdown: string, edit: () => void): string {
+ const editor = createHeadlessEditor({
+ namespace: 'astryx-list-nesting-test',
+ nodes: [...DEFAULT_NODES],
+ onError(error: Error) {
+ throw error;
+ },
+ });
+ editor.setEditorState(
+ editor.parseEditorState(markdownToEditorStateJSON(markdown)),
+ );
+ editor.update(edit, {discrete: true});
+ return editorStateJSONToMarkdown(
+ JSON.stringify(editor.getEditorState().toJSON()),
+ );
+}
+
+/** Every list item node, in document order. */
+function $listItems(): Array<LexicalNode> {
+ const items: Array<LexicalNode> = [];
+ const visit = (node: LexicalNode) => {
+ if ($isListItemNode(node)) {
+ items.push(node);
+ }
+ if ($isListNode(node) || $isListItemNode(node)) {
+ node.getChildren().forEach(visit);
+ }
+ };
+ $getRoot().getChildren().forEach(visit);
+ return items;
+}
+
+describe('list nesting (spec:AST-061 FR5)', () => {
+ it('nests an item under the item whose content it is indented to', () => {
+ expect(listItems('- a\n - b\n - c\n- d\n')).toEqual([
+ '0:a',
+ '1:b',
+ '2:c',
+ '0:d',
+ ]);
+ expect(listItems('1. a\n 1. b\n 2. c\n2. d\n')).toEqual([
+ '0:a',
+ '1:b',
+ '1:c',
+ '0:d',
+ ]);
+ expect(listItems('- a\n - four spaces\n')).toEqual([
+ '0:a',
+ '1:four spaces',
+ ]);
+ expect(listItems('- a\n\t- tab\n')).toEqual(['0:a', '1:tab']);
+ });
+
+ it('keeps an item that does not reach its parent content at the parent level', () => {
+ // One space is not enough to reach the content of `- a`.
+ expect(listItems('- a\n - b\n')).toEqual(['0:a', '0:b']);
+ // `10. ` puts its content at column four, so a three-space item starts
+ // a list of its own, and a four-space item does not reach that one's.
+ expect(listItems('10. a\n - b\n - c\n')).toEqual([
+ '0:a',
+ '0:b',
+ '0:c',
+ ]);
+ });
+
+ it('keeps nesting across blank lines in a loose list', () => {
+ expect(listItems('- a\n\n - b\n\n- c\n')).toEqual(['0:a', '1:b', '0:c']);
+ });
+
+ it('leaves non-list lines, fenced code, and the line count alone', () => {
+ const markdown =
+ '- a\n continued\n\n```\n - not a list\n```\n\nText\n - b\n';
+ const normalized = normalizeListIndentation(markdown);
+ expect(normalized.split('\n')).toHaveLength(markdown.split('\n').length);
+ expect(normalized).toContain('```\n - not a list\n```');
+ expect(normalized).toContain('- a\n continued');
+ });
+
+ it('measures the content column across a tab after the marker', () => {
+ expect(listItems('-\tparent\n\t-\tchild\n\t\t-\tgrandchild\n')).toEqual([
+ '0:parent',
+ '1:child',
+ '2:grandchild',
+ ]);
+ expect(listItems('1.\ta\n\t1.\tb\n')).toEqual(['0:a', '1:b']);
+ expect(listItems('- a\n -\tb\n - c\n')).toEqual([
+ '0:a',
+ '1:b',
+ '2:c',
+ ]);
+ expect(listItems('- a\r\n - b\r\n - c\r\n')).toEqual([
+ '0:a',
+ '1:b',
+ '2:c',
+ ]);
+ });
+
+ it('writes an edited nested list so it reads back at the same depths (spec:AST-062 FR3)', () => {
+ for (const parent of ['9.', '10.', '100.']) {
+ const child = ' '.repeat(parent.length + 1);
+ const markdown = `${parent} a\n${child}- one\n${child}- two\n`;
+ expect(listItems(markdown), parent).toEqual(['0:a', '1:one', '1:two']);
+ // Edit a child.
+ const edited = editAndExport(markdown, () => {
+ const one = $getRoot()
+ .getAllTextNodes()
+ .find(node => node.getTextContent() === 'one');
+ one?.setTextContent('one!');
+ });
+ expect(listItems(edited), `${parent} edited`).toEqual([
+ '0:a',
+ '1:one!',
+ '1:two',
+ ]);
+ // Insert a child after the last one.
+ const inserted = editAndExport(markdown, () => {
+ const two = $listItems().find(node => node.getTextContent() === 'two');
+ two?.insertAfter(
+ $createListItemNode().append($createTextNode('three')),
+ );
+ });
+ expect(listItems(inserted), `${parent} inserted`).toEqual([
+ '0:a',
+ '1:one',
+ '1:two',
+ '1:three',
+ ]);
+ // Move the second child before the first.
+ const moved = editAndExport(markdown, () => {
+ const items = $listItems();
+ const one = items.find(node => node.getTextContent() === 'one');
+ const two = items.find(node => node.getTextContent() === 'two');
+ if (one != null && two != null) {
+ one.insertBefore(two);
+ }
+ });
+ expect(listItems(moved), `${parent} moved`).toEqual([
+ '0:a',
+ '1:two',
+ '1:one',
+ ]);
+ }
+ // A list of another type nested under the item above, as CommonMark
+ // reads it, through the same edits.
+ for (const parent of ['9.', '100.']) {
+ const child = ' '.repeat(parent.length + 1);
+ const markdown = `${parent} a\n${child}- one\n${child}- two\n${parent.replace(/\d+/, n => String(Number(n) + 1))} b\n`;
+ expect(listItems(markdown), parent).toEqual([
+ '0:a',
+ '1:one',
+ '1:two',
+ '0:b',
+ ]);
+ const edited = editAndExport(markdown, () => {
+ const one = $getRoot()
+ .getAllTextNodes()
+ .find(node => node.getTextContent() === 'one');
+ one?.setTextContent('one!');
+ });
+ expect(listItems(edited), `${parent} mixed edited`).toEqual([
+ '0:a',
+ '1:one!',
+ '1:two',
+ '0:b',
+ ]);
+ }
+ // Three levels, and CRLF line endings.
+ for (const ending of ['\n', '\r\n']) {
+ const markdown = ['- a', ' - b', ' - c', ''].join(ending);
+ const edited = editAndExport(markdown, () => {
+ const c = $getRoot()
+ .getAllTextNodes()
+ .find(node => node.getTextContent() === 'c');
+ c?.setTextContent('c!');
+ });
+ expect(listItems(edited), JSON.stringify(ending)).toEqual([
+ '0:a',
+ '1:b',
+ '2:c!',
+ ]);
+ }
+ });
+
+ it('keeps the authored indentation of an untouched list (spec:AST-062)', () => {
+ const markdown = '- a\n - b\n\n1. a\n 1. b\n';
+ expect(editorStateJSONToMarkdown(markdownToEditorStateJSON(markdown))).toBe(
+ markdown,
+ );
+ });
+});
diff --git a/packages/richtext/src/markdownListIndentation.ts b/packages/richtext/src/markdownListIndentation.ts
new file mode 100644
index 0000000000..52cde72da1
--- /dev/null
+++ b/packages/richtext/src/markdownListIndentation.ts
@@ -0,0 +1,89 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file markdownListIndentation.ts
+ * @input Markdown source text.
+ * @output Exports normalizeListIndentation, which rewrites the indentation of
+ * every list item line so its nesting reads the same to Lexical as it does
+ * to CommonMark.
+ * @position Used by markdownSource.ts on the text it imports (never on the
+ * bytes it records). Lexical nests a list item one level per four spaces or
+ * one tab; CommonMark nests an item under the item whose content it is
+ * indented to, so `- a` / ` - b` (two spaces) and `1. a` / ` 1. b`
+ * (three) are nested lists that Lexical would flatten (spec:AST-061 FR5).
+ */
+
+/** A list item line: indentation, marker, and the spaces after it. */
+const LIST_ITEM = /^([ \t]*)([-+*]|\d{1,9}[.)])( {1,4}|\t|$)/;
+const FENCE_OPEN = /^[ \t]*(`{3,}|~{3,})/;
+const BLANK_LINE = /^[ \t]*\r?$/;
+
+/** Columns of leading whitespace, with tabs stopping every four columns. */
+function columnsOf(whitespace: string): number {
+ let columns = 0;
+ for (const character of whitespace) {
+ columns = character === '\t' ? columns + 4 - (columns % 4) : columns + 1;
+ }
+ return columns;
+}
+
+/**
+ * Rewrites each list item line's indentation as four spaces per CommonMark
+ * nesting level, leaving every other line, and every line inside fenced code,
+ * as it is. The line count and every line's content after its indentation
+ * are unchanged.
+ */
+export function normalizeListIndentation(markdown: string): string {
+ const lines = markdown.split('\n');
+ // The content column of each open list item, outermost first.
+ const open: Array<number> = [];
+ let fence: string | null = null;
+ let afterBlankLine = false;
+ for (let index = 0; index < lines.length; index++) {
+ const line = lines[index];
+ if (fence != null) {
+ if (
+ new RegExp(`^[ \\t]*${fence[0]}{${fence.length},}[ \\t]*\\r?$`).test(
+ line,
+ )
+ ) {
+ fence = null;
+ }
+ continue;
+ }
+ if (BLANK_LINE.test(line)) {
+ afterBlankLine = true;
+ continue;
+ }
+ const item = LIST_ITEM.exec(line);
+ const indent = columnsOf(/^[ \t]*/.exec(line)?.[0] ?? '');
+ if (item == null) {
+ const opening = FENCE_OPEN.exec(line);
+ if (opening != null) {
+ fence = opening[1];
+ }
+ // A line after a blank line belongs to the items it is indented into;
+ // without a blank line it continues the last item's paragraph lazily.
+ if (afterBlankLine) {
+ while (open.length > 0 && indent < open[open.length - 1]) {
+ open.pop();
+ }
+ }
+ afterBlankLine = false;
+ continue;
+ }
+ afterBlankLine = false;
+ // An item nests under every open item whose content it reaches.
+ while (open.length > 0 && indent < open[open.length - 1]) {
+ open.pop();
+ }
+ const depth = open.length;
+ // The content column, counted across the whole prefix so a tab after
+ // the marker stops at the right column; a marker that ends the line has
+ // its content one column after it.
+ const markerEnd = columnsOf(item[0]) + (item[3] === '' ? 1 : 0);
+ open.push(markerEnd);
+ lines[index] = ' '.repeat(depth * 4) + line.slice(item[1].length);
+ }
+ return lines.join('\n');
+}
diff --git a/packages/richtext/src/markdownSource.test.tsx b/packages/richtext/src/markdownSource.test.tsx
index 4a2cedc345..218a7509e1 100644
--- a/packages/richtext/src/markdownSource.test.tsx
+++ b/packages/richtext/src/markdownSource.test.tsx
@@ -23,6 +23,7 @@ import {
type SerializedLexicalNode,
} from 'lexical';
import {DEFAULT_NODES} from './editorNodes';
+import {normalizeListIndentation} from './markdownListIndentation';
import {DEFAULT_TRANSFORMERS} from './markdownTable';
import {
editorStateJSONToMarkdown,
@@ -30,6 +31,7 @@ import {
} from './markdownSerializers';
import {
$joinSoftLineBreaks,
+ $nestFollowingLists,
absentToken,
splitMarkdownChunks,
} from './markdownSource';
@@ -335,8 +337,9 @@ describe('Markdown source preservation (spec:AST-062)', () => {
it('imports the same structure as importing the whole document at once', () => {
// Node state aside, chunked import must build exactly the tree Lexical's
- // own import builds, soft breaks joined: lazy continuation lines, loose
- // lists, and line breaks all land in the same blocks.
+ // own import builds from the list-normalized document, soft breaks
+ // joined: lazy continuation lines, loose lists, and line breaks all land
+ // in the same blocks.
const structureOf = (json: string): unknown =>
JSON.parse(json, (key, value: unknown) =>
key === '$' ? undefined : value,
@@ -352,9 +355,12 @@ describe('Markdown source preservation (spec:AST-062)', () => {
editor.update(
() => {
$convertFromMarkdownString(
- markdown.replace(/^\uFEFF/, '').replace(/\r\n/g, '\n'),
+ normalizeListIndentation(
+ markdown.replace(/^\uFEFF/, '').replace(/\r\n/g, '\n'),
+ ),
[...DEFAULT_TRANSFORMERS],
);
+ $nestFollowingLists($getRoot());
$joinSoftLineBreaks($getRoot());
},
{discrete: true},
diff --git a/packages/richtext/src/markdownSource.ts b/packages/richtext/src/markdownSource.ts
index d8ccc53d47..37215303cb 100644
--- a/packages/richtext/src/markdownSource.ts
+++ b/packages/richtext/src/markdownSource.ts
@@ -32,6 +32,7 @@ import {
type Transformer,
} from '@lexical/markdown';
import {$isCodeNode} from '@lexical/code';
+import {$isListItemNode, $isListNode} from '@lexical/list';
import {
$createParagraphNode,
$createTextNode,
@@ -47,6 +48,7 @@ import {
type LexicalNode,
} from 'lexical';
import {isMarkedHardLineBreak} from './markdownHardLineBreak';
+import {normalizeListIndentation} from './markdownListIndentation';
/** The whitespace and content one chunk of Markdown source was split into. */
export interface MarkdownChunk {
@@ -290,6 +292,10 @@ export function importMarkdownKeepingSource(
const byteOrderMark = markdown.startsWith('\uFEFF');
const body = byteOrderMark ? markdown.slice(1) : markdown;
const chunks = splitMarkdownChunks(body);
+ // Lexical reads the same chunks with list nesting spelled its way; the
+ // records keep the authored bytes. Only list item indentation changes, so
+ // the chunks line up one for one.
+ const importChunks = splitMarkdownChunks(normalizeListIndentation(body));
const lineEnding = lineEndingOf(body, '\n');
editor.update(
() => {
@@ -304,7 +310,7 @@ export function importMarkdownKeepingSource(
root.append(holder);
// Lexical imports LF lines; the record keeps the authored endings.
$convertFromMarkdownString(
- withoutCarriageReturns(chunk.content),
+ withoutCarriageReturns(importChunks[index]?.content ?? chunk.content),
transformers,
holder,
);
@@ -317,6 +323,7 @@ export function importMarkdownKeepingSource(
}
holder.remove();
});
+ $nestFollowingLists(root);
if (root.getChildrenSize() === 0) {
// Nothing to import: keep one empty paragraph, as Lexical does.
root.append($createParagraphNode());
@@ -375,6 +382,35 @@ export function $joinSoftLineBreaks(element: ElementNode): void {
}
}
+/**
+ * Moves a list that begins nested right after another list into that list's
+ * last item. Lexical starts a new top-level list for an item of a different
+ * type (a bullet under `1. item`) and nests it there, while CommonMark nests
+ * it inside the item above; without this, writing it back would lose the
+ * nesting (spec:AST-061 FR5, spec:AST-062 FR3).
+ */
+export function $nestFollowingLists(root: ElementNode): void {
+ for (const node of root.getChildren()) {
+ const previous = node.getPreviousSibling();
+ if (!$isListNode(node) || !$isListNode(previous)) {
+ continue;
+ }
+ // Lexical keeps a nested list in an item of its own after its parent.
+ let first = node.getFirstChild();
+ while (
+ $isListItemNode(first) &&
+ first.getChildrenSize() === 1 &&
+ $isListNode(first.getFirstChild())
+ ) {
+ previous.append(first);
+ first = node.getFirstChild();
+ }
+ if (node.getChildrenSize() === 0) {
+ node.remove();
+ }
+ }
+}
+
/**
* Turns the first pass's chunk stamps into groups. Consecutive nodes stamped
* with one chunk form a group, and the group also covers every later chunk up
diff --git a/packages/richtext/src/markdownTable.ts b/packages/richtext/src/markdownTable.ts
index f22b0b65bb..1cc303c9e8 100644
--- a/packages/richtext/src/markdownTable.ts
+++ b/packages/richtext/src/markdownTable.ts
@@ -39,6 +39,7 @@ import {
import type {ElementFormatType} from 'lexical';
import {HARD_LINE_BREAK} from './markdownHardLineBreak';
import {THEMATIC_BREAK} from './markdownThematicBreak';
+import {LIST_EXPORT} from './markdownListExport';
/**
* Cells hold inline Markdown only, so they are imported and exported with the
@@ -236,13 +237,14 @@ export const TABLE: MultilineElementTransformer = {
};
/**
- * The editor's default Markdown transformers: thematic breaks, Lexical's
- * standard set, hard line breaks for breaks typed in the editor, and GFM
- * tables. Thematic breaks come first so a line such as `* * *` is a rule
- * rather than a list item.
+ * The editor's default Markdown transformers: thematic breaks, list export
+ * that nests to each parent's content column, Lexical's standard set, hard
+ * line breaks for breaks typed in the editor, and GFM tables. Thematic breaks
+ * come first so a line such as `* * *` is a rule rather than a list item.
*/
export const DEFAULT_TRANSFORMERS: ReadonlyArray<Transformer> = [
THEMATIC_BREAK,
+ LIST_EXPORT,
...TRANSFORMERS,
HARD_LINE_BREAK,
TABLE,