An open source design system that's fully customizable and agent ready
TypeScript
fix(Markdown, richtext): read a block quote marker as CommonMark does (#7101)
Cindy Zhangc2a49ec01d14
diff --git a/.changeset/markdown-blockquote-markers.md b/.changeset/markdown-blockquote-markers.mdnew file mode 100644index 0000000000..9fef8f9111--- /dev/null+++ b/.changeset/markdown-blockquote-markers.md@@ -0,0 +1,8 @@+---+'@astryxdesign/core': patch+---++[fix] Markdown: read a block quote marker as CommonMark does+@cixzhang++`Markdown` now reads a line that starts with up to three spaces and `>` as a block quote, whether or not a space follows the `>`: `>quote`, ` > quote`, and `>>> nested` are quotes, as CommonMark specifies. These lines used to show as plain text with their `>` marks.diff --git a/packages/core/src/Markdown/blockquoteMarkers.test.tsx b/packages/core/src/Markdown/blockquoteMarkers.test.tsxnew file mode 100644index 0000000000..795d9b7119--- /dev/null+++ b/packages/core/src/Markdown/blockquoteMarkers.test.tsx@@ -0,0 +1,69 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++import {describe, expect, it} from 'vitest';+import {+ createIncrementalState,+ parseMarkdown,+ parseMarkdownAst,+ parseMarkdownIncremental,+} from './parser';++type Json = {type: string; value?: string; children?: Json[]};++/** The document's blocks as compact markup. */+function blocks(markdown: string): string {+ const show = (node: Json): string =>+ node.type === 'text'+ ? JSON.stringify(node.value)+ : node.children == null+ ? node.type+ : `${node.type}[${node.children.map(show).join(', ')}]`;+ return (parseMarkdownAst(markdown).children as unknown as Json[])+ .map(show)+ .join(' | ');+}++describe('a block quote marker is up to three spaces, then `>` (CommonMark 0.31 §5.1)', () => {+ it.each([+ ['text right after the marker', '>a', 'blockquote[paragraph["a"]]'],+ ['a tab after the marker', '>\ta', 'blockquote[paragraph["a"]]'],+ ['three spaces of indentation', ' > a', 'blockquote[paragraph["a"]]'],+ [+ 'markers without spaces, nested',+ '>> a',+ 'blockquote[blockquote[paragraph["a"]]]',+ ],+ [+ 'three markers, nested three deep',+ '>>> a',+ 'blockquote[blockquote[blockquote[paragraph["a"]]]]',+ ],+ [+ 'consecutive lines without spaces',+ '>a\n>b',+ 'blockquote[paragraph["a\\nb"]]',+ ],+ [+ 'a quote interrupting a paragraph',+ 'text\n>quote',+ 'paragraph["text"] | blockquote[paragraph["quote"]]',+ ],+ ['the usual form, unchanged', '> a', 'blockquote[paragraph["a"]]'],+ ])('reads %s', (_, markdown, expected) => {+ expect(blocks(markdown)).toBe(expected);+ });++ it('reads four spaces of indentation as no marker', () => {+ expect(blocks(' > a')).toBe('paragraph[" > a"]');+ });++ it('streams quotes without spaces to the full parse at every character', () => {+ const markdown = '>a\n>b\n\n>> c\n\ntext\n > d';+ const state = createIncrementalState();+ let streamed = parseMarkdownIncremental('', state);+ for (let end = 1; end <= markdown.length; end++) {+ streamed = parseMarkdownIncremental(markdown.slice(0, end), state);+ }+ expect(streamed).toEqual(parseMarkdown(markdown));+ });+});diff --git a/packages/core/src/Markdown/parser.ts b/packages/core/src/Markdown/parser.tsindex cbdedfe74f..902a0643cd 100644--- a/packages/core/src/Markdown/parser.ts+++ b/packages/core/src/Markdown/parser.ts@@ -2932,6 +2932,17 @@ function blockExtensionColumn(line: string): number | null {return indentation <= 3 ? indentation : null;}+/**+ * A block quote marker: up to three spaces of indentation, then `>`+ * (CommonMark 0.31 §5.1). Text may follow the `>` directly.+ */+const QUOTE_MARKER = /^ {0,3}>/;++/** A quoted line's content: the marker and the one space or tab after it. */+function quotedContent(line: string): string {+ return line.replace(/^ {0,3}>[ \t]?/, '');+}+/*** Returns true when a line could start a new block — used to stop paragraph* continuation. Every regex here uses bounded or single-class quantifiers@@ -2947,7 +2958,7 @@ function isBlockStart(line: string): boolean {if (isHorizontalRule(line)) {return true;}- if (line.startsWith('> ') || line === '>') {+ if (QUOTE_MARKER.test(line)) {return true;}if (/^ {0,9}[-*+] /.test(line)) {@@ -2980,8 +2991,7 @@ function canContinueParagraphLazily(/^#{1,6} /.test(line) ||/^(`{3,}|~{3,})/.test(line) ||isHorizontalRule(line) ||- line.startsWith('> ') ||- line === '>' ||+ QUOTE_MARKER.test(line) ||/^ {0,9}[-*+] /.test(line) ||(insideList ? /^ {0,9}\d+[.)] /.test(line) : /^ {0,9}0*1[.)] /.test(line))) {@@ -3605,13 +3615,13 @@ function parseMarkdownImpl(}// --- Blockquote ---- if (line.startsWith('> ') || line === '>') {+ if (QUOTE_MARKER.test(line)) {const quoteLines: string[] = [];let lazyParagraphOpen: boolean | undefined;while (index < lines.length) {const quoteLine = lines[index];- if (quoteLine.startsWith('> ') || quoteLine === '>') {- const content = quoteLine.replace(/^> ?/, '');+ if (QUOTE_MARKER.test(quoteLine)) {+ const content = quotedContent(quoteLine);quoteLines.push(content);if (content.trim() === '') {lazyParagraphOpen = false;diff --git a/packages/richtext/src/markdownQuote.ts b/packages/richtext/src/markdownQuote.tsnew file mode 100644index 0000000000..e06d367e3d--- /dev/null+++ b/packages/richtext/src/markdownQuote.ts@@ -0,0 +1,35 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file markdownQuote.ts+ * @input Uses @lexical/markdown (Lexical's QUOTE element transformer).+ * @output Exports QUOTE_MARKERS, Lexical's quote transformer reading a block+ * quote marker as CommonMark does.+ * @position Part of DEFAULT_TRANSFORMERS (markdownTable.ts) in place of+ * Lexical's QUOTE, whose pattern needs a space after `>` and no indentation.+ * Matches core Markdown's block quote marker (CommonMark 0.31 §5.1). Typing+ * shortcuts keep Lexical's own trigger: `> ` makes a quote. Text that starts+ * like a marker is escaped on export by markdownSource.ts.+ */++import {QUOTE, type ElementTransformer} from '@lexical/markdown';++/** Lexical's own quote marker, which the typing shortcut keeps using. */+const SHORTCUT_MARKER = /^>\s/;++/**+ * A block quote marker: up to three spaces of indentation, then `>` and the+ * one space or tab that may follow it, so `>a` and ` > a` import as quotes,+ * as in core Markdown. A typed line becomes a quote only at Lexical's own+ * `> `, so typing `>` and Enter, or ` > `, changes nothing new. RichText+ * quotes do not nest: the marker of a quote inside a quote stays in the+ * outer quote's text.+ */+export const QUOTE_MARKERS: ElementTransformer = {+ ...QUOTE,+ regExp: /^ {0,3}>[ \t]?/,+ replace: (parentNode, children, match, isImport) =>+ isImport || SHORTCUT_MARKER.test(match[0])+ ? QUOTE.replace(parentNode, children, match, isImport)+ : false,+};diff --git a/packages/richtext/src/markdownQuoteMarkers.test.ts b/packages/richtext/src/markdownQuoteMarkers.test.tsnew file mode 100644index 0000000000..d8c14a4142--- /dev/null+++ b/packages/richtext/src/markdownQuoteMarkers.test.ts@@ -0,0 +1,206 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++import {describe, expect, it} from 'vitest';+import {createHeadlessEditor} from '@lexical/headless';+import {registerMarkdownShortcuts} from '@lexical/markdown';+import {+ $createLineBreakNode,+ $createParagraphNode,+ $createTextNode,+ $getRoot,+ $getSelection,+ $isRangeSelection,+ $isTextNode,+ KEY_ENTER_COMMAND,+ type LexicalEditor,+} from 'lexical';+import {parseMarkdownAst} from '@astryxdesign/core/Markdown/parser';+import {DEFAULT_NODES} from './editorNodes';+import {DEFAULT_TRANSFORMERS} from './markdownTable';+import {+ $exportMarkdownKeepingSource,+ importMarkdownKeepingSource,+} from './markdownSource';+import {markdownToEditorStateJSON} from './markdownSerializers';++type Json = {type: string; text?: string; value?: string; children?: Json[]};++/** The editor's top-level blocks: their types and text. */+function richBlocks(markdown: string): string[] {+ const text = (node: Json): string =>+ node.text ?? (node.children ?? []).map(text).join('');+ return (+ JSON.parse(markdownToEditorStateJSON(markdown)) as {root: Json}+ ).root.children!.map(node => `${node.type}:${text(node)}`);+}++/** Core's top-level blocks: their types and text. */+function coreBlocks(markdown: string): string[] {+ const text = (node: Json): string =>+ node.value ?? (node.children ?? []).map(text).join('');+ return (parseMarkdownAst(markdown).children as unknown as Json[]).map(+ node => `${node.type === 'blockquote' ? 'quote' : node.type}:${text(node)}`,+ );+}++describe('a block quote marker reads as core reads it', () => {+ it.each(['>a', '>\ta', ' > a', '> a'])(+ 'quotes %j, as core does',+ markdown => {+ expect(richBlocks(markdown)).toEqual(['quote:a']);+ expect(coreBlocks(markdown)).toEqual(['quote:a']);+ },+ );++ it('reads four spaces of indentation as no marker, as core does', () => {+ expect(richBlocks(' > a')).toEqual(['paragraph: > a']);+ expect(coreBlocks(' > a')).toEqual(['paragraph: > a']);+ });++ it('writes an edited quote back as one that reads as a quote', () => {+ const editor = createHeadlessEditor({+ nodes: [...DEFAULT_NODES],+ onError(error) {+ throw error;+ },+ });+ importMarkdownKeepingSource(editor, '>a\n', [...DEFAULT_TRANSFORMERS]);+ editor.update(+ () => {+ const last = $getRoot().getLastDescendant();+ if (!$isTextNode(last)) {+ throw new Error('No text to edit');+ }+ last.setTextContent(`${last.getTextContent()}x`);+ },+ {discrete: true},+ );+ const exported = editor+ .getEditorState()+ .read(() => $exportMarkdownKeepingSource([...DEFAULT_TRANSFORMERS]));+ expect(richBlocks(exported)).toEqual(['quote:ax']);+ });+});++/** A headless editor with the default typing shortcuts, caret in an empty paragraph. */+function shortcutEditor(): LexicalEditor {+ const editor = createHeadlessEditor({+ nodes: [...DEFAULT_NODES],+ onError(error) {+ throw error;+ },+ });+ registerMarkdownShortcuts(editor, [...DEFAULT_TRANSFORMERS]);+ editor.update(+ () => {+ const paragraph = $createParagraphNode();+ $getRoot().clear().append(paragraph);+ paragraph.select();+ },+ {discrete: true},+ );+ return editor;+}++/** Types `text` one character at a time, as a person would. */+function type(editor: LexicalEditor, text: string): void {+ for (const character of text) {+ editor.update(+ () => {+ const selection = $getSelection();+ if ($isRangeSelection(selection)) {+ selection.insertText(character);+ }+ },+ {discrete: true},+ );+ }+}++/** The editor's first block: its type and text, once queued updates ran. */+async function firstBlock(editor: LexicalEditor): Promise<string> {+ await Promise.resolve();+ editor.update(() => {}, {discrete: true});+ return editor.getEditorState().read(() => {+ const first = $getRoot().getFirstChild();+ return `${first?.getType()}:${first?.getTextContent()}`;+ });+}++describe('typing shortcuts keep the library quote trigger', () => {+ it('makes a quote from `> `, as before', async () => {+ const editor = shortcutEditor();+ type(editor, '> ');+ expect(await firstBlock(editor)).toBe('quote:');+ });++ it('leaves ` > ` typed with indentation as text', async () => {+ const editor = shortcutEditor();+ type(editor, ' > ');+ expect(await firstBlock(editor)).toBe('paragraph: > ');+ });++ it('leaves `>` and Enter as text', async () => {+ const editor = shortcutEditor();+ type(editor, '>');+ editor.dispatchCommand(KEY_ENTER_COMMAND, null);+ expect(await firstBlock(editor)).toBe('paragraph:>');+ });+});++describe('a paragraph line that starts like a quote marker stays text', () => {+ /** A one-paragraph document of `lines`, exported and read back. */+ function roundTrip(lines: ReadonlyArray<string>): {+ exported: string;+ blocks: string[];+ } {+ const editor = createHeadlessEditor({+ nodes: [...DEFAULT_NODES],+ onError(error) {+ throw error;+ },+ });+ editor.update(+ () => {+ const paragraph = $createParagraphNode();+ lines.forEach((line, index) => {+ if (index > 0) {+ paragraph.append($createLineBreakNode());+ }+ paragraph.append($createTextNode(line));+ });+ $getRoot().clear().append(paragraph);+ },+ {discrete: true},+ );+ const exported = editor+ .getEditorState()+ .read(() => $exportMarkdownKeepingSource([...DEFAULT_TRANSFORMERS]));+ return {exported, blocks: richBlocks(exported)};+ }++ it.each([+ [['>a'], '\\>a', ['paragraph:>a']],+ [[' >a'], ' \\>a', ['paragraph: >a']],+ [['> a'], '\\> a', ['paragraph:> a']],+ ])(+ 'writes %j with an escaped marker, read back as text',+ (lines, written, blocks) => {+ const result = roundTrip(lines);+ expect(result.exported.trimEnd()).toBe(written);+ expect(result.blocks).toEqual(blocks);+ },+ );++ it('escapes a marker after more spaces, as a list continuation line may add them', () => {+ const result = roundTrip([' >a']);+ expect(result.exported.trimEnd()).toBe(' \\>a');+ expect(result.blocks).toEqual(['paragraph: >a']);+ });++ it('escapes the line after a line break too', () => {+ const result = roundTrip(['a', '>b']);+ expect(result.exported.trimEnd()).toBe('a\\\n\\>b');+ expect(result.blocks).toEqual(['paragraph:a>b']);+ });+});diff --git a/packages/richtext/src/markdownSource.ts b/packages/richtext/src/markdownSource.tsindex 5596f1c71b..2560fa2769 100644--- a/packages/richtext/src/markdownSource.ts+++ b/packages/richtext/src/markdownSource.ts@@ -677,9 +677,8 @@ function $canonicalMarkdown(/** Line starts that would turn literal text into a block structure. */const LINE_START_SYNTAX: ReadonlyArray<RegExp> = [- // ATX heading, block quote, bullet list item.+ // ATX heading, bullet list item./^#{1,6}(?=[ \t]|$)/,- /^>/,/^[-+](?=[ \t]|$)/,// Setext underline or thematic break made of `=` or `-`./^[=-](?=[=\- \t]*$)/,@@ -688,6 +687,10 @@ const LINE_START_SYNTAX: ReadonlyArray<RegExp> = [];// An ordered list item escapes its delimiter, not its first character.const ORDERED_LIST_START = /^(\d{1,9})([.)])(?=[ \t]|$)/;+// A block quote marker may follow up to three spaces (CommonMark 0.31 §5.1),+// and a list item's continuation line adds its own indentation, so the escape+// goes before a `>` after any spaces.+const QUOTE_START = /^( *)>/;// Inline syntax Lexical's export leaves unescaped: link and image brackets and// character references.@@ -757,8 +760,11 @@ function markedView(const previous = node.getPreviousSibling();if (previous == null || $isLineBreakNode(previous)) {const ordered = ORDERED_LIST_START.exec(text);+ const quote = QUOTE_START.exec(text);if (ordered != null) {text = ordered[1] + token + text.slice(ordered[1].length);+ } else if (quote != null) {+ text = quote[1] + token + text.slice(quote[1].length);} else if (LINE_START_SYNTAX.some(pattern => pattern.test(text))) {text = token + text;}diff --git a/packages/richtext/src/markdownTable.ts b/packages/richtext/src/markdownTable.tsindex 84b5ebeab9..3b82b32ee9 100644--- a/packages/richtext/src/markdownTable.ts+++ b/packages/richtext/src/markdownTable.ts@@ -22,6 +22,7 @@ import {TEXT_MATCH_TRANSFORMERS,LINK,ORDERED_LIST,+ QUOTE,TRANSFORMERS,type MultilineElementTransformer,type Transformer,@@ -41,6 +42,7 @@ import {import type {ElementFormatType} from 'lexical';import {HARD_LINE_BREAK} from './markdownHardLineBreak';import {THEMATIC_BREAK} from './markdownThematicBreak';+import {QUOTE_MARKERS} from './markdownQuote';import {LIST_EXPORT} from './markdownListExport';import {TASK_LIST} from './markdownTaskList';import {ORDERED_LIST_KEEPING_START} from './markdownOrderedList';@@ -290,7 +292,9 @@ export const DEFAULT_TRANSFORMERS: ReadonlyArray<Transformer> = [? LINK_KEEPING_DESTINATIONS: transformer === CODE? BACKTICK_CODE- : transformer,+ : transformer === QUOTE+ ? QUOTE_MARKERS+ : transformer,),HARD_LINE_BREAK,TABLE,