An open source design system that's fully customizable and agent ready
TypeScript
fix(Markdown, richtext): read an ATX heading indented up to three spaces as a heading (#7103)
Cindy Zhang43b449565dc0
diff --git a/.changeset/markdown-indented-atx-headings.md b/.changeset/markdown-indented-atx-headings.mdnew file mode 100644index 0000000000..cf636be95f--- /dev/null+++ b/.changeset/markdown-indented-atx-headings.md@@ -0,0 +1,8 @@+---+'@astryxdesign/core': patch+---++[fix] Markdown: read an ATX heading indented up to three spaces as a heading+@cixzhang++`Markdown` now reads a heading line indented by up to three spaces, such as ` # Title`, as a heading, as CommonMark specifies. Such lines used to show as plain text with their `#` marks, except at the very start of a streamed message.diff --git a/packages/core/src/Markdown/atxHeadingIndentation.test.tsx b/packages/core/src/Markdown/atxHeadingIndentation.test.tsxnew file mode 100644index 0000000000..98ccc45c80--- /dev/null+++ b/packages/core/src/Markdown/atxHeadingIndentation.test.tsx@@ -0,0 +1,53 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++import {describe, expect, it} from 'vitest';+import {+ createIncrementalState,+ parseMarkdown,+ parseMarkdownAst,+ parseMarkdownIncremental,+} from './parser';++type Json = {type: string; value?: string; depth?: number; children?: Json[]};++/** The document's blocks as compact markup. */+function blocks(markdown: string): string {+ const show = (node: Json): string =>+ node.type === 'text'+ ? JSON.stringify(node.value)+ : node.children == null+ ? node.type+ : `${node.type}${node.depth ?? ''}[${node.children.map(show).join(', ')}]`;+ return (parseMarkdownAst(markdown).children as unknown as Json[])+ .map(show)+ .join(' | ');+}++describe('an ATX heading may be indented up to three spaces (CommonMark 0.31 §4.2)', () => {+ it.each([+ ['three spaces', ' # h', 'heading1["h"]'],+ ['one space, second level', ' ## h', 'heading2["h"]'],+ [+ 'two spaces, interrupting a paragraph',+ 'text\n # h',+ 'paragraph["text"] | heading1["h"]',+ ],+ ['no indentation, unchanged', '# h', 'heading1["h"]'],+ ])('reads %s as a heading', (_, markdown, expected) => {+ expect(blocks(markdown)).toBe(expected);+ });++ it('reads four spaces of indentation as no heading', () => {+ expect(blocks(' # h')).not.toContain('heading');+ });++ it('streams indented headings to the full parse at every character', () => {+ const markdown = 'a\n\n # one\n\ntext\n ## two\n';+ const state = createIncrementalState();+ let streamed = parseMarkdownIncremental('', state);+ for (let end = 1; end <= markdown.length; end++) {+ streamed = parseMarkdownIncremental(markdown.slice(0, end), state);+ }+ expect(streamed).toEqual(parseMarkdown(markdown));+ });+});diff --git a/packages/core/src/Markdown/parser.ts b/packages/core/src/Markdown/parser.tsindex 902a0643cd..8f197a9eb8 100644--- a/packages/core/src/Markdown/parser.ts+++ b/packages/core/src/Markdown/parser.ts@@ -2949,7 +2949,7 @@ function quotedContent(line: string): string {* to avoid ReDoS.*/function isBlockStart(line: string): boolean {- if (/^#{1,6} /.test(line)) {+ if (/^ {0,3}#{1,6} /.test(line)) {return true;}if (/^(`{3,}|~{3,})/.test(line)) {@@ -2988,7 +2988,7 @@ function canContinueParagraphLazily(return false;}if (- /^#{1,6} /.test(line) ||+ /^ {0,3}#{1,6} /.test(line) ||/^(`{3,}|~{3,})/.test(line) ||isHorizontalRule(line) ||QUOTE_MARKER.test(line) ||@@ -3563,7 +3563,8 @@ function parseMarkdownImpl(}// --- Heading ---- const headingMatch = line.match(/^(#{1,6}) +(.*)/);+ // An ATX heading may be indented up to three spaces (CommonMark 0.31 §4.2).+ const headingMatch = line.match(/^ {0,3}(#{1,6}) +(.*)/);if (headingMatch) {pushBlock({type: 'heading',diff --git a/packages/richtext/src/markdownHeading.ts b/packages/richtext/src/markdownHeading.tsnew file mode 100644index 0000000000..be7753b056--- /dev/null+++ b/packages/richtext/src/markdownHeading.ts@@ -0,0 +1,33 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file markdownHeading.ts+ * @input Uses @lexical/markdown (Lexical's HEADING element transformer).+ * @output Exports HEADING_MARKERS, Lexical's heading transformer reading an+ * ATX heading marker as CommonMark does.+ * @position Part of DEFAULT_TRANSFORMERS (markdownTable.ts) in place of+ * Lexical's HEADING, whose pattern allows no indentation. Matches core+ * Markdown's ATX headings (CommonMark 0.31 §4.2). Typing shortcuts keep+ * Lexical's own trigger: `# ` makes a heading. Text that starts like a+ * marker is escaped on export by markdownSource.ts.+ */++import {HEADING, type ElementTransformer} from '@lexical/markdown';++/** Lexical's own heading marker, which the typing shortcut keeps using. */+const SHORTCUT_MARKER = /^(#{1,6})\s/;++/**+ * An ATX heading marker: up to three spaces of indentation, then one to six+ * `#` and a space, so ` # h` imports as a heading, as in core Markdown. A+ * typed line becomes a heading only at Lexical's own `# `, so typing+ * ` # ` changes nothing new.+ */+export const HEADING_MARKERS: ElementTransformer = {+ ...HEADING,+ regExp: /^ {0,3}(#{1,6})\s/,+ replace: (parentNode, children, match, isImport) =>+ isImport || SHORTCUT_MARKER.test(match[0])+ ? HEADING.replace(parentNode, children, match, isImport)+ : false,+};diff --git a/packages/richtext/src/markdownHeadingMarkers.test.ts b/packages/richtext/src/markdownHeadingMarkers.test.tsnew file mode 100644index 0000000000..cf3be9d1ae--- /dev/null+++ b/packages/richtext/src/markdownHeadingMarkers.test.ts@@ -0,0 +1,132 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++import {describe, expect, it} from 'vitest';+import {createHeadlessEditor} from '@lexical/headless';+import {registerMarkdownShortcuts} from '@lexical/markdown';+import {+ $createParagraphNode,+ $createTextNode,+ $getRoot,+ $getSelection,+ $isRangeSelection,+ type LexicalEditor,+} from 'lexical';+import {parseMarkdownAst} from '@astryxdesign/core/Markdown/parser';+import {DEFAULT_NODES} from './editorNodes';+import {DEFAULT_TRANSFORMERS} from './markdownTable';+import {$exportMarkdownKeepingSource} from './markdownSource';+import {markdownToEditorStateJSON} from './markdownSerializers';++type Json = {+ type: string;+ tag?: string;+ depth?: number;+ text?: string;+ value?: string;+ children?: Json[];+};++const textOf = (node: Json): string =>+ node.text ?? node.value ?? (node.children ?? []).map(textOf).join('');++/** The editor's top-level blocks: heading level or type, and text. */+function richBlocks(markdown: string): string[] {+ return (+ JSON.parse(markdownToEditorStateJSON(markdown)) as {root: Json}+ ).root.children!.map(+ node => `${node.type === 'heading' ? node.tag : node.type}:${textOf(node)}`,+ );+}++/** Core's top-level blocks: heading level or type, and text. */+function coreBlocks(markdown: string): string[] {+ return (parseMarkdownAst(markdown).children as unknown as Json[]).map(+ node =>+ `${node.type === 'heading' ? `h${node.depth}` : node.type}:${textOf(node)}`,+ );+}++function newEditor(): LexicalEditor {+ return createHeadlessEditor({+ nodes: [...DEFAULT_NODES],+ onError(error) {+ throw error;+ },+ });+}++describe('an indented ATX heading reads as core reads it', () => {+ it.each([+ [' # h', ['h1:h']],+ [' ## h', ['h2:h']],+ ['# h', ['h1:h']],+ [' # h', ['paragraph: # h']],+ ])('imports %j as core reads it', (markdown, expected) => {+ expect(richBlocks(markdown)).toEqual(expected);+ expect(coreBlocks(markdown)).toEqual(expected);+ });+});++describe('typing shortcuts keep the library heading trigger', () => {+ /** Types `text` into an empty paragraph, then reads the first block. */+ async function typed(text: string): Promise<string> {+ const editor = newEditor();+ registerMarkdownShortcuts(editor, [...DEFAULT_TRANSFORMERS]);+ editor.update(+ () => {+ const paragraph = $createParagraphNode();+ $getRoot().clear().append(paragraph);+ paragraph.select();+ },+ {discrete: true},+ );+ for (const character of text) {+ editor.update(+ () => {+ const selection = $getSelection();+ if ($isRangeSelection(selection)) {+ selection.insertText(character);+ }+ },+ {discrete: true},+ );+ }+ await Promise.resolve();+ editor.update(() => {}, {discrete: true});+ return editor.getEditorState().read(() => {+ const first = $getRoot().getFirstChild();+ return `${first?.getType()}:${first?.getTextContent()}`;+ });+ }++ it('makes a heading from `# `, as before', async () => {+ expect(await typed('# ')).toBe('heading:');+ });++ it('leaves ` # ` typed with indentation as text', async () => {+ expect(await typed(' # ')).toBe('paragraph: # ');+ });+});++describe('paragraph text that starts like a heading marker stays text', () => {+ it.each([+ [' # h', ' \\# h'],+ [' # h', ' \\# h'],+ ['# h', '\\# h'],+ ])('writes %j escaped, read back as text', (text, written) => {+ const editor = newEditor();+ editor.update(+ () => {+ $getRoot()+ .clear()+ .append($createParagraphNode().append($createTextNode(text)));+ },+ {discrete: true},+ );+ const exported = editor+ .getEditorState()+ .read(() => $exportMarkdownKeepingSource([...DEFAULT_TRANSFORMERS]));+ expect(exported.trimEnd()).toBe(written);+ expect(richBlocks(exported)).toEqual([`paragraph:${text}`]);+ });+});diff --git a/packages/richtext/src/markdownSource.ts b/packages/richtext/src/markdownSource.tsindex 2560fa2769..c5a293250e 100644--- a/packages/richtext/src/markdownSource.ts+++ b/packages/richtext/src/markdownSource.ts@@ -677,8 +677,7 @@ function $canonicalMarkdown(/** Line starts that would turn literal text into a block structure. */const LINE_START_SYNTAX: ReadonlyArray<RegExp> = [- // ATX heading, bullet list item.- /^#{1,6}(?=[ \t]|$)/,+ // Bullet list item./^[-+](?=[ \t]|$)/,// Setext underline or thematic break made of `=` or `-`./^[=-](?=[=\- \t]*$)/,@@ -687,10 +686,10 @@ const LINE_START_SYNTAX: ReadonlyArray<RegExp> = [];// An ordered list item escapes its delimiter, not its first character.const ORDERED_LIST_START = /^(\d{1,9})([.)])(?=[ \t]|$)/;-// A block quote marker may follow up to three spaces (CommonMark 0.31 §5.1),-// and a list item's continuation line adds its own indentation, so the escape-// goes before a `>` after any spaces.-const QUOTE_START = /^( *)>/;+// A block quote marker or an ATX heading marker may follow up to three spaces+// (CommonMark 0.31 §5.1, §4.2), and a list item's continuation line adds its+// own indentation, so the escape goes before the marker after any spaces.+const SPACED_MARKER_START = /^( *)(?=>|#{1,6}(?:[ \t]|$))/;// Inline syntax Lexical's export leaves unescaped: link and image brackets and// character references.@@ -760,11 +759,11 @@ function markedView(const previous = node.getPreviousSibling();if (previous == null || $isLineBreakNode(previous)) {const ordered = ORDERED_LIST_START.exec(text);- const quote = QUOTE_START.exec(text);+ const marker = SPACED_MARKER_START.exec(text);if (ordered != null) {text = ordered[1] + token + text.slice(ordered[1].length);- } else if (quote != null) {- text = quote[1] + token + text.slice(quote[1].length);+ } else if (marker != null) {+ text = marker[1] + token + text.slice(marker[1].length);} else if (LINE_START_SYNTAX.some(pattern => pattern.test(text))) {text = token + text;}diff --git a/packages/richtext/src/markdownTable.ts b/packages/richtext/src/markdownTable.tsindex 3b82b32ee9..c0be205561 100644--- a/packages/richtext/src/markdownTable.ts+++ b/packages/richtext/src/markdownTable.ts@@ -18,6 +18,7 @@import {$convertFromMarkdownString,CODE,+ HEADING,TEXT_FORMAT_TRANSFORMERS,TEXT_MATCH_TRANSFORMERS,LINK,@@ -42,6 +43,7 @@ import {import type {ElementFormatType} from 'lexical';import {HARD_LINE_BREAK} from './markdownHardLineBreak';import {THEMATIC_BREAK} from './markdownThematicBreak';+import {HEADING_MARKERS} from './markdownHeading';import {QUOTE_MARKERS} from './markdownQuote';import {LIST_EXPORT} from './markdownListExport';import {TASK_LIST} from './markdownTaskList';@@ -294,7 +296,9 @@ export const DEFAULT_TRANSFORMERS: ReadonlyArray<Transformer> = [? BACKTICK_CODE: transformer === QUOTE? QUOTE_MARKERS- : transformer,+ : transformer === HEADING+ ? HEADING_MARKERS+ : transformer,),HARD_LINE_BREAK,TABLE,