Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

fix(Markdown, richtext): read an ATX heading indented up to three spaces as a heading (#7103)

Cindy Zhang43b449565dc0
diff --git a/.changeset/markdown-indented-atx-headings.md b/.changeset/markdown-indented-atx-headings.md
new file mode 100644
index 0000000000..cf636be95f
--- /dev/null
+++ b/.changeset/markdown-indented-atx-headings.md
@@ -0,0 +1,8 @@
+---
+'@astryxdesign/core': patch
+---
+
+[fix] Markdown: read an ATX heading indented up to three spaces as a heading
+@cixzhang
+
+`Markdown` now reads a heading line indented by up to three spaces, such as ` # Title`, as a heading, as CommonMark specifies. Such lines used to show as plain text with their `#` marks, except at the very start of a streamed message.
diff --git a/packages/core/src/Markdown/atxHeadingIndentation.test.tsx b/packages/core/src/Markdown/atxHeadingIndentation.test.tsx
new file mode 100644
index 0000000000..98ccc45c80
--- /dev/null
+++ b/packages/core/src/Markdown/atxHeadingIndentation.test.tsx
@@ -0,0 +1,53 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+import {describe, expect, it} from 'vitest';
+import {
+ createIncrementalState,
+ parseMarkdown,
+ parseMarkdownAst,
+ parseMarkdownIncremental,
+} from './parser';
+
+type Json = {type: string; value?: string; depth?: number; children?: Json[]};
+
+/** The document's blocks as compact markup. */
+function blocks(markdown: string): string {
+ const show = (node: Json): string =>
+ node.type === 'text'
+ ? JSON.stringify(node.value)
+ : node.children == null
+ ? node.type
+ : `${node.type}${node.depth ?? ''}[${node.children.map(show).join(', ')}]`;
+ return (parseMarkdownAst(markdown).children as unknown as Json[])
+ .map(show)
+ .join(' | ');
+}
+
+describe('an ATX heading may be indented up to three spaces (CommonMark 0.31 §4.2)', () => {
+ it.each([
+ ['three spaces', ' # h', 'heading1["h"]'],
+ ['one space, second level', ' ## h', 'heading2["h"]'],
+ [
+ 'two spaces, interrupting a paragraph',
+ 'text\n # h',
+ 'paragraph["text"] | heading1["h"]',
+ ],
+ ['no indentation, unchanged', '# h', 'heading1["h"]'],
+ ])('reads %s as a heading', (_, markdown, expected) => {
+ expect(blocks(markdown)).toBe(expected);
+ });
+
+ it('reads four spaces of indentation as no heading', () => {
+ expect(blocks(' # h')).not.toContain('heading');
+ });
+
+ it('streams indented headings to the full parse at every character', () => {
+ const markdown = 'a\n\n # one\n\ntext\n ## two\n';
+ const state = createIncrementalState();
+ let streamed = parseMarkdownIncremental('', state);
+ for (let end = 1; end <= markdown.length; end++) {
+ streamed = parseMarkdownIncremental(markdown.slice(0, end), state);
+ }
+ expect(streamed).toEqual(parseMarkdown(markdown));
+ });
+});
diff --git a/packages/core/src/Markdown/parser.ts b/packages/core/src/Markdown/parser.ts
index 902a0643cd..8f197a9eb8 100644
--- a/packages/core/src/Markdown/parser.ts
+++ b/packages/core/src/Markdown/parser.ts
@@ -2949,7 +2949,7 @@ function quotedContent(line: string): string {
* to avoid ReDoS.
*/
function isBlockStart(line: string): boolean {
- if (/^#{1,6} /.test(line)) {
+ if (/^ {0,3}#{1,6} /.test(line)) {
return true;
}
if (/^(`{3,}|~{3,})/.test(line)) {
@@ -2988,7 +2988,7 @@ function canContinueParagraphLazily(
return false;
}
if (
- /^#{1,6} /.test(line) ||
+ /^ {0,3}#{1,6} /.test(line) ||
/^(`{3,}|~{3,})/.test(line) ||
isHorizontalRule(line) ||
QUOTE_MARKER.test(line) ||
@@ -3563,7 +3563,8 @@ function parseMarkdownImpl(
}
// --- Heading ---
- const headingMatch = line.match(/^(#{1,6}) +(.*)/);
+ // An ATX heading may be indented up to three spaces (CommonMark 0.31 §4.2).
+ const headingMatch = line.match(/^ {0,3}(#{1,6}) +(.*)/);
if (headingMatch) {
pushBlock({
type: 'heading',
diff --git a/packages/richtext/src/markdownHeading.ts b/packages/richtext/src/markdownHeading.ts
new file mode 100644
index 0000000000..be7753b056
--- /dev/null
+++ b/packages/richtext/src/markdownHeading.ts
@@ -0,0 +1,33 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file markdownHeading.ts
+ * @input Uses @lexical/markdown (Lexical's HEADING element transformer).
+ * @output Exports HEADING_MARKERS, Lexical's heading transformer reading an
+ * ATX heading marker as CommonMark does.
+ * @position Part of DEFAULT_TRANSFORMERS (markdownTable.ts) in place of
+ * Lexical's HEADING, whose pattern allows no indentation. Matches core
+ * Markdown's ATX headings (CommonMark 0.31 §4.2). Typing shortcuts keep
+ * Lexical's own trigger: `# ` makes a heading. Text that starts like a
+ * marker is escaped on export by markdownSource.ts.
+ */
+
+import {HEADING, type ElementTransformer} from '@lexical/markdown';
+
+/** Lexical's own heading marker, which the typing shortcut keeps using. */
+const SHORTCUT_MARKER = /^(#{1,6})\s/;
+
+/**
+ * An ATX heading marker: up to three spaces of indentation, then one to six
+ * `#` and a space, so ` # h` imports as a heading, as in core Markdown. A
+ * typed line becomes a heading only at Lexical's own `# `, so typing
+ * ` # ` changes nothing new.
+ */
+export const HEADING_MARKERS: ElementTransformer = {
+ ...HEADING,
+ regExp: /^ {0,3}(#{1,6})\s/,
+ replace: (parentNode, children, match, isImport) =>
+ isImport || SHORTCUT_MARKER.test(match[0])
+ ? HEADING.replace(parentNode, children, match, isImport)
+ : false,
+};
diff --git a/packages/richtext/src/markdownHeadingMarkers.test.ts b/packages/richtext/src/markdownHeadingMarkers.test.ts
new file mode 100644
index 0000000000..cf3be9d1ae
--- /dev/null
+++ b/packages/richtext/src/markdownHeadingMarkers.test.ts
@@ -0,0 +1,132 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+import {describe, expect, it} from 'vitest';
+import {createHeadlessEditor} from '@lexical/headless';
+import {registerMarkdownShortcuts} from '@lexical/markdown';
+import {
+ $createParagraphNode,
+ $createTextNode,
+ $getRoot,
+ $getSelection,
+ $isRangeSelection,
+ type LexicalEditor,
+} from 'lexical';
+import {parseMarkdownAst} from '@astryxdesign/core/Markdown/parser';
+import {DEFAULT_NODES} from './editorNodes';
+import {DEFAULT_TRANSFORMERS} from './markdownTable';
+import {$exportMarkdownKeepingSource} from './markdownSource';
+import {markdownToEditorStateJSON} from './markdownSerializers';
+
+type Json = {
+ type: string;
+ tag?: string;
+ depth?: number;
+ text?: string;
+ value?: string;
+ children?: Json[];
+};
+
+const textOf = (node: Json): string =>
+ node.text ?? node.value ?? (node.children ?? []).map(textOf).join('');
+
+/** The editor's top-level blocks: heading level or type, and text. */
+function richBlocks(markdown: string): string[] {
+ return (
+ JSON.parse(markdownToEditorStateJSON(markdown)) as {root: Json}
+ ).root.children!.map(
+ node => `${node.type === 'heading' ? node.tag : node.type}:${textOf(node)}`,
+ );
+}
+
+/** Core's top-level blocks: heading level or type, and text. */
+function coreBlocks(markdown: string): string[] {
+ return (parseMarkdownAst(markdown).children as unknown as Json[]).map(
+ node =>
+ `${node.type === 'heading' ? `h${node.depth}` : node.type}:${textOf(node)}`,
+ );
+}
+
+function newEditor(): LexicalEditor {
+ return createHeadlessEditor({
+ nodes: [...DEFAULT_NODES],
+ onError(error) {
+ throw error;
+ },
+ });
+}
+
+describe('an indented ATX heading reads as core reads it', () => {
+ it.each([
+ [' # h', ['h1:h']],
+ [' ## h', ['h2:h']],
+ ['# h', ['h1:h']],
+ [' # h', ['paragraph: # h']],
+ ])('imports %j as core reads it', (markdown, expected) => {
+ expect(richBlocks(markdown)).toEqual(expected);
+ expect(coreBlocks(markdown)).toEqual(expected);
+ });
+});
+
+describe('typing shortcuts keep the library heading trigger', () => {
+ /** Types `text` into an empty paragraph, then reads the first block. */
+ async function typed(text: string): Promise<string> {
+ const editor = newEditor();
+ registerMarkdownShortcuts(editor, [...DEFAULT_TRANSFORMERS]);
+ editor.update(
+ () => {
+ const paragraph = $createParagraphNode();
+ $getRoot().clear().append(paragraph);
+ paragraph.select();
+ },
+ {discrete: true},
+ );
+ for (const character of text) {
+ editor.update(
+ () => {
+ const selection = $getSelection();
+ if ($isRangeSelection(selection)) {
+ selection.insertText(character);
+ }
+ },
+ {discrete: true},
+ );
+ }
+ await Promise.resolve();
+ editor.update(() => {}, {discrete: true});
+ return editor.getEditorState().read(() => {
+ const first = $getRoot().getFirstChild();
+ return `${first?.getType()}:${first?.getTextContent()}`;
+ });
+ }
+
+ it('makes a heading from `# `, as before', async () => {
+ expect(await typed('# ')).toBe('heading:');
+ });
+
+ it('leaves ` # ` typed with indentation as text', async () => {
+ expect(await typed(' # ')).toBe('paragraph: # ');
+ });
+});
+
+describe('paragraph text that starts like a heading marker stays text', () => {
+ it.each([
+ [' # h', ' \\# h'],
+ [' # h', ' \\# h'],
+ ['# h', '\\# h'],
+ ])('writes %j escaped, read back as text', (text, written) => {
+ const editor = newEditor();
+ editor.update(
+ () => {
+ $getRoot()
+ .clear()
+ .append($createParagraphNode().append($createTextNode(text)));
+ },
+ {discrete: true},
+ );
+ const exported = editor
+ .getEditorState()
+ .read(() => $exportMarkdownKeepingSource([...DEFAULT_TRANSFORMERS]));
+ expect(exported.trimEnd()).toBe(written);
+ expect(richBlocks(exported)).toEqual([`paragraph:${text}`]);
+ });
+});
diff --git a/packages/richtext/src/markdownSource.ts b/packages/richtext/src/markdownSource.ts
index 2560fa2769..c5a293250e 100644
--- a/packages/richtext/src/markdownSource.ts
+++ b/packages/richtext/src/markdownSource.ts
@@ -677,8 +677,7 @@ function $canonicalMarkdown(
/** Line starts that would turn literal text into a block structure. */
const LINE_START_SYNTAX: ReadonlyArray<RegExp> = [
- // ATX heading, bullet list item.
- /^#{1,6}(?=[ \t]|$)/,
+ // Bullet list item.
/^[-+](?=[ \t]|$)/,
// Setext underline or thematic break made of `=` or `-`.
/^[=-](?=[=\- \t]*$)/,
@@ -687,10 +686,10 @@ const LINE_START_SYNTAX: ReadonlyArray<RegExp> = [
];
// An ordered list item escapes its delimiter, not its first character.
const ORDERED_LIST_START = /^(\d{1,9})([.)])(?=[ \t]|$)/;
-// A block quote marker may follow up to three spaces (CommonMark 0.31 §5.1),
-// and a list item's continuation line adds its own indentation, so the escape
-// goes before a `>` after any spaces.
-const QUOTE_START = /^( *)>/;
+// A block quote marker or an ATX heading marker may follow up to three spaces
+// (CommonMark 0.31 §5.1, §4.2), and a list item's continuation line adds its
+// own indentation, so the escape goes before the marker after any spaces.
+const SPACED_MARKER_START = /^( *)(?=>|#{1,6}(?:[ \t]|$))/;
// Inline syntax Lexical's export leaves unescaped: link and image brackets and
// character references.
@@ -760,11 +759,11 @@ function markedView(
const previous = node.getPreviousSibling();
if (previous == null || $isLineBreakNode(previous)) {
const ordered = ORDERED_LIST_START.exec(text);
- const quote = QUOTE_START.exec(text);
+ const marker = SPACED_MARKER_START.exec(text);
if (ordered != null) {
text = ordered[1] + token + text.slice(ordered[1].length);
- } else if (quote != null) {
- text = quote[1] + token + text.slice(quote[1].length);
+ } else if (marker != null) {
+ text = marker[1] + token + text.slice(marker[1].length);
} else if (LINE_START_SYNTAX.some(pattern => pattern.test(text))) {
text = token + text;
}
diff --git a/packages/richtext/src/markdownTable.ts b/packages/richtext/src/markdownTable.ts
index 3b82b32ee9..c0be205561 100644
--- a/packages/richtext/src/markdownTable.ts
+++ b/packages/richtext/src/markdownTable.ts
@@ -18,6 +18,7 @@
import {
$convertFromMarkdownString,
CODE,
+ HEADING,
TEXT_FORMAT_TRANSFORMERS,
TEXT_MATCH_TRANSFORMERS,
LINK,
@@ -42,6 +43,7 @@ import {
import type {ElementFormatType} from 'lexical';
import {HARD_LINE_BREAK} from './markdownHardLineBreak';
import {THEMATIC_BREAK} from './markdownThematicBreak';
+import {HEADING_MARKERS} from './markdownHeading';
import {QUOTE_MARKERS} from './markdownQuote';
import {LIST_EXPORT} from './markdownListExport';
import {TASK_LIST} from './markdownTaskList';
@@ -294,7 +296,9 @@ export const DEFAULT_TRANSFORMERS: ReadonlyArray<Transformer> = [
? BACKTICK_CODE
: transformer === QUOTE
? QUOTE_MARKERS
- : transformer,
+ : transformer === HEADING
+ ? HEADING_MARKERS
+ : transformer,
),
HARD_LINE_BREAK,
TABLE,