An open source design system that's fully customizable and agent ready
TypeScript
fix(Markdown): keep content indented into a list item in the item after a blank line (#7092)
Cindy Zhang2e5bc96f8ddd
diff --git a/.changeset/markdown-list-item-blank-line-continuation.md b/.changeset/markdown-list-item-blank-line-continuation.mdnew file mode 100644index 0000000000..b8f68f40fb--- /dev/null+++ b/.changeset/markdown-list-item-blank-line-continuation.md@@ -0,0 +1,8 @@+---+'@astryxdesign/core': patch+---++[fix] Markdown: keep content indented into a list item in the item after a blank line+@cixzhang++`Markdown` now keeps lines indented to a list item's content in that item after a blank line, as CommonMark specifies: a nested list, a fenced code block, or another paragraph under a numbered step stays in the step, and the numbering continues after it. Such content used to end the list, so the sub-items showed as a separate list and a step's code block showed as raw text.diff --git a/packages/core/src/Markdown/listContinuation.test.tsx b/packages/core/src/Markdown/listContinuation.test.tsxnew file mode 100644index 0000000000..6c7a327c35--- /dev/null+++ b/packages/core/src/Markdown/listContinuation.test.tsx@@ -0,0 +1,94 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++import {describe, expect, it} from 'vitest';+import {+ createIncrementalState,+ parseMarkdown,+ parseMarkdownAst,+ parseMarkdownIncremental,+} from './parser';++type Json = {type: string; value?: string; lang?: string; children?: Json[]};++/** The document's blocks as compact markup. */+function blocks(markdown: string): string {+ const show = (node: Json): string =>+ node.type === 'text'+ ? JSON.stringify(node.value)+ : node.type === 'code'+ ? `code(${node.lang ?? ''})${JSON.stringify(node.value)}`+ : node.children == null+ ? node.type+ : `${node.type}[${node.children.map(show).join(', ')}]`;+ return (parseMarkdownAst(markdown).children as unknown as Json[])+ .map(show)+ .join(' | ');+}++const NESTED_AFTER_BLANK = '1. a\n\n - sub\n2. b';+const STEPS_WITH_CODE =+ '1. Step one\n\n ```bash\n cmd\n\n more\n ```\n\n2. Step two';++describe('a list item continues past blank lines (CommonMark 0.31 §5.2)', () => {+ it.each([+ [+ 'a nested list after a blank line, then the next item',+ NESTED_AFTER_BLANK,+ 'list[listItem[paragraph["a"], list[listItem[paragraph["sub"]]]], listItem[paragraph["b"]]]',+ ],+ [+ 'a nested ordered list after a blank line',+ '1. a\n\n 26. b',+ 'list[listItem[paragraph["a"], list[listItem[paragraph["b"]]]]]',+ ],+ [+ 'a second paragraph indented into the item',+ '- a\n\n b',+ 'list[listItem[paragraph["a"], paragraph["b"]]]',+ ],+ [+ 'nested items and a sibling, each after a blank line',+ '- a\n\n - b\n\n- c',+ 'list[listItem[paragraph["a"], list[listItem[paragraph["b"]]]], listItem[paragraph["c"]]]',+ ],+ [+ 'a fenced code block, blank lines inside it included',+ STEPS_WITH_CODE,+ 'list[listItem[paragraph["Step one"], code(bash)"cmd\\n\\nmore"], listItem[paragraph["Step two"]]]',+ ],+ ])('%s', (_, markdown, expected) => {+ expect(blocks(markdown)).toBe(expected);+ });++ it.each([+ [+ 'a paragraph at the margin',+ '- a\n\nb',+ 'list[listItem[paragraph["a"]]] | paragraph["b"]',+ ],+ [+ 'a line indented less than the content',+ '1. a\n\n b',+ 'list[listItem[paragraph["a"]]] | paragraph[" b"]',+ ],+ ])('ends the list at %s', (_, markdown, expected) => {+ expect(blocks(markdown)).toBe(expected);+ });++ it.each([+ NESTED_AFTER_BLANK,+ STEPS_WITH_CODE,+ '- a\n\n b\n\n - c\n\nd\n\n- e',+ ])('streams %j to the full parse at every character', markdown => {+ for (const sourceRanges of [false, true]) {+ const state = createIncrementalState();+ let streamed = parseMarkdownIncremental('', state, {sourceRanges});+ for (let end = 1; end <= markdown.length; end++) {+ streamed = parseMarkdownIncremental(markdown.slice(0, end), state, {+ sourceRanges,+ });+ }+ expect(streamed).toEqual(parseMarkdown(markdown, {sourceRanges}));+ }+ });+});diff --git a/packages/core/src/Markdown/parser.perf.test.ts b/packages/core/src/Markdown/parser.perf.test.tsindex bbe078ccb8..f385cee45c 100644--- a/packages/core/src/Markdown/parser.perf.test.ts+++ b/packages/core/src/Markdown/parser.perf.test.ts@@ -338,8 +338,10 @@ describe('parseMarkdownIncremental cache', () => {console.log(` tracked whole-prefix work (20/200 sections): ${JSON.stringify(short)} / ${JSON.stringify(long)}`,);- expect(short).toEqual({prefixCharacters: 13072, copiedEntries: 210});- expect(long).toEqual({prefixCharacters: 1372702, copiedEntries: 20100});+ // A blank line settles what precedes it once the next line shows it+ // starts at the margin, so the last section settles one call later.+ expect(short).toEqual({prefixCharacters: 11763, copiedEntries: 210});+ expect(long).toEqual({prefixCharacters: 1358973, copiedEntries: 20100});});it('builds a bounded number of blocks per chunk however long the document is', () => {diff --git a/packages/core/src/Markdown/parser.ts b/packages/core/src/Markdown/parser.tsindex f2e4ffc656..fe94c9f78c 100644--- a/packages/core/src/Markdown/parser.ts+++ b/packages/core/src/Markdown/parser.ts@@ -3096,6 +3096,12 @@ function parseList(const content = ordered? lines[index].replace(new RegExp(`^ *\\d+${escDelim} `), ''): lines[index].replace(/^ *[-*+] /, '');+ // Where the item's content starts: past the marker and the spaces after+ // it (one to four; more makes the content indented by one space).+ const markerEnd = lines[index].length - content.length;+ const spacesAfter = getIndent(content);+ const contentColumn =+ spacesAfter >= 4 ? markerEnd : markerEnd + spacesAfter;const taskMatch = content.match(/^\[([ xX])\] (.*)/);let checked: boolean | undefined;@@ -3155,6 +3161,35 @@ function parseList(index++;}+ // After blank lines, lines indented to the item's content still belong+ // to it (CommonMark 0.31 §5.2): a nested list, a code block, or another+ // paragraph. Lazy continuation stops at a blank line.+ for (;;) {+ let next = index;+ while (next < lines.length && lines[next].trim() === '') {+ next++;+ }+ if (+ next === index ||+ next >= lines.length ||+ getIndent(lines[next]) < contentColumn+ ) {+ break;+ }+ for (; index < next; index++) {+ subLines.push('');+ }+ while (+ index < lines.length &&+ lines[index].trim() !== '' &&+ getIndent(lines[index]) >= contentColumn+ ) {+ subLines.push(lines[index]);+ index++;+ }+ loose = true;+ }+const source = listItemSource(itemText, subLines, lazyLineIndexes);items.push({@@ -3926,7 +3961,13 @@ function findSettledBoundary(if (line.trim() === '') {suppressMathUntilBoundary = false;- if (lineIndex > 0 && lineIndex < lines.length - 1) {+ // A blank line settles only what precedes it when the next line starts+ // at the margin: an indented line may continue a list item across it.+ if (+ lineIndex > 0 &&+ lineIndex < lines.length - 1 &&+ /^\S/.test(lines[lineIndex + 1])+ ) {lastBoundary = lineIndex;}}diff --git a/packages/richtext/src/markdownListContinuation.test.ts b/packages/richtext/src/markdownListContinuation.test.tsnew file mode 100644index 0000000000..4e09663352--- /dev/null+++ b/packages/richtext/src/markdownListContinuation.test.ts@@ -0,0 +1,34 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++import {describe, expect, it} from 'vitest';+import {parseMarkdownAst} from '@astryxdesign/core/Markdown/parser';+import {markdownToEditorStateJSON} from './markdownSerializers';++type Json = {type: string; children?: Json[]};++/** The nesting of a document's lists: each list as the item texts' depth. */+function listShape(nodes: ReadonlyArray<Json>, type: string): string {+ return nodes+ .map(node =>+ node.type === type+ ? `[${listShape(node.children ?? [], type)}]`+ : node.children == null+ ? ''+ : listShape(node.children, type),+ )+ .join('');+}++describe('a nested list after a blank line, on both surfaces', () => {+ it.each([+ '1. a\n\n - sub\n2. b',+ '1. a\n\n 26. b',+ '- a\n\n - b\n\n- c',+ ])('nests %j the same way as core', markdown => {+ const rich = (+ JSON.parse(markdownToEditorStateJSON(markdown)) as {root: Json}+ ).root.children;+ const core = parseMarkdownAst(markdown).children as unknown as Json[];+ expect(listShape(rich ?? [], 'list')).toBe(listShape(core, 'list'));+ });+});