An open source design system that's fully customizable and agent ready
TypeScript
fix(Markdown): keep lazy continuations in containers (#6752)
Cindy Zhang8a64e957632b
diff --git a/.changeset/markdown-lazy-continuations.md b/.changeset/markdown-lazy-continuations.mdnew file mode 100644index 0000000000..f0f4e440d1--- /dev/null+++ b/.changeset/markdown-lazy-continuations.md@@ -0,0 +1,9 @@+---+'@astryxdesign/core': patch+---++[fix] Markdown: keep lazy continuation lines inside blockquotes and list items++Wrapped paragraph lines may omit repeated blockquote markers or list indentation without escaping their owning container. Nested, ordered, unordered, and task-list continuations now preserve their rendered structure, text projection, and source range.++@cixzhangdiff --git a/apps/storybook/stories/Markdown.stories.tsx b/apps/storybook/stories/Markdown.stories.tsxindex 65e96229cb..d4d5ea04c1 100644--- a/apps/storybook/stories/Markdown.stories.tsx+++ b/apps/storybook/stories/Markdown.stories.tsx@@ -200,6 +200,32 @@ export const AIResponse: Story = {},};+export const LazyContinuations: Story = {+ name: 'Lazy continuations',+ args: {+ children: [+ '## Wrapped container paragraphs',+ '',+ '> A quoted paragraph can wrap onto another source line',+ 'without repeating the quote marker.',+ '>',+ '> A later paragraph can wrap too',+ 'and remain in the same quote.',+ '',+ '- A list item can wrap the same way',+ 'without repeating its indentation.',+ '',+ '- [ ] A task item also keeps',+ 'its unindented continuation.',+ '',+ '> 1. > Nested quote text',+ 'continues in the deepest open paragraph.',+ '',+ 'This paragraph follows all containers.',+ ].join('\n'),+ },+};+export const ShiftedHeadings: Story = {name: 'Shifted Headings (start at h3)',args: {diff --git a/packages/core/src/Markdown/Markdown.spec.md b/packages/core/src/Markdown/Markdown.spec.mdindex 61c1e7c6c6..ba93f5715e 100644--- a/packages/core/src/Markdown/Markdown.spec.md+++ b/packages/core/src/Markdown/Markdown.spec.md@@ -177,7 +177,13 @@ unions. Enabled calls return the explicit `InlineNodeWithMath` and| FR24 | `createMarkdownFrontmatter()` recognizes only a leading `---` block of unique `key: value` lines, decodes it through the caller's typed parser, stores finite JSON-like metadata on the canonical document, and removes the syntax from rendered content. An unfinished leading block yields no visible Markdown while streaming; malformed or unfinished final input remains ordinary Markdown. Frontmatter has no renderer and requires no Remark compatibility. || FR25 | The canonical parser and plugin-construction subpaths remain server-safe and can run function-bearing plugins entirely within server or RSC code. The client-owned `Markdown` component supports traditional and streaming SSR, but plugin entries containing functions cannot be serialized from a Server Component into that client boundary; direct RSC rendering requires a future additive server renderer rather than weakening the plugin protocol. |-FR23 includes table-level escaping inside inline-code spans: `\|` keeps the pipe+FR23 includes CommonMark-compatible lazy paragraph continuation inside blockquotes+and ordered, unordered, or task-list items. Omitting a repeated quote marker or+item indentation does not close the deepest open paragraph; a blank line or an+interrupting block start does. The owning top-level block's source range includes+the lazy lines.++FR23 also includes table-level escaping inside inline-code spans: `\|` keeps the pipeinside its authored cell, contributes only `|` to the code value and renderedtext, and produces the same result in full and incremental parsing. A completedinline-code span excludes its delimiting backticks from the parsed value anddiff --git a/packages/core/src/Markdown/Markdown.test.tsx b/packages/core/src/Markdown/Markdown.test.tsxindex 80c19930dc..92eccf6cfc 100644--- a/packages/core/src/Markdown/Markdown.test.tsx+++ b/packages/core/src/Markdown/Markdown.test.tsx@@ -376,6 +376,21 @@ describe('Markdown', () => {expect(bq).toBeInTheDocument();});+ it('renders lazy continuations in the owning nested containers', () => {+ const {container} = render(+ <Markdown>{'> 1. > Quoted **text**\ncontinued lazily'}</Markdown>,+ );+ const root = container.querySelector('.astryx-markdown');+ const outerQuote = root?.querySelector('blockquote');+ const nestedQuote = outerQuote?.querySelector('ol blockquote');++ expect(root?.children).toHaveLength(1);+ expect(root?.querySelectorAll('blockquote')).toHaveLength(2);+ expect(root?.querySelectorAll('ol')).toHaveLength(1);+ expect(nestedQuote).toHaveTextContent('Quoted text');+ expect(nestedQuote).toHaveTextContent('continued lazily');+ });+it('renders unordered lists', () => {render(<Markdown>{'- A\n- B\n- C'}</Markdown>);const ul = document.querySelector('ul');diff --git a/packages/core/src/Markdown/ast.test.ts b/packages/core/src/Markdown/ast.test.tsindex 9ec45c40da..f42c966983 100644--- a/packages/core/src/Markdown/ast.test.ts+++ b/packages/core/src/Markdown/ast.test.ts@@ -6,6 +6,7 @@ import {parseMarkdown,parseMarkdownAst,} from './parser';+import {markdownAstText} from './ast';import {createMarkdownPlugin} from './plugins';import type {MarkdownExtensionNode,@@ -149,6 +150,72 @@ describe('canonical Markdown AST', () => {});});+ it('keeps lazy continuation text and source provenance in its owning block', () => {+ const source = '> quoted\ncontinued lazily';+ const ast = parseMarkdownAst(source);+ const quote = ast.children[0];++ expect(ast.children).toHaveLength(1);+ expect(quote).toMatchObject({+ type: 'blockquote',+ position: {start: {offset: 0}, end: {offset: source.length}},+ children: [+ {+ type: 'paragraph',+ children: [{type: 'text', value: 'quoted\ncontinued lazily'}],+ },+ ],+ });+ if (quote?.type !== 'blockquote') {+ throw new Error('Expected blockquote');+ }+ const paragraph = quote.children[0];+ if (paragraph?.type !== 'paragraph') {+ throw new Error('Expected paragraph');+ }+ expect(markdownAstText(paragraph.children, () => '')).toBe(+ 'quoted\ncontinued lazily',+ );+ expect(parseMarkdown(source, {sourceRanges: true})[0]?.range).toEqual({+ start: 0,+ end: source.length,+ });++ const listSource = '- listed\ncontinued lazily';+ const list = parseMarkdownAst(listSource).children[0];+ expect(list).toMatchObject({+ type: 'list',+ position: {start: {offset: 0}, end: {offset: listSource.length}},+ children: [+ {+ type: 'listItem',+ children: [+ {+ type: 'paragraph',+ children: [{type: 'text', value: 'listed\ncontinued lazily'}],+ },+ ],+ },+ ],+ });+ });++ it('lets block plugin syntax interrupt a lazy continuation', () => {+ const ast = parseMarkdownAst('> quoted\n:::note\nBody\n:::', {+ plugins: [demoNotes],+ });++ expect(ast.children).toMatchObject([+ {type: 'blockquote'},+ {+ type: 'extension',+ plugin: 'demo-notes',+ name: 'note',+ data: {body: 'Body'},+ },+ ]);+ });+it('keeps source provenance canonical while projecting the released range', () => {const source = '# Heading\n\nParagraph';const ast = parseMarkdownAst(source, {sourceRanges: true});diff --git a/packages/core/src/Markdown/incremental.test.ts b/packages/core/src/Markdown/incremental.test.tsindex be18a46145..d824f6a532 100644--- a/packages/core/src/Markdown/incremental.test.ts+++ b/packages/core/src/Markdown/incremental.test.ts@@ -268,6 +268,49 @@ describe('parseMarkdownIncremental', () => {}});+ it.each([+ ['blockquote', 'Intro.\n\n> quoted\ncontinued lazily'],+ ['unordered list', 'Intro.\n\n- listed\ncontinued lazily'],+ ['ordered list', 'Intro.\n\n1. listed\ncontinued lazily'],+ ['task list', 'Intro.\n\n- [ ] listed\ncontinued lazily'],+ ['CRLF blockquote', 'Intro.\r\n\r\n> quoted\r\ncontinued lazily'],+ [+ 'later blockquote paragraph',+ 'Intro.\n\n> para1\n>\n> para2\ncontinued lazily',+ ],+ ['nested containers', 'Intro.\n\n> 1. > quoted\ncontinued lazily'],+ ] as const)(+ 'keeps a lazy continuation in its %s at every stream split',+ (_label, text) => {+ const full = parseMarkdown(text, {sourceRanges: true});+ expect(full).toHaveLength(2);+ expect(JSON.stringify(full[1])).toContain('continued lazily');+ const characterState = createIncrementalState();+ let result: BlockNode[] = [];+ for (let end = 1; end <= text.length; end++) {+ result = parseMarkdownIncremental(text.slice(0, end), characterState, {+ sourceRanges: true,+ isFinal: false,+ });+ }+ expect(result).toEqual(full);++ for (let split = 0; split <= text.length; split++) {+ const state = createIncrementalState();+ parseMarkdownIncremental(text.slice(0, split), state, {+ sourceRanges: true,+ isFinal: false,+ });+ expect(+ parseMarkdownIncremental(text, state, {+ sourceRanges: true,+ isFinal: true,+ }),+ ).toEqual(full);+ }+ },+ );+it('handles 1-char chunks and matches full parse', () => {const text = '# Hello\n\nWorld';const {final} = simulateStreaming(text, 1);diff --git a/packages/core/src/Markdown/parser.test.ts b/packages/core/src/Markdown/parser.test.tsindex 20be607b8e..80bbe0c9d1 100644--- a/packages/core/src/Markdown/parser.test.ts+++ b/packages/core/src/Markdown/parser.test.ts@@ -422,6 +422,177 @@ describe('parseMarkdown', () => {expect(result[0].type).toBe('blockquote');});+ it('keeps lazy paragraph continuations inside blockquotes and list items', () => {+ expect(parseMarkdown('> quoted\ncontinued lazily')).toEqual([+ {+ type: 'blockquote',+ children: [+ {+ type: 'paragraph',+ children: [{type: 'text', content: 'quoted\ncontinued lazily'}],+ },+ ],+ },+ ]);++ for (const marker of ['-', '1.', '- [ ]']) {+ const [list] = parseMarkdown(`${marker} listed\ncontinued lazily`);+ expect(list).toMatchObject({+ type: 'list',+ items: [+ {+ children: [+ {+ type: 'paragraph',+ children: [{type: 'text', content: 'listed\ncontinued lazily'}],+ },+ ],+ },+ ],+ });+ }+ expect(parseMarkdown('> # Foo\n> bar\nbaz')).toMatchObject([+ {+ type: 'blockquote',+ children: [+ {type: 'heading', level: 1},+ {+ type: 'paragraph',+ children: [{type: 'text', content: 'bar\nbaz'}],+ },+ ],+ },+ ]);+ expect(parseMarkdown('> bar\nbaz\n> foo')).toMatchObject([+ {+ type: 'blockquote',+ children: [+ {+ type: 'paragraph',+ children: [{type: 'text', content: 'bar\nbaz\nfoo'}],+ },+ ],+ },+ ]);+ });++ it('keeps lazy continuations in the deepest open paragraph', () => {+ expect(parseMarkdown('> 1. > Blockquote\ncontinued here.')).toMatchObject([+ {+ type: 'blockquote',+ children: [+ {+ type: 'list',+ items: [+ {+ children: [+ {+ type: 'blockquote',+ children: [+ {+ type: 'paragraph',+ children: [+ {+ type: 'text',+ content: 'Blockquote\ncontinued here.',+ },+ ],+ },+ ],+ },+ ],+ },+ ],+ },+ ],+ },+ ]);++ expect(+ parseMarkdown('- Outer item\n > Nested quote\ncontinued here.'),+ ).toMatchObject([+ {+ type: 'list',+ items: [+ {+ children: [+ {type: 'paragraph'},+ {+ type: 'blockquote',+ children: [+ {+ type: 'paragraph',+ children: [+ {+ type: 'text',+ content: 'Nested quote\ncontinued here.',+ },+ ],+ },+ ],+ },+ ],+ },+ ],+ },+ ]);+ });++ it('resumes lazy continuation for a later blockquote paragraph', () => {+ expect(parseMarkdown('> para1\n>\n> para2\nlazy')).toMatchObject([+ {+ type: 'blockquote',+ children: [+ {+ type: 'paragraph',+ children: [{type: 'text', content: 'para1'}],+ },+ {+ type: 'paragraph',+ children: [{type: 'text', content: 'para2\nlazy'}],+ },+ ],+ },+ ]);++ expect(parseMarkdown('> > para1\n> >\n> > para2\nlazy')).toMatchObject([+ {+ type: 'blockquote',+ children: [+ {+ type: 'blockquote',+ children: [+ {+ type: 'paragraph',+ children: [{type: 'text', content: 'para1'}],+ },+ {+ type: 'paragraph',+ children: [{type: 'text', content: 'para2\nlazy'}],+ },+ ],+ },+ ],+ },+ ]);+ });++ it('does not continue a container paragraph across a blank or block start', () => {+ expect(+ parseMarkdown('> quoted\n\noutside').map(block => block.type),+ ).toEqual(['blockquote', 'paragraph']);+ expect(+ parseMarkdown('- listed\n\noutside').map(block => block.type),+ ).toEqual(['list', 'paragraph']);+ expect(+ parseMarkdown('> quoted\n# Outside').map(block => block.type),+ ).toEqual(['blockquote', 'heading']);+ expect(parseMarkdown('- listed\n```\noutside\n```')).toMatchObject([+ {type: 'list'},+ {type: 'codeblock', content: 'outside'},+ ]);+ });+it('parses horizontal rules', () => {const result = parseMarkdown('---');expect(result[0].type).toBe('hr');diff --git a/packages/core/src/Markdown/parser.ts b/packages/core/src/Markdown/parser.tsindex c22e4985b7..e31d3b7c1c 100644--- a/packages/core/src/Markdown/parser.ts+++ b/packages/core/src/Markdown/parser.ts@@ -2167,6 +2167,62 @@ function isBlockStart(line: string): boolean {return false;}+/**+ * A lazy continuation may omit an owning quote marker or list indentation only+ * while the deepest open leaf is a paragraph. A blank line or an interrupting+ * block start closes that opportunity.+ */+function canContinueParagraphLazily(+ lines: ReadonlyArray<string>,+ lineIndex: number,+ insideList = false,+): boolean {+ const line = lines[lineIndex];+ if (line == null || line.trim() === '') {+ return false;+ }+ if (+ /^#{1,6} /.test(line) ||+ /^(`{3,}|~{3,})/.test(line) ||+ isHorizontalRule(line) ||+ line.startsWith('> ') ||+ line === '>' ||+ /^ {0,9}[-*+] /.test(line) ||+ (insideList ? /^ {0,9}\d+[.)] /.test(line) : /^ {0,9}0*1[.)] /.test(line))+ ) {+ return false;+ }+ return !(+ line.includes('|') &&+ lineIndex + 1 < lines.length &&+ isTableSeparator(lines[lineIndex + 1])+ );+}++function endsInParagraph(+ blocks: ReadonlyArray<MarkdownAstBlockContent<RuntimeExtensionNode>>,+): boolean {+ const last = blocks[blocks.length - 1];+ if (last == null) {+ return false;+ }+ if (last.type === 'paragraph') {+ return true;+ }+ if (last.type === 'blockquote') {+ return endsInParagraph(last.children);+ }+ if (last.type === 'list') {+ const item = last.children[last.children.length - 1];+ return item != null && endsInParagraph(item.children);+ }+ return false;+}++function sourceEndsInParagraph(source: string, opts: ResolvedOptions): boolean {+ return endsInParagraph(parseMarkdownImpl(source, nested(opts)));+}+function splitTableRow(line: string): string[] {let start = 0;let end = line.length;@@ -2261,11 +2317,36 @@ function parseTable(};}+function listItemSource(+ itemText: string,+ subLines: ReadonlyArray<string>,+ lazyLineIndexes: ReadonlySet<number>,+): string {+ if (subLines.length === 0) {+ return itemText;+ }+ const nonBlankIndents = subLines+ .filter((line, index) => line.trim() !== '' && !lazyLineIndexes.has(index))+ .map(getIndent);+ const minSubIndent =+ nonBlankIndents.length === 0 ? 0 : Math.min(...nonBlankIndents);+ return `${itemText}\n${subLines+ .map((line, index) =>+ line.trim() === ''+ ? ''+ : line.slice(+ lazyLineIndexes.has(index) ? getIndent(line) : minSubIndent,+ ),+ )+ .join('\n')}`;+}+function parseList(lines: string[],startIndex: number,ordered: boolean,opts: ResolvedOptions,+ interruptsLazyContinuation: (lineIndex: number) => boolean,): {node: MarkdownAstBlockContent<RuntimeExtensionNode>; nextIndex: number} {const items: MarkdownAstListItem<RuntimeExtensionNode>[] = [];const baseIndent = getIndent(lines[startIndex]);@@ -2302,29 +2383,58 @@ function parseList(index++;- // Collect sub-content (nested items or continuation lines)+ // Collect sub-content. Lines indented past the marker remain ordinary+ // nested content. A less-indented nonblank line can still belong to the+ // item when the deepest open leaf is a paragraph: CommonMark's lazy+ // continuation rule permits deleting some or all of that indentation.const subLines: string[] = [];- while (- index < lines.length &&- lines[index].trim() !== '' &&- getIndent(lines[index]) > baseIndent- ) {+ const lazyLineIndexes = new Set<number>();+ let lazyParagraphOpen: boolean | undefined = canContinueParagraphLazily(+ [itemText],+ 0,+ true,+ )+ ? true+ : undefined;+ while (index < lines.length && lines[index].trim() !== '') {+ if (getIndent(lines[index]) > baseIndent) {+ subLines.push(lines[index]);+ if (+ lazyParagraphOpen === true &&+ !canContinueParagraphLazily([lines[index].trimStart()], 0)+ ) {+ lazyParagraphOpen = undefined;+ }+ index++;+ continue;+ }++ if (+ interruptsLazyContinuation(index) ||+ !canContinueParagraphLazily(lines, index, true)+ ) {+ break;+ }+ if (lazyParagraphOpen === undefined) {+ lazyParagraphOpen = sourceEndsInParagraph(+ listItemSource(itemText, subLines, lazyLineIndexes),+ opts,+ );+ }+ if (!lazyParagraphOpen) {+ break;+ }+ lazyLineIndexes.add(subLines.length);subLines.push(lines[index]);index++;}- if (subLines.length > 0) {- const minSubIndent = Math.min(- ...subLines.map(subLine => getIndent(subLine)),- );- const deindented = subLines.map(subLine => subLine.slice(minSubIndent));- itemText += '\n' + deindented.join('\n');- }+ const source = listItemSource(itemText, subLines, lazyLineIndexes);items.push({type: 'listItem',checked,- children: parseMarkdownImpl(itemText, nested(opts)),+ children: parseMarkdownImpl(source, nested(opts)),});// CommonMark loose list: blank line(s) between items of the same style@@ -2481,6 +2591,27 @@ function parseMarkdownImpl(}}const blockExtensionMatches = new Map<number, ExtensionMatch>();+ const interruptsWithBlockExtension = (lineIndex: number): boolean => {+ if (!hasBlockExtensionSyntax) {+ return false;+ }+ const extensionColumn = blockExtensionColumn(lines[lineIndex]);+ if (extensionColumn == null) {+ return false;+ }+ const extensionOffset = lineOffsets[lineIndex] + extensionColumn;+ const extension = matchExtensionSyntax(+ cleaned,+ extensionOffset,+ 'block',+ opts,+ );+ if (extension.status === 'none') {+ return false;+ }+ blockExtensionMatches.set(extensionOffset, extension);+ return true;+ };const blocks: MarkdownAstBlockContent<RuntimeExtensionNode>[] = [];// The line each block started on, parallel to `blocks`. Only collected when// ranges were asked for; a block's end is resolved after the loop, since the@@ -2607,11 +2738,41 @@ function parseMarkdownImpl(// --- Blockquote ---if (line.startsWith('> ') || line === '>') {const quoteLines: string[] = [];- while (- index < lines.length &&- (lines[index].startsWith('> ') || lines[index] === '>')- ) {- quoteLines.push(lines[index].replace(/^> ?/, ''));+ let lazyParagraphOpen: boolean | undefined;+ while (index < lines.length) {+ const quoteLine = lines[index];+ if (quoteLine.startsWith('> ') || quoteLine === '>') {+ const content = quoteLine.replace(/^> ?/, '');+ quoteLines.push(content);+ if (content.trim() === '') {+ lazyParagraphOpen = false;+ } else if (+ lazyParagraphOpen === false ||+ (lazyParagraphOpen === true &&+ !canContinueParagraphLazily([content], 0))+ ) {+ lazyParagraphOpen = undefined;+ }+ index++;+ continue;+ }+ if (+ quoteLines[quoteLines.length - 1]?.trim() === '' ||+ interruptsWithBlockExtension(index) ||+ !canContinueParagraphLazily(lines, index)+ ) {+ break;+ }+ if (lazyParagraphOpen === undefined) {+ lazyParagraphOpen = sourceEndsInParagraph(+ quoteLines.join('\n'),+ opts,+ );+ }+ if (!lazyParagraphOpen) {+ break;+ }+ quoteLines.push(quoteLine);index++;}pushBlock({@@ -2623,7 +2784,13 @@ function parseMarkdownImpl(// --- Unordered list ---if (/^ {0,9}[-*+] /.test(line)) {- const listResult = parseList(lines, index, false, opts);+ const listResult = parseList(+ lines,+ index,+ false,+ opts,+ interruptsWithBlockExtension,+ );pushBlock(listResult.node);index = listResult.nextIndex;continue;@@ -2631,7 +2798,13 @@ function parseMarkdownImpl(// --- Ordered list ---if (/^ {0,9}\d+[.)] /.test(line)) {- const listResult = parseList(lines, index, true, opts);+ const listResult = parseList(+ lines,+ index,+ true,+ opts,+ interruptsWithBlockExtension,+ );pushBlock(listResult.node);index = listResult.nextIndex;continue;@@ -2694,21 +2867,8 @@ function parseMarkdownImpl() {break;}- if (hasBlockExtensionSyntax) {- const extensionColumn = blockExtensionColumn(nextLine);- if (extensionColumn != null) {- const extensionOffset = lineOffsets[index] + extensionColumn;- const extension = matchExtensionSyntax(- cleaned,- extensionOffset,- 'block',- opts,- );- if (extension.status !== 'none') {- blockExtensionMatches.set(extensionOffset, extension);- break;- }- }+ if (interruptsWithBlockExtension(index)) {+ break;}paraLines.push(nextLine);index++;