An open source design system that's fully customizable and agent ready
TypeScript
fix(Markdown): keep two lists apart while streaming when their bullets or indentation differ (#7115)
Cindy Zhang28ea18bda53e
diff --git a/.changeset/markdown-streaming-list-shapes.md b/.changeset/markdown-streaming-list-shapes.mdnew file mode 100644index 0000000000..9222f3b6dc--- /dev/null+++ b/.changeset/markdown-streaming-list-shapes.md@@ -0,0 +1,8 @@+---+'@astryxdesign/core': patch+---++[fix] Markdown: keep two lists apart while streaming when their bullets or indentation differ+@cixzhang++While a message streamed, `Markdown` joined two lists a blank line apart whenever both were bulleted, or both numbered with the same delimiter — even when their bullets (`-` then `*`) or indentation differed, where the finished document shows two lists. The streamed render now keeps them apart, as the finished document does.diff --git a/packages/core/src/Markdown/parser.ts b/packages/core/src/Markdown/parser.tsindex 941aa92d84..996150cc6e 100644--- a/packages/core/src/Markdown/parser.ts+++ b/packages/core/src/Markdown/parser.ts@@ -3474,6 +3474,7 @@ function parseList(spread: loose || undefined,children: items,};+ listMarkerShapes.set(node, `${baseIndent}:${ordered ? delim : bullet}`);return {node, nextIndex: index};}@@ -3981,13 +3982,13 @@ function stampSourceRanges(// range that dropped it would slice to something that re-parses// differently.const end = lineStart(endLine) + lines[endLine].length;- blocks[i] = {+ blocks[i] = withMarkerShapeOf(blocks[i], {...blocks[i],position: {start: {offset: lineStart(startLine)},end: {offset: end},},- };+ });}}@@ -4562,6 +4563,48 @@ function atOffset(opts: ResolvedOptions, offset: number): ResolvedOptions {return opts.astPositions === true ? {...opts, baseOffset: offset} : opts;}+/**+ * Each list the block parser builds, keyed to its markers: their indentation+ * and the bullet or delimiter. The full parse continues a list only with+ * items at the same indentation with the same bullet or delimiter.+ */+const listMarkerShapes = new WeakMap<object, string>();++/**+ * Whether the full parse reads `next`, a blank line after `previous`, as more+ * of the same list: both numbered or both bulleted, with the same delimiter,+ * and — for lists the block parser built — markers at the same indentation+ * with the same bullet. So `- a⏎⏎* b` and ` 1. a⏎⏎2. b` stay two lists.+ */+function continuesList(+ previous: MarkdownAstList<RuntimeExtensionNode>,+ next: MarkdownAstList<RuntimeExtensionNode>,+): boolean {+ if (+ previous.ordered !== next.ordered ||+ previous.delimiter !== next.delimiter+ ) {+ return false;+ }+ const previousShape = listMarkerShapes.get(previous);+ const nextShape = listMarkerShapes.get(next);+ return (+ previousShape == null || nextShape == null || previousShape === nextShape+ );+}++/** `merged`, carrying the marker shape of the list it continues. */+function withMarkerShapeOf<Node extends object>(+ list: object,+ merged: Node,+): Node {+ const shape = listMarkerShapes.get(list);+ if (shape != null) {+ listMarkerShapes.set(merged, shape);+ }+ return merged;+}+/*** Concatenate freshly-parsed delta blocks with previously-settled blocks,* merging adjacent same-style lists into a single loose list. The boundary@@ -4581,8 +4624,7 @@ function mergeSettledBlocks(if (prevLast.type === 'list' &&deltaFirst.type === 'list' &&- prevLast.ordered === deltaFirst.ordered &&- prevLast.delimiter === deltaFirst.delimiter+ continuesList(prevLast, deltaFirst)) {const merged: MarkdownAstBlockContent<RuntimeExtensionNode> = {type: 'list',@@ -4601,7 +4643,11 @@ function mergeSettledBlocks(}: null),};- return [...prev.slice(0, -1), merged, ...delta.slice(1)];+ return [+ ...prev.slice(0, -1),+ withMarkerShapeOf(prevLast, merged),+ ...delta.slice(1),+ ];}return [...prev, ...delta];}@@ -4623,10 +4669,9 @@ function appendSettledBlocks(if (prevLast.type === 'list' &&deltaFirst.type === 'list' &&- prevLast.ordered === deltaFirst.ordered &&- prevLast.delimiter === deltaFirst.delimiter+ continuesList(prevLast, deltaFirst)) {- prev[prev.length - 1] = {+ prev[prev.length - 1] = withMarkerShapeOf(prevLast, {type: 'list',ordered: prevLast.ordered,start: prevLast.start,@@ -4641,7 +4686,7 @@ function appendSettledBlocks(},}: null),- };+ });prev.push(...delta.slice(1));return true;}diff --git a/packages/core/src/Markdown/streamingListShapes.test.tsx b/packages/core/src/Markdown/streamingListShapes.test.tsxnew file mode 100644index 0000000000..b6ced703b8--- /dev/null+++ b/packages/core/src/Markdown/streamingListShapes.test.tsx@@ -0,0 +1,93 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++import {describe, expect, it} from 'vitest';+import {+ createIncrementalState,+ parseMarkdown,+ parseMarkdownIncremental,+} from './parser';++/** The streamed parse of `markdown` in chunks of the given sizes, cycled. */+function streamInChunks(+ markdown: string,+ sizes: ReadonlyArray<number>,+ sourceRanges = false,+) {+ const state = createIncrementalState();+ let result = parseMarkdownIncremental('', state, {sourceRanges});+ let end = 0;+ for (let step = 0; end < markdown.length; step++) {+ end = Math.min(markdown.length, end + sizes[step % sizes.length]);+ result = parseMarkdownIncremental(markdown.slice(0, end), state, {+ sourceRanges,+ });+ }+ return result;+}++const listCount = (blocks: ReadonlyArray<{type: string}>) =>+ blocks.filter(block => block.type === 'list').length;++describe('lists a blank line apart, streamed', () => {+ it.each([+ ['a bullet change', '- a\n\n* b', 2],+ ['another bullet change', '* a\n\n- b', 2],+ ['two bullet changes', '- a\n\n+ b\n\n- c', 3],+ ['numbered items at another indentation', ' 1. a\n\n2. b', 2],+ ['bullets at another indentation', ' - a\n\n- b', 2],+ ['one list, then a bullet change', '- a\n\n- b\n\n* c', 2],+ ['one loose bulleted list', '- a\n\n- b\n\n- c', 1],+ ['one loose numbered list', '1. a\n\n3. b\n\n4. c', 1],+ ])('streams %s as the full parse reads it', (_, markdown, lists) => {+ expect(listCount(parseMarkdown(markdown))).toBe(lists);+ for (const sourceRanges of [false, true]) {+ expect(streamInChunks(markdown, [1], sourceRanges)).toEqual(+ parseMarkdown(markdown, {sourceRanges}),+ );+ }+ });++ it('streams 600 answers mixing list styles to the full parse', () => {+ let state = 5656;+ const random = () => {+ state = (state * 1103515245 + 12345) % 2147483648;+ return state / 2147483648;+ };+ const pick = <T,>(choices: ReadonlyArray<T>): T =>+ choices[Math.floor(random() * choices.length)];+ const answer = () => {+ const lines: string[] = [];+ const items = 2 + Math.floor(random() * 6);+ for (let item = 0; item < items; item++) {+ const indent = ' '.repeat(pick([0, 0, 0, 1, 2]));+ const marker = pick(['- ', '* ', '+ ', '1. ', '2) ', '3. ']);+ lines.push(`${indent}${marker}item ${item}`);+ if (random() < 0.2) {+ lines.push(`${indent}${' '.repeat(marker.length)}more`);+ }+ if (random() < 0.7) {+ lines.push('');+ }+ if (random() < 0.15) {+ lines.push('A paragraph.', '');+ }+ }+ return lines.join('\n').trimEnd();+ };+ const diverged: string[] = [];+ for (let round = 0; round < 600; round++) {+ const markdown = answer();+ const sizes =+ round % 10 === 0+ ? [1]+ : [1 + Math.floor(random() * 12), 1 + Math.floor(random() * 5)];+ if (+ JSON.stringify(streamInChunks(markdown, sizes)) !==+ JSON.stringify(parseMarkdown(markdown))+ ) {+ diverged.push(markdown);+ }+ }+ expect(diverged).toEqual([]);+ });+});