Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

fix(Markdown): keep two lists apart while streaming when their bullets or indentation differ (#7115)

Cindy Zhang28ea18bda53e
diff --git a/.changeset/markdown-streaming-list-shapes.md b/.changeset/markdown-streaming-list-shapes.md
new file mode 100644
index 0000000000..9222f3b6dc
--- /dev/null
+++ b/.changeset/markdown-streaming-list-shapes.md
@@ -0,0 +1,8 @@
+---
+'@astryxdesign/core': patch
+---
+
+[fix] Markdown: keep two lists apart while streaming when their bullets or indentation differ
+@cixzhang
+
+While a message streamed, `Markdown` joined two lists a blank line apart whenever both were bulleted, or both numbered with the same delimiter — even when their bullets (`-` then `*`) or indentation differed, where the finished document shows two lists. The streamed render now keeps them apart, as the finished document does.
diff --git a/packages/core/src/Markdown/parser.ts b/packages/core/src/Markdown/parser.ts
index 941aa92d84..996150cc6e 100644
--- a/packages/core/src/Markdown/parser.ts
+++ b/packages/core/src/Markdown/parser.ts
@@ -3474,6 +3474,7 @@ function parseList(
spread: loose || undefined,
children: items,
};
+ listMarkerShapes.set(node, `${baseIndent}:${ordered ? delim : bullet}`);
return {node, nextIndex: index};
}
@@ -3981,13 +3982,13 @@ function stampSourceRanges(
// range that dropped it would slice to something that re-parses
// differently.
const end = lineStart(endLine) + lines[endLine].length;
- blocks[i] = {
+ blocks[i] = withMarkerShapeOf(blocks[i], {
...blocks[i],
position: {
start: {offset: lineStart(startLine)},
end: {offset: end},
},
- };
+ });
}
}
@@ -4562,6 +4563,48 @@ function atOffset(opts: ResolvedOptions, offset: number): ResolvedOptions {
return opts.astPositions === true ? {...opts, baseOffset: offset} : opts;
}
+/**
+ * Each list the block parser builds, keyed to its markers: their indentation
+ * and the bullet or delimiter. The full parse continues a list only with
+ * items at the same indentation with the same bullet or delimiter.
+ */
+const listMarkerShapes = new WeakMap<object, string>();
+
+/**
+ * Whether the full parse reads `next`, a blank line after `previous`, as more
+ * of the same list: both numbered or both bulleted, with the same delimiter,
+ * and — for lists the block parser built — markers at the same indentation
+ * with the same bullet. So `- a⏎⏎* b` and ` 1. a⏎⏎2. b` stay two lists.
+ */
+function continuesList(
+ previous: MarkdownAstList<RuntimeExtensionNode>,
+ next: MarkdownAstList<RuntimeExtensionNode>,
+): boolean {
+ if (
+ previous.ordered !== next.ordered ||
+ previous.delimiter !== next.delimiter
+ ) {
+ return false;
+ }
+ const previousShape = listMarkerShapes.get(previous);
+ const nextShape = listMarkerShapes.get(next);
+ return (
+ previousShape == null || nextShape == null || previousShape === nextShape
+ );
+}
+
+/** `merged`, carrying the marker shape of the list it continues. */
+function withMarkerShapeOf<Node extends object>(
+ list: object,
+ merged: Node,
+): Node {
+ const shape = listMarkerShapes.get(list);
+ if (shape != null) {
+ listMarkerShapes.set(merged, shape);
+ }
+ return merged;
+}
+
/**
* Concatenate freshly-parsed delta blocks with previously-settled blocks,
* merging adjacent same-style lists into a single loose list. The boundary
@@ -4581,8 +4624,7 @@ function mergeSettledBlocks(
if (
prevLast.type === 'list' &&
deltaFirst.type === 'list' &&
- prevLast.ordered === deltaFirst.ordered &&
- prevLast.delimiter === deltaFirst.delimiter
+ continuesList(prevLast, deltaFirst)
) {
const merged: MarkdownAstBlockContent<RuntimeExtensionNode> = {
type: 'list',
@@ -4601,7 +4643,11 @@ function mergeSettledBlocks(
}
: null),
};
- return [...prev.slice(0, -1), merged, ...delta.slice(1)];
+ return [
+ ...prev.slice(0, -1),
+ withMarkerShapeOf(prevLast, merged),
+ ...delta.slice(1),
+ ];
}
return [...prev, ...delta];
}
@@ -4623,10 +4669,9 @@ function appendSettledBlocks(
if (
prevLast.type === 'list' &&
deltaFirst.type === 'list' &&
- prevLast.ordered === deltaFirst.ordered &&
- prevLast.delimiter === deltaFirst.delimiter
+ continuesList(prevLast, deltaFirst)
) {
- prev[prev.length - 1] = {
+ prev[prev.length - 1] = withMarkerShapeOf(prevLast, {
type: 'list',
ordered: prevLast.ordered,
start: prevLast.start,
@@ -4641,7 +4686,7 @@ function appendSettledBlocks(
},
}
: null),
- };
+ });
prev.push(...delta.slice(1));
return true;
}
diff --git a/packages/core/src/Markdown/streamingListShapes.test.tsx b/packages/core/src/Markdown/streamingListShapes.test.tsx
new file mode 100644
index 0000000000..b6ced703b8
--- /dev/null
+++ b/packages/core/src/Markdown/streamingListShapes.test.tsx
@@ -0,0 +1,93 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+import {describe, expect, it} from 'vitest';
+import {
+ createIncrementalState,
+ parseMarkdown,
+ parseMarkdownIncremental,
+} from './parser';
+
+/** The streamed parse of `markdown` in chunks of the given sizes, cycled. */
+function streamInChunks(
+ markdown: string,
+ sizes: ReadonlyArray<number>,
+ sourceRanges = false,
+) {
+ const state = createIncrementalState();
+ let result = parseMarkdownIncremental('', state, {sourceRanges});
+ let end = 0;
+ for (let step = 0; end < markdown.length; step++) {
+ end = Math.min(markdown.length, end + sizes[step % sizes.length]);
+ result = parseMarkdownIncremental(markdown.slice(0, end), state, {
+ sourceRanges,
+ });
+ }
+ return result;
+}
+
+const listCount = (blocks: ReadonlyArray<{type: string}>) =>
+ blocks.filter(block => block.type === 'list').length;
+
+describe('lists a blank line apart, streamed', () => {
+ it.each([
+ ['a bullet change', '- a\n\n* b', 2],
+ ['another bullet change', '* a\n\n- b', 2],
+ ['two bullet changes', '- a\n\n+ b\n\n- c', 3],
+ ['numbered items at another indentation', ' 1. a\n\n2. b', 2],
+ ['bullets at another indentation', ' - a\n\n- b', 2],
+ ['one list, then a bullet change', '- a\n\n- b\n\n* c', 2],
+ ['one loose bulleted list', '- a\n\n- b\n\n- c', 1],
+ ['one loose numbered list', '1. a\n\n3. b\n\n4. c', 1],
+ ])('streams %s as the full parse reads it', (_, markdown, lists) => {
+ expect(listCount(parseMarkdown(markdown))).toBe(lists);
+ for (const sourceRanges of [false, true]) {
+ expect(streamInChunks(markdown, [1], sourceRanges)).toEqual(
+ parseMarkdown(markdown, {sourceRanges}),
+ );
+ }
+ });
+
+ it('streams 600 answers mixing list styles to the full parse', () => {
+ let state = 5656;
+ const random = () => {
+ state = (state * 1103515245 + 12345) % 2147483648;
+ return state / 2147483648;
+ };
+ const pick = <T,>(choices: ReadonlyArray<T>): T =>
+ choices[Math.floor(random() * choices.length)];
+ const answer = () => {
+ const lines: string[] = [];
+ const items = 2 + Math.floor(random() * 6);
+ for (let item = 0; item < items; item++) {
+ const indent = ' '.repeat(pick([0, 0, 0, 1, 2]));
+ const marker = pick(['- ', '* ', '+ ', '1. ', '2) ', '3. ']);
+ lines.push(`${indent}${marker}item ${item}`);
+ if (random() < 0.2) {
+ lines.push(`${indent}${' '.repeat(marker.length)}more`);
+ }
+ if (random() < 0.7) {
+ lines.push('');
+ }
+ if (random() < 0.15) {
+ lines.push('A paragraph.', '');
+ }
+ }
+ return lines.join('\n').trimEnd();
+ };
+ const diverged: string[] = [];
+ for (let round = 0; round < 600; round++) {
+ const markdown = answer();
+ const sizes =
+ round % 10 === 0
+ ? [1]
+ : [1 + Math.floor(random() * 12), 1 + Math.floor(random() * 5)];
+ if (
+ JSON.stringify(streamInChunks(markdown, sizes)) !==
+ JSON.stringify(parseMarkdown(markdown))
+ ) {
+ diverged.push(markdown);
+ }
+ }
+ expect(diverged).toEqual([]);
+ });
+});