An open source design system that's fully customizable and agent ready
TypeScript
fix(Markdown, richtext): end a code block left open at the end on its last line of code (#7126)
Cindy Zhang9430d22578c0
diff --git a/.changeset/markdown-open-fence-final-newline.md b/.changeset/markdown-open-fence-final-newline.mdnew file mode 100644index 0000000000..b589c8b4cb--- /dev/null+++ b/.changeset/markdown-open-fence-final-newline.md@@ -0,0 +1,8 @@+---+'@astryxdesign/core': patch+---++[fix] Markdown: end a code block left open at the end of a message on its last line of code+@cixzhang++When a message ended inside a code block that was never closed, `Markdown` read the message's final line ending as one more, empty line of code, unlike a closed code block and unlike the streamed render of the same message. A real blank line before the end is still code, and while a message streams, an open code block now shows the blank lines already written instead of adding them later.diff --git a/packages/core/src/Markdown/openFenceFinalNewline.test.tsx b/packages/core/src/Markdown/openFenceFinalNewline.test.tsxnew file mode 100644index 0000000000..644caf1c75--- /dev/null+++ b/packages/core/src/Markdown/openFenceFinalNewline.test.tsx@@ -0,0 +1,63 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++import {describe, expect, it} from 'vitest';+import {+ createIncrementalState,+ parseMarkdown,+ parseMarkdownAst,+ parseMarkdownIncremental,+} from './parser';++type Json = {+ type: string;+ value?: string;+ position?: {end: {offset: number}};+ children?: Json[];+};++const codeBlocks = (markdown: string, sourceRanges = false) =>+ (+ parseMarkdownAst(markdown, {sourceRanges}).children as unknown as Json[]+ ).filter(node => node.type === 'code');++const CASES: ReadonlyArray<[string, string, string]> = [+ ['one line, then the final newline', '```js\ncode\n', 'code'],+ ['a blank line before the final newline', '```js\ncode\n\n', 'code\n'],+ ['only the opening line and its newline', '```js\n', ''],+ ['blank lines inside', 'a\n\n```\nx\n\ny\n', 'x\n\ny'],+ ['no final newline, unchanged', '```js\ncode', 'code'],+ ['a closed fence, unchanged', '```js\ncode\n```\n', 'code'],+];++describe('a fence left open at the end keeps its code, not the final line ending', () => {+ it.each(CASES)('reads %s', (_, markdown, value) => {+ expect(codeBlocks(markdown).map(node => node.value)).toEqual([value]);+ });++ it.each(CASES)(+ 'streams %s to the full parse at every character',+ (_, markdown) => {+ for (const sourceRanges of [false, true]) {+ const state = createIncrementalState();+ let streamed = parseMarkdownIncremental('', state, {sourceRanges});+ for (let end = 1; end <= markdown.length; end++) {+ streamed = parseMarkdownIncremental(markdown.slice(0, end), state, {+ sourceRanges,+ });+ }+ expect(streamed).toEqual(parseMarkdown(markdown, {sourceRanges}));+ }+ },+ );++ it.each(CASES)(+ 'gives %s a source range that re-parses to the same code',+ (_, markdown) => {+ const [block] = parseMarkdown(markdown, {sourceRanges: true});+ const {range, ...node} = block;+ expect(range).toBeDefined();+ const sliced = markdown.slice(range!.start, range!.end);+ expect(parseMarkdown(sliced)).toEqual([node]);+ },+ );+});diff --git a/packages/core/src/Markdown/parser.ts b/packages/core/src/Markdown/parser.tsindex 6ef7f39daf..a326d916fd 100644--- a/packages/core/src/Markdown/parser.ts+++ b/packages/core/src/Markdown/parser.ts@@ -3695,6 +3695,13 @@ function parseMarkdownImpl(codeLines.push(lines[index].replace(fenceIndentation, ''));index++;}+ const closed = index < lines.length;+ // A fence left open runs to the end of the input, whose final line+ // ending starts no line of code (CommonMark 0.31 §4.5) — as a closed+ // fence's code ends before its closing line.+ if (!closed && codeLines.length > 0 && lines[lines.length - 1] === '') {+ codeLines.pop();+ }index++; // skip closing fence// A fence owns its blank lines, and an unterminated one (mid-stream)// can end on them, so it states its own end rather than letting the@@ -4963,9 +4970,10 @@ function parseMarkdownIncrementalAstBlocks(: trimmedUnsettledInput;// Structural trimming holds back lines that look like an incomplete list or// table, which inside a fence is ordinary code: a TypeScript union or a `- `- // would disappear from the code block as it streams.+ // would disappear from the code block as it streams. And an open fence's+ // trailing blank lines are code, as in a full parse of the same text.const unsettledText = openFence- ? unsettledRaw+ ? unsettledInput.replace(/^(?:[ \t]*\r?\n)+/, ''): openMath? trimOpenDisplayMath(unsettledRaw): trimUnsettledStructural(unsettledRaw);diff --git a/packages/richtext/src/markdownCodeFence.ts b/packages/richtext/src/markdownCodeFence.tsindex 59f386475d..e7bce1df16 100644--- a/packages/richtext/src/markdownCodeFence.ts+++ b/packages/richtext/src/markdownCodeFence.ts@@ -119,6 +119,12 @@ function $importFencedCode(while (end < lines.length && !closing.test(lines[end])) {end++;}+ // A fence left open runs to the end of the document, whose final line+ // ending starts no line of code (CommonMark 0.31 §4.5), as core reads it.+ const codeEnd =+ end === lines.length && end > startLineIndex + 1 && lines[end - 1] === ''+ ? end - 1+ : end;// The code is the lines between the fences, exactly: Lexical's own// transformer trims a space from the first line and drops blank first and// last lines, adjustments meant for its own reading of the opening line.@@ -126,7 +132,7 @@ function $importFencedCode(block.append($createTextNode(lines- .slice(startLineIndex + 1, end)+ .slice(startLineIndex + 1, codeEnd).map(line => line.replace(fenceIndentation, '')).join('\n'),),diff --git a/packages/richtext/src/markdownOpenFence.test.ts b/packages/richtext/src/markdownOpenFence.test.tsnew file mode 100644index 0000000000..344e8f8bb6--- /dev/null+++ b/packages/richtext/src/markdownOpenFence.test.ts@@ -0,0 +1,35 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++import {describe, expect, it} from 'vitest';+import {parseMarkdownAst} from '@astryxdesign/core/Markdown/parser';+import {markdownToEditorStateJSON} from './markdownSerializers';++type Json = {type: string; text?: string; value?: string; children?: Json[]};++const textOf = (node: Json): string =>+ node.type === 'linebreak'+ ? '\n'+ : (node.text ?? (node.children ?? []).map(textOf).join(''));++describe('a fence left open at the end imports its code, as core reads it', () => {+ it.each([+ ['```js\ncode\n', 'code'],+ ['```js\ncode\n\n', 'code\n'],+ ['a\n\n```\nx\n\ny\n', 'x\n\ny'],+ ['```js\ncode', 'code'],+ ])('imports %j with code %j', (markdown, value) => {+ const root = (+ JSON.parse(markdownToEditorStateJSON(markdown)) as {+ root: {children: Json[]};+ }+ ).root.children;+ expect(+ root.filter(node => node.type === 'code').map(node => textOf(node)),+ ).toEqual([value]);+ expect(+ (parseMarkdownAst(markdown).children as unknown as Json[])+ .filter(node => node.type === 'code')+ .map(node => node.value),+ ).toEqual([value]);+ });+});