An open source design system that's fully customizable and agent ready
TypeScript
fix(Markdown, richtext): read a code fence indented up to three spaces as CommonMark does (#7111)
Cindy Zhang3a16495ea014
diff --git a/.changeset/markdown-indented-fences.md b/.changeset/markdown-indented-fences.mdnew file mode 100644index 0000000000..c0c6872d54--- /dev/null+++ b/.changeset/markdown-indented-fences.md@@ -0,0 +1,8 @@+---+'@astryxdesign/core': patch+---++[fix] Markdown: read a code fence indented up to three spaces as CommonMark does+@cixzhang++`Markdown` now reads a code fence indented by up to three spaces as a code block, and a closing fence may be indented the same way, as CommonMark specifies. Each code line loses as much indentation as the opening fence has. A closing fence has only spaces or tabs after it, so a line such as ` ```js ` inside an open block stays part of the code. Such fences used to show as raw backticks or tildes in a paragraph, and an indented closing fence left the block open to the end of the document.diff --git a/packages/core/src/Markdown/indentedFences.test.tsx b/packages/core/src/Markdown/indentedFences.test.tsxnew file mode 100644index 0000000000..354c5b63df--- /dev/null+++ b/packages/core/src/Markdown/indentedFences.test.tsx@@ -0,0 +1,374 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++import {describe, expect, it} from 'vitest';+import {+ createIncrementalState,+ parseMarkdown,+ parseMarkdownAst,+ parseMarkdownIncremental,+} from './parser';++type Json = {+ type: string;+ value?: string;+ lang?: string | null;+ children?: Json[];+};++/** The document's blocks as compact markup. */+function blocks(markdown: string): string {+ const show = (node: Json): string =>+ node.type === 'text'+ ? JSON.stringify(node.value)+ : node.type === 'code'+ ? `code(${node.lang ?? ''})${JSON.stringify(node.value)}`+ : node.children == null+ ? node.type+ : `${node.type}[${node.children.map(show).join(', ')}]`;+ return (parseMarkdownAst(markdown).children as unknown as Json[])+ .map(show)+ .join(' | ');+}++describe('a code fence may be indented up to three spaces (CommonMark 0.31 §4.5)', () => {+ it.each([+ [+ 'a backtick fence indented two spaces, its code losing two',+ ' ```js\n x\n y\n ```',+ 'code(js)"x\\n y"',+ ],+ ['a backtick fence indented one space', ' ```\na\n ```', 'code()"a"'],+ [+ 'a tilde fence indented three spaces',+ ' ~~~\n b\n ~~~',+ 'code()"b"',+ ],+ [+ 'an indented closing fence',+ '```\ncode\n ```\nafter',+ 'code()"code" | paragraph["after"]',+ ],+ [+ 'an indented fence interrupting a paragraph',+ 'Text\n ```\n code\n ```',+ 'paragraph["Text"] | code()"code"',+ ],+ [+ 'a fence in a list item, unchanged',+ '1. Step\n ```bash\n cmd\n ```',+ 'list[listItem[paragraph["Step"], code(bash)"cmd"]]',+ ],+ ])('reads %s', (_, markdown, expected) => {+ expect(blocks(markdown)).toBe(expected);+ });++ it('keeps a line indented four spaces inside the block, not as its closer', () => {+ expect(blocks('```\na\n ```\nb\n```')).toBe('code()"a\\n ```\\nb"');+ });++ it('reads a fence indented four spaces as no fence', () => {+ expect(blocks(' ```\n x\n ```')).not.toContain('code(');+ });++ it('collects no link definition inside an indented fence', () => {+ expect(blocks(' ```\n[x]: /u\n ```\n\n[x]')).toBe(+ 'code()"[x]: /u" | paragraph["[x]"]',+ );+ });++ it('streams indented fences with blank lines to the full parse at every character', () => {+ // `d` sits at the margin after a blank line inside a fence: that blank+ // line settles nothing, since the fence is still open.+ const markdown =+ 'a\n\n ```js\n x\n\n y\n ```\n\nb\n\n ~~~\n\n c\n\nd\n ~~~\n';+ for (const sourceRanges of [false, true]) {+ const state = createIncrementalState();+ let streamed = parseMarkdownIncremental('', state, {sourceRanges});+ for (let end = 1; end <= markdown.length; end++) {+ streamed = parseMarkdownIncremental(markdown.slice(0, end), state, {+ sourceRanges,+ });+ }+ expect(streamed).toEqual(parseMarkdown(markdown, {sourceRanges}));+ }+ });+});++/** The streamed parse of `markdown` in chunks of the given sizes, cycled. */+function streamInChunks(+ markdown: string,+ sizes: ReadonlyArray<number>,+ sourceRanges = false,+) {+ const state = createIncrementalState();+ let result = parseMarkdownIncremental('', state, {sourceRanges});+ let end = 0;+ for (let step = 0; end < markdown.length; step++) {+ end = Math.min(markdown.length, end + sizes[step % sizes.length]);+ result = parseMarkdownIncremental(markdown.slice(0, end), state, {+ sourceRanges,+ });+ }+ return result;+}++const STEP_CLOSED_AT_MARGIN = [+ '1. Install:',+ ' ```bash',+ ' npm i',+ '```',+ '',+ '2. Run:',+ ' ```bash',+ ' npm test',+ ' ```',+ '',+ 'Done.',+].join('\n');++describe('a fence inside a list step stays the step’s', () => {+ it('streams a step’s fence closed at the margin to the full parse at every character', () => {+ for (const sourceRanges of [false, true]) {+ expect(streamInChunks(STEP_CLOSED_AT_MARGIN, [1], sourceRanges)).toEqual(+ parseMarkdown(STEP_CLOSED_AT_MARGIN, {sourceRanges}),+ );+ }+ });++ it('reads the margin line after a step’s fence as the opening of a fence of its own', () => {+ expect(blocks('1. Step\n ```\n code\n```\nafter')).toBe(+ 'list[listItem[paragraph["Step"], code()"code"]] | code()"after"',+ );+ });++ it('keeps a definition-shaped line inside that margin fence as code', () => {+ expect(blocks('1. Step\n ```\n code\n```\n[x]: /u\n\n[x]')).toBe(+ 'list[listItem[paragraph["Step"], code()"code"]] | code()"[x]: /u\\n\\n[x]"',+ );+ });++ it('streams 600 step-by-step answers to the full parse, closers at the margin included', () => {+ let state = 7111;+ const random = () => {+ state = (state * 1103515245 + 12345) % 2147483648;+ return state / 2147483648;+ };+ const pick = <T,>(choices: ReadonlyArray<T>): T =>+ choices[Math.floor(random() * choices.length)];+ const answer = () => {+ const lines: string[] = [];+ if (random() < 0.5) {+ lines.push('Here are the steps:', '');+ }+ const steps = 1 + Math.floor(random() * 4);+ for (let step = 1; step <= steps; step++) {+ const marker = random() < 0.7 ? `${step}. ` : '- ';+ lines.push(`${marker}Step ${step}:`);+ if (random() < 0.3) {+ lines.push(`${' '.repeat(marker.length)}Some detail.`);+ }+ if (random() < 0.4) {+ lines.push('');+ }+ const indent = pick([0, 1, 2, 3, marker.length, marker.length + 1]);+ const fence = random() < 0.85 ? '```' : '~~~';+ lines.push(`${' '.repeat(indent)}${fence}${pick(['', 'bash', 'js'])}`);+ const codeLines = 1 + Math.floor(random() * 3);+ for (let line = 0; line < codeLines; line++) {+ lines.push(+ random() < 0.15+ ? ''+ : `${' '.repeat(pick([0, indent, indent + 2]))}code ${line}`,+ );+ }+ const closer = random();+ if (closer < 0.4) {+ lines.push(fence);+ } else if (closer < 0.9) {+ lines.push(`${' '.repeat(indent)}${fence}`);+ }+ if (random() < 0.6) {+ lines.push('');+ }+ }+ if (random() < 0.5) {+ lines.push('That is all.');+ }+ // A fence left open at the very end with a newline after it keeps a+ // final empty line in the full parse only — a separate difference.+ return lines.join('\n').trimEnd();+ };+ const diverged: string[] = [];+ for (let round = 0; round < 600; round++) {+ const markdown = answer();+ const sizes = [+ 1 + Math.floor(random() * 12),+ 1 + Math.floor(random() * 5),+ ];+ const full = parseMarkdown(markdown);+ if (+ JSON.stringify(streamInChunks(markdown, sizes)) !== JSON.stringify(full)+ ) {+ diverged.push(markdown);+ }+ }+ expect(diverged).toEqual([]);+ });+});++describe('fences around list steps, streamed', () => {+ it('streams 600 varied answers to the full parse: sub-steps, blank lines in fences, text after a step’s fence', () => {+ let state = 424242;+ const random = () => {+ state = (state * 1103515245 + 12345) % 2147483648;+ return state / 2147483648;+ };+ const pick = <T,>(choices: ReadonlyArray<T>): T =>+ choices[Math.floor(random() * choices.length)];+ const pushFence = (indent: number, lines: string[]) => {+ const fence = pick(['```', '```', '~~~', '````']);+ lines.push(`${' '.repeat(indent)}${fence}${pick(['', 'bash', ' py'])}`);+ const codeLines = Math.floor(random() * 4);+ for (let line = 0; line < codeLines; line++) {+ lines.push(+ random() < 0.2+ ? ''+ : `${' '.repeat(pick([0, indent, indent + 1, indent + 4]))}${pick(['code', '[x]: /u', '- item', '# title', '> q'])} ${line}`,+ );+ }+ const closer = random();+ if (closer < 0.35) {+ lines.push(fence);+ } else if (closer < 0.75) {+ lines.push(`${' '.repeat(indent)}${fence}`);+ } else if (closer < 0.85) {+ lines.push(`${' '.repeat(indent)}${fence}js`);+ } else if (closer < 0.9) {+ lines.push(`${' '.repeat(indent + 1)}${fence} `);+ }+ };+ const answer = () => {+ const lines: string[] = [];+ if (random() < 0.4) {+ lines.push(pick(['Steps:', 'Here you go']));+ if (random() < 0.6) {+ lines.push('');+ }+ }+ // Every list at the margin with a `-` bullet, as lines escaping a step's+ // fence are: two lists of different styles, a blank line apart, are a+ // separate case of the settled-list merge.+ const bullet = pick(['- ', '- ']);+ const delimiter = pick(['.', ')']);+ const base = 0;+ let number = pick([1, 1, 2]);+ const steps = 1 + Math.floor(random() * 5);+ for (let step = 0; step < steps; step++) {+ const marker = random() < 0.65 ? `${number++}${delimiter} ` : bullet;+ lines.push(`${' '.repeat(base)}${marker}Step ${step}`);+ const content = base + marker.length;+ const parts = Math.floor(random() * 3);+ for (let part = 0; part < parts; part++) {+ const roll = random();+ if (roll < 0.3) {+ lines.push('');+ }+ // A sub-step directly under its step: placed after the step has+ // ended, it would start a list of its own at another indentation.+ if (roll < 0.15 && part === 0) {+ lines.push(`${' '.repeat(content)}- sub ${part}`);+ } else if (roll < 0.45) {+ pushFence(pick([0, 1, 2, 3, content, content + 1]), lines);+ } else if (roll < 0.6) {+ lines.push(+ pick(['lazy text', `${' '.repeat(content)}more`, 'Then:']),+ );+ } else if (roll < 0.7) {+ lines.push(`${' '.repeat(pick([0, content]))}# Heading`);+ }+ }+ if (random() < 0.5) {+ lines.push('');+ }+ if (random() < 0.2) {+ lines.push('Between steps.');+ if (random() < 0.5) {+ lines.push('');+ }+ }+ }+ if (random() < 0.3) {+ pushFence(pick([0, 1, 2, 3]), lines);+ }+ if (random() < 0.4) {+ lines.push('That is all.');+ }+ return lines.join('\n').trimEnd();+ };+ const diverged: string[] = [];+ for (let round = 0; round < 600; round++) {+ const markdown = answer();+ const sizes =+ round % 10 === 0+ ? [1]+ : [1 + Math.floor(random() * 12), 1 + Math.floor(random() * 5)];+ if (+ JSON.stringify(streamInChunks(markdown, sizes)) !==+ JSON.stringify(parseMarkdown(markdown))+ ) {+ diverged.push(markdown);+ }+ }+ expect(diverged).toEqual([]);+ });+});++describe('a closing fence has only spaces or tabs after it (CommonMark 0.31 §4.5)', () => {+ it.each([+ [+ 'a backtick line with an info string',+ '```\na\n ```js\nb\n```',+ 'code()"a\\n ```js\\nb"',+ ],+ [+ 'a tilde line with an info string',+ '~~~\na\n ~~~ note\nb\n~~~',+ 'code()"a\\n ~~~ note\\nb"',+ ],+ [+ 'a closer followed by spaces and a tab',+ '```\na\n``` \t\nb',+ 'code()"a" | paragraph["b"]',+ ],+ ])(+ 'keeps %s inside the open block, or closes on spaces',+ (_, markdown, expected) => {+ expect(blocks(markdown)).toBe(expected);+ },+ );++ const STEP_TWO_OPENS = [+ '1. Step one',+ ' ```bash',+ ' cmd',+ '```',+ '2. Step two',+ ' ```js',+ ' x',+ ' ```',+ ].join('\n');++ it('reads step 1 closed at the margin, then step 2 opening, as CommonMark does', () => {+ expect(blocks(STEP_TWO_OPENS)).toBe(+ 'list[listItem[paragraph["Step one"], code(bash)"cmd"]] | code()"2. Step two\\n ```js\\n x"',+ );+ });++ it('streams that answer to the full parse at every character', () => {+ for (const sourceRanges of [false, true]) {+ expect(streamInChunks(STEP_TWO_OPENS, [1], sourceRanges)).toEqual(+ parseMarkdown(STEP_TWO_OPENS, {sourceRanges}),+ );+ }+ });+});diff --git a/packages/core/src/Markdown/parser.ts b/packages/core/src/Markdown/parser.tsindex f80369edd7..941aa92d84 100644--- a/packages/core/src/Markdown/parser.ts+++ b/packages/core/src/Markdown/parser.ts@@ -849,6 +849,172 @@ function matchLinkDefinition(};}+/**+ * A code fence line: up to three spaces of indentation, then three or more+ * backticks or tildes (CommonMark 0.31 §4.5). Group 1 is the indentation and+ * group 2 the fence.+ */+const FENCE_LINE = /^( {0,3})(`{3,}|~{3,})/;++/**+ * Whether `line` closes a code block opened with `fence`: a fence of the same+ * character, at least as long, after up to three spaces of indentation, with+ * only spaces or tabs after it (CommonMark 0.31 §4.5) — so ` ```js` inside+ * an open block is code, not its end.+ */+function closesFence(line: string, fence: string): boolean {+ const match = /^ {0,3}(`{3,}|~{3,})[ \t]*\r?$/.exec(line);+ return (+ match != null &&+ match[1].startsWith(fence[0]) &&+ match[1].length >= fence.length+ );+}++/** Where a list item's lines sit: its marker's indentation and content. */+interface ListItemScope {+ /** The marker's indentation. */+ readonly base: number;+ /** Where the item's content starts: past the marker and its spaces. */+ readonly content: number;+ readonly ordered: boolean;+}++/**+ * The list item `line` opens, read as the block parser reads one: a bullet or+ * a number with `.` or `)` after up to nine spaces, then a space. A thematic+ * break opens no item.+ */+function listItemScopeOf(line: string): ListItemScope | null {+ if (isHorizontalRule(line)) {+ return null;+ }+ const marker = /^( {0,9})([-*+]|\d+[.)]) /.exec(line);+ if (marker == null) {+ return null;+ }+ const markerEnd = marker[0].length;+ const spacesAfter = getIndent(line.slice(markerEnd));+ return {+ base: marker[1].length,+ content: spacesAfter >= 4 ? markerEnd : markerEnd + spacesAfter,+ ordered: marker[2] !== '-' && marker[2] !== '*' && marker[2] !== '+',+ };+}++/** What a line does to the top-level code fence. */+type TopLevelFenceEvent = 'open' | 'close' | 'inside' | null;++/**+ * Follows the code fences that open at the top level of a document, a line at+ * a time, as the block parser reads them. A fence indented into an open list+ * item belongs to that item, whose own parse reads it: it neither opens nor+ * closes a top-level fence, so a list step's fence closed at the margin leaves+ * the margin line to open a fence of its own, as the full parse does. Scanners+ * that run outside the block parser — streaming settlement, link definitions,+ * display-math trimming — share it so they pair fences the same way.+ */+function topLevelFences(): {+ readonly open: boolean;+ read(lines: ReadonlyArray<string>, index: number): TopLevelFenceEvent;+} {+ let fence = '';+ // The outermost open list item, and what it last read. After a blank line+ // inside the item, the parser keeps only lines indented to its content.+ let item: ListItemScope | null = null;+ let itemHadBlank = false;+ let itemFence = '';+ let itemEndsInParagraph = false;+ // A number other than 1 cannot interrupt an open top-level paragraph.+ let paragraphOpen = false;++ const isHeadingOrBreak = (text: string) =>+ /^ {0,3}#{1,6}(?:[ \t]|$)/.test(text) || isHorizontalRule(text);++ /** Reads a line of the open item, without the item's indentation. */+ const readItemLine = (text: string) => {+ if (itemFence !== '') {+ if (closesFence(text, itemFence)) {+ itemFence = '';+ }+ itemEndsInParagraph = false;+ return;+ }+ const opening = FENCE_LINE.exec(text);+ if (opening != null) {+ itemFence = opening[2];+ itemEndsInParagraph = false;+ return;+ }+ itemEndsInParagraph = text.trim() !== '' && !isHeadingOrBreak(text);+ };++ return {+ get open() {+ return fence !== '';+ },+ read(lines, index) {+ const line = lines[index];+ if (fence !== '') {+ if (closesFence(line, fence)) {+ fence = '';+ return 'close';+ }+ return 'inside';+ }+ if (line.trim() === '') {+ itemHadBlank = item != null;+ paragraphOpen = false;+ return null;+ }+ const indent = getIndent(line);+ if (+ item != null &&+ (itemHadBlank ? indent >= item.content : indent > item.base)+ ) {+ readItemLine(line.slice(Math.min(indent, item.content)));+ return null;+ }+ const scope = listItemScopeOf(line);+ if (+ scope != null &&+ !(+ paragraphOpen &&+ item == null &&+ scope.ordered &&+ !/^ {0,9}0*1[.)]/.test(line)+ )+ ) {+ item = scope;+ itemHadBlank = false;+ itemFence = '';+ paragraphOpen = false;+ readItemLine(line.slice(scope.content));+ return null;+ }+ const opening = FENCE_LINE.exec(line);+ if (opening != null) {+ item = null;+ paragraphOpen = false;+ fence = opening[2];+ return 'open';+ }+ if (+ item != null &&+ !itemHadBlank &&+ itemEndsInParagraph &&+ canContinueParagraphLazily(lines, index)+ ) {+ // A lazy continuation line of the item's paragraph.+ return null;+ }+ item = null;+ paragraphOpen = !isHeadingOrBreak(line);+ return null;+ },+ };+}+/*** Collect link reference definitions from the whole document and return the* input with the definition lines removed. A definition is recognized at a@@ -883,20 +1049,13 @@ function extractLinkDefinitions(const defs = new Map<string, string>();const keep = new Array<boolean>(lines.length).fill(true);let atBoundary = true;- let inFence = false;- let fenceMarker = '';+ const fences = topLevelFences();for (let index = 0; index < lines.length; index++) {const line = lines[index];- if (inFence) {- if (line.startsWith(fenceMarker)) {- inFence = false;- fenceMarker = '';- // The line after a closed fence begins a new block.- atBoundary = true;- } else {- atBoundary = false;- }+ if (fences.open) {+ // The line after a closed fence begins a new block.+ atBoundary = fences.read(lines, index) === 'close';continue;}if (math) {@@ -909,10 +1068,7 @@ function extractLinkDefinitions(continue;}}- const fenceMatch = line.match(/^(`{3,}|~{3,})/);- if (fenceMatch) {- inFence = true;- fenceMarker = fenceMatch[1];+ if (fences.read(lines, index) === 'open') {atBoundary = false;continue;}@@ -2952,7 +3108,7 @@ function isBlockStart(line: string): boolean {if (/^ {0,3}#{1,6} /.test(line)) {return true;}- if (/^(`{3,}|~{3,})/.test(line)) {+ if (FENCE_LINE.test(line)) {return true;}if (isHorizontalRule(line)) {@@ -2989,7 +3145,7 @@ function canContinueParagraphLazily(}if (/^ {0,3}#{1,6} /.test(line) ||- /^(`{3,}|~{3,})/.test(line) ||+ FENCE_LINE.test(line) ||isHorizontalRule(line) ||QUOTE_MARKER.test(line) ||/^ {0,9}[-*+] /.test(line) ||@@ -3512,12 +3668,12 @@ function parseMarkdownImpl(}// --- Fenced code block ---- const fenceMatch = line.match(/^(`{3,}|~{3,})/);+ const fenceMatch = FENCE_LINE.exec(line);if (fenceMatch) {- const fence = fenceMatch[1];+ const [opening, indentation, fence] = fenceMatch;// The rest of the line is the info string; its first word, after any// spaces, is the language (CommonMark 0.31 §4.5).- const info = line.slice(fence.length).trim();+ const info = line.slice(opening.length).trim();const language = info.match(/^(\S+)/)?.[1] ?? null;const legacyLanguage = info.match(/^(\w*)/)?.[1] || null;const meta =@@ -3525,9 +3681,11 @@ function parseMarkdownImpl(? undefined: info.slice(language.length).trim() || undefined;const codeLines: string[] = [];+ // Each code line loses as much indentation as the opening fence has.+ const fenceIndentation = new RegExp(`^ {0,${indentation.length}}`);index++;- while (index < lines.length && !lines[index].startsWith(fence)) {- codeLines.push(lines[index]);+ while (index < lines.length && !closesFence(lines[index], fence)) {+ codeLines.push(lines[index].replace(fenceIndentation, ''));index++;}index++; // skip closing fence@@ -3995,8 +4153,7 @@ function findSettledBoundary(openFence: boolean;openMath: boolean;} {- let inFence = false;- let fenceMarker = '';+ const fences = topLevelFences();let mathContainer: DisplayMathContainer | null = null;let suppressMathUntilBoundary = false;let lastBoundary = -1;@@ -4006,16 +4163,8 @@ function findSettledBoundary(for (let lineIndex = 0; lineIndex < lines.length; lineIndex++) {const line = lines[lineIndex];- if (inFence) {- const fenceMatch = line.match(/^(`{3,}|~{3,})/);- if (- fenceMatch &&- fenceMatch[1].startsWith(fenceMarker[0]) &&- fenceMatch[1].length >= fenceMarker.length- ) {- inFence = false;- fenceMarker = '';- }+ if (fences.open) {+ fences.read(lines, lineIndex);continue;}@@ -4048,10 +4197,7 @@ function findSettledBoundary(}}- const fenceMatch = line.match(/^(`{3,}|~{3,})/);- if (fenceMatch) {- inFence = true;- fenceMarker = fenceMatch[1];+ if (fences.read(lines, lineIndex) === 'open') {boundaryBeforeFence = lastBoundary;continue;}@@ -4071,12 +4217,12 @@ function findSettledBoundary(}return {- boundary: inFence+ boundary: fences.open? boundaryBeforeFence: mathContainer != null? boundaryBeforeMath: lastBoundary,- openFence: inFence,+ openFence: fences.open,openMath: mathContainer != null,};}@@ -4300,24 +4446,15 @@ export function trimStreamingArtifacts(*/function trimOpenDisplayMath(text: string): string {const lines = text.split('\n');- let inFence = false;- let fenceMarker = '';+ const fences = topLevelFences();let mathContainer: DisplayMathContainer | null = null;let suppressMathUntilBoundary = false;let mathStartLine = -1;for (let index = 0; index < lines.length; index++) {const line = lines[index];- if (inFence) {- const fence = line.match(/^(`{3,}|~{3,})/);- if (- fence != null &&- fence[1].startsWith(fenceMarker[0]) &&- fence[1].length >= fenceMarker.length- ) {- inFence = false;- fenceMarker = '';- }+ if (fences.open) {+ fences.read(lines, index);continue;}if (mathContainer != null) {@@ -4339,10 +4476,7 @@ function trimOpenDisplayMath(text: string): string {suppressMathUntilBoundary = false;}- const fence = line.match(/^(`{3,}|~{3,})/);- if (fence != null) {- inFence = true;- fenceMarker = fence[1];+ if (fences.read(lines, index) === 'open') {continue;}const container = suppressMathUntilBoundarydiff --git a/packages/richtext/src/markdownCodeFence.ts b/packages/richtext/src/markdownCodeFence.tsindex c9dd29fd9d..59f386475d 100644--- a/packages/richtext/src/markdownCodeFence.ts+++ b/packages/richtext/src/markdownCodeFence.ts@@ -13,8 +13,9 @@* spaces (so `~~~ c++` reads `c++`), as core and CommonMark read it. A* block closes at* a fence of its own character at least as long, or at the end of the- * document, and its code is exactly the lines between, indentation and- * blank lines included, as core reads it. Each keeps its fence and info+ * document, and its code is exactly the lines between, blank lines+ * included, each less as much indentation as the opening fence has (up to+ * three spaces), as core reads it. Each keeps its fence and info* string (spec:AST-062): an edited block exports them as written, with its* language first when that was changed, and its fence lengthened when its* code holds a run of the fence's character that long.@@ -65,17 +66,27 @@ function languageOf(info: string): string | null {return /^\S+/.exec(info.trimStart())?.[0] ?? null;}-/** An opening fence line: its fence and the info string after it. */-function readOpening(- line: string,-): {readonly fence: string; readonly info: string} | null {- const match = /^[ \t]*(`{3,}|~{3,})(.*)$/.exec(line);+/** An opening fence line: its indentation, its fence, and the info string. */+interface Opening {+ readonly indentation: number;+ readonly fence: string;+ readonly info: string;+}++/**+ * Reads an opening fence line: up to three spaces of indentation, then the+ * fence (CommonMark 0.31 §4.5), as core Markdown reads it.+ */+function readOpening(line: string): Opening | null {+ const match = /^( {0,3})(`{3,}|~{3,})(.*)$/.exec(line);if (match == null) {return null;}- const [, fence, info] = match;+ const [, indentation, fence, info] = match;// A backtick fence's info string holds no backtick.- return fence.startsWith('`') && info.includes('`') ? null : {fence, info};+ return fence.startsWith('`') && info.includes('`')+ ? null+ : {indentation: indentation.length, fence, info};}/** The longest run of three or more of `character` in `text`, or 0. */@@ -96,12 +107,14 @@ function $importFencedCode(rootNode: ElementNode,lines: ReadonlyArray<string>,startLineIndex: number,- opening: {readonly fence: string; readonly info: string},+ opening: Opening,): number {- const {fence, info} = opening;+ const {indentation, fence, info} = opening;const closing = new RegExp(- `^[ \\t]*${fence[0] === '`' ? '`' : '~'}{${fence.length},}[ \\t]*$`,+ `^ {0,3}${fence[0] === '`' ? '`' : '~'}{${fence.length},}[ \\t]*$`,);+ // Each code line loses as much indentation as the opening fence has.+ const fenceIndentation = new RegExp(`^ {0,${indentation}}`);let end = startLineIndex + 1;while (end < lines.length && !closing.test(lines[end])) {end++;@@ -111,7 +124,12 @@ function $importFencedCode(// last lines, adjustments meant for its own reading of the opening line.const block = $createCodeNode(languageOf(info) ?? undefined);block.append(- $createTextNode(lines.slice(startLineIndex + 1, end).join('\n')),+ $createTextNode(+ lines+ .slice(startLineIndex + 1, end)+ .map(line => line.replace(fenceIndentation, ''))+ .join('\n'),+ ),);rootNode.append(block);if (fence.startsWith('~')) {@@ -179,8 +197,8 @@ export const BACKTICK_CODE: MultilineElementTransformer = {export const TILDE_CODE: MultilineElementTransformer = {...CODE,- regExpStart: /^([ \t]*~{3,})(.*)$/,- regExpEnd: {optional: true, regExp: /^[ \t]*~{3,}[ \t]*$/},+ regExpStart: /^( {0,3}~{3,})(.*)$/,+ regExpEnd: {optional: true, regExp: /^ {0,3}~{3,}[ \t]*$/},handleImportAfterStartMatch: ({lines, rootNode, startLineIndex}) => {const opening = readOpening(lines[startLineIndex]);return opening == nulldiff --git a/packages/richtext/src/markdownIndentedFences.test.ts b/packages/richtext/src/markdownIndentedFences.test.tsnew file mode 100644index 0000000000..e3d8449017--- /dev/null+++ b/packages/richtext/src/markdownIndentedFences.test.ts@@ -0,0 +1,43 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++import {describe, expect, it} from 'vitest';+import {parseMarkdownAst} from '@astryxdesign/core/Markdown/parser';+import {markdownToEditorStateJSON} from './markdownSerializers';++type Json = {type: string; text?: string; value?: string; children?: Json[]};++/** The text of every code block the editor imports, in order. */+function richCode(markdown: string): string[] {+ const text = (node: Json): string =>+ node.type === 'linebreak'+ ? '\n'+ : (node.text ?? (node.children ?? []).map(text).join(''));+ return (JSON.parse(markdownToEditorStateJSON(markdown)) as {root: Json}).root+ .children!.filter(node => node.type === 'code')+ .map(text);+}++/** The text of every code block core reads, in order. */+function coreCode(markdown: string): string[] {+ return (parseMarkdownAst(markdown).children as unknown as Json[])+ .filter(node => node.type === 'code')+ .map(node => node.value ?? '');+}++describe('an indented code fence reads as core reads it', () => {+ it.each([+ [' ```js\n x\n y\n ```', ['x\n y']],+ [' ```\na\n ```', ['a']],+ [' ~~~\n b\n ~~~', ['b']],+ ['```\ncode\n ```\nafter', ['code']],+ ['Text\n ```\n code\n ```', ['code']],+ ])('imports %j with the code core reads', (markdown, expected) => {+ expect(richCode(markdown)).toEqual(expected);+ expect(coreCode(markdown)).toEqual(expected);+ });++ it('reads a fence indented four spaces as no fence, as core does', () => {+ expect(richCode(' ```\n x\n ```')).toEqual([]);+ expect(coreCode(' ```\n x\n ```')).toEqual([]);+ });+});