Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

fix(Markdown, richtext): read a code fence indented up to three spaces as CommonMark does (#7111)

Cindy Zhang3a16495ea014
diff --git a/.changeset/markdown-indented-fences.md b/.changeset/markdown-indented-fences.md
new file mode 100644
index 0000000000..c0c6872d54
--- /dev/null
+++ b/.changeset/markdown-indented-fences.md
@@ -0,0 +1,8 @@
+---
+'@astryxdesign/core': patch
+---
+
+[fix] Markdown: read a code fence indented up to three spaces as CommonMark does
+@cixzhang
+
+`Markdown` now reads a code fence indented by up to three spaces as a code block, and a closing fence may be indented the same way, as CommonMark specifies. Each code line loses as much indentation as the opening fence has. A closing fence has only spaces or tabs after it, so a line such as ` ```js ` inside an open block stays part of the code. Such fences used to show as raw backticks or tildes in a paragraph, and an indented closing fence left the block open to the end of the document.
diff --git a/packages/core/src/Markdown/indentedFences.test.tsx b/packages/core/src/Markdown/indentedFences.test.tsx
new file mode 100644
index 0000000000..354c5b63df
--- /dev/null
+++ b/packages/core/src/Markdown/indentedFences.test.tsx
@@ -0,0 +1,374 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+import {describe, expect, it} from 'vitest';
+import {
+ createIncrementalState,
+ parseMarkdown,
+ parseMarkdownAst,
+ parseMarkdownIncremental,
+} from './parser';
+
+type Json = {
+ type: string;
+ value?: string;
+ lang?: string | null;
+ children?: Json[];
+};
+
+/** The document's blocks as compact markup. */
+function blocks(markdown: string): string {
+ const show = (node: Json): string =>
+ node.type === 'text'
+ ? JSON.stringify(node.value)
+ : node.type === 'code'
+ ? `code(${node.lang ?? ''})${JSON.stringify(node.value)}`
+ : node.children == null
+ ? node.type
+ : `${node.type}[${node.children.map(show).join(', ')}]`;
+ return (parseMarkdownAst(markdown).children as unknown as Json[])
+ .map(show)
+ .join(' | ');
+}
+
+describe('a code fence may be indented up to three spaces (CommonMark 0.31 §4.5)', () => {
+ it.each([
+ [
+ 'a backtick fence indented two spaces, its code losing two',
+ ' ```js\n x\n y\n ```',
+ 'code(js)"x\\n y"',
+ ],
+ ['a backtick fence indented one space', ' ```\na\n ```', 'code()"a"'],
+ [
+ 'a tilde fence indented three spaces',
+ ' ~~~\n b\n ~~~',
+ 'code()"b"',
+ ],
+ [
+ 'an indented closing fence',
+ '```\ncode\n ```\nafter',
+ 'code()"code" | paragraph["after"]',
+ ],
+ [
+ 'an indented fence interrupting a paragraph',
+ 'Text\n ```\n code\n ```',
+ 'paragraph["Text"] | code()"code"',
+ ],
+ [
+ 'a fence in a list item, unchanged',
+ '1. Step\n ```bash\n cmd\n ```',
+ 'list[listItem[paragraph["Step"], code(bash)"cmd"]]',
+ ],
+ ])('reads %s', (_, markdown, expected) => {
+ expect(blocks(markdown)).toBe(expected);
+ });
+
+ it('keeps a line indented four spaces inside the block, not as its closer', () => {
+ expect(blocks('```\na\n ```\nb\n```')).toBe('code()"a\\n ```\\nb"');
+ });
+
+ it('reads a fence indented four spaces as no fence', () => {
+ expect(blocks(' ```\n x\n ```')).not.toContain('code(');
+ });
+
+ it('collects no link definition inside an indented fence', () => {
+ expect(blocks(' ```\n[x]: /u\n ```\n\n[x]')).toBe(
+ 'code()"[x]: /u" | paragraph["[x]"]',
+ );
+ });
+
+ it('streams indented fences with blank lines to the full parse at every character', () => {
+ // `d` sits at the margin after a blank line inside a fence: that blank
+ // line settles nothing, since the fence is still open.
+ const markdown =
+ 'a\n\n ```js\n x\n\n y\n ```\n\nb\n\n ~~~\n\n c\n\nd\n ~~~\n';
+ for (const sourceRanges of [false, true]) {
+ const state = createIncrementalState();
+ let streamed = parseMarkdownIncremental('', state, {sourceRanges});
+ for (let end = 1; end <= markdown.length; end++) {
+ streamed = parseMarkdownIncremental(markdown.slice(0, end), state, {
+ sourceRanges,
+ });
+ }
+ expect(streamed).toEqual(parseMarkdown(markdown, {sourceRanges}));
+ }
+ });
+});
+
+/** The streamed parse of `markdown` in chunks of the given sizes, cycled. */
+function streamInChunks(
+ markdown: string,
+ sizes: ReadonlyArray<number>,
+ sourceRanges = false,
+) {
+ const state = createIncrementalState();
+ let result = parseMarkdownIncremental('', state, {sourceRanges});
+ let end = 0;
+ for (let step = 0; end < markdown.length; step++) {
+ end = Math.min(markdown.length, end + sizes[step % sizes.length]);
+ result = parseMarkdownIncremental(markdown.slice(0, end), state, {
+ sourceRanges,
+ });
+ }
+ return result;
+}
+
+const STEP_CLOSED_AT_MARGIN = [
+ '1. Install:',
+ ' ```bash',
+ ' npm i',
+ '```',
+ '',
+ '2. Run:',
+ ' ```bash',
+ ' npm test',
+ ' ```',
+ '',
+ 'Done.',
+].join('\n');
+
+describe('a fence inside a list step stays the step’s', () => {
+ it('streams a step’s fence closed at the margin to the full parse at every character', () => {
+ for (const sourceRanges of [false, true]) {
+ expect(streamInChunks(STEP_CLOSED_AT_MARGIN, [1], sourceRanges)).toEqual(
+ parseMarkdown(STEP_CLOSED_AT_MARGIN, {sourceRanges}),
+ );
+ }
+ });
+
+ it('reads the margin line after a step’s fence as the opening of a fence of its own', () => {
+ expect(blocks('1. Step\n ```\n code\n```\nafter')).toBe(
+ 'list[listItem[paragraph["Step"], code()"code"]] | code()"after"',
+ );
+ });
+
+ it('keeps a definition-shaped line inside that margin fence as code', () => {
+ expect(blocks('1. Step\n ```\n code\n```\n[x]: /u\n\n[x]')).toBe(
+ 'list[listItem[paragraph["Step"], code()"code"]] | code()"[x]: /u\\n\\n[x]"',
+ );
+ });
+
+ it('streams 600 step-by-step answers to the full parse, closers at the margin included', () => {
+ let state = 7111;
+ const random = () => {
+ state = (state * 1103515245 + 12345) % 2147483648;
+ return state / 2147483648;
+ };
+ const pick = <T,>(choices: ReadonlyArray<T>): T =>
+ choices[Math.floor(random() * choices.length)];
+ const answer = () => {
+ const lines: string[] = [];
+ if (random() < 0.5) {
+ lines.push('Here are the steps:', '');
+ }
+ const steps = 1 + Math.floor(random() * 4);
+ for (let step = 1; step <= steps; step++) {
+ const marker = random() < 0.7 ? `${step}. ` : '- ';
+ lines.push(`${marker}Step ${step}:`);
+ if (random() < 0.3) {
+ lines.push(`${' '.repeat(marker.length)}Some detail.`);
+ }
+ if (random() < 0.4) {
+ lines.push('');
+ }
+ const indent = pick([0, 1, 2, 3, marker.length, marker.length + 1]);
+ const fence = random() < 0.85 ? '```' : '~~~';
+ lines.push(`${' '.repeat(indent)}${fence}${pick(['', 'bash', 'js'])}`);
+ const codeLines = 1 + Math.floor(random() * 3);
+ for (let line = 0; line < codeLines; line++) {
+ lines.push(
+ random() < 0.15
+ ? ''
+ : `${' '.repeat(pick([0, indent, indent + 2]))}code ${line}`,
+ );
+ }
+ const closer = random();
+ if (closer < 0.4) {
+ lines.push(fence);
+ } else if (closer < 0.9) {
+ lines.push(`${' '.repeat(indent)}${fence}`);
+ }
+ if (random() < 0.6) {
+ lines.push('');
+ }
+ }
+ if (random() < 0.5) {
+ lines.push('That is all.');
+ }
+ // A fence left open at the very end with a newline after it keeps a
+ // final empty line in the full parse only — a separate difference.
+ return lines.join('\n').trimEnd();
+ };
+ const diverged: string[] = [];
+ for (let round = 0; round < 600; round++) {
+ const markdown = answer();
+ const sizes = [
+ 1 + Math.floor(random() * 12),
+ 1 + Math.floor(random() * 5),
+ ];
+ const full = parseMarkdown(markdown);
+ if (
+ JSON.stringify(streamInChunks(markdown, sizes)) !== JSON.stringify(full)
+ ) {
+ diverged.push(markdown);
+ }
+ }
+ expect(diverged).toEqual([]);
+ });
+});
+
+describe('fences around list steps, streamed', () => {
+ it('streams 600 varied answers to the full parse: sub-steps, blank lines in fences, text after a step’s fence', () => {
+ let state = 424242;
+ const random = () => {
+ state = (state * 1103515245 + 12345) % 2147483648;
+ return state / 2147483648;
+ };
+ const pick = <T,>(choices: ReadonlyArray<T>): T =>
+ choices[Math.floor(random() * choices.length)];
+ const pushFence = (indent: number, lines: string[]) => {
+ const fence = pick(['```', '```', '~~~', '````']);
+ lines.push(`${' '.repeat(indent)}${fence}${pick(['', 'bash', ' py'])}`);
+ const codeLines = Math.floor(random() * 4);
+ for (let line = 0; line < codeLines; line++) {
+ lines.push(
+ random() < 0.2
+ ? ''
+ : `${' '.repeat(pick([0, indent, indent + 1, indent + 4]))}${pick(['code', '[x]: /u', '- item', '# title', '> q'])} ${line}`,
+ );
+ }
+ const closer = random();
+ if (closer < 0.35) {
+ lines.push(fence);
+ } else if (closer < 0.75) {
+ lines.push(`${' '.repeat(indent)}${fence}`);
+ } else if (closer < 0.85) {
+ lines.push(`${' '.repeat(indent)}${fence}js`);
+ } else if (closer < 0.9) {
+ lines.push(`${' '.repeat(indent + 1)}${fence} `);
+ }
+ };
+ const answer = () => {
+ const lines: string[] = [];
+ if (random() < 0.4) {
+ lines.push(pick(['Steps:', 'Here you go']));
+ if (random() < 0.6) {
+ lines.push('');
+ }
+ }
+ // Every list at the margin with a `-` bullet, as lines escaping a step's
+ // fence are: two lists of different styles, a blank line apart, are a
+ // separate case of the settled-list merge.
+ const bullet = pick(['- ', '- ']);
+ const delimiter = pick(['.', ')']);
+ const base = 0;
+ let number = pick([1, 1, 2]);
+ const steps = 1 + Math.floor(random() * 5);
+ for (let step = 0; step < steps; step++) {
+ const marker = random() < 0.65 ? `${number++}${delimiter} ` : bullet;
+ lines.push(`${' '.repeat(base)}${marker}Step ${step}`);
+ const content = base + marker.length;
+ const parts = Math.floor(random() * 3);
+ for (let part = 0; part < parts; part++) {
+ const roll = random();
+ if (roll < 0.3) {
+ lines.push('');
+ }
+ // A sub-step directly under its step: placed after the step has
+ // ended, it would start a list of its own at another indentation.
+ if (roll < 0.15 && part === 0) {
+ lines.push(`${' '.repeat(content)}- sub ${part}`);
+ } else if (roll < 0.45) {
+ pushFence(pick([0, 1, 2, 3, content, content + 1]), lines);
+ } else if (roll < 0.6) {
+ lines.push(
+ pick(['lazy text', `${' '.repeat(content)}more`, 'Then:']),
+ );
+ } else if (roll < 0.7) {
+ lines.push(`${' '.repeat(pick([0, content]))}# Heading`);
+ }
+ }
+ if (random() < 0.5) {
+ lines.push('');
+ }
+ if (random() < 0.2) {
+ lines.push('Between steps.');
+ if (random() < 0.5) {
+ lines.push('');
+ }
+ }
+ }
+ if (random() < 0.3) {
+ pushFence(pick([0, 1, 2, 3]), lines);
+ }
+ if (random() < 0.4) {
+ lines.push('That is all.');
+ }
+ return lines.join('\n').trimEnd();
+ };
+ const diverged: string[] = [];
+ for (let round = 0; round < 600; round++) {
+ const markdown = answer();
+ const sizes =
+ round % 10 === 0
+ ? [1]
+ : [1 + Math.floor(random() * 12), 1 + Math.floor(random() * 5)];
+ if (
+ JSON.stringify(streamInChunks(markdown, sizes)) !==
+ JSON.stringify(parseMarkdown(markdown))
+ ) {
+ diverged.push(markdown);
+ }
+ }
+ expect(diverged).toEqual([]);
+ });
+});
+
+describe('a closing fence has only spaces or tabs after it (CommonMark 0.31 §4.5)', () => {
+ it.each([
+ [
+ 'a backtick line with an info string',
+ '```\na\n ```js\nb\n```',
+ 'code()"a\\n ```js\\nb"',
+ ],
+ [
+ 'a tilde line with an info string',
+ '~~~\na\n ~~~ note\nb\n~~~',
+ 'code()"a\\n ~~~ note\\nb"',
+ ],
+ [
+ 'a closer followed by spaces and a tab',
+ '```\na\n``` \t\nb',
+ 'code()"a" | paragraph["b"]',
+ ],
+ ])(
+ 'keeps %s inside the open block, or closes on spaces',
+ (_, markdown, expected) => {
+ expect(blocks(markdown)).toBe(expected);
+ },
+ );
+
+ const STEP_TWO_OPENS = [
+ '1. Step one',
+ ' ```bash',
+ ' cmd',
+ '```',
+ '2. Step two',
+ ' ```js',
+ ' x',
+ ' ```',
+ ].join('\n');
+
+ it('reads step 1 closed at the margin, then step 2 opening, as CommonMark does', () => {
+ expect(blocks(STEP_TWO_OPENS)).toBe(
+ 'list[listItem[paragraph["Step one"], code(bash)"cmd"]] | code()"2. Step two\\n ```js\\n x"',
+ );
+ });
+
+ it('streams that answer to the full parse at every character', () => {
+ for (const sourceRanges of [false, true]) {
+ expect(streamInChunks(STEP_TWO_OPENS, [1], sourceRanges)).toEqual(
+ parseMarkdown(STEP_TWO_OPENS, {sourceRanges}),
+ );
+ }
+ });
+});
diff --git a/packages/core/src/Markdown/parser.ts b/packages/core/src/Markdown/parser.ts
index f80369edd7..941aa92d84 100644
--- a/packages/core/src/Markdown/parser.ts
+++ b/packages/core/src/Markdown/parser.ts
@@ -849,6 +849,172 @@ function matchLinkDefinition(
};
}
+/**
+ * A code fence line: up to three spaces of indentation, then three or more
+ * backticks or tildes (CommonMark 0.31 §4.5). Group 1 is the indentation and
+ * group 2 the fence.
+ */
+const FENCE_LINE = /^( {0,3})(`{3,}|~{3,})/;
+
+/**
+ * Whether `line` closes a code block opened with `fence`: a fence of the same
+ * character, at least as long, after up to three spaces of indentation, with
+ * only spaces or tabs after it (CommonMark 0.31 §4.5) — so ` ```js` inside
+ * an open block is code, not its end.
+ */
+function closesFence(line: string, fence: string): boolean {
+ const match = /^ {0,3}(`{3,}|~{3,})[ \t]*\r?$/.exec(line);
+ return (
+ match != null &&
+ match[1].startsWith(fence[0]) &&
+ match[1].length >= fence.length
+ );
+}
+
+/** Where a list item's lines sit: its marker's indentation and content. */
+interface ListItemScope {
+ /** The marker's indentation. */
+ readonly base: number;
+ /** Where the item's content starts: past the marker and its spaces. */
+ readonly content: number;
+ readonly ordered: boolean;
+}
+
+/**
+ * The list item `line` opens, read as the block parser reads one: a bullet or
+ * a number with `.` or `)` after up to nine spaces, then a space. A thematic
+ * break opens no item.
+ */
+function listItemScopeOf(line: string): ListItemScope | null {
+ if (isHorizontalRule(line)) {
+ return null;
+ }
+ const marker = /^( {0,9})([-*+]|\d+[.)]) /.exec(line);
+ if (marker == null) {
+ return null;
+ }
+ const markerEnd = marker[0].length;
+ const spacesAfter = getIndent(line.slice(markerEnd));
+ return {
+ base: marker[1].length,
+ content: spacesAfter >= 4 ? markerEnd : markerEnd + spacesAfter,
+ ordered: marker[2] !== '-' && marker[2] !== '*' && marker[2] !== '+',
+ };
+}
+
+/** What a line does to the top-level code fence. */
+type TopLevelFenceEvent = 'open' | 'close' | 'inside' | null;
+
+/**
+ * Follows the code fences that open at the top level of a document, a line at
+ * a time, as the block parser reads them. A fence indented into an open list
+ * item belongs to that item, whose own parse reads it: it neither opens nor
+ * closes a top-level fence, so a list step's fence closed at the margin leaves
+ * the margin line to open a fence of its own, as the full parse does. Scanners
+ * that run outside the block parser — streaming settlement, link definitions,
+ * display-math trimming — share it so they pair fences the same way.
+ */
+function topLevelFences(): {
+ readonly open: boolean;
+ read(lines: ReadonlyArray<string>, index: number): TopLevelFenceEvent;
+} {
+ let fence = '';
+ // The outermost open list item, and what it last read. After a blank line
+ // inside the item, the parser keeps only lines indented to its content.
+ let item: ListItemScope | null = null;
+ let itemHadBlank = false;
+ let itemFence = '';
+ let itemEndsInParagraph = false;
+ // A number other than 1 cannot interrupt an open top-level paragraph.
+ let paragraphOpen = false;
+
+ const isHeadingOrBreak = (text: string) =>
+ /^ {0,3}#{1,6}(?:[ \t]|$)/.test(text) || isHorizontalRule(text);
+
+ /** Reads a line of the open item, without the item's indentation. */
+ const readItemLine = (text: string) => {
+ if (itemFence !== '') {
+ if (closesFence(text, itemFence)) {
+ itemFence = '';
+ }
+ itemEndsInParagraph = false;
+ return;
+ }
+ const opening = FENCE_LINE.exec(text);
+ if (opening != null) {
+ itemFence = opening[2];
+ itemEndsInParagraph = false;
+ return;
+ }
+ itemEndsInParagraph = text.trim() !== '' && !isHeadingOrBreak(text);
+ };
+
+ return {
+ get open() {
+ return fence !== '';
+ },
+ read(lines, index) {
+ const line = lines[index];
+ if (fence !== '') {
+ if (closesFence(line, fence)) {
+ fence = '';
+ return 'close';
+ }
+ return 'inside';
+ }
+ if (line.trim() === '') {
+ itemHadBlank = item != null;
+ paragraphOpen = false;
+ return null;
+ }
+ const indent = getIndent(line);
+ if (
+ item != null &&
+ (itemHadBlank ? indent >= item.content : indent > item.base)
+ ) {
+ readItemLine(line.slice(Math.min(indent, item.content)));
+ return null;
+ }
+ const scope = listItemScopeOf(line);
+ if (
+ scope != null &&
+ !(
+ paragraphOpen &&
+ item == null &&
+ scope.ordered &&
+ !/^ {0,9}0*1[.)]/.test(line)
+ )
+ ) {
+ item = scope;
+ itemHadBlank = false;
+ itemFence = '';
+ paragraphOpen = false;
+ readItemLine(line.slice(scope.content));
+ return null;
+ }
+ const opening = FENCE_LINE.exec(line);
+ if (opening != null) {
+ item = null;
+ paragraphOpen = false;
+ fence = opening[2];
+ return 'open';
+ }
+ if (
+ item != null &&
+ !itemHadBlank &&
+ itemEndsInParagraph &&
+ canContinueParagraphLazily(lines, index)
+ ) {
+ // A lazy continuation line of the item's paragraph.
+ return null;
+ }
+ item = null;
+ paragraphOpen = !isHeadingOrBreak(line);
+ return null;
+ },
+ };
+}
+
/**
* Collect link reference definitions from the whole document and return the
* input with the definition lines removed. A definition is recognized at a
@@ -883,20 +1049,13 @@ function extractLinkDefinitions(
const defs = new Map<string, string>();
const keep = new Array<boolean>(lines.length).fill(true);
let atBoundary = true;
- let inFence = false;
- let fenceMarker = '';
+ const fences = topLevelFences();
for (let index = 0; index < lines.length; index++) {
const line = lines[index];
- if (inFence) {
- if (line.startsWith(fenceMarker)) {
- inFence = false;
- fenceMarker = '';
- // The line after a closed fence begins a new block.
- atBoundary = true;
- } else {
- atBoundary = false;
- }
+ if (fences.open) {
+ // The line after a closed fence begins a new block.
+ atBoundary = fences.read(lines, index) === 'close';
continue;
}
if (math) {
@@ -909,10 +1068,7 @@ function extractLinkDefinitions(
continue;
}
}
- const fenceMatch = line.match(/^(`{3,}|~{3,})/);
- if (fenceMatch) {
- inFence = true;
- fenceMarker = fenceMatch[1];
+ if (fences.read(lines, index) === 'open') {
atBoundary = false;
continue;
}
@@ -2952,7 +3108,7 @@ function isBlockStart(line: string): boolean {
if (/^ {0,3}#{1,6} /.test(line)) {
return true;
}
- if (/^(`{3,}|~{3,})/.test(line)) {
+ if (FENCE_LINE.test(line)) {
return true;
}
if (isHorizontalRule(line)) {
@@ -2989,7 +3145,7 @@ function canContinueParagraphLazily(
}
if (
/^ {0,3}#{1,6} /.test(line) ||
- /^(`{3,}|~{3,})/.test(line) ||
+ FENCE_LINE.test(line) ||
isHorizontalRule(line) ||
QUOTE_MARKER.test(line) ||
/^ {0,9}[-*+] /.test(line) ||
@@ -3512,12 +3668,12 @@ function parseMarkdownImpl(
}
// --- Fenced code block ---
- const fenceMatch = line.match(/^(`{3,}|~{3,})/);
+ const fenceMatch = FENCE_LINE.exec(line);
if (fenceMatch) {
- const fence = fenceMatch[1];
+ const [opening, indentation, fence] = fenceMatch;
// The rest of the line is the info string; its first word, after any
// spaces, is the language (CommonMark 0.31 §4.5).
- const info = line.slice(fence.length).trim();
+ const info = line.slice(opening.length).trim();
const language = info.match(/^(\S+)/)?.[1] ?? null;
const legacyLanguage = info.match(/^(\w*)/)?.[1] || null;
const meta =
@@ -3525,9 +3681,11 @@ function parseMarkdownImpl(
? undefined
: info.slice(language.length).trim() || undefined;
const codeLines: string[] = [];
+ // Each code line loses as much indentation as the opening fence has.
+ const fenceIndentation = new RegExp(`^ {0,${indentation.length}}`);
index++;
- while (index < lines.length && !lines[index].startsWith(fence)) {
- codeLines.push(lines[index]);
+ while (index < lines.length && !closesFence(lines[index], fence)) {
+ codeLines.push(lines[index].replace(fenceIndentation, ''));
index++;
}
index++; // skip closing fence
@@ -3995,8 +4153,7 @@ function findSettledBoundary(
openFence: boolean;
openMath: boolean;
} {
- let inFence = false;
- let fenceMarker = '';
+ const fences = topLevelFences();
let mathContainer: DisplayMathContainer | null = null;
let suppressMathUntilBoundary = false;
let lastBoundary = -1;
@@ -4006,16 +4163,8 @@ function findSettledBoundary(
for (let lineIndex = 0; lineIndex < lines.length; lineIndex++) {
const line = lines[lineIndex];
- if (inFence) {
- const fenceMatch = line.match(/^(`{3,}|~{3,})/);
- if (
- fenceMatch &&
- fenceMatch[1].startsWith(fenceMarker[0]) &&
- fenceMatch[1].length >= fenceMarker.length
- ) {
- inFence = false;
- fenceMarker = '';
- }
+ if (fences.open) {
+ fences.read(lines, lineIndex);
continue;
}
@@ -4048,10 +4197,7 @@ function findSettledBoundary(
}
}
- const fenceMatch = line.match(/^(`{3,}|~{3,})/);
- if (fenceMatch) {
- inFence = true;
- fenceMarker = fenceMatch[1];
+ if (fences.read(lines, lineIndex) === 'open') {
boundaryBeforeFence = lastBoundary;
continue;
}
@@ -4071,12 +4217,12 @@ function findSettledBoundary(
}
return {
- boundary: inFence
+ boundary: fences.open
? boundaryBeforeFence
: mathContainer != null
? boundaryBeforeMath
: lastBoundary,
- openFence: inFence,
+ openFence: fences.open,
openMath: mathContainer != null,
};
}
@@ -4300,24 +4446,15 @@ export function trimStreamingArtifacts(
*/
function trimOpenDisplayMath(text: string): string {
const lines = text.split('\n');
- let inFence = false;
- let fenceMarker = '';
+ const fences = topLevelFences();
let mathContainer: DisplayMathContainer | null = null;
let suppressMathUntilBoundary = false;
let mathStartLine = -1;
for (let index = 0; index < lines.length; index++) {
const line = lines[index];
- if (inFence) {
- const fence = line.match(/^(`{3,}|~{3,})/);
- if (
- fence != null &&
- fence[1].startsWith(fenceMarker[0]) &&
- fence[1].length >= fenceMarker.length
- ) {
- inFence = false;
- fenceMarker = '';
- }
+ if (fences.open) {
+ fences.read(lines, index);
continue;
}
if (mathContainer != null) {
@@ -4339,10 +4476,7 @@ function trimOpenDisplayMath(text: string): string {
suppressMathUntilBoundary = false;
}
- const fence = line.match(/^(`{3,}|~{3,})/);
- if (fence != null) {
- inFence = true;
- fenceMarker = fence[1];
+ if (fences.read(lines, index) === 'open') {
continue;
}
const container = suppressMathUntilBoundary
diff --git a/packages/richtext/src/markdownCodeFence.ts b/packages/richtext/src/markdownCodeFence.ts
index c9dd29fd9d..59f386475d 100644
--- a/packages/richtext/src/markdownCodeFence.ts
+++ b/packages/richtext/src/markdownCodeFence.ts
@@ -13,8 +13,9 @@
* spaces (so `~~~ c++` reads `c++`), as core and CommonMark read it. A
* block closes at
* a fence of its own character at least as long, or at the end of the
- * document, and its code is exactly the lines between, indentation and
- * blank lines included, as core reads it. Each keeps its fence and info
+ * document, and its code is exactly the lines between, blank lines
+ * included, each less as much indentation as the opening fence has (up to
+ * three spaces), as core reads it. Each keeps its fence and info
* string (spec:AST-062): an edited block exports them as written, with its
* language first when that was changed, and its fence lengthened when its
* code holds a run of the fence's character that long.
@@ -65,17 +66,27 @@ function languageOf(info: string): string | null {
return /^\S+/.exec(info.trimStart())?.[0] ?? null;
}
-/** An opening fence line: its fence and the info string after it. */
-function readOpening(
- line: string,
-): {readonly fence: string; readonly info: string} | null {
- const match = /^[ \t]*(`{3,}|~{3,})(.*)$/.exec(line);
+/** An opening fence line: its indentation, its fence, and the info string. */
+interface Opening {
+ readonly indentation: number;
+ readonly fence: string;
+ readonly info: string;
+}
+
+/**
+ * Reads an opening fence line: up to three spaces of indentation, then the
+ * fence (CommonMark 0.31 §4.5), as core Markdown reads it.
+ */
+function readOpening(line: string): Opening | null {
+ const match = /^( {0,3})(`{3,}|~{3,})(.*)$/.exec(line);
if (match == null) {
return null;
}
- const [, fence, info] = match;
+ const [, indentation, fence, info] = match;
// A backtick fence's info string holds no backtick.
- return fence.startsWith('`') && info.includes('`') ? null : {fence, info};
+ return fence.startsWith('`') && info.includes('`')
+ ? null
+ : {indentation: indentation.length, fence, info};
}
/** The longest run of three or more of `character` in `text`, or 0. */
@@ -96,12 +107,14 @@ function $importFencedCode(
rootNode: ElementNode,
lines: ReadonlyArray<string>,
startLineIndex: number,
- opening: {readonly fence: string; readonly info: string},
+ opening: Opening,
): number {
- const {fence, info} = opening;
+ const {indentation, fence, info} = opening;
const closing = new RegExp(
- `^[ \\t]*${fence[0] === '`' ? '`' : '~'}{${fence.length},}[ \\t]*$`,
+ `^ {0,3}${fence[0] === '`' ? '`' : '~'}{${fence.length},}[ \\t]*$`,
);
+ // Each code line loses as much indentation as the opening fence has.
+ const fenceIndentation = new RegExp(`^ {0,${indentation}}`);
let end = startLineIndex + 1;
while (end < lines.length && !closing.test(lines[end])) {
end++;
@@ -111,7 +124,12 @@ function $importFencedCode(
// last lines, adjustments meant for its own reading of the opening line.
const block = $createCodeNode(languageOf(info) ?? undefined);
block.append(
- $createTextNode(lines.slice(startLineIndex + 1, end).join('\n')),
+ $createTextNode(
+ lines
+ .slice(startLineIndex + 1, end)
+ .map(line => line.replace(fenceIndentation, ''))
+ .join('\n'),
+ ),
);
rootNode.append(block);
if (fence.startsWith('~')) {
@@ -179,8 +197,8 @@ export const BACKTICK_CODE: MultilineElementTransformer = {
export const TILDE_CODE: MultilineElementTransformer = {
...CODE,
- regExpStart: /^([ \t]*~{3,})(.*)$/,
- regExpEnd: {optional: true, regExp: /^[ \t]*~{3,}[ \t]*$/},
+ regExpStart: /^( {0,3}~{3,})(.*)$/,
+ regExpEnd: {optional: true, regExp: /^ {0,3}~{3,}[ \t]*$/},
handleImportAfterStartMatch: ({lines, rootNode, startLineIndex}) => {
const opening = readOpening(lines[startLineIndex]);
return opening == null
diff --git a/packages/richtext/src/markdownIndentedFences.test.ts b/packages/richtext/src/markdownIndentedFences.test.ts
new file mode 100644
index 0000000000..e3d8449017
--- /dev/null
+++ b/packages/richtext/src/markdownIndentedFences.test.ts
@@ -0,0 +1,43 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+import {describe, expect, it} from 'vitest';
+import {parseMarkdownAst} from '@astryxdesign/core/Markdown/parser';
+import {markdownToEditorStateJSON} from './markdownSerializers';
+
+type Json = {type: string; text?: string; value?: string; children?: Json[]};
+
+/** The text of every code block the editor imports, in order. */
+function richCode(markdown: string): string[] {
+ const text = (node: Json): string =>
+ node.type === 'linebreak'
+ ? '\n'
+ : (node.text ?? (node.children ?? []).map(text).join(''));
+ return (JSON.parse(markdownToEditorStateJSON(markdown)) as {root: Json}).root
+ .children!.filter(node => node.type === 'code')
+ .map(text);
+}
+
+/** The text of every code block core reads, in order. */
+function coreCode(markdown: string): string[] {
+ return (parseMarkdownAst(markdown).children as unknown as Json[])
+ .filter(node => node.type === 'code')
+ .map(node => node.value ?? '');
+}
+
+describe('an indented code fence reads as core reads it', () => {
+ it.each([
+ [' ```js\n x\n y\n ```', ['x\n y']],
+ [' ```\na\n ```', ['a']],
+ [' ~~~\n b\n ~~~', ['b']],
+ ['```\ncode\n ```\nafter', ['code']],
+ ['Text\n ```\n code\n ```', ['code']],
+ ])('imports %j with the code core reads', (markdown, expected) => {
+ expect(richCode(markdown)).toEqual(expected);
+ expect(coreCode(markdown)).toEqual(expected);
+ });
+
+ it('reads a fence indented four spaces as no fence, as core does', () => {
+ expect(richCode(' ```\n x\n ```')).toEqual([]);
+ expect(coreCode(' ```\n x\n ```')).toEqual([]);
+ });
+});