Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

fix(richtext): read tilde code fences, and every fence's info string as core does (#7061)

Cindy Zhang6260001bfec3
diff --git a/packages/richtext/src/markdownCodeFence.test.ts b/packages/richtext/src/markdownCodeFence.test.ts
new file mode 100644
index 0000000000..32ab59d719
--- /dev/null
+++ b/packages/richtext/src/markdownCodeFence.test.ts
@@ -0,0 +1,142 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+import {describe, expect, it} from 'vitest';
+import {createHeadlessEditor} from '@lexical/headless';
+import {$isCodeNode, type CodeNode} from '@lexical/code';
+import {$getRoot, $isTextNode} from 'lexical';
+import {parseMarkdownAst} from '@astryxdesign/core/Markdown/parser';
+import {DEFAULT_NODES} from './editorNodes';
+import {DEFAULT_TRANSFORMERS} from './markdownTable';
+import {
+ $exportMarkdownKeepingSource,
+ importMarkdownKeepingSource,
+} from './markdownSource';
+import {
+ editorStateJSONToMarkdown,
+ markdownToEditorStateJSON,
+} from './markdownSerializers';
+
+type Json = Record<string, unknown>;
+
+/** Each top-level block: a code block's language and code, or its type. */
+function blocks(markdown: string): Array<unknown> {
+ const root = (JSON.parse(markdownToEditorStateJSON(markdown)) as {root: Json})
+ .root;
+ return (root.children as Array<Json>).map(node =>
+ node.type === 'code'
+ ? {
+ language: node.language ?? null,
+ code: ((node.children as Array<Json>) ?? [])
+ .map(child =>
+ child.type === 'linebreak' ? '\n' : (child.text as string),
+ )
+ .join(''),
+ }
+ : node.type,
+ );
+}
+
+/** The same, as core Markdown reads it. */
+function coreBlocks(markdown: string): Array<unknown> {
+ return (parseMarkdownAst(markdown).children as unknown as Array<Json>).map(
+ node =>
+ node.type === 'code'
+ ? {language: node.lang ?? null, code: node.value as string}
+ : node.type,
+ );
+}
+
+/** Imports `markdown`, edits its code block with `edit`, and exports. */
+function editAndExport(
+ markdown: string,
+ edit: (code: CodeNode) => void,
+): string {
+ const editor = createHeadlessEditor({
+ nodes: [...DEFAULT_NODES],
+ onError(error) {
+ throw error;
+ },
+ });
+ importMarkdownKeepingSource(editor, markdown, [...DEFAULT_TRANSFORMERS]);
+ editor.update(
+ () => {
+ const code = $getRoot().getFirstChild();
+ if (!$isCodeNode(code)) {
+ throw new Error('No code block');
+ }
+ edit(code);
+ },
+ {discrete: true},
+ );
+ return editor
+ .getEditorState()
+ .read(() => $exportMarkdownKeepingSource([...DEFAULT_TRANSFORMERS]));
+}
+
+const setCode = (text: string) => (code: CodeNode) => {
+ const first = code.getFirstChild();
+ if (!$isTextNode(first)) {
+ throw new Error('No code text');
+ }
+ first.setTextContent(text);
+};
+
+describe.each(['~~~', '```'])('%s fences (CommonMark 0.31 §4.5)', fence => {
+ it.each([
+ `${fence} js\nx\n${fence}\n`,
+ `${fence}python title="a.py"\nx = 1\n${fence}\n`,
+ `${fence} {.haskell .numberLines}\nmain = pure ()\n${fence}\n`,
+ `${fence}c++\nint x;\n${fence}\n`,
+ `${fence}\nplain\n${fence}\n`,
+ ])(
+ 'reads %j with the language and code core reads, and round-trips it',
+ markdown => {
+ expect(blocks(markdown)).toEqual(coreBlocks(markdown));
+ expect(
+ editorStateJSONToMarkdown(markdownToEditorStateJSON(markdown)),
+ ).toBe(markdown);
+ },
+ );
+
+ it('keeps the info string through an edit', () => {
+ const edited = editAndExport(
+ `${fence}python title="a.py"\nx = 1\n${fence}\n`,
+ setCode('x = 2'),
+ );
+ expect(edited).toBe(`${fence}python title="a.py"\nx = 2\n${fence}\n`);
+ expect(blocks(edited)).toEqual(coreBlocks(edited));
+ });
+
+ it('writes a changed language first, before the rest of the info string', () => {
+ const edited = editAndExport(
+ `${fence}python title="a.py"\nx = 1\n${fence}\n`,
+ code => code.setLanguage('ruby'),
+ );
+ expect(edited).toBe(`${fence}ruby title="a.py"\nx = 1\n${fence}\n`);
+ });
+});
+
+describe('tilde code fences', () => {
+ it.each([
+ '~~~~\n~~~\ninner\n~~~\n~~~~\n',
+ '~~~\n```\nbackticks\n```\n~~~\n',
+ 'Before\n\n~~~js\nx();\n~~~\n\nAfter\n',
+ ])('reads %j as core does, and round-trips it', markdown => {
+ expect(blocks(markdown)).toEqual(coreBlocks(markdown));
+ expect(editorStateJSONToMarkdown(markdownToEditorStateJSON(markdown))).toBe(
+ markdown,
+ );
+ });
+
+ it('runs an unclosed tilde block to the end of the document', () => {
+ expect(blocks('~~~\nopen\ncode\n')).toEqual([
+ {language: null, code: 'open\ncode'},
+ ]);
+ });
+
+ it('exports an edited tilde block with tildes, longer than any tilde fence in its code', () => {
+ const edited = editAndExport('~~~ts\nconst a = 1;\n~~~\n', setCode('~~~'));
+ expect(edited).toBe('~~~~ts\n~~~\n~~~~\n');
+ expect(blocks(edited)).toEqual([{language: 'ts', code: '~~~'}]);
+ });
+});
diff --git a/packages/richtext/src/markdownCodeFence.ts b/packages/richtext/src/markdownCodeFence.ts
new file mode 100644
index 0000000000..03b26f1ec3
--- /dev/null
+++ b/packages/richtext/src/markdownCodeFence.ts
@@ -0,0 +1,176 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file markdownCodeFence.ts
+ * @input Uses @lexical/markdown (CODE) and @lexical/code.
+ * @output Exports BACKTICK_CODE and TILDE_CODE, the transformers for fenced
+ * code blocks opened with backticks and with tildes.
+ * @position Part of DEFAULT_TRANSFORMERS (markdownTable.ts): BACKTICK_CODE
+ * stands in for Lexical's CODE, and TILDE_CODE reads the `~~~` fences CODE
+ * does not (CommonMark 0.31 §4.5, spec:AST-061 FR5). Both read the opening
+ * line as core Markdown does: the rest of the line after the fence is the
+ * info string, never code; the language is the text right after the fence
+ * up to the first space (so `c++` stays `c++`), and a block whose info
+ * string starts with a space has none, as core reads it. A block closes at
+ * a fence of its own character at least as long, or at the end of the
+ * document. Each keeps its fence and info string (spec:AST-062): an edited
+ * block exports them as written, with its language first when that was
+ * changed, and a tilde fence lengthened when its code holds one that long.
+ * A one-line backtick block (```` ```code``` ````) stays Lexical's.
+ */
+
+import {$isCodeNode} from '@lexical/code';
+import {
+ CODE,
+ type ElementTransformer,
+ type MultilineElementTransformer,
+} from '@lexical/markdown';
+import {$getState, $setState, createState, type ElementNode} from 'lexical';
+
+/**
+ * The tilde fence a code block was opened with; null for a backtick block.
+ * Lexical's own fence state holds only backtick fences.
+ */
+const tildeFence = createState('astryxTildeFence', {
+ parse: (value: unknown): string | null =>
+ typeof value === 'string' && /^~{3,}$/.test(value) ? value : null,
+});
+
+/** The info string a code block's opening fence carried, as written. */
+const fenceInfo = createState('astryxFenceInfo', {
+ parse: (value: unknown): string | null =>
+ typeof value === 'string' && value !== '' ? value : null,
+});
+
+/** The language core Markdown reads from an info string, or null. */
+function languageOf(info: string): string | null {
+ return /^\S+/.exec(info)?.[0] ?? null;
+}
+
+/** An opening fence line: its fence and the info string after it. */
+function readOpening(
+ line: string,
+): {readonly fence: string; readonly info: string} | null {
+ const match = /^[ \t]*(`{3,}|~{3,})(.*)$/.exec(line);
+ if (match == null) {
+ return null;
+ }
+ const [, fence, info] = match;
+ // A backtick fence's info string holds no backtick.
+ return fence.startsWith('`') && info.includes('`') ? null : {fence, info};
+}
+
+/** The longest run of three or more tildes in `text`, or 0. */
+function longestTildeRun(text: string): number {
+ return Math.max(0, ...(text.match(/~{3,}/g) ?? []).map(run => run.length));
+}
+
+/**
+ * Builds the code block an opening line starts, from the lines after it up
+ * to its closing fence, and returns the index of the last line it used.
+ */
+function $importFencedCode(
+ rootNode: ElementNode,
+ lines: ReadonlyArray<string>,
+ startLineIndex: number,
+ opening: {readonly fence: string; readonly info: string},
+): number {
+ const {fence, info} = opening;
+ const closing = new RegExp(
+ `^[ \\t]*${fence[0] === '`' ? '`' : '~'}{${fence.length},}[ \\t]*$`,
+ );
+ let end = startLineIndex + 1;
+ while (end < lines.length && !closing.test(lines[end])) {
+ end++;
+ }
+ const language = languageOf(info);
+ const startMatch = Object.assign(
+ [lines[startLineIndex], fence, language ?? undefined],
+ {index: 0, input: lines[startLineIndex]},
+ ) as unknown as RegExpMatchArray;
+ const endMatch = end < lines.length ? lines[end].match(closing) : null;
+ CODE.replace(
+ rootNode,
+ null,
+ startMatch,
+ endMatch,
+ lines.slice(startLineIndex + 1, end),
+ true,
+ );
+ const block = rootNode.getLastChild();
+ if ($isCodeNode(block)) {
+ if (fence.startsWith('~')) {
+ $setState(block, tildeFence, fence);
+ }
+ $setState(block, fenceInfo, info.trimEnd());
+ }
+ return Math.min(end, lines.length - 1);
+}
+
+/** Writes a code block that keeps its fence or info string; null otherwise. */
+const exportFencedCode: ElementTransformer['export'] = (
+ node,
+ exportChildren,
+) => {
+ if (!$isCodeNode(node)) {
+ return null;
+ }
+ const tilde = $getState(node, tildeFence);
+ const info = $getState(node, fenceInfo);
+ if (tilde == null && info == null) {
+ return null;
+ }
+ const text = node.getTextContent();
+ const language = node.getLanguage() ?? '';
+ let written = info ?? '';
+ if ((languageOf(written) ?? '') !== language) {
+ // The language changed: it leads, before the rest of the info string.
+ const rest =
+ languageOf(written) == null
+ ? ''
+ : written.slice(languageOf(written)?.length).trim();
+ written = rest === '' ? language : `${language} ${rest}`;
+ }
+ const fence =
+ tilde != null
+ ? '~'.repeat(Math.max(tilde.length, longestTildeRun(text) + 1))
+ : (/^`+/.exec(CODE.export?.(node, exportChildren) ?? '')?.[0] ?? '```');
+ return `${fence}${written}${text === '' ? '' : `\n${text}`}\n${fence}`;
+};
+
+export const BACKTICK_CODE: MultilineElementTransformer = {
+ ...CODE,
+ handleImportAfterStartMatch: args => {
+ const opening = readOpening(args.lines[args.startLineIndex]);
+ if (opening == null) {
+ return CODE.handleImportAfterStartMatch?.(args) ?? null;
+ }
+ return [
+ true,
+ $importFencedCode(
+ args.rootNode,
+ args.lines,
+ args.startLineIndex,
+ opening,
+ ),
+ ];
+ },
+ export: (node, exportChildren) =>
+ exportFencedCode(node, exportChildren) ??
+ CODE.export?.(node, exportChildren) ??
+ null,
+};
+
+export const TILDE_CODE: MultilineElementTransformer = {
+ ...CODE,
+ regExpStart: /^([ \t]*~{3,})(.*)$/,
+ regExpEnd: {optional: true, regExp: /^[ \t]*~{3,}[ \t]*$/},
+ handleImportAfterStartMatch: ({lines, rootNode, startLineIndex}) => {
+ const opening = readOpening(lines[startLineIndex]);
+ return opening == null
+ ? null
+ : [true, $importFencedCode(rootNode, lines, startLineIndex, opening)];
+ },
+ export: (node, exportChildren) =>
+ exportFencedCode(node, exportChildren) ?? null,
+};
diff --git a/packages/richtext/src/markdownTable.ts b/packages/richtext/src/markdownTable.ts
index 30a0ad894a..9e233a3791 100644
--- a/packages/richtext/src/markdownTable.ts
+++ b/packages/richtext/src/markdownTable.ts
@@ -18,6 +18,7 @@
import {
$convertFromMarkdownString,
$convertToMarkdownString,
+ CODE,
TEXT_FORMAT_TRANSFORMERS,
TEXT_MATCH_TRANSFORMERS,
LINK,
@@ -45,6 +46,7 @@ import {LIST_EXPORT} from './markdownListExport';
import {TASK_LIST} from './markdownTaskList';
import {ORDERED_LIST_KEEPING_START} from './markdownOrderedList';
import {LINK_KEEPING_DESTINATIONS} from './markdownLink';
+import {BACKTICK_CODE, TILDE_CODE} from './markdownCodeFence';
/**
* Cells hold inline Markdown only, so they are imported and exported with the
@@ -246,7 +248,9 @@ export const TABLE: MultilineElementTransformer = {
/**
* The editor's default Markdown transformers: thematic breaks, list export
* that nests to each parent's content column, GFM task list items, Lexical's
- * standard set (its ordered list keeping each nested list's start, and its
+ * standard set (with `~~~` fences beside its backtick ones, both reading the
+ * opening line's info string as core does, its ordered list keeping each
+ * nested list's start, and its
* link writing an unbalanced destination's parentheses escaped), hard line
* breaks for breaks typed in the editor, and GFM tables. Thematic breaks come first so a line such as `* * *` is a rule
* rather than a list item, and task items before bullets so `- [ ] text` is a
@@ -256,12 +260,15 @@ export const DEFAULT_TRANSFORMERS: ReadonlyArray<Transformer> = [
THEMATIC_BREAK,
LIST_EXPORT,
TASK_LIST,
+ TILDE_CODE,
...TRANSFORMERS.map(transformer =>
transformer === ORDERED_LIST
? ORDERED_LIST_KEEPING_START
: transformer === LINK
? LINK_KEEPING_DESTINATIONS
- : transformer,
+ : transformer === CODE
+ ? BACKTICK_CODE
+ : transformer,
),
HARD_LINE_BREAK,
TABLE,