An open source design system that's fully customizable and agent ready
TypeScript
fix(richtext): read tilde code fences, and every fence's info string as core does (#7061)
Cindy Zhang6260001bfec3
diff --git a/packages/richtext/src/markdownCodeFence.test.ts b/packages/richtext/src/markdownCodeFence.test.tsnew file mode 100644index 0000000000..32ab59d719--- /dev/null+++ b/packages/richtext/src/markdownCodeFence.test.ts@@ -0,0 +1,142 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++import {describe, expect, it} from 'vitest';+import {createHeadlessEditor} from '@lexical/headless';+import {$isCodeNode, type CodeNode} from '@lexical/code';+import {$getRoot, $isTextNode} from 'lexical';+import {parseMarkdownAst} from '@astryxdesign/core/Markdown/parser';+import {DEFAULT_NODES} from './editorNodes';+import {DEFAULT_TRANSFORMERS} from './markdownTable';+import {+ $exportMarkdownKeepingSource,+ importMarkdownKeepingSource,+} from './markdownSource';+import {+ editorStateJSONToMarkdown,+ markdownToEditorStateJSON,+} from './markdownSerializers';++type Json = Record<string, unknown>;++/** Each top-level block: a code block's language and code, or its type. */+function blocks(markdown: string): Array<unknown> {+ const root = (JSON.parse(markdownToEditorStateJSON(markdown)) as {root: Json})+ .root;+ return (root.children as Array<Json>).map(node =>+ node.type === 'code'+ ? {+ language: node.language ?? null,+ code: ((node.children as Array<Json>) ?? [])+ .map(child =>+ child.type === 'linebreak' ? '\n' : (child.text as string),+ )+ .join(''),+ }+ : node.type,+ );+}++/** The same, as core Markdown reads it. */+function coreBlocks(markdown: string): Array<unknown> {+ return (parseMarkdownAst(markdown).children as unknown as Array<Json>).map(+ node =>+ node.type === 'code'+ ? {language: node.lang ?? null, code: node.value as string}+ : node.type,+ );+}++/** Imports `markdown`, edits its code block with `edit`, and exports. */+function editAndExport(+ markdown: string,+ edit: (code: CodeNode) => void,+): string {+ const editor = createHeadlessEditor({+ nodes: [...DEFAULT_NODES],+ onError(error) {+ throw error;+ },+ });+ importMarkdownKeepingSource(editor, markdown, [...DEFAULT_TRANSFORMERS]);+ editor.update(+ () => {+ const code = $getRoot().getFirstChild();+ if (!$isCodeNode(code)) {+ throw new Error('No code block');+ }+ edit(code);+ },+ {discrete: true},+ );+ return editor+ .getEditorState()+ .read(() => $exportMarkdownKeepingSource([...DEFAULT_TRANSFORMERS]));+}++const setCode = (text: string) => (code: CodeNode) => {+ const first = code.getFirstChild();+ if (!$isTextNode(first)) {+ throw new Error('No code text');+ }+ first.setTextContent(text);+};++describe.each(['~~~', '```'])('%s fences (CommonMark 0.31 §4.5)', fence => {+ it.each([+ `${fence} js\nx\n${fence}\n`,+ `${fence}python title="a.py"\nx = 1\n${fence}\n`,+ `${fence} {.haskell .numberLines}\nmain = pure ()\n${fence}\n`,+ `${fence}c++\nint x;\n${fence}\n`,+ `${fence}\nplain\n${fence}\n`,+ ])(+ 'reads %j with the language and code core reads, and round-trips it',+ markdown => {+ expect(blocks(markdown)).toEqual(coreBlocks(markdown));+ expect(+ editorStateJSONToMarkdown(markdownToEditorStateJSON(markdown)),+ ).toBe(markdown);+ },+ );++ it('keeps the info string through an edit', () => {+ const edited = editAndExport(+ `${fence}python title="a.py"\nx = 1\n${fence}\n`,+ setCode('x = 2'),+ );+ expect(edited).toBe(`${fence}python title="a.py"\nx = 2\n${fence}\n`);+ expect(blocks(edited)).toEqual(coreBlocks(edited));+ });++ it('writes a changed language first, before the rest of the info string', () => {+ const edited = editAndExport(+ `${fence}python title="a.py"\nx = 1\n${fence}\n`,+ code => code.setLanguage('ruby'),+ );+ expect(edited).toBe(`${fence}ruby title="a.py"\nx = 1\n${fence}\n`);+ });+});++describe('tilde code fences', () => {+ it.each([+ '~~~~\n~~~\ninner\n~~~\n~~~~\n',+ '~~~\n```\nbackticks\n```\n~~~\n',+ 'Before\n\n~~~js\nx();\n~~~\n\nAfter\n',+ ])('reads %j as core does, and round-trips it', markdown => {+ expect(blocks(markdown)).toEqual(coreBlocks(markdown));+ expect(editorStateJSONToMarkdown(markdownToEditorStateJSON(markdown))).toBe(+ markdown,+ );+ });++ it('runs an unclosed tilde block to the end of the document', () => {+ expect(blocks('~~~\nopen\ncode\n')).toEqual([+ {language: null, code: 'open\ncode'},+ ]);+ });++ it('exports an edited tilde block with tildes, longer than any tilde fence in its code', () => {+ const edited = editAndExport('~~~ts\nconst a = 1;\n~~~\n', setCode('~~~'));+ expect(edited).toBe('~~~~ts\n~~~\n~~~~\n');+ expect(blocks(edited)).toEqual([{language: 'ts', code: '~~~'}]);+ });+});diff --git a/packages/richtext/src/markdownCodeFence.ts b/packages/richtext/src/markdownCodeFence.tsnew file mode 100644index 0000000000..03b26f1ec3--- /dev/null+++ b/packages/richtext/src/markdownCodeFence.ts@@ -0,0 +1,176 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file markdownCodeFence.ts+ * @input Uses @lexical/markdown (CODE) and @lexical/code.+ * @output Exports BACKTICK_CODE and TILDE_CODE, the transformers for fenced+ * code blocks opened with backticks and with tildes.+ * @position Part of DEFAULT_TRANSFORMERS (markdownTable.ts): BACKTICK_CODE+ * stands in for Lexical's CODE, and TILDE_CODE reads the `~~~` fences CODE+ * does not (CommonMark 0.31 §4.5, spec:AST-061 FR5). Both read the opening+ * line as core Markdown does: the rest of the line after the fence is the+ * info string, never code; the language is the text right after the fence+ * up to the first space (so `c++` stays `c++`), and a block whose info+ * string starts with a space has none, as core reads it. A block closes at+ * a fence of its own character at least as long, or at the end of the+ * document. Each keeps its fence and info string (spec:AST-062): an edited+ * block exports them as written, with its language first when that was+ * changed, and a tilde fence lengthened when its code holds one that long.+ * A one-line backtick block (```` ```code``` ````) stays Lexical's.+ */++import {$isCodeNode} from '@lexical/code';+import {+ CODE,+ type ElementTransformer,+ type MultilineElementTransformer,+} from '@lexical/markdown';+import {$getState, $setState, createState, type ElementNode} from 'lexical';++/**+ * The tilde fence a code block was opened with; null for a backtick block.+ * Lexical's own fence state holds only backtick fences.+ */+const tildeFence = createState('astryxTildeFence', {+ parse: (value: unknown): string | null =>+ typeof value === 'string' && /^~{3,}$/.test(value) ? value : null,+});++/** The info string a code block's opening fence carried, as written. */+const fenceInfo = createState('astryxFenceInfo', {+ parse: (value: unknown): string | null =>+ typeof value === 'string' && value !== '' ? value : null,+});++/** The language core Markdown reads from an info string, or null. */+function languageOf(info: string): string | null {+ return /^\S+/.exec(info)?.[0] ?? null;+}++/** An opening fence line: its fence and the info string after it. */+function readOpening(+ line: string,+): {readonly fence: string; readonly info: string} | null {+ const match = /^[ \t]*(`{3,}|~{3,})(.*)$/.exec(line);+ if (match == null) {+ return null;+ }+ const [, fence, info] = match;+ // A backtick fence's info string holds no backtick.+ return fence.startsWith('`') && info.includes('`') ? null : {fence, info};+}++/** The longest run of three or more tildes in `text`, or 0. */+function longestTildeRun(text: string): number {+ return Math.max(0, ...(text.match(/~{3,}/g) ?? []).map(run => run.length));+}++/**+ * Builds the code block an opening line starts, from the lines after it up+ * to its closing fence, and returns the index of the last line it used.+ */+function $importFencedCode(+ rootNode: ElementNode,+ lines: ReadonlyArray<string>,+ startLineIndex: number,+ opening: {readonly fence: string; readonly info: string},+): number {+ const {fence, info} = opening;+ const closing = new RegExp(+ `^[ \\t]*${fence[0] === '`' ? '`' : '~'}{${fence.length},}[ \\t]*$`,+ );+ let end = startLineIndex + 1;+ while (end < lines.length && !closing.test(lines[end])) {+ end++;+ }+ const language = languageOf(info);+ const startMatch = Object.assign(+ [lines[startLineIndex], fence, language ?? undefined],+ {index: 0, input: lines[startLineIndex]},+ ) as unknown as RegExpMatchArray;+ const endMatch = end < lines.length ? lines[end].match(closing) : null;+ CODE.replace(+ rootNode,+ null,+ startMatch,+ endMatch,+ lines.slice(startLineIndex + 1, end),+ true,+ );+ const block = rootNode.getLastChild();+ if ($isCodeNode(block)) {+ if (fence.startsWith('~')) {+ $setState(block, tildeFence, fence);+ }+ $setState(block, fenceInfo, info.trimEnd());+ }+ return Math.min(end, lines.length - 1);+}++/** Writes a code block that keeps its fence or info string; null otherwise. */+const exportFencedCode: ElementTransformer['export'] = (+ node,+ exportChildren,+) => {+ if (!$isCodeNode(node)) {+ return null;+ }+ const tilde = $getState(node, tildeFence);+ const info = $getState(node, fenceInfo);+ if (tilde == null && info == null) {+ return null;+ }+ const text = node.getTextContent();+ const language = node.getLanguage() ?? '';+ let written = info ?? '';+ if ((languageOf(written) ?? '') !== language) {+ // The language changed: it leads, before the rest of the info string.+ const rest =+ languageOf(written) == null+ ? ''+ : written.slice(languageOf(written)?.length).trim();+ written = rest === '' ? language : `${language} ${rest}`;+ }+ const fence =+ tilde != null+ ? '~'.repeat(Math.max(tilde.length, longestTildeRun(text) + 1))+ : (/^`+/.exec(CODE.export?.(node, exportChildren) ?? '')?.[0] ?? '```');+ return `${fence}${written}${text === '' ? '' : `\n${text}`}\n${fence}`;+};++export const BACKTICK_CODE: MultilineElementTransformer = {+ ...CODE,+ handleImportAfterStartMatch: args => {+ const opening = readOpening(args.lines[args.startLineIndex]);+ if (opening == null) {+ return CODE.handleImportAfterStartMatch?.(args) ?? null;+ }+ return [+ true,+ $importFencedCode(+ args.rootNode,+ args.lines,+ args.startLineIndex,+ opening,+ ),+ ];+ },+ export: (node, exportChildren) =>+ exportFencedCode(node, exportChildren) ??+ CODE.export?.(node, exportChildren) ??+ null,+};++export const TILDE_CODE: MultilineElementTransformer = {+ ...CODE,+ regExpStart: /^([ \t]*~{3,})(.*)$/,+ regExpEnd: {optional: true, regExp: /^[ \t]*~{3,}[ \t]*$/},+ handleImportAfterStartMatch: ({lines, rootNode, startLineIndex}) => {+ const opening = readOpening(lines[startLineIndex]);+ return opening == null+ ? null+ : [true, $importFencedCode(rootNode, lines, startLineIndex, opening)];+ },+ export: (node, exportChildren) =>+ exportFencedCode(node, exportChildren) ?? null,+};diff --git a/packages/richtext/src/markdownTable.ts b/packages/richtext/src/markdownTable.tsindex 30a0ad894a..9e233a3791 100644--- a/packages/richtext/src/markdownTable.ts+++ b/packages/richtext/src/markdownTable.ts@@ -18,6 +18,7 @@import {$convertFromMarkdownString,$convertToMarkdownString,+ CODE,TEXT_FORMAT_TRANSFORMERS,TEXT_MATCH_TRANSFORMERS,LINK,@@ -45,6 +46,7 @@ import {LIST_EXPORT} from './markdownListExport';import {TASK_LIST} from './markdownTaskList';import {ORDERED_LIST_KEEPING_START} from './markdownOrderedList';import {LINK_KEEPING_DESTINATIONS} from './markdownLink';+import {BACKTICK_CODE, TILDE_CODE} from './markdownCodeFence';/*** Cells hold inline Markdown only, so they are imported and exported with the@@ -246,7 +248,9 @@ export const TABLE: MultilineElementTransformer = {/*** The editor's default Markdown transformers: thematic breaks, list export* that nests to each parent's content column, GFM task list items, Lexical's- * standard set (its ordered list keeping each nested list's start, and its+ * standard set (with `~~~` fences beside its backtick ones, both reading the+ * opening line's info string as core does, its ordered list keeping each+ * nested list's start, and its* link writing an unbalanced destination's parentheses escaped), hard line* breaks for breaks typed in the editor, and GFM tables. Thematic breaks come first so a line such as `* * *` is a rule* rather than a list item, and task items before bullets so `- [ ] text` is a@@ -256,12 +260,15 @@ export const DEFAULT_TRANSFORMERS: ReadonlyArray<Transformer> = [THEMATIC_BREAK,LIST_EXPORT,TASK_LIST,+ TILDE_CODE,...TRANSFORMERS.map(transformer =>transformer === ORDERED_LIST? ORDERED_LIST_KEEPING_START: transformer === LINK? LINK_KEEPING_DESTINATIONS- : transformer,+ : transformer === CODE+ ? BACKTICK_CODE+ : transformer,),HARD_LINE_BREAK,TABLE,