An open source design system that's fully customizable and agent ready
TypeScript
feat(richtext): color fenced code with core CodeBlock's tokenizer and syntax color tokens (#7031)
Cindy Zhang822d4527fc5b
diff --git a/packages/richtext/src/CodeSyntaxPlugin.test.tsx b/packages/richtext/src/CodeSyntaxPlugin.test.tsxnew file mode 100644index 0000000000..b2419b6536--- /dev/null+++ b/packages/richtext/src/CodeSyntaxPlugin.test.tsx@@ -0,0 +1,40 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++import {describe, it, expect} from 'vitest';+import {render, screen} from '@testing-library/react';+import {tokenize} from '@astryxdesign/core/CodeBlock';+import {renderedTokens} from './CodeSyntaxPlugin';+import {RichTextView} from './RichTextView';+import {markdownToEditorStateJSON} from './markdownSerializers';++describe('code block syntax colors (spec:AST-061 FR1)', () => {+ const code = 'let a = 1;\nreturn a;';++ it("uses core CodeBlock's tokens line by line when the text keeps its line breaks", () => {+ expect(renderedTokens(code, 'ts', code)).toEqual(tokenize(code, 'ts'));+ });++ it('lays the tokens over the text as one line when line breaks render without text', () => {+ const rendered = code.replace('\n', '');+ const [line] = renderedTokens(code, 'ts', rendered) ?? [];+ const texts = (line ?? []).map(+ token => `${token.type}:${rendered.slice(token.start, token.end)}`,+ );+ expect(texts).toContain('keyword:let');+ expect(texts).toContain('keyword:return');+ });++ it('colors nothing while the rendered text is not the code, or for a language core does not know', () => {+ expect(renderedTokens(code, 'ts', 'let a')).toBeNull();+ expect(renderedTokens(code, 'notalanguage', code)).toEqual([]);+ });++ it('renders code blocks where the Highlight API is missing', async () => {+ render(+ <RichTextView+ value={markdownToEditorStateJSON('```ts\nlet plain = 1;\n```\n')}+ />,+ );+ expect(await screen.findByText('let plain = 1;')).toBeInTheDocument();+ });+});diff --git a/packages/richtext/src/CodeSyntaxPlugin.tsx b/packages/richtext/src/CodeSyntaxPlugin.tsxnew file mode 100644index 0000000000..6ec0560735--- /dev/null+++ b/packages/richtext/src/CodeSyntaxPlugin.tsx@@ -0,0 +1,193 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++'use client';++/**+ * @file CodeSyntaxPlugin.tsx+ * @input Uses @lexical/react (composer context), @lexical/code (CodeNode),+ * lexical, and core CodeBlock's tokenizer, highlight ranges, and token+ * types.+ * @output Exports CodeSyntaxPlugin, which colors the code in every fenced code+ * block with core CodeBlock's tokenizer and syntax color tokens;+ * CODE_SYNTAX_CLASS, the class the theme gives code blocks for it; and+ * renderedTokens, which lays tokens out over a block's rendered text.+ * @position Rendered by RichTextEditor and RichTextView. A fenced code block+ * with a language core CodeBlock knows is colored as core Markdown colors+ * it: the same tokenizer finds the same tokens, and each token type takes+ * its `--color-syntax-*` token (spec:AST-061 FR1) — no palette of its own.+ * The colors are CSS Custom Highlight ranges over the text Lexical renders,+ * the way core CodeBlock colors editable code, so the document and its DOM+ * are untouched; the ranges are rebuilt when a block's code or language+ * changes. Browsers without the Highlight API show plain code.+ */++import {useEffect} from 'react';+import {useLexicalComposerContext} from '@lexical/react/LexicalComposerContext';+import {$isCodeNode, CodeNode} from '@lexical/code';+import {$getNodeByKey, type NodeKey} from 'lexical';+import {+ applyHighlightRangesFlat,+ TOKEN_TYPES,+ tokenize,+ type TokenLine,+} from '@astryxdesign/core/CodeBlock';++/** The class the theme adds to every code block (editorTheme.ts). */+export const CODE_SYNTAX_CLASS = 'astryx-richtext-code';++let hasInjectedRules = false;++/**+ * The highlight rules for code blocks, injected once: each token type takes+ * the same syntax color token core CodeBlock's rules use.+ */+function ensureRules(): void {+ if (hasInjectedRules || typeof document === 'undefined') {+ return;+ }+ hasInjectedRules = true;+ const style = document.createElement('style');+ style.setAttribute('data-astryx-richtext-code-syntax', '');+ style.textContent = TOKEN_TYPES.map(+ type =>+ `.${CODE_SYNTAX_CLASS}::highlight(astryx-${type}) { color: var(--color-syntax-${type}); }`,+ ).join('\n');+ document.head.append(style);+}++function supportsHighlights(): boolean {+ return (+ typeof CSS !== 'undefined' &&+ 'highlights' in CSS &&+ typeof Highlight !== 'undefined'+ );+}++/**+ * The tokens of `code`, laid out for the rendered text of its block: line by+ * line when the text keeps the line breaks, or as one line of offsets when+ * each line break is an element with no text. Null when the rendered text is+ * not the code (it is mid-update), so nothing is colored out of place.+ */+export function renderedTokens(+ code: string,+ language: string,+ rendered: string,+): TokenLine[] | null {+ const lines = tokenize(code, language);+ if (rendered === code) {+ return lines;+ }+ if (rendered !== code.replaceAll('\n', '')) {+ return null;+ }+ const flat: TokenLine = [];+ let offset = 0;+ code.split('\n').forEach((line, index) => {+ for (const token of lines[index] ?? []) {+ flat.push({+ type: token.type,+ start: offset + token.start,+ end: offset + token.end,+ });+ }+ offset += line.length;+ });+ return [flat];+}++interface Colored {+ readonly element: HTMLElement;+ readonly code: string;+ readonly language: string;+ readonly cleanup: () => void;+}++export function CodeSyntaxPlugin(): null {+ const [editor] = useLexicalComposerContext();++ useEffect(() => {+ if (!supportsHighlights()) {+ return undefined;+ }+ ensureRules();+ // Every code block in the editor, and the ones colored, by node key.+ const blocks = new Set<NodeKey>();+ const colored = new Map<NodeKey, Colored>();+ const uncolor = (key: NodeKey) => {+ colored.get(key)?.cleanup();+ colored.delete(key);+ };+ const color = (key: NodeKey) => {+ const block = editor.getEditorState().read(() => {+ const node = $getNodeByKey(key);+ return $isCodeNode(node)+ ? {code: node.getTextContent(), language: node.getLanguage() ?? ''}+ : null;+ });+ const element = editor.getElementByKey(key);+ if (block == null || element == null) {+ uncolor(key);+ return;+ }+ const previous = colored.get(key);+ if (+ previous != null &&+ previous.element === element &&+ previous.code === block.code &&+ previous.language === block.language+ ) {+ return;+ }+ previous?.cleanup();+ const tokens = renderedTokens(+ block.code,+ block.language,+ element.textContent ?? '',+ );+ if (tokens == null) {+ // Not rendered yet; the next update tries again.+ colored.delete(key);+ return;+ }+ colored.set(key, {+ element,+ ...block,+ cleanup: applyHighlightRangesFlat(element, tokens),+ });+ };+ const unregisterMutations = editor.registerMutationListener(+ CodeNode,+ mutations => {+ for (const [key, mutation] of mutations) {+ if (mutation === 'destroyed') {+ blocks.delete(key);+ uncolor(key);+ } else {+ blocks.add(key);+ color(key);+ }+ }+ },+ {skipInitialization: false},+ );+ // Typing in a block changes its text nodes, not always the block itself:+ // recheck every block after each update (unchanged ones stop at the+ // comparison above).+ const unregisterUpdates = editor.registerUpdateListener(() => {+ for (const key of blocks) {+ color(key);+ }+ });+ return () => {+ unregisterMutations();+ unregisterUpdates();+ for (const key of [...colored.keys()]) {+ uncolor(key);+ }+ blocks.clear();+ };+ }, [editor]);++ return null;+}diff --git a/packages/richtext/src/RichTextEditor.tsx b/packages/richtext/src/RichTextEditor.tsxindex ecc61ac7a9..629c09e931 100644--- a/packages/richtext/src/RichTextEditor.tsx+++ b/packages/richtext/src/RichTextEditor.tsx@@ -76,6 +76,7 @@ import {HorizontalRuleExtension} from '@lexical/extension';import {StrikethroughDeletionExtension} from './strikethroughDeletion';import {TableColumnFloorPlugin} from './TableColumnFloorPlugin';import {CodeBlockHeaderPlugin} from './CodeBlockHeaderPlugin';+import {CodeSyntaxPlugin} from './CodeSyntaxPlugin';import {TaskCheckboxPlugin} from './TaskCheckboxPlugin';import {type Transformer} from '@lexical/markdown';export type {Transformer} from '@lexical/markdown';@@ -694,6 +695,7 @@ export const RichTextEditor = forwardRef</><TableColumnFloorPlugin /><CodeBlockHeaderPlugin />+ <CodeSyntaxPlugin /><TabIndentationPlugin /><TabFocusEscapePlugin />{hasMarkdownShortcuts && (diff --git a/packages/richtext/src/RichTextView.tsx b/packages/richtext/src/RichTextView.tsxindex 9587323179..db3f6b3b0f 100644--- a/packages/richtext/src/RichTextView.tsx+++ b/packages/richtext/src/RichTextView.tsx@@ -33,6 +33,7 @@ import {TablePlugin} from '@lexical/react/LexicalTablePlugin';import {TableScrollRegionPlugin} from './TableScrollRegionPlugin';import {TableColumnFloorPlugin} from './TableColumnFloorPlugin';import {CodeBlockHeaderPlugin} from './CodeBlockHeaderPlugin';+import {CodeSyntaxPlugin} from './CodeSyntaxPlugin';import {TaskCheckboxPlugin} from './TaskCheckboxPlugin';import type {AnyLexicalExtension,@@ -296,6 +297,7 @@ export function RichTextView({{/* After the content, so the copy buttons and checkboxes follow itin tab order. */}<CodeBlockHeaderPlugin />+ <CodeSyntaxPlugin /><TaskCheckboxPlugin isReadOnly />{plugins}</LexicalExtensionComposer>diff --git a/packages/richtext/src/__tests__/MarkdownParitySandbox.a11y.chromium.spec.ts b/packages/richtext/src/__tests__/MarkdownParitySandbox.a11y.chromium.spec.tsindex 2244b1bd13..48d944ff35 100644--- a/packages/richtext/src/__tests__/MarkdownParitySandbox.a11y.chromium.spec.ts+++ b/packages/richtext/src/__tests__/MarkdownParitySandbox.a11y.chromium.spec.ts@@ -1083,6 +1083,115 @@ for (const globals of [});}+// spec:AST-061 FR1: fenced code takes the same syntax colors on both+// surfaces — the same tokenizer finds the same tokens, and each token type+// takes its `--color-syntax-*` token.+/** Each block's highlighted tokens on one surface, as `type:text`. */+async function syntaxTokens(+ page: Page,+ surface: string,+): Promise<Record<string, Array<string>>> {+ return page.evaluate(selector => {+ const tokens: Record<string, Array<string>> = {};+ for (const [name, highlight] of CSS.highlights) {+ if (!name.startsWith('astryx-')) {+ continue;+ }+ for (const range of highlight) {+ const element = range.startContainer.parentElement;+ const block = element?.closest(selector)?.contains(element)+ ? element.closest('[data-parity-block]')+ : null;+ if (block == null || !(range instanceof Range)) {+ continue;+ }+ const key = block.getAttribute('data-parity-block') ?? '';+ (tokens[key] ??= []).push(+ `${name.slice('astryx-'.length)}:${range.toString()}`,+ );+ }+ }+ for (const list of Object.values(tokens)) {+ list.sort();+ }+ return tokens;+ }, surface);+}++for (const globals of [+ 'colorMode:light;direction:ltr',+ 'colorMode:dark;direction:rtl',+]) {+ test(`side by side: fenced code takes core Markdown's syntax colors (${globals})`, async ({+ page,+ }) => {+ const errors = await openStory(page, STORY.sideBySide, DESKTOP, globals);+ await waitForDocument(page, MARKDOWN);+ await waitForDocument(page, RICH_TEXT);+ await expect+ .poll(async () => (await syntaxTokens(page, RICH_TEXT))['code-fence'])+ .toBeDefined();+ const markdown = await syntaxTokens(page, MARKDOWN);+ const richText = await syntaxTokens(page, RICH_TEXT);+ for (const key of ['code-fence', 'code-long']) {+ expect(markdown[key]?.length ?? 0, key).toBeGreaterThan(0);+ expect(richText[key], key).toEqual(markdown[key]);+ }+ // A fence with no language core CodeBlock knows stays plain on both.+ for (const key of ['code-plain', 'code-plaintext', 'code-unknown']) {+ expect(richText[key], key).toBeUndefined();+ expect(markdown[key], key).toBeUndefined();+ }+ // Each token type takes the same syntax color token as core CodeBlock.+ const rules = await page.evaluate(+ () =>+ document.querySelector('style[data-astryx-richtext-code-syntax]')+ ?.textContent ?? '',+ );+ expect(rules).toContain(+ '::highlight(astryx-keyword) { color: var(--color-syntax-keyword); }',+ );+ expect(rules).toContain(+ '::highlight(astryx-string) { color: var(--color-syntax-string); }',+ );+ expect(errors).toEqual([]);+ });+}++test('edited code is colored again as it changes', async ({page}) => {+ await page.setViewportSize(DESKTOP);+ await page.goto(+ `${storybook.origin}/iframe.html?id=lab-richtexteditor--markdown-serializers&viewMode=story&globals=astryxTheme:neutral;colorMode:light;direction:ltr`,+ {waitUntil: 'load'},+ );+ await page.locator('textarea').fill('```ts\nlet value = 1;\n```');+ const editor = page.locator('[data-lexical-editor][contenteditable="true"]');+ const editorTokens = () =>+ page.evaluate(() => {+ const root = document.querySelector(+ '[data-lexical-editor][contenteditable="true"]',+ );+ const tokens: Array<string> = [];+ for (const [name, highlight] of CSS.highlights) {+ for (const range of highlight) {+ if (range instanceof Range && root?.contains(range.startContainer)) {+ tokens.push(`${name}:${range.toString()}`);+ }+ }+ }+ return tokens.sort();+ });+ await expect.poll(editorTokens).toContain('astryx-keyword:let');+ // Type a keyword on a new line: it is colored, and the first line keeps+ // its colors.+ await editor.getByText('let value = 1;').click();+ await page.keyboard.press('End');+ await page.keyboard.press('Enter');+ await page.keyboard.type('return value;');+ await expect.poll(editorTokens).toContain('astryx-keyword:return');+ expect(await editorTokens()).toContain('astryx-keyword:let');+});+for (const viewport of [PHONE, {width: 1280, height: 900}] as const) {for (const globals of ['colorMode:light;direction:ltr',diff --git a/packages/richtext/src/editorTheme.ts b/packages/richtext/src/editorTheme.tsindex 6238bc67bb..f2719143f3 100644--- a/packages/richtext/src/editorTheme.ts+++ b/packages/richtext/src/editorTheme.ts@@ -24,6 +24,7 @@ import {focusVars,sizeVars,} from '@astryxdesign/core/theme/tokens.stylex';+import {CODE_SYNTAX_CLASS} from './CodeSyntaxPlugin';import type {EditorThemeClasses} from 'lexical';// Document blocks follow core Markdown's typography, spacing, and measure@@ -446,7 +447,8 @@ export function sharedEditorTheme(): EditorThemeClasses {strikethrough: stylex.props(editorTheme.textStrikethrough).className,code: stylex.props(editorTheme.textCode).className,},- code: block(editorTheme.code),+ // The plain class lets CodeSyntaxPlugin's highlight rules find code blocks.+ code: `${block(editorTheme.code)} ${CODE_SYNTAX_CLASS}`,hr: stylex.props(editorTheme.hr).className,hrSelected: stylex.props(editorTheme.hrSelected).className,table: block(editorTheme.table),