Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

feat(richtext): color fenced code with core CodeBlock's tokenizer and syntax color tokens (#7031)

Cindy Zhang822d4527fc5b
diff --git a/packages/richtext/src/CodeSyntaxPlugin.test.tsx b/packages/richtext/src/CodeSyntaxPlugin.test.tsx
new file mode 100644
index 0000000000..b2419b6536
--- /dev/null
+++ b/packages/richtext/src/CodeSyntaxPlugin.test.tsx
@@ -0,0 +1,40 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+import {describe, it, expect} from 'vitest';
+import {render, screen} from '@testing-library/react';
+import {tokenize} from '@astryxdesign/core/CodeBlock';
+import {renderedTokens} from './CodeSyntaxPlugin';
+import {RichTextView} from './RichTextView';
+import {markdownToEditorStateJSON} from './markdownSerializers';
+
+describe('code block syntax colors (spec:AST-061 FR1)', () => {
+ const code = 'let a = 1;\nreturn a;';
+
+ it("uses core CodeBlock's tokens line by line when the text keeps its line breaks", () => {
+ expect(renderedTokens(code, 'ts', code)).toEqual(tokenize(code, 'ts'));
+ });
+
+ it('lays the tokens over the text as one line when line breaks render without text', () => {
+ const rendered = code.replace('\n', '');
+ const [line] = renderedTokens(code, 'ts', rendered) ?? [];
+ const texts = (line ?? []).map(
+ token => `${token.type}:${rendered.slice(token.start, token.end)}`,
+ );
+ expect(texts).toContain('keyword:let');
+ expect(texts).toContain('keyword:return');
+ });
+
+ it('colors nothing while the rendered text is not the code, or for a language core does not know', () => {
+ expect(renderedTokens(code, 'ts', 'let a')).toBeNull();
+ expect(renderedTokens(code, 'notalanguage', code)).toEqual([]);
+ });
+
+ it('renders code blocks where the Highlight API is missing', async () => {
+ render(
+ <RichTextView
+ value={markdownToEditorStateJSON('```ts\nlet plain = 1;\n```\n')}
+ />,
+ );
+ expect(await screen.findByText('let plain = 1;')).toBeInTheDocument();
+ });
+});
diff --git a/packages/richtext/src/CodeSyntaxPlugin.tsx b/packages/richtext/src/CodeSyntaxPlugin.tsx
new file mode 100644
index 0000000000..6ec0560735
--- /dev/null
+++ b/packages/richtext/src/CodeSyntaxPlugin.tsx
@@ -0,0 +1,193 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+'use client';
+
+/**
+ * @file CodeSyntaxPlugin.tsx
+ * @input Uses @lexical/react (composer context), @lexical/code (CodeNode),
+ * lexical, and core CodeBlock's tokenizer, highlight ranges, and token
+ * types.
+ * @output Exports CodeSyntaxPlugin, which colors the code in every fenced code
+ * block with core CodeBlock's tokenizer and syntax color tokens;
+ * CODE_SYNTAX_CLASS, the class the theme gives code blocks for it; and
+ * renderedTokens, which lays tokens out over a block's rendered text.
+ * @position Rendered by RichTextEditor and RichTextView. A fenced code block
+ * with a language core CodeBlock knows is colored as core Markdown colors
+ * it: the same tokenizer finds the same tokens, and each token type takes
+ * its `--color-syntax-*` token (spec:AST-061 FR1) — no palette of its own.
+ * The colors are CSS Custom Highlight ranges over the text Lexical renders,
+ * the way core CodeBlock colors editable code, so the document and its DOM
+ * are untouched; the ranges are rebuilt when a block's code or language
+ * changes. Browsers without the Highlight API show plain code.
+ */
+
+import {useEffect} from 'react';
+import {useLexicalComposerContext} from '@lexical/react/LexicalComposerContext';
+import {$isCodeNode, CodeNode} from '@lexical/code';
+import {$getNodeByKey, type NodeKey} from 'lexical';
+import {
+ applyHighlightRangesFlat,
+ TOKEN_TYPES,
+ tokenize,
+ type TokenLine,
+} from '@astryxdesign/core/CodeBlock';
+
+/** The class the theme adds to every code block (editorTheme.ts). */
+export const CODE_SYNTAX_CLASS = 'astryx-richtext-code';
+
+let hasInjectedRules = false;
+
+/**
+ * The highlight rules for code blocks, injected once: each token type takes
+ * the same syntax color token core CodeBlock's rules use.
+ */
+function ensureRules(): void {
+ if (hasInjectedRules || typeof document === 'undefined') {
+ return;
+ }
+ hasInjectedRules = true;
+ const style = document.createElement('style');
+ style.setAttribute('data-astryx-richtext-code-syntax', '');
+ style.textContent = TOKEN_TYPES.map(
+ type =>
+ `.${CODE_SYNTAX_CLASS}::highlight(astryx-${type}) { color: var(--color-syntax-${type}); }`,
+ ).join('\n');
+ document.head.append(style);
+}
+
+function supportsHighlights(): boolean {
+ return (
+ typeof CSS !== 'undefined' &&
+ 'highlights' in CSS &&
+ typeof Highlight !== 'undefined'
+ );
+}
+
+/**
+ * The tokens of `code`, laid out for the rendered text of its block: line by
+ * line when the text keeps the line breaks, or as one line of offsets when
+ * each line break is an element with no text. Null when the rendered text is
+ * not the code (it is mid-update), so nothing is colored out of place.
+ */
+export function renderedTokens(
+ code: string,
+ language: string,
+ rendered: string,
+): TokenLine[] | null {
+ const lines = tokenize(code, language);
+ if (rendered === code) {
+ return lines;
+ }
+ if (rendered !== code.replaceAll('\n', '')) {
+ return null;
+ }
+ const flat: TokenLine = [];
+ let offset = 0;
+ code.split('\n').forEach((line, index) => {
+ for (const token of lines[index] ?? []) {
+ flat.push({
+ type: token.type,
+ start: offset + token.start,
+ end: offset + token.end,
+ });
+ }
+ offset += line.length;
+ });
+ return [flat];
+}
+
+interface Colored {
+ readonly element: HTMLElement;
+ readonly code: string;
+ readonly language: string;
+ readonly cleanup: () => void;
+}
+
+export function CodeSyntaxPlugin(): null {
+ const [editor] = useLexicalComposerContext();
+
+ useEffect(() => {
+ if (!supportsHighlights()) {
+ return undefined;
+ }
+ ensureRules();
+ // Every code block in the editor, and the ones colored, by node key.
+ const blocks = new Set<NodeKey>();
+ const colored = new Map<NodeKey, Colored>();
+ const uncolor = (key: NodeKey) => {
+ colored.get(key)?.cleanup();
+ colored.delete(key);
+ };
+ const color = (key: NodeKey) => {
+ const block = editor.getEditorState().read(() => {
+ const node = $getNodeByKey(key);
+ return $isCodeNode(node)
+ ? {code: node.getTextContent(), language: node.getLanguage() ?? ''}
+ : null;
+ });
+ const element = editor.getElementByKey(key);
+ if (block == null || element == null) {
+ uncolor(key);
+ return;
+ }
+ const previous = colored.get(key);
+ if (
+ previous != null &&
+ previous.element === element &&
+ previous.code === block.code &&
+ previous.language === block.language
+ ) {
+ return;
+ }
+ previous?.cleanup();
+ const tokens = renderedTokens(
+ block.code,
+ block.language,
+ element.textContent ?? '',
+ );
+ if (tokens == null) {
+ // Not rendered yet; the next update tries again.
+ colored.delete(key);
+ return;
+ }
+ colored.set(key, {
+ element,
+ ...block,
+ cleanup: applyHighlightRangesFlat(element, tokens),
+ });
+ };
+ const unregisterMutations = editor.registerMutationListener(
+ CodeNode,
+ mutations => {
+ for (const [key, mutation] of mutations) {
+ if (mutation === 'destroyed') {
+ blocks.delete(key);
+ uncolor(key);
+ } else {
+ blocks.add(key);
+ color(key);
+ }
+ }
+ },
+ {skipInitialization: false},
+ );
+ // Typing in a block changes its text nodes, not always the block itself:
+ // recheck every block after each update (unchanged ones stop at the
+ // comparison above).
+ const unregisterUpdates = editor.registerUpdateListener(() => {
+ for (const key of blocks) {
+ color(key);
+ }
+ });
+ return () => {
+ unregisterMutations();
+ unregisterUpdates();
+ for (const key of [...colored.keys()]) {
+ uncolor(key);
+ }
+ blocks.clear();
+ };
+ }, [editor]);
+
+ return null;
+}
diff --git a/packages/richtext/src/RichTextEditor.tsx b/packages/richtext/src/RichTextEditor.tsx
index ecc61ac7a9..629c09e931 100644
--- a/packages/richtext/src/RichTextEditor.tsx
+++ b/packages/richtext/src/RichTextEditor.tsx
@@ -76,6 +76,7 @@ import {HorizontalRuleExtension} from '@lexical/extension';
import {StrikethroughDeletionExtension} from './strikethroughDeletion';
import {TableColumnFloorPlugin} from './TableColumnFloorPlugin';
import {CodeBlockHeaderPlugin} from './CodeBlockHeaderPlugin';
+import {CodeSyntaxPlugin} from './CodeSyntaxPlugin';
import {TaskCheckboxPlugin} from './TaskCheckboxPlugin';
import {type Transformer} from '@lexical/markdown';
export type {Transformer} from '@lexical/markdown';
@@ -694,6 +695,7 @@ export const RichTextEditor = forwardRef<
/>
<TableColumnFloorPlugin />
<CodeBlockHeaderPlugin />
+ <CodeSyntaxPlugin />
<TabIndentationPlugin />
<TabFocusEscapePlugin />
{hasMarkdownShortcuts && (
diff --git a/packages/richtext/src/RichTextView.tsx b/packages/richtext/src/RichTextView.tsx
index 9587323179..db3f6b3b0f 100644
--- a/packages/richtext/src/RichTextView.tsx
+++ b/packages/richtext/src/RichTextView.tsx
@@ -33,6 +33,7 @@ import {TablePlugin} from '@lexical/react/LexicalTablePlugin';
import {TableScrollRegionPlugin} from './TableScrollRegionPlugin';
import {TableColumnFloorPlugin} from './TableColumnFloorPlugin';
import {CodeBlockHeaderPlugin} from './CodeBlockHeaderPlugin';
+import {CodeSyntaxPlugin} from './CodeSyntaxPlugin';
import {TaskCheckboxPlugin} from './TaskCheckboxPlugin';
import type {
AnyLexicalExtension,
@@ -296,6 +297,7 @@ export function RichTextView({
{/* After the content, so the copy buttons and checkboxes follow it
in tab order. */}
<CodeBlockHeaderPlugin />
+ <CodeSyntaxPlugin />
<TaskCheckboxPlugin isReadOnly />
{plugins}
</LexicalExtensionComposer>
diff --git a/packages/richtext/src/__tests__/MarkdownParitySandbox.a11y.chromium.spec.ts b/packages/richtext/src/__tests__/MarkdownParitySandbox.a11y.chromium.spec.ts
index 2244b1bd13..48d944ff35 100644
--- a/packages/richtext/src/__tests__/MarkdownParitySandbox.a11y.chromium.spec.ts
+++ b/packages/richtext/src/__tests__/MarkdownParitySandbox.a11y.chromium.spec.ts
@@ -1083,6 +1083,115 @@ for (const globals of [
});
}
+// spec:AST-061 FR1: fenced code takes the same syntax colors on both
+// surfaces — the same tokenizer finds the same tokens, and each token type
+// takes its `--color-syntax-*` token.
+/** Each block's highlighted tokens on one surface, as `type:text`. */
+async function syntaxTokens(
+ page: Page,
+ surface: string,
+): Promise<Record<string, Array<string>>> {
+ return page.evaluate(selector => {
+ const tokens: Record<string, Array<string>> = {};
+ for (const [name, highlight] of CSS.highlights) {
+ if (!name.startsWith('astryx-')) {
+ continue;
+ }
+ for (const range of highlight) {
+ const element = range.startContainer.parentElement;
+ const block = element?.closest(selector)?.contains(element)
+ ? element.closest('[data-parity-block]')
+ : null;
+ if (block == null || !(range instanceof Range)) {
+ continue;
+ }
+ const key = block.getAttribute('data-parity-block') ?? '';
+ (tokens[key] ??= []).push(
+ `${name.slice('astryx-'.length)}:${range.toString()}`,
+ );
+ }
+ }
+ for (const list of Object.values(tokens)) {
+ list.sort();
+ }
+ return tokens;
+ }, surface);
+}
+
+for (const globals of [
+ 'colorMode:light;direction:ltr',
+ 'colorMode:dark;direction:rtl',
+]) {
+ test(`side by side: fenced code takes core Markdown's syntax colors (${globals})`, async ({
+ page,
+ }) => {
+ const errors = await openStory(page, STORY.sideBySide, DESKTOP, globals);
+ await waitForDocument(page, MARKDOWN);
+ await waitForDocument(page, RICH_TEXT);
+ await expect
+ .poll(async () => (await syntaxTokens(page, RICH_TEXT))['code-fence'])
+ .toBeDefined();
+ const markdown = await syntaxTokens(page, MARKDOWN);
+ const richText = await syntaxTokens(page, RICH_TEXT);
+ for (const key of ['code-fence', 'code-long']) {
+ expect(markdown[key]?.length ?? 0, key).toBeGreaterThan(0);
+ expect(richText[key], key).toEqual(markdown[key]);
+ }
+ // A fence with no language core CodeBlock knows stays plain on both.
+ for (const key of ['code-plain', 'code-plaintext', 'code-unknown']) {
+ expect(richText[key], key).toBeUndefined();
+ expect(markdown[key], key).toBeUndefined();
+ }
+ // Each token type takes the same syntax color token as core CodeBlock.
+ const rules = await page.evaluate(
+ () =>
+ document.querySelector('style[data-astryx-richtext-code-syntax]')
+ ?.textContent ?? '',
+ );
+ expect(rules).toContain(
+ '::highlight(astryx-keyword) { color: var(--color-syntax-keyword); }',
+ );
+ expect(rules).toContain(
+ '::highlight(astryx-string) { color: var(--color-syntax-string); }',
+ );
+ expect(errors).toEqual([]);
+ });
+}
+
+test('edited code is colored again as it changes', async ({page}) => {
+ await page.setViewportSize(DESKTOP);
+ await page.goto(
+ `${storybook.origin}/iframe.html?id=lab-richtexteditor--markdown-serializers&viewMode=story&globals=astryxTheme:neutral;colorMode:light;direction:ltr`,
+ {waitUntil: 'load'},
+ );
+ await page.locator('textarea').fill('```ts\nlet value = 1;\n```');
+ const editor = page.locator('[data-lexical-editor][contenteditable="true"]');
+ const editorTokens = () =>
+ page.evaluate(() => {
+ const root = document.querySelector(
+ '[data-lexical-editor][contenteditable="true"]',
+ );
+ const tokens: Array<string> = [];
+ for (const [name, highlight] of CSS.highlights) {
+ for (const range of highlight) {
+ if (range instanceof Range && root?.contains(range.startContainer)) {
+ tokens.push(`${name}:${range.toString()}`);
+ }
+ }
+ }
+ return tokens.sort();
+ });
+ await expect.poll(editorTokens).toContain('astryx-keyword:let');
+ // Type a keyword on a new line: it is colored, and the first line keeps
+ // its colors.
+ await editor.getByText('let value = 1;').click();
+ await page.keyboard.press('End');
+ await page.keyboard.press('Enter');
+ await page.keyboard.type('return value;');
+ await expect.poll(editorTokens).toContain('astryx-keyword:return');
+ expect(await editorTokens()).toContain('astryx-keyword:let');
+});
+
for (const viewport of [PHONE, {width: 1280, height: 900}] as const) {
for (const globals of [
'colorMode:light;direction:ltr',
diff --git a/packages/richtext/src/editorTheme.ts b/packages/richtext/src/editorTheme.ts
index 6238bc67bb..f2719143f3 100644
--- a/packages/richtext/src/editorTheme.ts
+++ b/packages/richtext/src/editorTheme.ts
@@ -24,6 +24,7 @@ import {
focusVars,
sizeVars,
} from '@astryxdesign/core/theme/tokens.stylex';
+import {CODE_SYNTAX_CLASS} from './CodeSyntaxPlugin';
import type {EditorThemeClasses} from 'lexical';
// Document blocks follow core Markdown's typography, spacing, and measure
@@ -446,7 +447,8 @@ export function sharedEditorTheme(): EditorThemeClasses {
strikethrough: stylex.props(editorTheme.textStrikethrough).className,
code: stylex.props(editorTheme.textCode).className,
},
- code: block(editorTheme.code),
+ // The plain class lets CodeSyntaxPlugin's highlight rules find code blocks.
+ code: `${block(editorTheme.code)} ${CODE_SYNTAX_CLASS}`,
hr: stylex.props(editorTheme.hr).className,
hrSelected: stylex.props(editorTheme.hrSelected).className,
table: block(editorTheme.table),