An open source design system that's fully customizable and agent ready
TypeScript
feat(code): scoped selectors, streaming tokenizer, spans fallback (#805)
Cindy Zhang21abcb5e413e
diff --git a/packages/lab/src/CodeBlock/XDSCodeBlock.tsx b/packages/lab/src/CodeBlock/XDSCodeBlock.tsxindex 70e5aef257..6d3f3323eb 100644--- a/packages/lab/src/CodeBlock/XDSCodeBlock.tsx+++ b/packages/lab/src/CodeBlock/XDSCodeBlock.tsx@@ -12,7 +12,14 @@'use client';-import {useLayoutEffect, useRef, useState, useCallback} from 'react';+import {+ useLayoutEffect,+ useEffect,+ useRef,+ useState,+ useCallback,+ useMemo,+} from 'react';import type {XDSBaseProps} from '@xds/core/XDSBaseProps';import * as stylex from '@stylexjs/stylex';import {@@ -25,7 +32,7 @@ import {lineHeightVars,} from '@xds/core/theme/tokens.stylex';import {xdsClassName, mergeProps} from '@xds/core/utils';-import {tokenize} from './tokenizer';+import {tokenize, tokenizeAsync, SYNC_TOKENIZE_THRESHOLD} from './tokenizer';import type {Token} from './tokenizer';import {ensureHighlightStyles, TOKEN_TYPES} from './highlightStyles';@@ -176,6 +183,14 @@ export interface XDSCodeBlockProps extends XDSBaseProps<HTMLPreElement> {code: string,language: string,) => Array<{type: string; start: number; end: number}>;+ /**+ * How to apply syntax highlighting.+ * - 'css-highlight': Uses CSS Custom Highlight API (zero DOM overhead).+ * Falls back to 'spans' if the API is not available.+ * - 'spans': Renders `<span>` elements with CSS classes per token.+ * @default 'css-highlight'+ */+ highlightMode?: 'css-highlight' | 'spans';}// ---------------------------------------------------------------------------@@ -199,6 +214,62 @@ function hasHighlightAPI(): boolean {*/let instanceCounter = 0;+/**+ * Build span-based highlighted line content from tokens.+ *+ * Splits a line into segments: plain text between tokens gets rendered+ * as bare text nodes, tokens get wrapped in <span className="xds-token-{type}">.+ */+function buildSpanLine(+ lineText: string,+ lineStart: number,+ tokens: Token[],+): React.ReactNode {+ if (tokens.length === 0) {+ return lineText || '\u200b';+ }++ const lineEnd = lineStart + lineText.length;++ // Filter tokens that overlap this line+ const lineTokens = tokens.filter(t => t.start < lineEnd && t.end > lineStart);++ if (lineTokens.length === 0) {+ return lineText || '\u200b';+ }++ const parts: React.ReactNode[] = [];+ let cursor = lineStart;++ for (const token of lineTokens) {+ const tStart = Math.max(token.start, lineStart);+ const tEnd = Math.min(token.end, lineEnd);++ // Plain text before this token+ if (tStart > cursor) {+ parts.push(lineText.slice(cursor - lineStart, tStart - lineStart));+ }++ // Token span+ parts.push(+ <span+ key={`${tStart}-${token.type}`}+ className={`xds-token-${token.type}`}>+ {lineText.slice(tStart - lineStart, tEnd - lineStart)}+ </span>,+ );++ cursor = tEnd;+ }++ // Remaining plain text after last token+ if (cursor < lineEnd) {+ parts.push(lineText.slice(cursor - lineStart));+ }++ return parts.length > 0 ? parts : '\u200b';+}+// ---------------------------------------------------------------------------// Component// ---------------------------------------------------------------------------@@ -207,7 +278,8 @@ let instanceCounter = 0;* A read-only syntax-highlighted code block.** Uses the CSS Custom Highlight API for zero-DOM-overhead syntax coloring.- * Falls back to plain text display in browsers without support.+ * Falls back to span-based rendering in browsers without support, or when+ * `highlightMode="spans"` is explicitly set.** @example* ```tsx@@ -230,6 +302,7 @@ export function XDSCodeBlock({maxHeight,size = 'md',tokenizer: customTokenizer,+ highlightMode: highlightModeProp = 'css-highlight',xstyle,className,style,@@ -240,6 +313,13 @@ export function XDSCodeBlock({const [instanceId] = useState(() => ++instanceCounter);const [copied, setCopied] = useState(false);+ // Resolve effective highlight mode — fall back to spans if API unavailable+ const useSpans = highlightModeProp === 'spans' || !hasHighlightAPI();++ // For span mode: we need tokens to render. Small code is tokenized sync,+ // large code is tokenized async.+ const [asyncTokens, setAsyncTokens] = useState<Token[] | null>(null);+const lines = code.split('\n');// Remove trailing empty line from code that ends with newlineif (lines.length > 1 && lines[lines.length - 1] === '') {@@ -256,8 +336,39 @@ export function XDSCodeBlock({});}, [code, onCopy]);- // Apply CSS Custom Highlight API ranges+ // Compute tokens for span mode+ const syncTokens = useMemo(() => {+ if (!useSpans) return null;+ if (code.length >= SYNC_TOKENIZE_THRESHOLD) return null;+ const tok = customTokenizer ?? tokenize;+ return tok(code, language);+ }, [useSpans, code, language, customTokenizer]);++ // Async tokenization for span mode with large code+ useEffect(() => {+ if (!useSpans) return;+ if (code.length < SYNC_TOKENIZE_THRESHOLD) return;++ const abortController = new AbortController();++ tokenizeAsync(code, language, abortController.signal).then(tokens => {+ if (!abortController.signal.aborted) {+ setAsyncTokens(tokens);+ }+ });++ return () => {+ abortController.abort();+ setAsyncTokens(null);+ };+ }, [useSpans, code, language, customTokenizer]);++ const spanTokens = syncTokens ?? asyncTokens ?? [];++ // Apply CSS Custom Highlight API ranges — small code (sync, layout effect)useLayoutEffect(() => {+ if (useSpans) return;+ if (code.length >= SYNC_TOKENIZE_THRESHOLD) return;if (!hasHighlightAPI()) return;ensureHighlightStyles();@@ -269,103 +380,41 @@ export function XDSCodeBlock({const tokens = tok(code, language);if (tokens.length === 0) return;- // Group tokens by type- const tokensByType = new Map<string, Token[]>();- for (const token of tokens) {- const existing = tokensByType.get(token.type);- if (existing) {- existing.push(token);- } else {- tokensByType.set(token.type, [token]);- }- }+ return applyHighlightRanges(codeEl, tokens);+ }, [useSpans, code, language, customTokenizer, instanceId]);- // Build text node mapping using TreeWalker — same approach as CodeEditor.- // Finds all text nodes and maps code offsets to DOM positions.- // Code string has \n between lines; DOM has separate text nodes per line- // with no \n, so we add 1 after each line's text node to stay in sync.- const walker = document.createTreeWalker(codeEl, NodeFilter.SHOW_TEXT);- const textNodes: Array<{node: Text; start: number}> = [];- let charOffset = 0;-- let currentNode = walker.nextNode();- while (currentNode) {- const text = currentNode as Text;- const content = text.textContent ?? '';- // Skip zero-width space placeholders for empty lines- if (content === '\u200b') {- // Empty line — still accounts for the \n in the code string- charOffset += 1; // just the \n- } else {- textNodes.push({node: text, start: charOffset});- charOffset += content.length + 1; // +1 for the \n after this line- }- currentNode = walker.nextNode();- }+ // Apply CSS Custom Highlight API ranges — large code (async, regular effect)+ useEffect(() => {+ if (useSpans) return;+ if (code.length < SYNC_TOKENIZE_THRESHOLD) return;+ if (!hasHighlightAPI()) return;- /**- * Find the text node and local offset for a given code offset.- */- function findPosition(offset: number): {node: Text; offset: number} | null {- for (let i = textNodes.length - 1; i >= 0; i--) {- const entry = textNodes[i];- if (offset >= entry.start) {- const localOffset = offset - entry.start;- if (localOffset <= (entry.node.textContent?.length ?? 0)) {- return {node: entry.node, offset: localOffset};- }- return null;- }- }- return null;- }+ ensureHighlightStyles();- // Create highlight ranges for each token type.- // Multiple CodeBlock instances share the same highlight names —- // we add our ranges to the existing Highlight (or create a new one).- // On cleanup, we remove only our ranges.- const myRanges: Range[] = [];-- for (const tokenType of TOKEN_TYPES) {- const typedTokens = tokensByType.get(tokenType);- if (!typedTokens || typedTokens.length === 0) continue;-- const name = `xds-${tokenType}`;- let highlight = CSS.highlights.get(name);- if (!highlight) {- highlight = new Highlight();- CSS.highlights.set(name, highlight);- }+ const codeEl = codeRef.current;+ if (!codeEl) return;- for (const token of typedTokens) {- const startPos = findPosition(token.start);- const endPos = findPosition(token.end);- if (!startPos || !endPos) continue;-- try {- const range = new Range();- range.setStart(startPos.node, startPos.offset);- range.setEnd(endPos.node, endPos.offset);- highlight.add(range);- myRanges.push(range);- } catch {- // Skip invalid ranges- }- }- }+ const abortController = new AbortController();+ let cleanup: (() => void) | undefined;++ tokenizeAsync(code, language, abortController.signal).then(tokens => {+ if (abortController.signal.aborted) return;+ if (tokens.length === 0) return;+ cleanup = applyHighlightRanges(codeEl, tokens);+ });return () => {- // Remove only this instance's ranges from the shared highlights- for (const range of myRanges) {- for (const tokenType of TOKEN_TYPES) {- const highlight = CSS.highlights.get(`xds-${tokenType}`);- if (highlight) {- highlight.delete(range);- }- }- }+ abortController.abort();+ cleanup?.();};- }, [code, language, customTokenizer, instanceId]);+ }, [useSpans, code, language, customTokenizer, instanceId]);++ // Ensure styles are injected for span mode too+ useLayoutEffect(() => {+ if (useSpans) {+ ensureHighlightStyles();+ }+ }, [useSpans]);const sizeStyle = size === 'sm' ? styles.sizeSm : styles.sizeMd;const gutterSizeStyle = size === 'sm' ? styles.gutterSm : styles.gutterMd;@@ -384,10 +433,25 @@ export function XDSCodeBlock({styles.copyButton,!showHeader && styles.copyButtonAbsolute,)}>- {copied ? '✓' : '⎘'}+ {copied ? '\u2713' : '\u2398'}</button>) : null;+ // Build line content: spans mode renders tokens inline, highlight mode+ // renders plain text (CSS Highlight API colors it via ranges).+ function renderLineContent(line: string, lineIndex: number): React.ReactNode {+ if (!useSpans) {+ return line || '\u200b';+ }+ // Calculate the character offset of this line in the full code string.+ // Each previous line + 1 for the \n separator.+ let lineStart = 0;+ for (let i = 0; i < lineIndex; i++) {+ lineStart += lines[i].length + 1;+ }+ return buildSpanLine(line, lineStart, spanTokens);+ }+return (<preref={ref}@@ -432,7 +496,7 @@ export function XDSCodeBlock({styles.line,highlightSet?.has(i + 1) && styles.lineHighlighted,)}>- {line || '\u200b'}+ {renderLineContent(line, i)}</div>))}</code>@@ -444,3 +508,100 @@ export function XDSCodeBlock({}XDSCodeBlock.displayName = 'XDSCodeBlock';++// ---------------------------------------------------------------------------+// Shared highlight range application+// ---------------------------------------------------------------------------++/**+ * Apply CSS Custom Highlight API ranges to a code element.+ * Returns a cleanup function that removes the ranges.+ */+function applyHighlightRanges(+ codeEl: HTMLElement,+ tokens: Token[],+): () => void {+ // Group tokens by type+ const tokensByType = new Map<string, Token[]>();+ for (const token of tokens) {+ const existing = tokensByType.get(token.type);+ if (existing) {+ existing.push(token);+ } else {+ tokensByType.set(token.type, [token]);+ }+ }++ // Build text node mapping using TreeWalker — same approach as CodeEditor.+ const walker = document.createTreeWalker(codeEl, NodeFilter.SHOW_TEXT);+ const textNodes: Array<{node: Text; start: number}> = [];+ let charOffset = 0;++ let currentNode = walker.nextNode();+ while (currentNode) {+ const text = currentNode as Text;+ const content = text.textContent ?? '';+ if (content === '\u200b') {+ charOffset += 1;+ } else {+ textNodes.push({node: text, start: charOffset});+ charOffset += content.length + 1;+ }+ currentNode = walker.nextNode();+ }++ function findPosition(offset: number): {node: Text; offset: number} | null {+ for (let i = textNodes.length - 1; i >= 0; i--) {+ const entry = textNodes[i];+ if (offset >= entry.start) {+ const localOffset = offset - entry.start;+ if (localOffset <= (entry.node.textContent?.length ?? 0)) {+ return {node: entry.node, offset: localOffset};+ }+ return null;+ }+ }+ return null;+ }++ const myRanges: Range[] = [];++ for (const tokenType of TOKEN_TYPES) {+ const typedTokens = tokensByType.get(tokenType);+ if (!typedTokens || typedTokens.length === 0) continue;++ const name = `xds-${tokenType}`;+ let highlight = CSS.highlights.get(name);+ if (!highlight) {+ highlight = new Highlight();+ CSS.highlights.set(name, highlight);+ }++ for (const token of typedTokens) {+ const startPos = findPosition(token.start);+ const endPos = findPosition(token.end);+ if (!startPos || !endPos) continue;++ try {+ const range = new Range();+ range.setStart(startPos.node, startPos.offset);+ range.setEnd(endPos.node, endPos.offset);+ highlight.add(range);+ myRanges.push(range);+ } catch {+ // Skip invalid ranges+ }+ }+ }++ return () => {+ for (const range of myRanges) {+ for (const tokenType of TOKEN_TYPES) {+ const highlight = CSS.highlights.get(`xds-${tokenType}`);+ if (highlight) {+ highlight.delete(range);+ }+ }+ }+ };+}diff --git a/packages/lab/src/CodeBlock/highlightStyles.ts b/packages/lab/src/CodeBlock/highlightStyles.tsindex 41ed990751..d255b3d41d 100644--- a/packages/lab/src/CodeBlock/highlightStyles.ts+++ b/packages/lab/src/CodeBlock/highlightStyles.ts@@ -19,22 +19,56 @@ const FALLBACK_TOKENS = `:root {\n${Object.entries(syntaxTokenDefaults).map(([name, value]) => ` ${name}: ${value};`).join('\n')}\n}`;+/**+ * Scoped ::highlight() rules — constrained to .xds-codeblock and .xds-codeeditor+ * so the browser only checks highlight ranges within code components, not the+ * entire document tree. See: Fabio's perf feedback on bare ::highlight() selectors.+ */const HIGHLIGHT_STYLES = `${FALLBACK_TOKENS}-::highlight(xds-keyword) { color: var(--color-syntax-keyword); }-::highlight(xds-string) { color: var(--color-syntax-string); }-::highlight(xds-comment) { color: var(--color-syntax-comment); }-::highlight(xds-number) { color: var(--color-syntax-number); }-::highlight(xds-function) { color: var(--color-syntax-function); }-::highlight(xds-type) { color: var(--color-syntax-type); }-::highlight(xds-tag) { color: var(--color-syntax-tag); }-::highlight(xds-attribute) { color: var(--color-syntax-attribute); }-::highlight(xds-property) { color: var(--color-syntax-property); }-::highlight(xds-operator) { color: var(--color-syntax-operator); }-::highlight(xds-constant) { color: var(--color-syntax-constant); }-::highlight(xds-punctuation) { color: var(--color-syntax-punctuation); }-::highlight(xds-variable) { color: var(--color-syntax-variable); }+.xds-codeblock ::highlight(xds-keyword),+.xds-codeeditor ::highlight(xds-keyword) { color: var(--color-syntax-keyword); }+.xds-codeblock ::highlight(xds-string),+.xds-codeeditor ::highlight(xds-string) { color: var(--color-syntax-string); }+.xds-codeblock ::highlight(xds-comment),+.xds-codeeditor ::highlight(xds-comment) { color: var(--color-syntax-comment); }+.xds-codeblock ::highlight(xds-number),+.xds-codeeditor ::highlight(xds-number) { color: var(--color-syntax-number); }+.xds-codeblock ::highlight(xds-function),+.xds-codeeditor ::highlight(xds-function) { color: var(--color-syntax-function); }+.xds-codeblock ::highlight(xds-type),+.xds-codeeditor ::highlight(xds-type) { color: var(--color-syntax-type); }+.xds-codeblock ::highlight(xds-tag),+.xds-codeeditor ::highlight(xds-tag) { color: var(--color-syntax-tag); }+.xds-codeblock ::highlight(xds-attribute),+.xds-codeeditor ::highlight(xds-attribute) { color: var(--color-syntax-attribute); }+.xds-codeblock ::highlight(xds-property),+.xds-codeeditor ::highlight(xds-property) { color: var(--color-syntax-property); }+.xds-codeblock ::highlight(xds-operator),+.xds-codeeditor ::highlight(xds-operator) { color: var(--color-syntax-operator); }+.xds-codeblock ::highlight(xds-constant),+.xds-codeeditor ::highlight(xds-constant) { color: var(--color-syntax-constant); }+.xds-codeblock ::highlight(xds-punctuation),+.xds-codeeditor ::highlight(xds-punctuation) { color: var(--color-syntax-punctuation); }+.xds-codeblock ::highlight(xds-variable),+.xds-codeeditor ::highlight(xds-variable) { color: var(--color-syntax-variable); }++/* Span-based fallback classes — used when highlightMode='spans' or+ when the CSS Custom Highlight API is not available. */+.xds-token-keyword { color: var(--color-syntax-keyword); }+.xds-token-string { color: var(--color-syntax-string); }+.xds-token-comment { color: var(--color-syntax-comment); }+.xds-token-number { color: var(--color-syntax-number); }+.xds-token-function { color: var(--color-syntax-function); }+.xds-token-type { color: var(--color-syntax-type); }+.xds-token-tag { color: var(--color-syntax-tag); }+.xds-token-attribute { color: var(--color-syntax-attribute); }+.xds-token-property { color: var(--color-syntax-property); }+.xds-token-operator { color: var(--color-syntax-operator); }+.xds-token-constant { color: var(--color-syntax-constant); }+.xds-token-punctuation { color: var(--color-syntax-punctuation); }+.xds-token-variable { color: var(--color-syntax-variable); }`;let inserted = false;diff --git a/packages/lab/src/CodeBlock/tokenizer.test.ts b/packages/lab/src/CodeBlock/tokenizer.test.tsindex 4c396dd4b9..377e1dd7c9 100644--- a/packages/lab/src/CodeBlock/tokenizer.test.ts+++ b/packages/lab/src/CodeBlock/tokenizer.test.ts@@ -1,5 +1,5 @@import {describe, it, expect} from 'vitest';-import {tokenize} from './tokenizer';+import {tokenize, tokenizeAsync, SYNC_TOKENIZE_THRESHOLD} from './tokenizer';function tokenTypes(code: string, lang: string) {return tokenize(code, lang).map(t => ({@@ -278,4 +278,46 @@ const y = "hello";}});});++ describe('tokenizeAsync', () => {+ it('produces same results as sync tokenize for small code', async () => {+ const code = 'const x = 42;\nreturn "hello";';+ const syncTokens = tokenize(code, 'typescript');+ const asyncTokens = await tokenizeAsync(code, 'typescript');+ expect(asyncTokens).toEqual(syncTokens);+ });++ it('produces same results for larger code', async () => {+ // Generate code larger than SYNC_TOKENIZE_THRESHOLD+ const line = 'const x = 42; // comment\n';+ const code = line.repeat(+ Math.ceil(SYNC_TOKENIZE_THRESHOLD / line.length) + 10,+ );+ const syncTokens = tokenize(code, 'typescript');+ const asyncTokens = await tokenizeAsync(code, 'typescript');+ expect(asyncTokens).toEqual(syncTokens);+ });++ it('returns empty for unknown language', async () => {+ const tokens = await tokenizeAsync('const x = 1;', 'brainfuck');+ expect(tokens).toEqual([]);+ });++ it('respects abort signal', async () => {+ const controller = new AbortController();+ controller.abort();+ const tokens = await tokenizeAsync(+ 'const x = 42;',+ 'typescript',+ controller.signal,+ );+ // Aborted before processing — tokens may be empty or partial+ expect(Array.isArray(tokens)).toBe(true);+ });++ it('exports SYNC_TOKENIZE_THRESHOLD as a number', () => {+ expect(typeof SYNC_TOKENIZE_THRESHOLD).toBe('number');+ expect(SYNC_TOKENIZE_THRESHOLD).toBe(2000);+ });+ });});diff --git a/packages/lab/src/CodeBlock/tokenizer.ts b/packages/lab/src/CodeBlock/tokenizer.tsindex 9305691d7b..3e65043ad5 100644--- a/packages/lab/src/CodeBlock/tokenizer.ts+++ b/packages/lab/src/CodeBlock/tokenizer.ts@@ -287,24 +287,44 @@ function buildLanguagePatterns(// Core tokenizer// ---------------------------------------------------------------------------+/** Threshold below which we tokenize synchronously (characters). */+export const SYNC_TOKENIZE_THRESHOLD = 2000;++/** Characters to process per chunk in async tokenization. */+const ASYNC_CHUNK_SIZE = 5000;+/**- * Tokenizes a code string into an array of typed tokens with character offsets.- *- * Uses a greedy first-match strategy: at each position, try all patterns in- * order and emit the first match. Skips characters that don't match any pattern.- *- * @param code - The source code string to tokenize- * @param language - Language identifier (e.g. 'typescript', 'python')- * @returns Array of tokens sorted by start position+ * Yield to the main thread. Uses `scheduler.yield()` when available,+ * falling back to `setTimeout(resolve, 0)`.*/-export function tokenize(code: string, language: string): Token[] {- const langDef = buildLanguage(language);- if (!langDef) return [];-- const tokens: Token[] = [];- let pos = 0;+function yieldToMain(): Promise<void> {+ if (+ typeof scheduler !== 'undefined' &&+ // eslint-disable-next-line @typescript-eslint/no-explicit-any+ typeof (scheduler as any).yield === 'function'+ ) {+ // eslint-disable-next-line @typescript-eslint/no-explicit-any+ return (scheduler as any).yield();+ }+ return new Promise(resolve => setTimeout(resolve, 0));+}- while (pos < code.length) {+/**+ * Internal tokenization loop. Processes from `startPos` up to `endPos`+ * (or end of string), pushing tokens into the provided array.+ * Returns the final position.+ */+function tokenizeRange(+ code: string,+ langDef: LangDef,+ tokens: Token[],+ startPos: number,+ endPos: number,+): number {+ let pos = startPos;+ const limit = Math.min(endPos, code.length);++ while (pos < limit) {let matched = false;for (const pattern of langDef.patterns) {@@ -328,5 +348,67 @@ export function tokenize(code: string, language: string): Token[] {}}+ return pos;+}++/**+ * Tokenizes a code string into an array of typed tokens with character offsets.+ *+ * Uses a greedy first-match strategy: at each position, try all patterns in+ * order and emit the first match. Skips characters that don't match any pattern.+ *+ * Best for small code strings (< 2000 chars). For large code, use+ * `tokenizeAsync()` which yields to the main thread between chunks.+ *+ * @param code - The source code string to tokenize+ * @param language - Language identifier (e.g. 'typescript', 'python')+ * @returns Array of tokens sorted by start position+ */+export function tokenize(code: string, language: string): Token[] {+ const langDef = buildLanguage(language);+ if (!langDef) return [];++ const tokens: Token[] = [];+ tokenizeRange(code, langDef, tokens, 0, code.length);+ return tokens;+}++/**+ * Async streaming tokenizer that yields to the main thread between chunks.+ *+ * Processes ~5000 characters at a time, then yields via `scheduler.yield()`+ * (or `setTimeout`) to avoid blocking the main thread on large files.+ *+ * Supports cancellation via AbortSignal — returns tokens accumulated so far+ * if aborted (callers should discard the result).+ *+ * @param code - The source code string to tokenize+ * @param language - Language identifier (e.g. 'typescript', 'python')+ * @param signal - Optional AbortSignal for cancellation+ * @returns Promise resolving to array of tokens sorted by start position+ */+export async function tokenizeAsync(+ code: string,+ language: string,+ signal?: AbortSignal,+): Promise<Token[]> {+ const langDef = buildLanguage(language);+ if (!langDef) return [];++ const tokens: Token[] = [];+ let pos = 0;++ while (pos < code.length) {+ if (signal?.aborted) return tokens;++ // Process one chunk+ pos = tokenizeRange(code, langDef, tokens, pos, pos + ASYNC_CHUNK_SIZE);++ // Yield to main thread if there's more to process+ if (pos < code.length) {+ await yieldToMain();+ }+ }+return tokens;}diff --git a/packages/lab/src/CodeEditor/XDSCodeEditor.tsx b/packages/lab/src/CodeEditor/XDSCodeEditor.tsxindex c9b7cb6eab..950e20d57b 100644--- a/packages/lab/src/CodeEditor/XDSCodeEditor.tsx+++ b/packages/lab/src/CodeEditor/XDSCodeEditor.tsx@@ -12,7 +12,14 @@'use client';-import {useEffect, useLayoutEffect, useRef, useCallback, useState} from 'react';+import {+ useEffect,+ useLayoutEffect,+ useRef,+ useCallback,+ useState,+ useMemo,+} from 'react';import type {XDSBaseProps} from '@xds/core/XDSBaseProps';import * as stylex from '@stylexjs/stylex';import {@@ -24,7 +31,11 @@ import {lineHeightVars,} from '@xds/core/theme/tokens.stylex';import {xdsClassName, mergeProps} from '@xds/core/utils';-import {tokenize} from '../CodeBlock/tokenizer';+import {+ tokenize,+ tokenizeAsync,+ SYNC_TOKENIZE_THRESHOLD,+} from '../CodeBlock/tokenizer';import type {Token} from '../CodeBlock/tokenizer';import {ensureHighlightStyles, TOKEN_TYPES} from '../CodeBlock/highlightStyles';@@ -134,6 +145,15 @@ export interface XDSCodeEditorProps extends Omit<code: string,language: string,) => Array<{type: string; start: number; end: number}>;+ /**+ * How to apply syntax highlighting.+ * - 'css-highlight': Uses CSS Custom Highlight API (zero DOM overhead).+ * Falls back to 'spans' if the API is not available.+ * - 'spans': Renders `<span>` elements with CSS classes per token.+ * In the editor, uses a transparent-text overlay approach.+ * @default 'css-highlight'+ */+ highlightMode?: 'css-highlight' | 'spans';}// ---------------------------------------------------------------------------@@ -159,6 +179,157 @@ const AUTO_CLOSE_PAIRS: Record<string, string> = {let editorInstanceCounter = 0;+// ---------------------------------------------------------------------------+// Span-based rendering helpers+// ---------------------------------------------------------------------------++/**+ * Build span-based highlighted line content from tokens.+ */+function buildSpanLine(+ lineText: string,+ lineStart: number,+ tokens: Token[],+): React.ReactNode {+ if (!lineText) return '\u200b';++ const lineEnd = lineStart + lineText.length;+ const lineTokens = tokens.filter(t => t.start < lineEnd && t.end > lineStart);++ if (lineTokens.length === 0) return lineText;++ const parts: React.ReactNode[] = [];+ let cursor = lineStart;++ for (const token of lineTokens) {+ const tStart = Math.max(token.start, lineStart);+ const tEnd = Math.min(token.end, lineEnd);++ if (tStart > cursor) {+ parts.push(lineText.slice(cursor - lineStart, tStart - lineStart));+ }++ parts.push(+ <span+ key={`${tStart}-${token.type}`}+ className={`xds-token-${token.type}`}>+ {lineText.slice(tStart - lineStart, tEnd - lineStart)}+ </span>,+ );++ cursor = tEnd;+ }++ if (cursor < lineEnd) {+ parts.push(lineText.slice(cursor - lineStart));+ }++ return parts.length > 0 ? parts : lineText;+}++/**+ * Build span-based content for all lines.+ */+function buildSpanContent(lines: string[], tokens: Token[]): React.ReactNode {+ return lines.map((line, i) => {+ let lineStart = 0;+ for (let j = 0; j < i; j++) {+ lineStart += lines[j].length + 1;+ }+ return (+ <div key={i}>+ {buildSpanLine(line, lineStart, tokens)}+ {i < lines.length - 1 ? '\n' : null}+ </div>+ );+ });+}++// ---------------------------------------------------------------------------+// Shared highlight range application+// ---------------------------------------------------------------------------++function applyEditorHighlightRanges(+ el: HTMLElement,+ tokens: Token[],+): () => void {+ const tokensByType = new Map<string, Token[]>();+ for (const token of tokens) {+ const existing = tokensByType.get(token.type);+ if (existing) {+ existing.push(token);+ } else {+ tokensByType.set(token.type, [token]);+ }+ }++ const walker = document.createTreeWalker(el, NodeFilter.SHOW_TEXT);+ const textNodes: Array<{node: Text; start: number}> = [];+ let totalOffset = 0;++ let node = walker.nextNode();+ while (node) {+ textNodes.push({node: node as Text, start: totalOffset});+ totalOffset += (node as Text).length;+ node = walker.nextNode();+ }++ function findPosition(offset: number): {node: Text; offset: number} | null {+ for (let i = textNodes.length - 1; i >= 0; i--) {+ const entry = textNodes[i];+ if (offset >= entry.start) {+ const localOffset = offset - entry.start;+ if (localOffset <= entry.node.length) {+ return {node: entry.node, offset: localOffset};+ }+ return null;+ }+ }+ return null;+ }++ const myRanges: Range[] = [];++ for (const tokenType of TOKEN_TYPES) {+ const typedTokens = tokensByType.get(tokenType);+ if (!typedTokens || typedTokens.length === 0) continue;++ const name = `xds-${tokenType}`;+ let highlight = CSS.highlights.get(name);+ if (!highlight) {+ highlight = new Highlight();+ CSS.highlights.set(name, highlight);+ }++ for (const token of typedTokens) {+ const startPos = findPosition(token.start);+ const endPos = findPosition(token.end);+ if (!startPos || !endPos) continue;++ try {+ const range = new Range();+ range.setStart(startPos.node, startPos.offset);+ range.setEnd(endPos.node, endPos.offset);+ highlight.add(range);+ myRanges.push(range);+ } catch {+ // Skip invalid ranges+ }+ }+ }++ return () => {+ for (const range of myRanges) {+ for (const tokenType of TOKEN_TYPES) {+ const highlight = CSS.highlights.get(`xds-${tokenType}`);+ if (highlight) {+ highlight.delete(range);+ }+ }+ }+ };+}+// ---------------------------------------------------------------------------// Component// ---------------------------------------------------------------------------@@ -168,6 +339,8 @@ let editorInstanceCounter = 0;** Uses CSS Custom Highlight API for syntax coloring. Supports* auto-indent, tab insertion, and bracket auto-closing.+ * Falls back to span-based rendering when the API is unavailable+ * or `highlightMode="spans"` is set.** @example* ```tsx@@ -190,6 +363,7 @@ export function XDSCodeEditor({maxHeight,size = 'md',tokenizer: customTokenizer,+ highlightMode: highlightModeProp = 'css-highlight',xstyle,className,style,@@ -201,23 +375,62 @@ export function XDSCodeEditor({const [focused, setFocused] = useState(false);const isComposingRef = useRef(false);+ // Resolve effective highlight mode+ const useSpans = highlightModeProp === 'spans' || !hasHighlightAPI();++ // Async tokens for span mode overlay+ const [asyncTokens, setAsyncTokens] = useState<Token[] | null>(null);+const lines = value.split('\n');// Sync textContent with controlled valueuseLayoutEffect(() => {const el = editorRef.current;if (!el) return;- // Only update DOM if it differs (avoids clobbering cursor)if (el.textContent !== value) {el.textContent = value;}}, [value]);- // Apply CSS Custom Highlight API ranges+ // Compute sync tokens for span mode (small code)+ const syncTokens = useMemo(() => {+ if (!useSpans) return null;+ if (value.length >= SYNC_TOKENIZE_THRESHOLD) return null;+ const tok = customTokenizer ?? tokenize;+ return tok(value, language);+ }, [useSpans, value, language, customTokenizer]);++ // Async tokenization for span mode with large codeuseEffect(() => {- if (!hasHighlightAPI()) return;+ if (!useSpans) return;+ if (value.length < SYNC_TOKENIZE_THRESHOLD) return;+ const abortController = new AbortController();++ tokenizeAsync(value, language, abortController.signal).then(tokens => {+ if (!abortController.signal.aborted) {+ setAsyncTokens(tokens);+ }+ });++ return () => {+ abortController.abort();+ setAsyncTokens(null);+ };+ }, [useSpans, value, language, customTokenizer]);++ const spanTokens = syncTokens ?? asyncTokens ?? [];++ // Ensure styles are always injected+ useLayoutEffect(() => {ensureHighlightStyles();+ }, []);++ // Apply CSS Custom Highlight API ranges — small code+ useLayoutEffect(() => {+ if (useSpans) return;+ if (value.length >= SYNC_TOKENIZE_THRESHOLD) return;+ if (!hasHighlightAPI()) return;const el = editorRef.current;if (!el) return;@@ -226,84 +439,32 @@ export function XDSCodeEditor({const tokens = tok(value, language);if (tokens.length === 0) return;- // Group tokens by type- const tokensByType = new Map<string, Token[]>();- for (const token of tokens) {- const existing = tokensByType.get(token.type);- if (existing) {- existing.push(token);- } else {- tokensByType.set(token.type, [token]);- }- }-- // Build text node mapping from the contenteditable element- const walker = document.createTreeWalker(el, NodeFilter.SHOW_TEXT);- const textNodes: Array<{node: Text; start: number}> = [];- let totalOffset = 0;+ return applyEditorHighlightRanges(el, tokens);+ }, [useSpans, value, language, customTokenizer, instanceId]);- let node = walker.nextNode();- while (node) {- textNodes.push({node: node as Text, start: totalOffset});- totalOffset += (node as Text).length;- node = walker.nextNode();- }-- function findPosition(offset: number): {node: Text; offset: number} | null {- for (let i = textNodes.length - 1; i >= 0; i--) {- const entry = textNodes[i];- if (offset >= entry.start) {- const localOffset = offset - entry.start;- if (localOffset <= entry.node.length) {- return {node: entry.node, offset: localOffset};- }- return null;- }- }- return null;- }+ // Apply CSS Custom Highlight API ranges — large code (async)+ useEffect(() => {+ if (useSpans) return;+ if (value.length < SYNC_TOKENIZE_THRESHOLD) return;+ if (!hasHighlightAPI()) return;- const myRanges: Range[] = [];+ const el = editorRef.current;+ if (!el) return;- for (const tokenType of TOKEN_TYPES) {- const typedTokens = tokensByType.get(tokenType);- if (!typedTokens || typedTokens.length === 0) continue;+ const abortController = new AbortController();+ let cleanup: (() => void) | undefined;- const name = `xds-${tokenType}`;- let highlight = CSS.highlights.get(name);- if (!highlight) {- highlight = new Highlight();- CSS.highlights.set(name, highlight);- }-- for (const token of typedTokens) {- const startPos = findPosition(token.start);- const endPos = findPosition(token.end);- if (!startPos || !endPos) continue;-- try {- const range = new Range();- range.setStart(startPos.node, startPos.offset);- range.setEnd(endPos.node, endPos.offset);- highlight.add(range);- myRanges.push(range);- } catch {- // Skip invalid ranges- }- }- }+ tokenizeAsync(value, language, abortController.signal).then(tokens => {+ if (abortController.signal.aborted) return;+ if (tokens.length === 0) return;+ cleanup = applyEditorHighlightRanges(el, tokens);+ });return () => {- for (const range of myRanges) {- for (const tokenType of TOKEN_TYPES) {- const highlight = CSS.highlights.get(`xds-${tokenType}`);- if (highlight) {- highlight.delete(range);- }- }- }+ abortController.abort();+ cleanup?.();};- }, [value, language, customTokenizer, instanceId]);+ }, [useSpans, value, language, customTokenizer, instanceId]);const handleInput = useCallback(() => {if (isComposingRef.current) return;@@ -414,6 +575,30 @@ export function XDSCodeEditor({? {maxHeight: typeof maxHeight === 'number' ? `${maxHeight}px` : maxHeight}: undefined;+ // For span mode in the editor, we render a non-interactive overlay with+ // colored spans on top of the contentEditable. The contentEditable text+ // is made transparent so the overlay provides the visual coloring while+ // the contentEditable handles input/caret.+ const spanOverlay =+ useSpans && spanTokens.length > 0 ? (+ <div+ aria-hidden="true"+ style={{+ position: 'absolute',+ top: 0,+ left: 0,+ right: 0,+ bottom: 0,+ pointerEvents: 'none',+ whiteSpace: 'pre',+ wordBreak: 'normal',+ overflowWrap: 'normal',+ }}+ {...stylex.props(styles.editor, sizeStyle)}>+ {buildSpanContent(lines, spanTokens)}+ </div>+ ) : null;+return (<divref={ref}@@ -435,7 +620,9 @@ export function XDSCodeEditor({))}</div>)}- <div {...stylex.props(styles.editorContainer)} style={containerStyle}>+ <div+ {...stylex.props(styles.editorContainer)}+ style={{...containerStyle, position: 'relative'}}>{showPlaceholder && (<div {...stylex.props(styles.placeholder, sizeStyle)}>{placeholder}@@ -456,7 +643,13 @@ export function XDSCodeEditor({onCompositionStart={handleCompositionStart}onCompositionEnd={handleCompositionEnd}{...stylex.props(styles.editor, sizeStyle)}+ style={+ useSpans && spanTokens.length > 0+ ? {color: 'transparent', caretColor: 'inherit'}+ : undefined+ }/>+ {spanOverlay}</div></div>);diff --git a/packages/lab/src/index.ts b/packages/lab/src/index.tsindex 41d71addb9..a4e30663b6 100644--- a/packages/lab/src/index.ts+++ b/packages/lab/src/index.ts@@ -30,4 +30,9 @@ export {// Code components — syntax highlighting domainexport {XDSCodeBlock, type XDSCodeBlockProps} from './CodeBlock';export {XDSCodeEditor, type XDSCodeEditorProps} from './CodeEditor';-export {tokenize, type Token} from './CodeBlock/tokenizer';+export {+ tokenize,+ tokenizeAsync,+ SYNC_TOKENIZE_THRESHOLD,+ type Token,+} from './CodeBlock/tokenizer';