Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

feat(code): scoped selectors, streaming tokenizer, spans fallback (#805)

Cindy Zhang21abcb5e413e
diff --git a/packages/lab/src/CodeBlock/XDSCodeBlock.tsx b/packages/lab/src/CodeBlock/XDSCodeBlock.tsx
index 70e5aef257..6d3f3323eb 100644
--- a/packages/lab/src/CodeBlock/XDSCodeBlock.tsx
+++ b/packages/lab/src/CodeBlock/XDSCodeBlock.tsx
@@ -12,7 +12,14 @@
'use client';
-import {useLayoutEffect, useRef, useState, useCallback} from 'react';
+import {
+ useLayoutEffect,
+ useEffect,
+ useRef,
+ useState,
+ useCallback,
+ useMemo,
+} from 'react';
import type {XDSBaseProps} from '@xds/core/XDSBaseProps';
import * as stylex from '@stylexjs/stylex';
import {
@@ -25,7 +32,7 @@ import {
lineHeightVars,
} from '@xds/core/theme/tokens.stylex';
import {xdsClassName, mergeProps} from '@xds/core/utils';
-import {tokenize} from './tokenizer';
+import {tokenize, tokenizeAsync, SYNC_TOKENIZE_THRESHOLD} from './tokenizer';
import type {Token} from './tokenizer';
import {ensureHighlightStyles, TOKEN_TYPES} from './highlightStyles';
@@ -176,6 +183,14 @@ export interface XDSCodeBlockProps extends XDSBaseProps<HTMLPreElement> {
code: string,
language: string,
) => Array<{type: string; start: number; end: number}>;
+ /**
+ * How to apply syntax highlighting.
+ * - 'css-highlight': Uses CSS Custom Highlight API (zero DOM overhead).
+ * Falls back to 'spans' if the API is not available.
+ * - 'spans': Renders `<span>` elements with CSS classes per token.
+ * @default 'css-highlight'
+ */
+ highlightMode?: 'css-highlight' | 'spans';
}
// ---------------------------------------------------------------------------
@@ -199,6 +214,62 @@ function hasHighlightAPI(): boolean {
*/
let instanceCounter = 0;
+/**
+ * Build span-based highlighted line content from tokens.
+ *
+ * Splits a line into segments: plain text between tokens gets rendered
+ * as bare text nodes, tokens get wrapped in <span className="xds-token-{type}">.
+ */
+function buildSpanLine(
+ lineText: string,
+ lineStart: number,
+ tokens: Token[],
+): React.ReactNode {
+ if (tokens.length === 0) {
+ return lineText || '\u200b';
+ }
+
+ const lineEnd = lineStart + lineText.length;
+
+ // Filter tokens that overlap this line
+ const lineTokens = tokens.filter(t => t.start < lineEnd && t.end > lineStart);
+
+ if (lineTokens.length === 0) {
+ return lineText || '\u200b';
+ }
+
+ const parts: React.ReactNode[] = [];
+ let cursor = lineStart;
+
+ for (const token of lineTokens) {
+ const tStart = Math.max(token.start, lineStart);
+ const tEnd = Math.min(token.end, lineEnd);
+
+ // Plain text before this token
+ if (tStart > cursor) {
+ parts.push(lineText.slice(cursor - lineStart, tStart - lineStart));
+ }
+
+ // Token span
+ parts.push(
+ <span
+ key={`${tStart}-${token.type}`}
+ className={`xds-token-${token.type}`}>
+ {lineText.slice(tStart - lineStart, tEnd - lineStart)}
+ </span>,
+ );
+
+ cursor = tEnd;
+ }
+
+ // Remaining plain text after last token
+ if (cursor < lineEnd) {
+ parts.push(lineText.slice(cursor - lineStart));
+ }
+
+ return parts.length > 0 ? parts : '\u200b';
+}
+
// ---------------------------------------------------------------------------
// Component
// ---------------------------------------------------------------------------
@@ -207,7 +278,8 @@ let instanceCounter = 0;
* A read-only syntax-highlighted code block.
*
* Uses the CSS Custom Highlight API for zero-DOM-overhead syntax coloring.
- * Falls back to plain text display in browsers without support.
+ * Falls back to span-based rendering in browsers without support, or when
+ * `highlightMode="spans"` is explicitly set.
*
* @example
* ```tsx
@@ -230,6 +302,7 @@ export function XDSCodeBlock({
maxHeight,
size = 'md',
tokenizer: customTokenizer,
+ highlightMode: highlightModeProp = 'css-highlight',
xstyle,
className,
style,
@@ -240,6 +313,13 @@ export function XDSCodeBlock({
const [instanceId] = useState(() => ++instanceCounter);
const [copied, setCopied] = useState(false);
+ // Resolve effective highlight mode — fall back to spans if API unavailable
+ const useSpans = highlightModeProp === 'spans' || !hasHighlightAPI();
+
+ // For span mode: we need tokens to render. Small code is tokenized sync,
+ // large code is tokenized async.
+ const [asyncTokens, setAsyncTokens] = useState<Token[] | null>(null);
+
const lines = code.split('\n');
// Remove trailing empty line from code that ends with newline
if (lines.length > 1 && lines[lines.length - 1] === '') {
@@ -256,8 +336,39 @@ export function XDSCodeBlock({
});
}, [code, onCopy]);
- // Apply CSS Custom Highlight API ranges
+ // Compute tokens for span mode
+ const syncTokens = useMemo(() => {
+ if (!useSpans) return null;
+ if (code.length >= SYNC_TOKENIZE_THRESHOLD) return null;
+ const tok = customTokenizer ?? tokenize;
+ return tok(code, language);
+ }, [useSpans, code, language, customTokenizer]);
+
+ // Async tokenization for span mode with large code
+ useEffect(() => {
+ if (!useSpans) return;
+ if (code.length < SYNC_TOKENIZE_THRESHOLD) return;
+
+ const abortController = new AbortController();
+
+ tokenizeAsync(code, language, abortController.signal).then(tokens => {
+ if (!abortController.signal.aborted) {
+ setAsyncTokens(tokens);
+ }
+ });
+
+ return () => {
+ abortController.abort();
+ setAsyncTokens(null);
+ };
+ }, [useSpans, code, language, customTokenizer]);
+
+ const spanTokens = syncTokens ?? asyncTokens ?? [];
+
+ // Apply CSS Custom Highlight API ranges — small code (sync, layout effect)
useLayoutEffect(() => {
+ if (useSpans) return;
+ if (code.length >= SYNC_TOKENIZE_THRESHOLD) return;
if (!hasHighlightAPI()) return;
ensureHighlightStyles();
@@ -269,103 +380,41 @@ export function XDSCodeBlock({
const tokens = tok(code, language);
if (tokens.length === 0) return;
- // Group tokens by type
- const tokensByType = new Map<string, Token[]>();
- for (const token of tokens) {
- const existing = tokensByType.get(token.type);
- if (existing) {
- existing.push(token);
- } else {
- tokensByType.set(token.type, [token]);
- }
- }
+ return applyHighlightRanges(codeEl, tokens);
+ }, [useSpans, code, language, customTokenizer, instanceId]);
- // Build text node mapping using TreeWalker — same approach as CodeEditor.
- // Finds all text nodes and maps code offsets to DOM positions.
- // Code string has \n between lines; DOM has separate text nodes per line
- // with no \n, so we add 1 after each line's text node to stay in sync.
- const walker = document.createTreeWalker(codeEl, NodeFilter.SHOW_TEXT);
- const textNodes: Array<{node: Text; start: number}> = [];
- let charOffset = 0;
-
- let currentNode = walker.nextNode();
- while (currentNode) {
- const text = currentNode as Text;
- const content = text.textContent ?? '';
- // Skip zero-width space placeholders for empty lines
- if (content === '\u200b') {
- // Empty line — still accounts for the \n in the code string
- charOffset += 1; // just the \n
- } else {
- textNodes.push({node: text, start: charOffset});
- charOffset += content.length + 1; // +1 for the \n after this line
- }
- currentNode = walker.nextNode();
- }
+ // Apply CSS Custom Highlight API ranges — large code (async, regular effect)
+ useEffect(() => {
+ if (useSpans) return;
+ if (code.length < SYNC_TOKENIZE_THRESHOLD) return;
+ if (!hasHighlightAPI()) return;
- /**
- * Find the text node and local offset for a given code offset.
- */
- function findPosition(offset: number): {node: Text; offset: number} | null {
- for (let i = textNodes.length - 1; i >= 0; i--) {
- const entry = textNodes[i];
- if (offset >= entry.start) {
- const localOffset = offset - entry.start;
- if (localOffset <= (entry.node.textContent?.length ?? 0)) {
- return {node: entry.node, offset: localOffset};
- }
- return null;
- }
- }
- return null;
- }
+ ensureHighlightStyles();
- // Create highlight ranges for each token type.
- // Multiple CodeBlock instances share the same highlight names —
- // we add our ranges to the existing Highlight (or create a new one).
- // On cleanup, we remove only our ranges.
- const myRanges: Range[] = [];
-
- for (const tokenType of TOKEN_TYPES) {
- const typedTokens = tokensByType.get(tokenType);
- if (!typedTokens || typedTokens.length === 0) continue;
-
- const name = `xds-${tokenType}`;
- let highlight = CSS.highlights.get(name);
- if (!highlight) {
- highlight = new Highlight();
- CSS.highlights.set(name, highlight);
- }
+ const codeEl = codeRef.current;
+ if (!codeEl) return;
- for (const token of typedTokens) {
- const startPos = findPosition(token.start);
- const endPos = findPosition(token.end);
- if (!startPos || !endPos) continue;
-
- try {
- const range = new Range();
- range.setStart(startPos.node, startPos.offset);
- range.setEnd(endPos.node, endPos.offset);
- highlight.add(range);
- myRanges.push(range);
- } catch {
- // Skip invalid ranges
- }
- }
- }
+ const abortController = new AbortController();
+ let cleanup: (() => void) | undefined;
+
+ tokenizeAsync(code, language, abortController.signal).then(tokens => {
+ if (abortController.signal.aborted) return;
+ if (tokens.length === 0) return;
+ cleanup = applyHighlightRanges(codeEl, tokens);
+ });
return () => {
- // Remove only this instance's ranges from the shared highlights
- for (const range of myRanges) {
- for (const tokenType of TOKEN_TYPES) {
- const highlight = CSS.highlights.get(`xds-${tokenType}`);
- if (highlight) {
- highlight.delete(range);
- }
- }
- }
+ abortController.abort();
+ cleanup?.();
};
- }, [code, language, customTokenizer, instanceId]);
+ }, [useSpans, code, language, customTokenizer, instanceId]);
+
+ // Ensure styles are injected for span mode too
+ useLayoutEffect(() => {
+ if (useSpans) {
+ ensureHighlightStyles();
+ }
+ }, [useSpans]);
const sizeStyle = size === 'sm' ? styles.sizeSm : styles.sizeMd;
const gutterSizeStyle = size === 'sm' ? styles.gutterSm : styles.gutterMd;
@@ -384,10 +433,25 @@ export function XDSCodeBlock({
styles.copyButton,
!showHeader && styles.copyButtonAbsolute,
)}>
- {copied ? '✓' : '⎘'}
+ {copied ? '\u2713' : '\u2398'}
</button>
) : null;
+ // Build line content: spans mode renders tokens inline, highlight mode
+ // renders plain text (CSS Highlight API colors it via ranges).
+ function renderLineContent(line: string, lineIndex: number): React.ReactNode {
+ if (!useSpans) {
+ return line || '\u200b';
+ }
+ // Calculate the character offset of this line in the full code string.
+ // Each previous line + 1 for the \n separator.
+ let lineStart = 0;
+ for (let i = 0; i < lineIndex; i++) {
+ lineStart += lines[i].length + 1;
+ }
+ return buildSpanLine(line, lineStart, spanTokens);
+ }
+
return (
<pre
ref={ref}
@@ -432,7 +496,7 @@ export function XDSCodeBlock({
styles.line,
highlightSet?.has(i + 1) && styles.lineHighlighted,
)}>
- {line || '\u200b'}
+ {renderLineContent(line, i)}
</div>
))}
</code>
@@ -444,3 +508,100 @@ export function XDSCodeBlock({
}
XDSCodeBlock.displayName = 'XDSCodeBlock';
+
+// ---------------------------------------------------------------------------
+// Shared highlight range application
+// ---------------------------------------------------------------------------
+
+/**
+ * Apply CSS Custom Highlight API ranges to a code element.
+ * Returns a cleanup function that removes the ranges.
+ */
+function applyHighlightRanges(
+ codeEl: HTMLElement,
+ tokens: Token[],
+): () => void {
+ // Group tokens by type
+ const tokensByType = new Map<string, Token[]>();
+ for (const token of tokens) {
+ const existing = tokensByType.get(token.type);
+ if (existing) {
+ existing.push(token);
+ } else {
+ tokensByType.set(token.type, [token]);
+ }
+ }
+
+ // Build text node mapping using TreeWalker — same approach as CodeEditor.
+ const walker = document.createTreeWalker(codeEl, NodeFilter.SHOW_TEXT);
+ const textNodes: Array<{node: Text; start: number}> = [];
+ let charOffset = 0;
+
+ let currentNode = walker.nextNode();
+ while (currentNode) {
+ const text = currentNode as Text;
+ const content = text.textContent ?? '';
+ if (content === '\u200b') {
+ charOffset += 1;
+ } else {
+ textNodes.push({node: text, start: charOffset});
+ charOffset += content.length + 1;
+ }
+ currentNode = walker.nextNode();
+ }
+
+ function findPosition(offset: number): {node: Text; offset: number} | null {
+ for (let i = textNodes.length - 1; i >= 0; i--) {
+ const entry = textNodes[i];
+ if (offset >= entry.start) {
+ const localOffset = offset - entry.start;
+ if (localOffset <= (entry.node.textContent?.length ?? 0)) {
+ return {node: entry.node, offset: localOffset};
+ }
+ return null;
+ }
+ }
+ return null;
+ }
+
+ const myRanges: Range[] = [];
+
+ for (const tokenType of TOKEN_TYPES) {
+ const typedTokens = tokensByType.get(tokenType);
+ if (!typedTokens || typedTokens.length === 0) continue;
+
+ const name = `xds-${tokenType}`;
+ let highlight = CSS.highlights.get(name);
+ if (!highlight) {
+ highlight = new Highlight();
+ CSS.highlights.set(name, highlight);
+ }
+
+ for (const token of typedTokens) {
+ const startPos = findPosition(token.start);
+ const endPos = findPosition(token.end);
+ if (!startPos || !endPos) continue;
+
+ try {
+ const range = new Range();
+ range.setStart(startPos.node, startPos.offset);
+ range.setEnd(endPos.node, endPos.offset);
+ highlight.add(range);
+ myRanges.push(range);
+ } catch {
+ // Skip invalid ranges
+ }
+ }
+ }
+
+ return () => {
+ for (const range of myRanges) {
+ for (const tokenType of TOKEN_TYPES) {
+ const highlight = CSS.highlights.get(`xds-${tokenType}`);
+ if (highlight) {
+ highlight.delete(range);
+ }
+ }
+ }
+ };
+}
diff --git a/packages/lab/src/CodeBlock/highlightStyles.ts b/packages/lab/src/CodeBlock/highlightStyles.ts
index 41ed990751..d255b3d41d 100644
--- a/packages/lab/src/CodeBlock/highlightStyles.ts
+++ b/packages/lab/src/CodeBlock/highlightStyles.ts
@@ -19,22 +19,56 @@ const FALLBACK_TOKENS = `:root {\n${Object.entries(syntaxTokenDefaults)
.map(([name, value]) => ` ${name}: ${value};`)
.join('\n')}\n}`;
+/**
+ * Scoped ::highlight() rules — constrained to .xds-codeblock and .xds-codeeditor
+ * so the browser only checks highlight ranges within code components, not the
+ * entire document tree. See: Fabio's perf feedback on bare ::highlight() selectors.
+ */
const HIGHLIGHT_STYLES = `
${FALLBACK_TOKENS}
-::highlight(xds-keyword) { color: var(--color-syntax-keyword); }
-::highlight(xds-string) { color: var(--color-syntax-string); }
-::highlight(xds-comment) { color: var(--color-syntax-comment); }
-::highlight(xds-number) { color: var(--color-syntax-number); }
-::highlight(xds-function) { color: var(--color-syntax-function); }
-::highlight(xds-type) { color: var(--color-syntax-type); }
-::highlight(xds-tag) { color: var(--color-syntax-tag); }
-::highlight(xds-attribute) { color: var(--color-syntax-attribute); }
-::highlight(xds-property) { color: var(--color-syntax-property); }
-::highlight(xds-operator) { color: var(--color-syntax-operator); }
-::highlight(xds-constant) { color: var(--color-syntax-constant); }
-::highlight(xds-punctuation) { color: var(--color-syntax-punctuation); }
-::highlight(xds-variable) { color: var(--color-syntax-variable); }
+.xds-codeblock ::highlight(xds-keyword),
+.xds-codeeditor ::highlight(xds-keyword) { color: var(--color-syntax-keyword); }
+.xds-codeblock ::highlight(xds-string),
+.xds-codeeditor ::highlight(xds-string) { color: var(--color-syntax-string); }
+.xds-codeblock ::highlight(xds-comment),
+.xds-codeeditor ::highlight(xds-comment) { color: var(--color-syntax-comment); }
+.xds-codeblock ::highlight(xds-number),
+.xds-codeeditor ::highlight(xds-number) { color: var(--color-syntax-number); }
+.xds-codeblock ::highlight(xds-function),
+.xds-codeeditor ::highlight(xds-function) { color: var(--color-syntax-function); }
+.xds-codeblock ::highlight(xds-type),
+.xds-codeeditor ::highlight(xds-type) { color: var(--color-syntax-type); }
+.xds-codeblock ::highlight(xds-tag),
+.xds-codeeditor ::highlight(xds-tag) { color: var(--color-syntax-tag); }
+.xds-codeblock ::highlight(xds-attribute),
+.xds-codeeditor ::highlight(xds-attribute) { color: var(--color-syntax-attribute); }
+.xds-codeblock ::highlight(xds-property),
+.xds-codeeditor ::highlight(xds-property) { color: var(--color-syntax-property); }
+.xds-codeblock ::highlight(xds-operator),
+.xds-codeeditor ::highlight(xds-operator) { color: var(--color-syntax-operator); }
+.xds-codeblock ::highlight(xds-constant),
+.xds-codeeditor ::highlight(xds-constant) { color: var(--color-syntax-constant); }
+.xds-codeblock ::highlight(xds-punctuation),
+.xds-codeeditor ::highlight(xds-punctuation) { color: var(--color-syntax-punctuation); }
+.xds-codeblock ::highlight(xds-variable),
+.xds-codeeditor ::highlight(xds-variable) { color: var(--color-syntax-variable); }
+
+/* Span-based fallback classes — used when highlightMode='spans' or
+ when the CSS Custom Highlight API is not available. */
+.xds-token-keyword { color: var(--color-syntax-keyword); }
+.xds-token-string { color: var(--color-syntax-string); }
+.xds-token-comment { color: var(--color-syntax-comment); }
+.xds-token-number { color: var(--color-syntax-number); }
+.xds-token-function { color: var(--color-syntax-function); }
+.xds-token-type { color: var(--color-syntax-type); }
+.xds-token-tag { color: var(--color-syntax-tag); }
+.xds-token-attribute { color: var(--color-syntax-attribute); }
+.xds-token-property { color: var(--color-syntax-property); }
+.xds-token-operator { color: var(--color-syntax-operator); }
+.xds-token-constant { color: var(--color-syntax-constant); }
+.xds-token-punctuation { color: var(--color-syntax-punctuation); }
+.xds-token-variable { color: var(--color-syntax-variable); }
`;
let inserted = false;
diff --git a/packages/lab/src/CodeBlock/tokenizer.test.ts b/packages/lab/src/CodeBlock/tokenizer.test.ts
index 4c396dd4b9..377e1dd7c9 100644
--- a/packages/lab/src/CodeBlock/tokenizer.test.ts
+++ b/packages/lab/src/CodeBlock/tokenizer.test.ts
@@ -1,5 +1,5 @@
import {describe, it, expect} from 'vitest';
-import {tokenize} from './tokenizer';
+import {tokenize, tokenizeAsync, SYNC_TOKENIZE_THRESHOLD} from './tokenizer';
function tokenTypes(code: string, lang: string) {
return tokenize(code, lang).map(t => ({
@@ -278,4 +278,46 @@ const y = "hello";
}
});
});
+
+ describe('tokenizeAsync', () => {
+ it('produces same results as sync tokenize for small code', async () => {
+ const code = 'const x = 42;\nreturn "hello";';
+ const syncTokens = tokenize(code, 'typescript');
+ const asyncTokens = await tokenizeAsync(code, 'typescript');
+ expect(asyncTokens).toEqual(syncTokens);
+ });
+
+ it('produces same results for larger code', async () => {
+ // Generate code larger than SYNC_TOKENIZE_THRESHOLD
+ const line = 'const x = 42; // comment\n';
+ const code = line.repeat(
+ Math.ceil(SYNC_TOKENIZE_THRESHOLD / line.length) + 10,
+ );
+ const syncTokens = tokenize(code, 'typescript');
+ const asyncTokens = await tokenizeAsync(code, 'typescript');
+ expect(asyncTokens).toEqual(syncTokens);
+ });
+
+ it('returns empty for unknown language', async () => {
+ const tokens = await tokenizeAsync('const x = 1;', 'brainfuck');
+ expect(tokens).toEqual([]);
+ });
+
+ it('respects abort signal', async () => {
+ const controller = new AbortController();
+ controller.abort();
+ const tokens = await tokenizeAsync(
+ 'const x = 42;',
+ 'typescript',
+ controller.signal,
+ );
+ // Aborted before processing — tokens may be empty or partial
+ expect(Array.isArray(tokens)).toBe(true);
+ });
+
+ it('exports SYNC_TOKENIZE_THRESHOLD as a number', () => {
+ expect(typeof SYNC_TOKENIZE_THRESHOLD).toBe('number');
+ expect(SYNC_TOKENIZE_THRESHOLD).toBe(2000);
+ });
+ });
});
diff --git a/packages/lab/src/CodeBlock/tokenizer.ts b/packages/lab/src/CodeBlock/tokenizer.ts
index 9305691d7b..3e65043ad5 100644
--- a/packages/lab/src/CodeBlock/tokenizer.ts
+++ b/packages/lab/src/CodeBlock/tokenizer.ts
@@ -287,24 +287,44 @@ function buildLanguagePatterns(
// Core tokenizer
// ---------------------------------------------------------------------------
+/** Threshold below which we tokenize synchronously (characters). */
+export const SYNC_TOKENIZE_THRESHOLD = 2000;
+
+/** Characters to process per chunk in async tokenization. */
+const ASYNC_CHUNK_SIZE = 5000;
+
/**
- * Tokenizes a code string into an array of typed tokens with character offsets.
- *
- * Uses a greedy first-match strategy: at each position, try all patterns in
- * order and emit the first match. Skips characters that don't match any pattern.
- *
- * @param code - The source code string to tokenize
- * @param language - Language identifier (e.g. 'typescript', 'python')
- * @returns Array of tokens sorted by start position
+ * Yield to the main thread. Uses `scheduler.yield()` when available,
+ * falling back to `setTimeout(resolve, 0)`.
*/
-export function tokenize(code: string, language: string): Token[] {
- const langDef = buildLanguage(language);
- if (!langDef) return [];
-
- const tokens: Token[] = [];
- let pos = 0;
+function yieldToMain(): Promise<void> {
+ if (
+ typeof scheduler !== 'undefined' &&
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ typeof (scheduler as any).yield === 'function'
+ ) {
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ return (scheduler as any).yield();
+ }
+ return new Promise(resolve => setTimeout(resolve, 0));
+}
- while (pos < code.length) {
+/**
+ * Internal tokenization loop. Processes from `startPos` up to `endPos`
+ * (or end of string), pushing tokens into the provided array.
+ * Returns the final position.
+ */
+function tokenizeRange(
+ code: string,
+ langDef: LangDef,
+ tokens: Token[],
+ startPos: number,
+ endPos: number,
+): number {
+ let pos = startPos;
+ const limit = Math.min(endPos, code.length);
+
+ while (pos < limit) {
let matched = false;
for (const pattern of langDef.patterns) {
@@ -328,5 +348,67 @@ export function tokenize(code: string, language: string): Token[] {
}
}
+ return pos;
+}
+
+/**
+ * Tokenizes a code string into an array of typed tokens with character offsets.
+ *
+ * Uses a greedy first-match strategy: at each position, try all patterns in
+ * order and emit the first match. Skips characters that don't match any pattern.
+ *
+ * Best for small code strings (< 2000 chars). For large code, use
+ * `tokenizeAsync()` which yields to the main thread between chunks.
+ *
+ * @param code - The source code string to tokenize
+ * @param language - Language identifier (e.g. 'typescript', 'python')
+ * @returns Array of tokens sorted by start position
+ */
+export function tokenize(code: string, language: string): Token[] {
+ const langDef = buildLanguage(language);
+ if (!langDef) return [];
+
+ const tokens: Token[] = [];
+ tokenizeRange(code, langDef, tokens, 0, code.length);
+ return tokens;
+}
+
+/**
+ * Async streaming tokenizer that yields to the main thread between chunks.
+ *
+ * Processes ~5000 characters at a time, then yields via `scheduler.yield()`
+ * (or `setTimeout`) to avoid blocking the main thread on large files.
+ *
+ * Supports cancellation via AbortSignal — returns tokens accumulated so far
+ * if aborted (callers should discard the result).
+ *
+ * @param code - The source code string to tokenize
+ * @param language - Language identifier (e.g. 'typescript', 'python')
+ * @param signal - Optional AbortSignal for cancellation
+ * @returns Promise resolving to array of tokens sorted by start position
+ */
+export async function tokenizeAsync(
+ code: string,
+ language: string,
+ signal?: AbortSignal,
+): Promise<Token[]> {
+ const langDef = buildLanguage(language);
+ if (!langDef) return [];
+
+ const tokens: Token[] = [];
+ let pos = 0;
+
+ while (pos < code.length) {
+ if (signal?.aborted) return tokens;
+
+ // Process one chunk
+ pos = tokenizeRange(code, langDef, tokens, pos, pos + ASYNC_CHUNK_SIZE);
+
+ // Yield to main thread if there's more to process
+ if (pos < code.length) {
+ await yieldToMain();
+ }
+ }
+
return tokens;
}
diff --git a/packages/lab/src/CodeEditor/XDSCodeEditor.tsx b/packages/lab/src/CodeEditor/XDSCodeEditor.tsx
index c9b7cb6eab..950e20d57b 100644
--- a/packages/lab/src/CodeEditor/XDSCodeEditor.tsx
+++ b/packages/lab/src/CodeEditor/XDSCodeEditor.tsx
@@ -12,7 +12,14 @@
'use client';
-import {useEffect, useLayoutEffect, useRef, useCallback, useState} from 'react';
+import {
+ useEffect,
+ useLayoutEffect,
+ useRef,
+ useCallback,
+ useState,
+ useMemo,
+} from 'react';
import type {XDSBaseProps} from '@xds/core/XDSBaseProps';
import * as stylex from '@stylexjs/stylex';
import {
@@ -24,7 +31,11 @@ import {
lineHeightVars,
} from '@xds/core/theme/tokens.stylex';
import {xdsClassName, mergeProps} from '@xds/core/utils';
-import {tokenize} from '../CodeBlock/tokenizer';
+import {
+ tokenize,
+ tokenizeAsync,
+ SYNC_TOKENIZE_THRESHOLD,
+} from '../CodeBlock/tokenizer';
import type {Token} from '../CodeBlock/tokenizer';
import {ensureHighlightStyles, TOKEN_TYPES} from '../CodeBlock/highlightStyles';
@@ -134,6 +145,15 @@ export interface XDSCodeEditorProps extends Omit<
code: string,
language: string,
) => Array<{type: string; start: number; end: number}>;
+ /**
+ * How to apply syntax highlighting.
+ * - 'css-highlight': Uses CSS Custom Highlight API (zero DOM overhead).
+ * Falls back to 'spans' if the API is not available.
+ * - 'spans': Renders `<span>` elements with CSS classes per token.
+ * In the editor, uses a transparent-text overlay approach.
+ * @default 'css-highlight'
+ */
+ highlightMode?: 'css-highlight' | 'spans';
}
// ---------------------------------------------------------------------------
@@ -159,6 +179,157 @@ const AUTO_CLOSE_PAIRS: Record<string, string> = {
let editorInstanceCounter = 0;
+// ---------------------------------------------------------------------------
+// Span-based rendering helpers
+// ---------------------------------------------------------------------------
+
+/**
+ * Build span-based highlighted line content from tokens.
+ */
+function buildSpanLine(
+ lineText: string,
+ lineStart: number,
+ tokens: Token[],
+): React.ReactNode {
+ if (!lineText) return '\u200b';
+
+ const lineEnd = lineStart + lineText.length;
+ const lineTokens = tokens.filter(t => t.start < lineEnd && t.end > lineStart);
+
+ if (lineTokens.length === 0) return lineText;
+
+ const parts: React.ReactNode[] = [];
+ let cursor = lineStart;
+
+ for (const token of lineTokens) {
+ const tStart = Math.max(token.start, lineStart);
+ const tEnd = Math.min(token.end, lineEnd);
+
+ if (tStart > cursor) {
+ parts.push(lineText.slice(cursor - lineStart, tStart - lineStart));
+ }
+
+ parts.push(
+ <span
+ key={`${tStart}-${token.type}`}
+ className={`xds-token-${token.type}`}>
+ {lineText.slice(tStart - lineStart, tEnd - lineStart)}
+ </span>,
+ );
+
+ cursor = tEnd;
+ }
+
+ if (cursor < lineEnd) {
+ parts.push(lineText.slice(cursor - lineStart));
+ }
+
+ return parts.length > 0 ? parts : lineText;
+}
+
+/**
+ * Build span-based content for all lines.
+ */
+function buildSpanContent(lines: string[], tokens: Token[]): React.ReactNode {
+ return lines.map((line, i) => {
+ let lineStart = 0;
+ for (let j = 0; j < i; j++) {
+ lineStart += lines[j].length + 1;
+ }
+ return (
+ <div key={i}>
+ {buildSpanLine(line, lineStart, tokens)}
+ {i < lines.length - 1 ? '\n' : null}
+ </div>
+ );
+ });
+}
+
+// ---------------------------------------------------------------------------
+// Shared highlight range application
+// ---------------------------------------------------------------------------
+
+function applyEditorHighlightRanges(
+ el: HTMLElement,
+ tokens: Token[],
+): () => void {
+ const tokensByType = new Map<string, Token[]>();
+ for (const token of tokens) {
+ const existing = tokensByType.get(token.type);
+ if (existing) {
+ existing.push(token);
+ } else {
+ tokensByType.set(token.type, [token]);
+ }
+ }
+
+ const walker = document.createTreeWalker(el, NodeFilter.SHOW_TEXT);
+ const textNodes: Array<{node: Text; start: number}> = [];
+ let totalOffset = 0;
+
+ let node = walker.nextNode();
+ while (node) {
+ textNodes.push({node: node as Text, start: totalOffset});
+ totalOffset += (node as Text).length;
+ node = walker.nextNode();
+ }
+
+ function findPosition(offset: number): {node: Text; offset: number} | null {
+ for (let i = textNodes.length - 1; i >= 0; i--) {
+ const entry = textNodes[i];
+ if (offset >= entry.start) {
+ const localOffset = offset - entry.start;
+ if (localOffset <= entry.node.length) {
+ return {node: entry.node, offset: localOffset};
+ }
+ return null;
+ }
+ }
+ return null;
+ }
+
+ const myRanges: Range[] = [];
+
+ for (const tokenType of TOKEN_TYPES) {
+ const typedTokens = tokensByType.get(tokenType);
+ if (!typedTokens || typedTokens.length === 0) continue;
+
+ const name = `xds-${tokenType}`;
+ let highlight = CSS.highlights.get(name);
+ if (!highlight) {
+ highlight = new Highlight();
+ CSS.highlights.set(name, highlight);
+ }
+
+ for (const token of typedTokens) {
+ const startPos = findPosition(token.start);
+ const endPos = findPosition(token.end);
+ if (!startPos || !endPos) continue;
+
+ try {
+ const range = new Range();
+ range.setStart(startPos.node, startPos.offset);
+ range.setEnd(endPos.node, endPos.offset);
+ highlight.add(range);
+ myRanges.push(range);
+ } catch {
+ // Skip invalid ranges
+ }
+ }
+ }
+
+ return () => {
+ for (const range of myRanges) {
+ for (const tokenType of TOKEN_TYPES) {
+ const highlight = CSS.highlights.get(`xds-${tokenType}`);
+ if (highlight) {
+ highlight.delete(range);
+ }
+ }
+ }
+ };
+}
+
// ---------------------------------------------------------------------------
// Component
// ---------------------------------------------------------------------------
@@ -168,6 +339,8 @@ let editorInstanceCounter = 0;
*
* Uses CSS Custom Highlight API for syntax coloring. Supports
* auto-indent, tab insertion, and bracket auto-closing.
+ * Falls back to span-based rendering when the API is unavailable
+ * or `highlightMode="spans"` is set.
*
* @example
* ```tsx
@@ -190,6 +363,7 @@ export function XDSCodeEditor({
maxHeight,
size = 'md',
tokenizer: customTokenizer,
+ highlightMode: highlightModeProp = 'css-highlight',
xstyle,
className,
style,
@@ -201,23 +375,62 @@ export function XDSCodeEditor({
const [focused, setFocused] = useState(false);
const isComposingRef = useRef(false);
+ // Resolve effective highlight mode
+ const useSpans = highlightModeProp === 'spans' || !hasHighlightAPI();
+
+ // Async tokens for span mode overlay
+ const [asyncTokens, setAsyncTokens] = useState<Token[] | null>(null);
+
const lines = value.split('\n');
// Sync textContent with controlled value
useLayoutEffect(() => {
const el = editorRef.current;
if (!el) return;
- // Only update DOM if it differs (avoids clobbering cursor)
if (el.textContent !== value) {
el.textContent = value;
}
}, [value]);
- // Apply CSS Custom Highlight API ranges
+ // Compute sync tokens for span mode (small code)
+ const syncTokens = useMemo(() => {
+ if (!useSpans) return null;
+ if (value.length >= SYNC_TOKENIZE_THRESHOLD) return null;
+ const tok = customTokenizer ?? tokenize;
+ return tok(value, language);
+ }, [useSpans, value, language, customTokenizer]);
+
+ // Async tokenization for span mode with large code
useEffect(() => {
- if (!hasHighlightAPI()) return;
+ if (!useSpans) return;
+ if (value.length < SYNC_TOKENIZE_THRESHOLD) return;
+ const abortController = new AbortController();
+
+ tokenizeAsync(value, language, abortController.signal).then(tokens => {
+ if (!abortController.signal.aborted) {
+ setAsyncTokens(tokens);
+ }
+ });
+
+ return () => {
+ abortController.abort();
+ setAsyncTokens(null);
+ };
+ }, [useSpans, value, language, customTokenizer]);
+
+ const spanTokens = syncTokens ?? asyncTokens ?? [];
+
+ // Ensure styles are always injected
+ useLayoutEffect(() => {
ensureHighlightStyles();
+ }, []);
+
+ // Apply CSS Custom Highlight API ranges — small code
+ useLayoutEffect(() => {
+ if (useSpans) return;
+ if (value.length >= SYNC_TOKENIZE_THRESHOLD) return;
+ if (!hasHighlightAPI()) return;
const el = editorRef.current;
if (!el) return;
@@ -226,84 +439,32 @@ export function XDSCodeEditor({
const tokens = tok(value, language);
if (tokens.length === 0) return;
- // Group tokens by type
- const tokensByType = new Map<string, Token[]>();
- for (const token of tokens) {
- const existing = tokensByType.get(token.type);
- if (existing) {
- existing.push(token);
- } else {
- tokensByType.set(token.type, [token]);
- }
- }
-
- // Build text node mapping from the contenteditable element
- const walker = document.createTreeWalker(el, NodeFilter.SHOW_TEXT);
- const textNodes: Array<{node: Text; start: number}> = [];
- let totalOffset = 0;
+ return applyEditorHighlightRanges(el, tokens);
+ }, [useSpans, value, language, customTokenizer, instanceId]);
- let node = walker.nextNode();
- while (node) {
- textNodes.push({node: node as Text, start: totalOffset});
- totalOffset += (node as Text).length;
- node = walker.nextNode();
- }
-
- function findPosition(offset: number): {node: Text; offset: number} | null {
- for (let i = textNodes.length - 1; i >= 0; i--) {
- const entry = textNodes[i];
- if (offset >= entry.start) {
- const localOffset = offset - entry.start;
- if (localOffset <= entry.node.length) {
- return {node: entry.node, offset: localOffset};
- }
- return null;
- }
- }
- return null;
- }
+ // Apply CSS Custom Highlight API ranges — large code (async)
+ useEffect(() => {
+ if (useSpans) return;
+ if (value.length < SYNC_TOKENIZE_THRESHOLD) return;
+ if (!hasHighlightAPI()) return;
- const myRanges: Range[] = [];
+ const el = editorRef.current;
+ if (!el) return;
- for (const tokenType of TOKEN_TYPES) {
- const typedTokens = tokensByType.get(tokenType);
- if (!typedTokens || typedTokens.length === 0) continue;
+ const abortController = new AbortController();
+ let cleanup: (() => void) | undefined;
- const name = `xds-${tokenType}`;
- let highlight = CSS.highlights.get(name);
- if (!highlight) {
- highlight = new Highlight();
- CSS.highlights.set(name, highlight);
- }
-
- for (const token of typedTokens) {
- const startPos = findPosition(token.start);
- const endPos = findPosition(token.end);
- if (!startPos || !endPos) continue;
-
- try {
- const range = new Range();
- range.setStart(startPos.node, startPos.offset);
- range.setEnd(endPos.node, endPos.offset);
- highlight.add(range);
- myRanges.push(range);
- } catch {
- // Skip invalid ranges
- }
- }
- }
+ tokenizeAsync(value, language, abortController.signal).then(tokens => {
+ if (abortController.signal.aborted) return;
+ if (tokens.length === 0) return;
+ cleanup = applyEditorHighlightRanges(el, tokens);
+ });
return () => {
- for (const range of myRanges) {
- for (const tokenType of TOKEN_TYPES) {
- const highlight = CSS.highlights.get(`xds-${tokenType}`);
- if (highlight) {
- highlight.delete(range);
- }
- }
- }
+ abortController.abort();
+ cleanup?.();
};
- }, [value, language, customTokenizer, instanceId]);
+ }, [useSpans, value, language, customTokenizer, instanceId]);
const handleInput = useCallback(() => {
if (isComposingRef.current) return;
@@ -414,6 +575,30 @@ export function XDSCodeEditor({
? {maxHeight: typeof maxHeight === 'number' ? `${maxHeight}px` : maxHeight}
: undefined;
+ // For span mode in the editor, we render a non-interactive overlay with
+ // colored spans on top of the contentEditable. The contentEditable text
+ // is made transparent so the overlay provides the visual coloring while
+ // the contentEditable handles input/caret.
+ const spanOverlay =
+ useSpans && spanTokens.length > 0 ? (
+ <div
+ aria-hidden="true"
+ style={{
+ position: 'absolute',
+ top: 0,
+ left: 0,
+ right: 0,
+ bottom: 0,
+ pointerEvents: 'none',
+ whiteSpace: 'pre',
+ wordBreak: 'normal',
+ overflowWrap: 'normal',
+ }}
+ {...stylex.props(styles.editor, sizeStyle)}>
+ {buildSpanContent(lines, spanTokens)}
+ </div>
+ ) : null;
+
return (
<div
ref={ref}
@@ -435,7 +620,9 @@ export function XDSCodeEditor({
))}
</div>
)}
- <div {...stylex.props(styles.editorContainer)} style={containerStyle}>
+ <div
+ {...stylex.props(styles.editorContainer)}
+ style={{...containerStyle, position: 'relative'}}>
{showPlaceholder && (
<div {...stylex.props(styles.placeholder, sizeStyle)}>
{placeholder}
@@ -456,7 +643,13 @@ export function XDSCodeEditor({
onCompositionStart={handleCompositionStart}
onCompositionEnd={handleCompositionEnd}
{...stylex.props(styles.editor, sizeStyle)}
+ style={
+ useSpans && spanTokens.length > 0
+ ? {color: 'transparent', caretColor: 'inherit'}
+ : undefined
+ }
/>
+ {spanOverlay}
</div>
</div>
);
diff --git a/packages/lab/src/index.ts b/packages/lab/src/index.ts
index 41d71addb9..a4e30663b6 100644
--- a/packages/lab/src/index.ts
+++ b/packages/lab/src/index.ts
@@ -30,4 +30,9 @@ export {
// Code components — syntax highlighting domain
export {XDSCodeBlock, type XDSCodeBlockProps} from './CodeBlock';
export {XDSCodeEditor, type XDSCodeEditorProps} from './CodeEditor';
-export {tokenize, type Token} from './CodeBlock/tokenizer';
+export {
+ tokenize,
+ tokenizeAsync,
+ SYNC_TOKENIZE_THRESHOLD,
+ type Token,
+} from './CodeBlock/tokenizer';