Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

perf(richtext): code highlighting re-tokenizes only the lines an edit changes (#8585)

Ernest540e9dfdfa4e
diff --git a/.changeset/typing-in-a-richtexteditor-code-block-no-longer--4xw9.md b/.changeset/typing-in-a-richtexteditor-code-block-no-longer--4xw9.md
new file mode 100644
index 0000000000..de4bb21dcc
--- /dev/null
+++ b/.changeset/typing-in-a-richtexteditor-code-block-no-longer--4xw9.md
@@ -0,0 +1,6 @@
+---
+'@astryxdesign/richtext': patch
+---
+
+[perf] Typing in a `RichTextEditor` code block no longer re-tokenizes the whole block. In TypeScript, JavaScript, Python, and shell code, an edit inside one line re-tokenizes that line, plus at most the line above it, and replaces only the color ranges inside the text node it edited, so a keystroke in a 2,000-line block tokenizes as little code as one in a short block. Edits that open or close a string or comment across lines, and other languages, still re-tokenize the whole block. Colors are unchanged, except that a block whose text nodes are replaced by others with the same text is now recolored instead of left uncolored.
+@emilyhorsman @ernestt
diff --git a/packages/richtext/src/CodeSyntaxPlugin.test.tsx b/packages/richtext/src/CodeSyntaxPlugin.test.tsx
index 69569d5b52..acee23ae4d 100644
--- a/packages/richtext/src/CodeSyntaxPlugin.test.tsx
+++ b/packages/richtext/src/CodeSyntaxPlugin.test.tsx
@@ -1,29 +1,292 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
-import {describe, it, expect} from 'vitest';
-import {render, screen} from '@testing-library/react';
-import {tokenize} from '@astryxdesign/core/CodeBlock';
+import {describe, it, expect, vi, afterEach, beforeAll} from 'vitest';
+import {act, cleanup, render, screen, waitFor} from '@testing-library/react';
+import {useEffect} from 'react';
+import {useLexicalComposerContext} from '@lexical/react/LexicalComposerContext';
+import {$isCodeNode} from '@lexical/code';
+import {
+ $createLineBreakNode,
+ $createRangeSelection,
+ $createTextNode,
+ $getRoot,
+ $getSelection,
+ $isRangeSelection,
+ $isTextNode,
+ $setSelection,
+ ElementNode,
+ type LexicalEditor,
+} from 'lexical';
+import type * as codeBlock from '@astryxdesign/core/CodeBlock';
+import {
+ applyHighlightRangesFlat,
+ tokenize,
+ type CodeLanguage,
+ type TokenLine,
+} from '@astryxdesign/core/CodeBlock';
import {thrift} from '@astryxdesign/core/CodeBlock/languages/thrift';
import {listedLanguage, sameLanguages} from './codeLanguages';
-import {renderedTokens} from './CodeSyntaxPlugin';
+import {RichTextEditor} from './RichTextEditor';
import {RichTextView} from './RichTextView';
import {markdownToEditorStateJSON} from './markdownSerializers';
+vi.mock('@astryxdesign/core/CodeBlock', async importOriginal => {
+ const actual = await importOriginal<typeof codeBlock>();
+ return {...actual, tokenize: vi.fn(actual.tokenize)};
+});
+
+let rangesAdded = 0;
+
+/** jsdom has no CSS Custom Highlight API: named sets of ranges stand in. */
+class TestHighlight extends Set<AbstractRange> {
+ override add(range: AbstractRange): this {
+ rangesAdded++;
+ return super.add(range);
+ }
+}
+
+function stubHighlights(): Map<string, TestHighlight> {
+ const registry = new Map<string, TestHighlight>();
+ vi.stubGlobal('CSS', {...globalThis.CSS, highlights: registry});
+ vi.stubGlobal('Highlight', TestHighlight);
+ return registry;
+}
+
+afterEach(() => {
+ cleanup();
+ vi.unstubAllGlobals();
+});
+
+// jsdom's Range has no getBoundingClientRect, which Lexical calls to scroll
+// a reconciled selection into view while the editor has focus.
+beforeAll(() => {
+ if (typeof Range.prototype.getBoundingClientRect !== 'function') {
+ Range.prototype.getBoundingClientRect = () => new DOMRect();
+ }
+});
+
+function CaptureEditor({onReady}: {onReady: (editor: LexicalEditor) => void}) {
+ const [editor] = useLexicalComposerContext();
+ useEffect(() => {
+ onReady(editor);
+ }, [editor, onReady]);
+ return null;
+}
+
+async function mountEditor(
+ markdown: string,
+ codeLanguages?: ReadonlyArray<CodeLanguage>,
+): Promise<LexicalEditor> {
+ let editor: LexicalEditor | undefined;
+ render(
+ <RichTextEditor
+ label="Code"
+ codeLanguages={codeLanguages}
+ defaultValue={markdownToEditorStateJSON(markdown)}
+ plugins={<CaptureEditor onReady={live => (editor = live)} />}
+ />,
+ );
+ await waitFor(() => expect(editor).toBeDefined());
+ return editor!;
+}
+
+function update(editor: LexicalEditor, fn: () => void) {
+ act(() => editor.update(fn, {discrete: true}));
+}
+
+function $block() {
+ const block = $getRoot().getFirstChild();
+ if (!$isCodeNode(block)) {
+ throw new Error('missing code block');
+ }
+ return block;
+}
+
+/** Rewrites the code block the way typing and pasting build one: a text node
+ * per line, with a line break node between lines. */
+function $writeLines(lines: ReadonlyArray<string>) {
+ const block = $block();
+ block.clear();
+ lines.forEach((line, i) => {
+ if (i > 0) {
+ block.append($createLineBreakNode());
+ }
+ if (line !== '') {
+ block.append($createTextNode(line));
+ }
+ });
+}
+
+/** The text node and offset at `offset` in the code block's text. */
+function $pointAt(offset: number): [string, number] {
+ let start = 0;
+ for (const child of $block().getChildren()) {
+ const size = child.getTextContentSize();
+ if ($isTextNode(child) && offset <= start + size) {
+ return [child.getKey(), offset - start];
+ }
+ start += size;
+ }
+ throw new Error(`no text node at ${offset}`);
+}
+
+/** Selects `length` characters at `offset` in the code block's text. */
+function $selectCode(offset: number, length = 0) {
+ const [anchorKey, anchorOffset] = $pointAt(offset);
+ const [focusKey, focusOffset] = $pointAt(offset + length);
+ const selection = $createRangeSelection();
+ selection.anchor.set(anchorKey, anchorOffset, 'text');
+ selection.focus.set(focusKey, focusOffset, 'text');
+ $setSelection(selection);
+}
+
+function $type(text: string) {
+ const selection = $getSelection();
+ if (!$isRangeSelection(selection)) {
+ throw new Error('missing selection');
+ }
+ selection.insertText(text);
+}
+
+function codeOf(editor: LexicalEditor): {code: string; language: string} {
+ return editor.getEditorState().read(() => ({
+ code: $block().getTextContent(),
+ language: $block().getLanguage() ?? '',
+ }));
+}
+
+function codeElement(editor: LexicalEditor): HTMLElement {
+ const key = editor.getEditorState().read(() => $block().getKey());
+ return editor.getElementByKey(key)!;
+}
+
+interface Span {
+ type: string;
+ start: number;
+ end: number;
+}
+
+/** Every range in `registry`, as a token type over offsets in the code
+ * element's rendered text (-1 for a range outside it). */
+function spansIn(
+ registry: ReadonlyMap<string, Set<AbstractRange>>,
+ element: HTMLElement,
+): Span[] {
+ const starts = new Map<Node, number>();
+ let offset = 0;
+ const walker = document.createTreeWalker(element, NodeFilter.SHOW_TEXT);
+ for (let node = walker.nextNode(); node != null; node = walker.nextNode()) {
+ starts.set(node, offset);
+ offset += (node as Text).length;
+ }
+ const at = (node: Node, nodeOffset: number) =>
+ starts.has(node) ? starts.get(node)! + nodeOffset : -1;
+ const spans: Span[] = [];
+ for (const [name, highlight] of registry) {
+ for (const range of highlight) {
+ spans.push({
+ type: name.replace(/^astryx-/, ''),
+ start: at(range.startContainer, range.startOffset),
+ end: at(range.endContainer, range.endOffset),
+ });
+ }
+ }
+ return spans.sort(
+ (a, b) =>
+ a.start - b.start ||
+ a.end - b.end ||
+ (a.type < b.type ? -1 : a.type > b.type ? 1 : 0),
+ );
+}
+
+/**
+ * What coloring the block from scratch paints: core CodeBlock's tokens for
+ * the whole code, laid over the rendered text line by line when it keeps the
+ * line breaks or as one line of offsets when it drops them (nothing when it
+ * is neither), and written by core CodeBlock's own range writer.
+ */
+function fullRepaint(
+ element: HTMLElement,
+ code: string,
+ language: string | CodeLanguage,
+): Span[] {
+ const rendered = element.textContent ?? '';
+ const lines = tokenize(code, language);
+ let laid: TokenLine[] | null = null;
+ if (rendered === code) {
+ laid = lines;
+ } else if (rendered === code.replaceAll('\n', '')) {
+ const flat: TokenLine = [];
+ let offset = 0;
+ code.split('\n').forEach((line, i) => {
+ for (const token of lines[i] ?? []) {
+ flat.push({
+ ...token,
+ start: offset + token.start,
+ end: offset + token.end,
+ });
+ }
+ offset += line.length;
+ });
+ laid = [flat];
+ }
+ if (laid == null) {
+ return [];
+ }
+ const live = CSS.highlights;
+ const registry = new Map<string, TestHighlight>();
+ Object.assign(CSS, {highlights: registry});
+ try {
+ const remove = applyHighlightRangesFlat(element, laid);
+ const spans = spansIn(registry, element);
+ remove();
+ return spans;
+ } finally {
+ Object.assign(CSS, {highlights: live});
+ }
+}
+
+function painted(
+ registry: ReadonlyMap<string, Set<AbstractRange>>,
+ type: string,
+): string[] {
+ return [...(registry.get(`astryx-${type}`) ?? [])].map(range =>
+ (range as Range).toString(),
+ );
+}
+
describe('code block syntax colors (spec:AST-061 FR1)', () => {
const code = 'let a = 1;\nreturn a;';
- it("uses core CodeBlock's tokens line by line when the text keeps its line breaks", () => {
- expect(renderedTokens(code, 'ts', code)).toEqual(tokenize(code, 'ts'));
+ it("colors with core CodeBlock's tokens, line by line where the text keeps its line breaks", async () => {
+ const registry = stubHighlights();
+ const editor = await mountEditor('```ts\n' + code + '\n```\n');
+ expect(codeElement(editor).textContent).toBe(code);
+ expect(painted(registry, 'keyword')).toEqual(['let', 'return']);
+ expect(spansIn(registry, codeElement(editor))).toEqual(
+ fullRepaint(codeElement(editor), code, 'ts'),
+ );
});
- it('paints a listed language value, as Markdown does (CodeBlock FR13)', () => {
- const idl = 'struct User {}';
- expect(renderedTokens(idl, 'thrift', idl)?.flat()).toEqual([]);
- expect(
- renderedTokens(idl, listedLanguage([thrift], 'thrift') ?? 'thrift', idl)
- ?.flat()
- .map(t => t.type),
- ).toContain('keyword');
+ it('lays the tokens over the text as one line where line breaks render without text', async () => {
+ const registry = stubHighlights();
+ const editor = await mountEditor('```ts\nx\n```\n');
+ update(editor, () => $writeLines(code.split('\n')));
+ expect(codeElement(editor).textContent).toBe(code.replace('\n', ''));
+ expect(painted(registry, 'keyword')).toEqual(['let', 'return']);
+ expect(spansIn(registry, codeElement(editor))).toEqual(
+ fullRepaint(codeElement(editor), code, 'ts'),
+ );
+ });
+
+ it('paints a listed language value, as Markdown does (CodeBlock FR13)', async () => {
+ const registry = stubHighlights();
+ await mountEditor('```thrift\nstruct User {}\n```\n');
+ expect(painted(registry, 'keyword')).toEqual([]);
+ cleanup();
+ await mountEditor('```thrift\nstruct User {}\n```\n', [thrift]);
+ expect(painted(registry, 'keyword')).toEqual(['struct']);
+ expect(listedLanguage([thrift], 'thrift')).toBe(thrift);
expect(listedLanguage([thrift], 'graphql')).toBeUndefined();
});
@@ -34,19 +297,24 @@ describe('code block syntax colors (spec:AST-061 FR1)', () => {
expect(sameLanguages([thrift], undefined)).toBe(false);
});
- it('lays the tokens over the text as one line when line breaks render without text', () => {
- const rendered = code.replace('\n', '');
- const [line] = renderedTokens(code, 'ts', rendered) ?? [];
- const texts = (line ?? []).map(
- token => `${token.type}:${rendered.slice(token.start, token.end)}`,
+ it('colors nothing for a language core does not know, or while the rendered text is not the code', async () => {
+ const registry = stubHighlights();
+ const editor = await mountEditor('```notalanguage\n' + code + '\n```\n');
+ expect([...registry.values()].flatMap(highlight => [...highlight])).toEqual(
+ [],
);
- expect(texts).toContain('keyword:let');
- expect(texts).toContain('keyword:return');
- });
-
- it('colors nothing while the rendered text is not the code, or for a language core does not know', () => {
- expect(renderedTokens(code, 'ts', 'let a')).toBeNull();
- expect(renderedTokens(code, 'notalanguage', code)).toEqual([]);
+ update(editor, () => $block().setLanguage('ts'));
+ expect(painted(registry, 'keyword')).toEqual(['let', 'return']);
+ // A pasted line break node beside the imported text's own line breaks
+ // leaves the rendered text neither the code nor the code without breaks.
+ update(editor, () => {
+ $selectCode(code.length);
+ const selection = $getSelection();
+ if ($isRangeSelection(selection)) {
+ selection.insertRawText('\nlet b;');
+ }
+ });
+ expect(spansIn(registry, codeElement(editor))).toEqual([]);
});
it('renders code blocks where the Highlight API is missing', async () => {
@@ -58,3 +326,369 @@ describe('code block syntax colors (spec:AST-061 FR1)', () => {
expect(await screen.findByText('let plain = 1;')).toBeInTheDocument();
});
});
+
+type Edit = [find: (code: string) => number, removed: number, inserted: string];
+
+/** Applies each edit, then checks the colors against a full repaint. */
+async function expectEditsMatchFullRepaint(
+ language: string,
+ lines: ReadonlyArray<string>,
+ layout: 'text node per line' | 'one text node',
+ edits: ReadonlyArray<Edit>,
+) {
+ const registry = stubHighlights();
+ const editor = await mountEditor(
+ '```' + language + '\n' + lines.join('\n') + '\n```\n',
+ );
+ if (layout === 'text node per line') {
+ update(editor, () => $writeLines(lines));
+ }
+ const check = (step: string) => {
+ const {code, language: current} = codeOf(editor);
+ expect(
+ spansIn(registry, codeElement(editor)),
+ `${layout}, ${step}: ${JSON.stringify(code)}`,
+ ).toEqual(fullRepaint(codeElement(editor), code, current));
+ };
+ check('as loaded');
+ for (const [find, removed, inserted] of edits) {
+ const at = find(codeOf(editor).code);
+ update(editor, () => {
+ $selectCode(at, removed);
+ $type(inserted);
+ });
+ check(`replace ${removed} at ${at} with ${JSON.stringify(inserted)}`);
+ }
+ cleanup();
+}
+
+function indexIn(code: string, text: string): number {
+ const at = code.indexOf(text);
+ if (at < 0) {
+ throw new Error(`missing ${JSON.stringify(text)}`);
+ }
+ return at;
+}
+const before = (text: string) => (code: string) => indexIn(code, text);
+const after = (text: string) => (code: string) =>
+ indexIn(code, text) + text.length;
+
+const TS_LINES = [
+ 'const a = call(1, "row"); // note',
+ 'let b = `two`;',
+ '/* first */ const c = 3;',
+ 'function run() {',
+ ' return a + b;',
+ '}',
+];
+
+describe('edited code blocks color exactly as a full repaint does', () => {
+ const layouts = ['text node per line', 'one text node'] as const;
+
+ it.each(layouts)(
+ 'TypeScript, opening and closing strings, comments, and template literals (%s)',
+ async layout => {
+ await expectEditsMatchFullRepaint('ts', TS_LINES, layout, [
+ [after('// note'), 0, 'x'],
+ [after('let b = '), 0, '"'],
+ [after('let b = "'), 0, 'q'],
+ [before('"q'), 2, ''],
+ [before('function'), 0, '/*'],
+ [after('return a + b;'), 0, ' */'],
+ [before('/*function'), 2, ''],
+ [after('let b = `two`'), 0, '`'],
+ [before('`;'), 1, ''],
+ [after('const c = 3;'), 0, ' call'],
+ [before('function'), 0, '('],
+ [before('(function'), 1, ''],
+ [after('row'), 0, 'x'],
+ [after('rowx'), 0, '\\'],
+ [before('1, "row'), 1, '42'],
+ [after('call(42'), 0, '\n'],
+ [after('return a'), 0, 'x'],
+ ]);
+ },
+ );
+
+ it.each(layouts)(
+ 'Python, opening and closing a triple-quoted string (%s)',
+ async layout => {
+ await expectEditsMatchFullRepaint(
+ 'python',
+ ['def run():', ' x = call(1)', ' return x', 'y = 2'],
+ layout,
+ [
+ [after('x = '), 0, '"""'],
+ [after('return x'), 0, '"""'],
+ [before('"""call'), 3, ''],
+ [after('y = 2'), 0, ' # done'],
+ [after('def run'), 0, 'ning'],
+ ],
+ );
+ },
+ );
+
+ it.each(layouts)('shell, opening and closing ${ (%s)', async layout => {
+ await expectEditsMatchFullRepaint(
+ 'sh',
+ ['echo "row"', 'printf 1', 'exit 0'],
+ layout,
+ [
+ [after('echo '), 0, '${value'],
+ [after('printf 1'), 0, '}'],
+ [before('${value'), 7, ''],
+ [after('exit 0'), 0, ' # ok'],
+ ],
+ );
+ });
+
+ it.each(layouts)(
+ 'a language without the line cache re-tokenizes each edit (%s)',
+ async layout => {
+ await expectEditsMatchFullRepaint(
+ 'css',
+ ['.a {', ' color: red;', '}', '/* end */'],
+ layout,
+ [
+ [after('color: red'), 0, ' !important'],
+ [before('}'), 0, '/*'],
+ [after('/* end */'), 0, 'x'],
+ ],
+ );
+ },
+ );
+
+ it('follows line breaks added and removed, and keeps later lines exact', async () => {
+ const registry = stubHighlights();
+ const editor = await mountEditor('```ts\nx\n```\n');
+ update(editor, () => $writeLines(TS_LINES));
+ const check = () => {
+ const {code} = codeOf(editor);
+ expect(spansIn(registry, codeElement(editor))).toEqual(
+ fullRepaint(codeElement(editor), code, 'ts'),
+ );
+ };
+ update(editor, () => {
+ $selectCode(after('call(1, ')(codeOf(editor).code));
+ const selection = $getSelection();
+ if ($isRangeSelection(selection)) {
+ selection.insertLineBreak();
+ }
+ });
+ check();
+ update(editor, () => {
+ $selectCode(after('let b = ')(codeOf(editor).code));
+ $type('x + ');
+ });
+ check();
+ update(editor, () => {
+ $selectCode(before('\n"row"')(codeOf(editor).code), 1);
+ $type('');
+ });
+ check();
+ update(editor, () => {
+ $selectCode(after('return a')(codeOf(editor).code));
+ const selection = $getSelection();
+ if ($isRangeSelection(selection)) {
+ selection.insertRawText(' +\n\t`more`');
+ }
+ });
+ check();
+ update(editor, () => {
+ $selectCode(after('`more')(codeOf(editor).code));
+ $type('!');
+ });
+ check();
+ });
+
+ it('recolors a block whose text nodes are replaced by others with the same text', async () => {
+ const registry = stubHighlights();
+ const editor = await mountEditor(
+ '```ts\n' + TS_LINES.join('\n') + '\n```\n',
+ );
+ update(editor, () => $writeLines(TS_LINES));
+ expect(painted(registry, 'keyword')).toContain('const');
+ expect(spansIn(registry, codeElement(editor))).toEqual(
+ fullRepaint(codeElement(editor), TS_LINES.join('\n'), 'ts'),
+ );
+ });
+
+ it('follows the language and restored editor states', async () => {
+ const registry = stubHighlights();
+ const editor = await mountEditor('```ts\nx\n```\n');
+ update(editor, () => $writeLines(TS_LINES));
+ const check = () => {
+ const {code, language} = codeOf(editor);
+ expect(spansIn(registry, codeElement(editor))).toEqual(
+ fullRepaint(codeElement(editor), code, language),
+ );
+ };
+ const loaded = editor.getEditorState();
+ update(editor, () => {
+ $selectCode(after('const c = ')(codeOf(editor).code), 1);
+ $type('"three"');
+ });
+ check();
+ const edited = editor.getEditorState();
+ act(() => editor.setEditorState(loaded));
+ check();
+ expect(painted(registry, 'number')).toContain('3');
+ act(() => editor.setEditorState(edited));
+ check();
+ expect(painted(registry, 'string')).toContain('"three"');
+ update(editor, () => $block().setLanguage('python'));
+ check();
+ expect(painted(registry, 'keyword')).not.toContain('const');
+ });
+});
+
+interface Work {
+ tokenizeCalls: number;
+ linesTokenized: number;
+ charactersTokenized: number;
+ textNodesScanned: number;
+ blockTextReads: number;
+ rangesAdded: number;
+}
+
+/** Counts what `run` makes the plugin do: core tokenizer calls and the lines
+ * and characters they read, the DOM text nodes the plugin reads (through
+ * any getter or tree walk), whole-block text reads from the editor state,
+ * and ranges added. */
+function measure(run: () => void): Work {
+ // The plugin's own modules, not this test file's frames below them.
+ const byPlugin = () =>
+ /CodeSyntaxPlugin\.tsx|codeLineTokens\.ts/.test(new Error().stack ?? '');
+ const scanned = new Set<Node>();
+ let blockTextReads = 0;
+ const visit = (node: Node) => {
+ if (node.nodeType === Node.TEXT_NODE) {
+ scanned.add(node);
+ } else {
+ node.childNodes.forEach(visit);
+ }
+ };
+ const restores: Array<() => void> = [];
+ const watch = (proto: object, name: string) => {
+ const descriptor = Object.getOwnPropertyDescriptor(proto, name)!;
+ Object.defineProperty(proto, name, {
+ ...descriptor,
+ get(this: Node) {
+ if (byPlugin()) {
+ visit(this);
+ }
+ return descriptor.get!.call(this);
+ },
+ });
+ restores.push(() => Object.defineProperty(proto, name, descriptor));
+ };
+ watch(CharacterData.prototype, 'data');
+ watch(CharacterData.prototype, 'length');
+ watch(Node.prototype, 'nodeValue');
+ watch(Node.prototype, 'textContent');
+ const createTreeWalker = document.createTreeWalker;
+ document.createTreeWalker = function (
+ this: Document,
+ ...args: Parameters<Document['createTreeWalker']>
+ ) {
+ const walker = createTreeWalker.apply(this, args);
+ const nextNode = walker.nextNode.bind(walker);
+ walker.nextNode = () => {
+ const node = nextNode();
+ if (node != null && byPlugin()) {
+ visit(node);
+ }
+ return node;
+ };
+ return walker;
+ };
+ restores.push(() => {
+ document.createTreeWalker = createTreeWalker;
+ });
+ const getTextContent = ElementNode.prototype.getTextContent;
+ ElementNode.prototype.getTextContent = function (this: ElementNode) {
+ if ($isCodeNode(this) && byPlugin()) {
+ blockTextReads++;
+ }
+ return getTextContent.call(this);
+ };
+ restores.push(() => {
+ ElementNode.prototype.getTextContent = getTextContent;
+ });
+ const stackTraceLimit = Error.stackTraceLimit;
+ Error.stackTraceLimit = 100;
+ restores.push(() => {
+ Error.stackTraceLimit = stackTraceLimit;
+ });
+ const tokenizer = vi.mocked(tokenize);
+ tokenizer.mockClear();
+ rangesAdded = 0;
+ try {
+ run();
+ } finally {
+ restores.reverse().forEach(restore => restore());
+ }
+ const inputs = tokenizer.mock.calls.map(([text]) => text);
+ return {
+ tokenizeCalls: inputs.length,
+ linesTokenized: inputs.reduce(
+ (sum, text) => sum + text.split('\n').length,
+ 0,
+ ),
+ charactersTokenized: inputs.reduce((sum, text) => sum + text.length, 0),
+ textNodesScanned: scanned.size,
+ blockTextReads,
+ rangesAdded,
+ };
+}
+
+const LINE = 'const a = 1;';
+
+/** The work one character typed at the end of the middle line costs in a
+ * TypeScript block of `lines` copies of LINE. */
+async function typingWork(
+ lines: number,
+ layout: 'text node per line' | 'one text node',
+): Promise<Work> {
+ stubHighlights();
+ const source = Array.from({length: lines}, () => LINE);
+ const editor = await mountEditor('```ts\n' + source.join('\n') + '\n```\n');
+ if (layout === 'text node per line') {
+ update(editor, () => $writeLines(source));
+ }
+ const middle = Math.floor(lines / 2);
+ update(editor, () => $selectCode(middle * (LINE.length + 1) + LINE.length));
+ const work = measure(() => update(editor, () => $type('x')));
+ cleanup();
+ return work;
+}
+
+describe('typing in a long code block re-tokenizes only what it changed', () => {
+ // The typed line now ends in a name, which is a function name if the next
+ // visible character is `(`, so the next line's first character is
+ // tokenized with it: two lines, 15 characters.
+ const ONE_LINE = {
+ tokenizeCalls: 1,
+ linesTokenized: 2,
+ charactersTokenized: LINE.length + 3,
+ textNodesScanned: 1,
+ blockTextReads: 0,
+ };
+
+ it('costs the same in 200 and 2,000 lines when each line is a text node', async () => {
+ const small = await typingWork(200, 'text node per line');
+ const large = await typingWork(2000, 'text node per line');
+ expect(large).toEqual(small);
+ // The typed line's four tokens: `const`, `=`, `1`, `;`.
+ expect(large).toEqual({...ONE_LINE, rangesAdded: 4});
+ }, 60000);
+
+ it('tokenizes and reads the same in 200 and 2,000 lines imported as one text node', async () => {
+ const small = await typingWork(200, 'one text node');
+ const large = await typingWork(2000, 'one text node');
+ // Lexical rewrites that one text node whole, so every range in it is
+ // replaced; nothing is tokenized or read again.
+ expect(small).toEqual({...ONE_LINE, rangesAdded: 200 * 4});
+ expect(large).toEqual({...ONE_LINE, rangesAdded: 2000 * 4});
+ }, 60000);
+});
diff --git a/packages/richtext/src/CodeSyntaxPlugin.tsx b/packages/richtext/src/CodeSyntaxPlugin.tsx
index 14fa7a3b8c..b7d6609990 100644
--- a/packages/richtext/src/CodeSyntaxPlugin.tsx
+++ b/packages/richtext/src/CodeSyntaxPlugin.tsx
@@ -5,34 +5,43 @@
/**
* @file CodeSyntaxPlugin.tsx
* @input Uses @lexical/react (composer context), @lexical/code (CodeNode),
- * lexical, and core CodeBlock's tokenizer, highlight ranges, and token
- * types.
+ * lexical, core CodeBlock's highlight styles and token types, and
+ * CodeLineTokens (codeLineTokens.ts), core CodeBlock's tokens kept line by
+ * line.
* @output Exports CodeSyntaxPlugin, which colors the code in every fenced code
- * block with core CodeBlock's tokenizer and syntax color tokens;
- * CODE_SYNTAX_CLASS, the class the theme gives code blocks for it; and
- * renderedTokens, which lays tokens out over a block's rendered text.
+ * block with core CodeBlock's tokenizer and syntax color tokens; and
+ * CODE_SYNTAX_CLASS, the class the theme gives code blocks for it.
* @position Rendered by RichTextEditor and RichTextView. A fenced code block
* with a language core CodeBlock knows is colored as core Markdown colors
* it: the same tokenizer finds the same tokens, and each token type takes
* its `--color-syntax-*` token (spec:AST-061 FR1) — no palette of its own.
* The colors are CSS Custom Highlight ranges over the text Lexical renders,
* the way core CodeBlock colors editable code, so the document and its DOM
- * are untouched; the ranges are rebuilt when a block's code or language
- * changes. Browsers without the Highlight API show plain code.
+ * are untouched. A block is colored in full when it appears, or when its
+ * language, its element, or its set of text nodes changes. An edit that
+ * only rewrites one text node's text re-tokenizes just the lines
+ * CodeLineTokens says it can change, and replaces only the ranges in that
+ * text node, reading no other. Browsers without the Highlight API show
+ * plain code.
*/
import {useEffect, useState} from 'react';
import {useLexicalComposerContext} from '@lexical/react/LexicalComposerContext';
import {$isCodeNode, CodeNode} from '@lexical/code';
-import {$getNodeByKey, type NodeKey} from 'lexical';
import {
- applyHighlightRangesFlat,
+ $getNodeByKey,
+ $isTextNode,
+ type LexicalEditor,
+ type NodeKey,
+ type NodeMutation,
+} from 'lexical';
+import {
+ ensureHighlightStyles,
TOKEN_TYPES,
- tokenize,
type CodeLanguage,
- type TokenLine,
} from '@astryxdesign/core/CodeBlock';
import {listedLanguage, sameLanguages} from './codeLanguages';
+import {CodeLineTokens} from './codeLineTokens';
/** The class the theme adds to every code block (editorTheme.ts). */
export const CODE_SYNTAX_CLASS = 'astryx-richtext-code';
@@ -65,44 +74,166 @@ function supportsHighlights(): boolean {
);
}
+/** A colored token: its range, and the highlight that holds it. */
+interface Painted {
+ readonly range: Range;
+ readonly highlight: Highlight;
+}
+
/**
- * The tokens of `code`, laid out for the rendered text of its block: line by
- * line when the text keeps the line breaks, or as one line of offsets when
- * each line break is an element with no text. Null when the rendered text is
- * not the code (it is mid-update), so nothing is colored out of place.
+ * A colored code block, with what an edit needs to recolor part of it.
+ * Offsets in the rendered text are offsets in the block's DOM text nodes
+ * laid end to end, which is the code itself when its line breaks render as
+ * text, and the code without them when each renders as an element (`flat`).
*/
-export function renderedTokens(
- code: string,
- language: string | CodeLanguage,
- rendered: string,
-): TokenLine[] | null {
- const lines = tokenize(code, language);
- if (rendered === code) {
- return lines;
+interface Colored {
+ readonly element: HTMLElement;
+ readonly language: string;
+ readonly lines: CodeLineTokens;
+ readonly flat: boolean;
+ /** The block's DOM text nodes in order. */
+ readonly texts: Text[];
+ /** Where each DOM text node starts in the rendered text. */
+ readonly starts: number[];
+ /** Where each leaf's DOM text node starts in the code (NaN for others). */
+ readonly codeStarts: number[];
+ /** The rendered text's length. */
+ length: number;
+ /** The Lexical text nodes that render as one DOM text node each, by key:
+ * that node's index in `texts`, and the text last colored. */
+ readonly leaves: Map<NodeKey, {readonly index: number; text: string}>;
+ /** Each line's colored tokens. */
+ ranges: Painted[][];
+}
+
+function highlightFor(type: string): Highlight {
+ const name = `astryx-${type}`;
+ let highlight = CSS.highlights.get(name);
+ if (highlight == null) {
+ highlight = new Highlight();
+ CSS.highlights.set(name, highlight);
}
- if (rendered !== code.replaceAll('\n', '')) {
+ return highlight;
+}
+
+/**
+ * The DOM position of `offset` in a block's rendered text: in the text node
+ * that holds the character there when `inside`, else in the first text node
+ * that reaches it. Null past the end of the text.
+ */
+function pointAt(
+ block: Colored,
+ offset: number,
+ inside: boolean,
+): [Text, number] | null {
+ const {texts, starts, length} = block;
+ if (texts.length === 0 || offset > length) {
return null;
}
- const flat: TokenLine = [];
- let offset = 0;
- code.split('\n').forEach((line, index) => {
- for (const token of lines[index] ?? []) {
- flat.push({
- type: token.type,
- start: offset + token.start,
- end: offset + token.end,
- });
+ let low = 0;
+ let high = texts.length - 1;
+ while (low < high) {
+ const mid = (low + high) >>> 1;
+ const end = mid + 1 < texts.length ? starts[mid + 1] : length;
+ if (inside ? end > offset : end >= offset) {
+ high = mid;
+ } else {
+ low = mid + 1;
}
- offset += line.length;
- });
- return [flat];
+ }
+ return [texts[low], offset - starts[low]];
}
-interface Colored {
- readonly element: HTMLElement;
- readonly code: string;
- readonly language: string;
- readonly cleanup: () => void;
+/** Colors one line's tokens. */
+function paintLine(block: Colored, line: number): void {
+ // A flat block's rendered text has no line breaks before this line.
+ const base = block.lines.starts[line] - (block.flat ? line : 0);
+ const painted: Painted[] = [];
+ for (const token of block.lines.tokens[line] ?? []) {
+ const start = base + token.start;
+ const end = base + token.end;
+ const from = pointAt(block, start, start < end);
+ const to = pointAt(block, end, false);
+ if (from == null || to == null) {
+ continue;
+ }
+ const highlight = highlightFor(token.type);
+ try {
+ const range = new Range();
+ range.setStart(...from);
+ range.setEnd(...to);
+ highlight.add(range);
+ painted.push({range, highlight});
+ } catch {
+ // A range the DOM refuses colors nothing.
+ }
+ }
+ block.ranges[line] = painted;
+}
+
+function unpaintLine(block: Colored, line: number): void {
+ for (const {range, highlight} of block.ranges[line] ?? []) {
+ highlight.delete(range);
+ }
+ block.ranges[line] = [];
+}
+
+function unpaintAll(block: Colored): void {
+ for (let line = 0; line < block.ranges.length; line++) {
+ unpaintLine(block, line);
+ }
+ block.ranges = [];
+}
+
+function repaintAll(block: Colored): void {
+ unpaintAll(block);
+ for (let line = 0; line < block.lines.tokens.length; line++) {
+ paintLine(block, line);
+ }
+}
+
+/**
+ * Lists a block's DOM text nodes in order with their offsets, and the
+ * Lexical text nodes that render as exactly one of them.
+ */
+function $indexText(
+ editor: LexicalEditor,
+ node: CodeNode,
+ block: Colored,
+): void {
+ const walker = document.createTreeWalker(block.element, NodeFilter.SHOW_TEXT);
+ while (walker.nextNode() != null) {
+ const text = walker.currentNode as Text;
+ block.texts.push(text);
+ block.starts.push(block.length);
+ block.codeStarts.push(NaN);
+ block.length += text.length;
+ }
+ const indexes = new Map(block.texts.map((text, index) => [text, index]));
+ let codeStart = 0;
+ for (const child of node.getChildren()) {
+ const dom = $isTextNode(child)
+ ? editor.getElementByKey(child.getKey())
+ : null;
+ const index =
+ dom?.childNodes.length === 1
+ ? indexes.get(dom.firstChild as Text)
+ : undefined;
+ if (index != null) {
+ block.leaves.set(child.getKey(), {index, text: child.getTextContent()});
+ block.codeStarts[index] = codeStart;
+ }
+ codeStart += child.getTextContentSize();
+ }
+}
+
+/** The key of the code block that holds the node `key`, if any. */
+function $codeBlockOf(key: NodeKey): NodeKey | null {
+ let node = $getNodeByKey(key)?.getParent() ?? null;
+ while (node != null && !$isCodeNode(node)) {
+ node = node.getParent();
+ }
+ return node?.getKey() ?? null;
}
export function CodeSyntaxPlugin({
@@ -121,81 +252,233 @@ export function CodeSyntaxPlugin({
}
// Synchronizes the editor's code blocks with the CSS highlight registry:
- // Lexical update and mutation listeners colour each block, and the cleanup
+ // Lexical mutation and update listeners colour each block, and the cleanup
// removes the listeners and every highlight range they added.
useEffect(() => {
if (!supportsHighlights()) {
return undefined;
}
ensureRules();
- // Every code block in the editor, and the ones colored, by node key.
+ // Every code block in the editor, the ones colored, and the ones colored
+ // in full during the current update.
const blocks = new Set<NodeKey>();
const colored = new Map<NodeKey, Colored>();
+ const repainted = new Set<NodeKey>();
const uncolor = (key: NodeKey) => {
- colored.get(key)?.cleanup();
- colored.delete(key);
+ const block = colored.get(key);
+ if (block != null) {
+ unpaintAll(block);
+ colored.delete(key);
+ }
};
- const color = (key: NodeKey) => {
- const block = editor.getEditorState().read(() => {
- const node = $getNodeByKey(key);
- return $isCodeNode(node)
- ? {code: node.getTextContent(), language: node.getLanguage() ?? ''}
- : null;
- });
+ // Colors a block in full, unless `force` is false and its element,
+ // language, and code are as last colored.
+ const $color = (key: NodeKey, force: boolean) => {
+ const node = $getNodeByKey(key);
const element = editor.getElementByKey(key);
- if (block == null || element == null) {
+ if (!$isCodeNode(node) || element == null) {
uncolor(key);
return;
}
+ const code = node.getTextContent();
+ const language = node.getLanguage() ?? '';
const previous = colored.get(key);
if (
+ !force &&
previous != null &&
previous.element === element &&
- previous.code === block.code &&
- previous.language === block.language
+ previous.language === language &&
+ previous.lines.text === code
) {
return;
}
- previous?.cleanup();
- const tokens = renderedTokens(
- block.code,
- listedLanguage(languages, block.language) ?? block.language,
- element.textContent ?? '',
- );
- if (tokens == null) {
- // Not rendered yet; the next update tries again.
- colored.delete(key);
+ uncolor(key);
+ const rendered = element.textContent ?? '';
+ if (rendered !== code && rendered !== code.replaceAll('\n', '')) {
+ // Not rendered yet; the block's next change tries again.
return;
}
- colored.set(key, {
+ ensureHighlightStyles();
+ const block: Colored = {
element,
- ...block,
- cleanup: applyHighlightRangesFlat(element, tokens),
- });
+ language,
+ lines: new CodeLineTokens(
+ code,
+ listedLanguage(languages, language) ?? language,
+ ),
+ flat: rendered !== code,
+ texts: [],
+ starts: [],
+ codeStarts: [],
+ length: 0,
+ leaves: new Map(),
+ ranges: [],
+ };
+ $indexText(editor, node, block);
+ repaintAll(block);
+ colored.set(key, block);
+ repainted.add(key);
+ };
+ // Recolors a block after an update that rewrote the text of its leaf
+ // `leafKey` and changed nothing else in it. False when the block must
+ // be colored in full instead.
+ const $recolorLeaf = (key: NodeKey, leafKey: NodeKey): boolean => {
+ const block = colored.get(key);
+ const leaf = block?.leaves.get(leafKey);
+ const node = $getNodeByKey(leafKey);
+ const parent = node?.getParent();
+ if (
+ block == null ||
+ leaf == null ||
+ !$isTextNode(node) ||
+ !$isCodeNode(parent) ||
+ parent.getKey() !== key ||
+ (parent.getLanguage() ?? '') !== block.language ||
+ editor.getElementByKey(key) !== block.element
+ ) {
+ return false;
+ }
+ const text = block.texts[leaf.index];
+ const dom = editor.getElementByKey(leafKey);
+ const after = node.getTextContent();
+ if (
+ dom?.childNodes.length !== 1 ||
+ dom.firstChild !== text ||
+ text.data !== after
+ ) {
+ return false;
+ }
+ const before = leaf.text;
+ if (after === before) {
+ return true;
+ }
+ let first = 0;
+ while (
+ first < before.length &&
+ first < after.length &&
+ before[first] === after[first]
+ ) {
+ first++;
+ }
+ let beforeEnd = before.length;
+ let afterEnd = after.length;
+ while (
+ beforeEnd > first &&
+ afterEnd > first &&
+ before[beforeEnd - 1] === after[afterEnd - 1]
+ ) {
+ beforeEnd--;
+ afterEnd--;
+ }
+ const inserted = after.slice(first, afterEnd);
+ // A flat block's text holds no line breaks; one typed in would no
+ // longer lay the code over its rendered text.
+ if (block.flat && inserted.includes('\n')) {
+ return false;
+ }
+ const leafStart = block.codeStarts[leaf.index];
+ const lines = block.lines.replace(
+ leafStart + first,
+ beforeEnd - first,
+ inserted,
+ );
+ const delta = after.length - before.length;
+ for (let i = leaf.index + 1; i < block.texts.length; i++) {
+ block.starts[i] += delta;
+ block.codeStarts[i] += delta;
+ }
+ block.length += delta;
+ leaf.text = after;
+ if (lines == null) {
+ repaintAll(block);
+ return true;
+ }
+ // Lexical rewrites a changed DOM text node whole, which leaves every
+ // range in it stale: recolor each line the node holds, and any other
+ // line the edit re-tokenized. Every other range sits in a text node of
+ // its own line and is still right.
+ const recolor = new Set(lines);
+ const last = block.lines.lineAt(
+ leafStart + Math.max(after.length - 1, 0),
+ );
+ for (let i = block.lines.lineAt(leafStart); i <= last; i++) {
+ recolor.add(i);
+ }
+ for (const i of recolor) {
+ unpaintLine(block, i);
+ paintLine(block, i);
+ }
+ return true;
};
const unregisterMutations = editor.registerMutationListener(
CodeNode,
mutations => {
- for (const [key, mutation] of mutations) {
- if (mutation === 'destroyed') {
- blocks.delete(key);
- uncolor(key);
- } else {
- blocks.add(key);
- color(key);
+ editor.getEditorState().read(() => {
+ for (const [key, mutation] of mutations) {
+ if (mutation === 'destroyed') {
+ blocks.delete(key);
+ uncolor(key);
+ } else {
+ blocks.add(key);
+ $color(key, false);
+ }
}
- }
+ });
},
{skipInitialization: false},
);
- // Typing in a block changes its text nodes, not always the block itself:
- // recheck every block after each update (unchanged ones stop at the
- // comparison above).
- const unregisterUpdates = editor.registerUpdateListener(() => {
- for (const key of blocks) {
- color(key);
- }
- });
+ repainted.clear();
+ // Typing in a block changes its text nodes, not the block itself: after
+ // each update, recolor the blocks whose text nodes changed. A block whose
+ // only change is one leaf's text takes the partial path; any other
+ // change colors it in full, unless the block listener just did.
+ const unregisterUpdates = editor.registerUpdateListener(
+ ({editorState, mutatedNodes, prevEditorState}) => {
+ if (mutatedNodes != null && blocks.size > 0) {
+ const mutations: Array<[NodeKey, NodeMutation]> = [];
+ for (const nodes of mutatedNodes.values()) {
+ for (const mutation of nodes) {
+ mutations.push(mutation);
+ }
+ }
+ const before = prevEditorState.read(() =>
+ mutations.map(([key, mutation]) =>
+ mutation === 'created' ? null : $codeBlockOf(key),
+ ),
+ );
+ editorState.read(() => {
+ // Each changed block, with the one leaf whose text an update
+ // rewrote, or null when anything else in it changed.
+ const changed = new Map<NodeKey, NodeKey | null>();
+ const note = (key: NodeKey | null, leafKey: NodeKey | null) => {
+ if (key != null && blocks.has(key)) {
+ changed.set(key, changed.has(key) ? null : leafKey);
+ }
+ };
+ mutations.forEach(([key, mutation], i) => {
+ const after = mutation === 'destroyed' ? null : $codeBlockOf(key);
+ const rewritten =
+ mutation === 'updated' &&
+ after === before[i] &&
+ $isTextNode($getNodeByKey(key));
+ note(after, rewritten ? key : null);
+ if (before[i] !== after) {
+ note(before[i], null);
+ }
+ });
+ for (const [key, leafKey] of changed) {
+ if (
+ !repainted.has(key) &&
+ (leafKey == null || !$recolorLeaf(key, leafKey))
+ ) {
+ $color(key, true);
+ }
+ }
+ });
+ }
+ repainted.clear();
+ },
+ );
return () => {
unregisterMutations();
unregisterUpdates();
diff --git a/packages/richtext/src/codeLineTokens.test.ts b/packages/richtext/src/codeLineTokens.test.ts
new file mode 100644
index 0000000000..897e416b59
--- /dev/null
+++ b/packages/richtext/src/codeLineTokens.test.ts
@@ -0,0 +1,234 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+import {describe, it, expect, vi, beforeEach} from 'vitest';
+import type * as codeBlock from '@astryxdesign/core/CodeBlock';
+import {tokenize} from '@astryxdesign/core/CodeBlock';
+import {CodeLineTokens} from './codeLineTokens';
+
+vi.mock('@astryxdesign/core/CodeBlock', async importOriginal => {
+ const actual = await importOriginal<typeof codeBlock>();
+ return {...actual, tokenize: vi.fn(actual.tokenize)};
+});
+
+const tokenizer = vi.mocked(tokenize);
+
+beforeEach(() => {
+ tokenizer.mockClear();
+});
+
+/** What the cache must always hold: a full `tokenize` of its code. */
+function expectWhole(lines: CodeLineTokens, language: string) {
+ const text = lines.text;
+ let start = 0;
+ expect({tokens: lines.tokens, starts: lines.starts}).toEqual({
+ tokens: tokenize(text, language),
+ starts: text.split('\n').map(line => {
+ const lineStart = start;
+ start += line.length + 1;
+ return lineStart;
+ }),
+ });
+}
+
+/** The characters the tokenizer read since the last check. */
+function charactersTokenized(): number {
+ const characters = tokenizer.mock.calls.reduce(
+ (sum, [text]) => sum + text.length,
+ 0,
+ );
+ tokenizer.mockClear();
+ return characters;
+}
+
+const DELIMITERS = [
+ '"',
+ "'",
+ '`',
+ '"""',
+ '/*',
+ '*/',
+ '//',
+ '#',
+ '${',
+ '}',
+ '<<-',
+ '\r\n',
+ '\\',
+ '\n',
+ '(',
+ ')',
+ ' ',
+ 'x',
+];
+
+describe('CodeLineTokens', () => {
+ // An edit can recolor lines above it as well as below: removing a closing
+ // delimiter reopens a string or comment that started earlier.
+ it.each([
+ [
+ 'typescript',
+ 'function run() {\n const a = call(1, "row"); // line\n return a;\n}',
+ ],
+ ['ts', 'const a = call(1); // line\nresult'],
+ ['ts', '/* first\n\nlast */\nconst a = `one\ntwo`;\ncall\n (a);'],
+ ['js', 'const a = 1;\n+// "open\nclose";\nconst b = 2;'],
+ ['ts', 'const a = 1; /* note\nstill note */\nconst b = 2;'],
+ ['tsx', 'const view = list\n\n .map(item => item);\nrender\n\n(view)'],
+ ['hack', '<?hh\n$x->call();\n/* first\nlast */'],
+ ['python', 'def run():\n x = """first\nlast"""\n return x'],
+ ['python', 'def run():\n x = call(1, "row") # note\n return 1\nresult'],
+ ['py', 'x = 1\npass\n@decorator\ncall\n (1)'],
+ ['python', 'x = "a\nb"\nreturn 1'],
+ ['bash', 'echo "row"; # note\nprintf 1\nresult'],
+ ['sh', 'echo ${value\n}\necho "row";'],
+ ['shell', 'cat <<EOF\necho "body"\nEOF\necho "after";'],
+ ['zsh', 'echo $(printf "row")\necho `printf row`;'],
+ ['markdown', '# Heading\n```ts\nconst a = 1;\n```\n**bold**'],
+ ['yaml', 'name: first\nvalue: |\n next line\nend: true'],
+ ['json', '{\n "name"\n : "first",\n "value": 1\n}'],
+ ])(
+ 'matches a whole %s tokenize through delimiters typed and deleted anywhere',
+ (language, text) => {
+ const lines = new CodeLineTokens(text, language);
+ expectWhole(lines, language);
+ for (let i = 0; i <= text.length; i++) {
+ for (const inserted of DELIMITERS) {
+ lines.replace(i, 0, inserted);
+ expectWhole(lines, language);
+ lines.replace(i, inserted.length, '');
+ expect(lines.text).toBe(text);
+ expectWhole(lines, language);
+ }
+ if (i < text.length) {
+ lines.replace(i, 1, '');
+ expectWhole(lines, language);
+ lines.replace(i, 0, text[i]);
+ expectWhole(lines, language);
+ }
+ }
+ },
+ );
+
+ it.each(['ts', 'python', 'bash'])(
+ 'matches a whole %s tokenize through edits at the end of the code',
+ language => {
+ const lines = new CodeLineTokens('call(1);\nresult', language);
+ for (const suffix of [
+ 'Value',
+ '(',
+ ' ',
+ '\n(1);',
+ '\n\n (1);',
+ '\n/* next\nline */',
+ ]) {
+ const at = lines.text.length;
+ lines.replace(at, 0, suffix);
+ expectWhole(lines, language);
+ lines.replace(at, suffix.length, '');
+ expectWhole(lines, language);
+ }
+ },
+ );
+
+ it.each(['', 'r', 'b', 'u', 'f', 'fr', 'rf'])(
+ 'keeps Python triple quotes first behind a "%s" prefix',
+ prefix => {
+ for (const spelling of new Set([prefix, prefix.toUpperCase()])) {
+ for (const quote of ['"', "'"]) {
+ const text = `x = ${spelling}${quote}${quote}\ny = ${quote.repeat(3)}ok${quote.repeat(3)}`;
+ const lines = new CodeLineTokens(text, 'python');
+ const at = text.indexOf(quote);
+ for (const inserted of [quote, quote.repeat(2), quote.repeat(3)]) {
+ lines.replace(at, 0, inserted);
+ expectWhole(lines, 'python');
+ lines.replace(at, inserted.length, '');
+ expectWhole(lines, 'python');
+ }
+ }
+ }
+ },
+ );
+
+ it('recolors a name ending an earlier line when the next visible character becomes "("', () => {
+ const text = 'const a = call\n\n \n value;\nnext();';
+ const lines = new CodeLineTokens(text, 'ts');
+ const at = text.indexOf('value');
+ const types = () => lines.tokens[0].map(token => token.type);
+ expect(types()).not.toContain('function');
+ charactersTokenized();
+ expect(lines.replace(at, 0, '(')).toEqual([0, 3]);
+ expectWhole(lines, 'ts');
+ expect(types()).toContain('function');
+ expect(lines.replace(at, 1, '')).toEqual([0, 3]);
+ expectWhole(lines, 'ts');
+ expect(types()).not.toContain('function');
+ // Typing elsewhere on the line leaves the name above alone.
+ expect(lines.replace(at + 2, 0, 'x')).toEqual([3]);
+ });
+
+ it.each([
+ ['python', 'value = """closed"""'],
+ ['bash', 'echo ${value};'],
+ ['sh', 'cat <<-EOF'],
+ ['shell', 'echo `date`;'],
+ ['zsh', 'echo $(date);'],
+ ['ts', 'const a = /* open'],
+ ['notalanguage', 'value'],
+ ])(
+ 'tokenizes all of a %s block holding `%s`, whose tokens may reach other lines',
+ (language, unsafe) => {
+ const ordinary =
+ language === 'python' ? 'value = call(1)' : 'echo "row";';
+ const text = `${unsafe}\n${ordinary}\nresult`;
+ const lines = new CodeLineTokens(text, language);
+ charactersTokenized();
+ expect(lines.replace(text.indexOf(ordinary), 0, 'x')).toBeNull();
+ expect(charactersTokenized()).toBe(text.length + 1);
+ expectWhole(lines, language);
+ },
+ );
+
+ it.each(
+ [200, 2000].flatMap(count =>
+ [
+ ['ts', 'const value = call(1, "row"); // note'],
+ ['python', 'value = call(1) # note'],
+ ['bash', 'echo "row" $value # note'],
+ ].map(([language, line]) => ({count, language, line})),
+ ),
+ )(
+ 'tokenizes only the edited $language line in a $count-line block',
+ ({count, language, line}) => {
+ const text = Array.from({length: count}, () => line).join('\n');
+ const lines = new CodeLineTokens(text, language);
+ const middle = lines.starts[count / 2];
+ charactersTokenized();
+ for (const at of [middle, middle + line.length]) {
+ expect(lines.replace(at, 0, 'x')).toEqual([count / 2]);
+ expect(charactersTokenized()).toBeLessThan(2 * line.length);
+ expectWhole(lines, language);
+ lines.replace(at, 1, '');
+ expectWhole(lines, language);
+ charactersTokenized();
+ }
+ },
+ );
+
+ it('keeps offsets exact through edits in a 2,000-line block', () => {
+ const text = Array.from(
+ {length: 2000},
+ (_, i) => `const value_${i} = call(${i}, "row"); // line ${i}`,
+ ).join('\n');
+ const lines = new CodeLineTokens(text, 'ts');
+ const offset = text.indexOf('value_1000');
+ for (const inserted of ['x', '']) {
+ lines.replace(offset, inserted ? 0 : 1, inserted);
+ expectWhole(lines, 'ts');
+ }
+ lines.replace(offset, 0, '"');
+ expectWhole(lines, 'ts');
+ lines.replace(offset, 1, '');
+ expectWhole(lines, 'ts');
+ expect(lines.text).toBe(text);
+ });
+});
diff --git a/packages/richtext/src/codeLineTokens.ts b/packages/richtext/src/codeLineTokens.ts
new file mode 100644
index 0000000000..f3ba853c9f
--- /dev/null
+++ b/packages/richtext/src/codeLineTokens.ts
@@ -0,0 +1,241 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file codeLineTokens.ts
+ * @input Core CodeBlock's tokenizer (`tokenize`) and token line type.
+ * @output Exports CodeLineTokens, which keeps core CodeBlock's tokens for a
+ * block of code line by line and re-tokenizes only the lines an edit can
+ * change.
+ * @position Used by CodeSyntaxPlugin. Its tokens always equal what `tokenize`
+ * returns for the whole code. An edit re-tokenizes only its own line, and
+ * at most the line above it, for the built-in TypeScript, JavaScript,
+ * Python, and shell grammars while every line passes `standsAlone`;
+ * anything else re-tokenizes all of it. The checks follow those grammars'
+ * patterns in core CodeBlock's tokenizer.ts, and codeLineTokens.test.ts
+ * compares edits against a full `tokenize`, so a grammar change that
+ * breaks them fails there.
+ */
+
+import {
+ tokenize,
+ type CodeLanguage,
+ type TokenLine,
+} from '@astryxdesign/core/CodeBlock';
+
+type Family = 'javascript' | 'python' | 'shell';
+
+const FAMILIES: ReadonlyMap<string, Family> = new Map([
+ ['typescript', 'javascript'],
+ ['javascript', 'javascript'],
+ ['tsx', 'javascript'],
+ ['jsx', 'javascript'],
+ ['ts', 'javascript'],
+ ['js', 'javascript'],
+ ['python', 'python'],
+ ['py', 'python'],
+ ['bash', 'shell'],
+ ['sh', 'shell'],
+ ['zsh', 'shell'],
+ ['shell', 'shell'],
+]);
+
+/** What can open a string or comment that runs on past its line. */
+const OPENERS: Readonly<Record<Family, RegExp>> = {
+ javascript: /['"`]|\/\*/,
+ python: /['"]/,
+ shell: /['"`]|\$\{|\$\(|<</,
+};
+
+/** A line whose last visible character can end a function name. */
+const NAME_END = /[\w$]\s*$/;
+
+/**
+ * Whether the name that ends a line can change color with the code after
+ * it: its last visible character is a name character outside every string,
+ * comment, number, keyword, and constant, whose patterns come before
+ * function names.
+ */
+function endsInOpenName(text: string, tokens: TokenLine): boolean {
+ const last = text.trimEnd().length - 1;
+ return (
+ NAME_END.test(text) &&
+ !tokens.some(
+ token =>
+ token.start <= last &&
+ token.end > last &&
+ token.type !== 'function' &&
+ token.type !== 'type',
+ )
+ );
+}
+
+/** A line's first visible character, or '' when it has none. */
+function firstVisible(line: string): string {
+ return /\S/.exec(line)?.[0] ?? '';
+}
+
+/**
+ * Whether a line's strings and comments all close on it. Their patterns
+ * match line breaks, so an opener left open can run on to a later line.
+ * Only a string or comment the tokenizer emitted shields an opener: one that
+ * another token swallowed, such as `/*` in the operator `+/*`, still counts
+ * as open. Python tries triple quotes first, so `"""` can pair with another
+ * line even where its quotes close as single strings on this one.
+ */
+function standsAlone(text: string, tokens: TokenLine, family: Family): boolean {
+ if (family === 'python' && /'''|"""/.test(text)) {
+ return false;
+ }
+ const opener = OPENERS[family];
+ let position = 0;
+ for (const token of tokens) {
+ if (token.end > text.length) {
+ return false;
+ }
+ if (token.type === 'string' || token.type === 'comment') {
+ if (opener.test(text.slice(position, token.start))) {
+ return false;
+ }
+ position = token.end;
+ }
+ }
+ return !opener.test(text.slice(position));
+}
+
+/**
+ * Core CodeBlock's tokens for a block of code, kept line by line with
+ * line-relative offsets, exactly as `tokenize` returns them for the whole
+ * code.
+ */
+export class CodeLineTokens {
+ readonly language: string | CodeLanguage;
+ /** Where each line starts in the code. */
+ starts: number[] = [];
+ /** Each line's tokens. */
+ tokens: TokenLine[] = [];
+ private lines: string[] = [];
+ private readonly family: Family | null;
+ /** Whether every line stands alone, so an edit re-tokenizes lines alone. */
+ private linesStandAlone = false;
+
+ constructor(text: string, language: string | CodeLanguage) {
+ this.language = language;
+ this.family =
+ typeof language === 'string' ? (FAMILIES.get(language) ?? null) : null;
+ this.tokenizeAll(text);
+ }
+
+ /** The code. */
+ get text(): string {
+ return this.lines.join('\n');
+ }
+
+ /** The line that holds `offset` in the code. */
+ lineAt(offset: number): number {
+ let low = 0;
+ let high = this.starts.length - 1;
+ while (low < high) {
+ const mid = (low + high + 1) >>> 1;
+ if (this.starts[mid] <= offset) {
+ low = mid;
+ } else {
+ high = mid - 1;
+ }
+ }
+ return low;
+ }
+
+ /**
+ * Replaces `removed` characters at `offset` in the code with `inserted`.
+ * Returns the lines it re-tokenized when the edit stays inside one line
+ * and every line still stands alone; otherwise it re-tokenizes all the
+ * code and returns null.
+ */
+ replace(offset: number, removed: number, inserted: string): number[] | null {
+ const line = this.lineAt(offset);
+ const before = this.lines[line];
+ const column = offset - this.starts[line];
+ if (
+ this.family == null ||
+ !this.linesStandAlone ||
+ column + removed > before.length ||
+ inserted.includes('\n')
+ ) {
+ const text = this.text;
+ this.tokenizeAll(
+ text.slice(0, offset) + inserted + text.slice(offset + removed),
+ );
+ return null;
+ }
+ const after =
+ before.slice(0, column) + inserted + before.slice(column + removed);
+ this.lines[line] = after;
+ const tokens = this.tokenizeLine(line);
+ if (!standsAlone(after, tokens, this.family)) {
+ this.tokenizeAll(this.text);
+ return null;
+ }
+ this.tokens[line] = tokens;
+ for (let i = line + 1; i < this.starts.length; i++) {
+ this.starts[i] += inserted.length - removed;
+ }
+ const changed = [line];
+ // A name that ends an earlier line looks past blank lines for `(`, so a
+ // new first visible character here can change that name's color.
+ if (firstVisible(before) !== firstVisible(after)) {
+ let above = line - 1;
+ while (above >= 0 && firstVisible(this.lines[above]) === '') {
+ above--;
+ }
+ if (
+ above >= 0 &&
+ endsInOpenName(this.lines[above], this.tokens[above] ?? [])
+ ) {
+ this.tokens[above] = this.tokenizeLine(above);
+ changed.unshift(above);
+ }
+ }
+ return changed;
+ }
+
+ /**
+ * A line's tokens as `tokenize` finds them in the whole code. A function
+ * name is a name followed by `(` past any whitespace, line breaks
+ * included, so when the line ends in a name, the code up to the next
+ * visible character is tokenized with it.
+ */
+ private tokenizeLine(line: number): TokenLine {
+ let text = this.lines[line];
+ if (NAME_END.test(text)) {
+ for (let next = line + 1; next < this.lines.length; next++) {
+ const visible = /\S/.exec(this.lines[next]);
+ text +=
+ '\n' +
+ (visible == null
+ ? this.lines[next]
+ : this.lines[next].slice(0, visible.index + 1));
+ if (visible != null) {
+ break;
+ }
+ }
+ }
+ return tokenize(text, this.language)[0] ?? [];
+ }
+
+ private tokenizeAll(text: string): void {
+ this.lines = text.split('\n');
+ let start = 0;
+ this.starts = this.lines.map(line => {
+ const lineStart = start;
+ start += line.length + 1;
+ return lineStart;
+ });
+ this.tokens = tokenize(text, this.language);
+ const family = this.family;
+ this.linesStandAlone =
+ family != null &&
+ this.lines.every((line, i) =>
+ standsAlone(line, this.tokens[i] ?? [], family),
+ );
+ }
+}