An open source design system that's fully customizable and agent ready
TypeScript
perf(richtext): code highlighting re-tokenizes only the lines an edit changes (#8585)
Ernest540e9dfdfa4e
diff --git a/.changeset/typing-in-a-richtexteditor-code-block-no-longer--4xw9.md b/.changeset/typing-in-a-richtexteditor-code-block-no-longer--4xw9.mdnew file mode 100644index 0000000000..de4bb21dcc--- /dev/null+++ b/.changeset/typing-in-a-richtexteditor-code-block-no-longer--4xw9.md@@ -0,0 +1,6 @@+---+'@astryxdesign/richtext': patch+---++[perf] Typing in a `RichTextEditor` code block no longer re-tokenizes the whole block. In TypeScript, JavaScript, Python, and shell code, an edit inside one line re-tokenizes that line, plus at most the line above it, and replaces only the color ranges inside the text node it edited, so a keystroke in a 2,000-line block tokenizes as little code as one in a short block. Edits that open or close a string or comment across lines, and other languages, still re-tokenize the whole block. Colors are unchanged, except that a block whose text nodes are replaced by others with the same text is now recolored instead of left uncolored.+@emilyhorsman @ernesttdiff --git a/packages/richtext/src/CodeSyntaxPlugin.test.tsx b/packages/richtext/src/CodeSyntaxPlugin.test.tsxindex 69569d5b52..acee23ae4d 100644--- a/packages/richtext/src/CodeSyntaxPlugin.test.tsx+++ b/packages/richtext/src/CodeSyntaxPlugin.test.tsx@@ -1,29 +1,292 @@// Copyright (c) Meta Platforms, Inc. and affiliates.-import {describe, it, expect} from 'vitest';-import {render, screen} from '@testing-library/react';-import {tokenize} from '@astryxdesign/core/CodeBlock';+import {describe, it, expect, vi, afterEach, beforeAll} from 'vitest';+import {act, cleanup, render, screen, waitFor} from '@testing-library/react';+import {useEffect} from 'react';+import {useLexicalComposerContext} from '@lexical/react/LexicalComposerContext';+import {$isCodeNode} from '@lexical/code';+import {+ $createLineBreakNode,+ $createRangeSelection,+ $createTextNode,+ $getRoot,+ $getSelection,+ $isRangeSelection,+ $isTextNode,+ $setSelection,+ ElementNode,+ type LexicalEditor,+} from 'lexical';+import type * as codeBlock from '@astryxdesign/core/CodeBlock';+import {+ applyHighlightRangesFlat,+ tokenize,+ type CodeLanguage,+ type TokenLine,+} from '@astryxdesign/core/CodeBlock';import {thrift} from '@astryxdesign/core/CodeBlock/languages/thrift';import {listedLanguage, sameLanguages} from './codeLanguages';-import {renderedTokens} from './CodeSyntaxPlugin';+import {RichTextEditor} from './RichTextEditor';import {RichTextView} from './RichTextView';import {markdownToEditorStateJSON} from './markdownSerializers';+vi.mock('@astryxdesign/core/CodeBlock', async importOriginal => {+ const actual = await importOriginal<typeof codeBlock>();+ return {...actual, tokenize: vi.fn(actual.tokenize)};+});++let rangesAdded = 0;++/** jsdom has no CSS Custom Highlight API: named sets of ranges stand in. */+class TestHighlight extends Set<AbstractRange> {+ override add(range: AbstractRange): this {+ rangesAdded++;+ return super.add(range);+ }+}++function stubHighlights(): Map<string, TestHighlight> {+ const registry = new Map<string, TestHighlight>();+ vi.stubGlobal('CSS', {...globalThis.CSS, highlights: registry});+ vi.stubGlobal('Highlight', TestHighlight);+ return registry;+}++afterEach(() => {+ cleanup();+ vi.unstubAllGlobals();+});++// jsdom's Range has no getBoundingClientRect, which Lexical calls to scroll+// a reconciled selection into view while the editor has focus.+beforeAll(() => {+ if (typeof Range.prototype.getBoundingClientRect !== 'function') {+ Range.prototype.getBoundingClientRect = () => new DOMRect();+ }+});++function CaptureEditor({onReady}: {onReady: (editor: LexicalEditor) => void}) {+ const [editor] = useLexicalComposerContext();+ useEffect(() => {+ onReady(editor);+ }, [editor, onReady]);+ return null;+}++async function mountEditor(+ markdown: string,+ codeLanguages?: ReadonlyArray<CodeLanguage>,+): Promise<LexicalEditor> {+ let editor: LexicalEditor | undefined;+ render(+ <RichTextEditor+ label="Code"+ codeLanguages={codeLanguages}+ defaultValue={markdownToEditorStateJSON(markdown)}+ plugins={<CaptureEditor onReady={live => (editor = live)} />}+ />,+ );+ await waitFor(() => expect(editor).toBeDefined());+ return editor!;+}++function update(editor: LexicalEditor, fn: () => void) {+ act(() => editor.update(fn, {discrete: true}));+}++function $block() {+ const block = $getRoot().getFirstChild();+ if (!$isCodeNode(block)) {+ throw new Error('missing code block');+ }+ return block;+}++/** Rewrites the code block the way typing and pasting build one: a text node+ * per line, with a line break node between lines. */+function $writeLines(lines: ReadonlyArray<string>) {+ const block = $block();+ block.clear();+ lines.forEach((line, i) => {+ if (i > 0) {+ block.append($createLineBreakNode());+ }+ if (line !== '') {+ block.append($createTextNode(line));+ }+ });+}++/** The text node and offset at `offset` in the code block's text. */+function $pointAt(offset: number): [string, number] {+ let start = 0;+ for (const child of $block().getChildren()) {+ const size = child.getTextContentSize();+ if ($isTextNode(child) && offset <= start + size) {+ return [child.getKey(), offset - start];+ }+ start += size;+ }+ throw new Error(`no text node at ${offset}`);+}++/** Selects `length` characters at `offset` in the code block's text. */+function $selectCode(offset: number, length = 0) {+ const [anchorKey, anchorOffset] = $pointAt(offset);+ const [focusKey, focusOffset] = $pointAt(offset + length);+ const selection = $createRangeSelection();+ selection.anchor.set(anchorKey, anchorOffset, 'text');+ selection.focus.set(focusKey, focusOffset, 'text');+ $setSelection(selection);+}++function $type(text: string) {+ const selection = $getSelection();+ if (!$isRangeSelection(selection)) {+ throw new Error('missing selection');+ }+ selection.insertText(text);+}++function codeOf(editor: LexicalEditor): {code: string; language: string} {+ return editor.getEditorState().read(() => ({+ code: $block().getTextContent(),+ language: $block().getLanguage() ?? '',+ }));+}++function codeElement(editor: LexicalEditor): HTMLElement {+ const key = editor.getEditorState().read(() => $block().getKey());+ return editor.getElementByKey(key)!;+}++interface Span {+ type: string;+ start: number;+ end: number;+}++/** Every range in `registry`, as a token type over offsets in the code+ * element's rendered text (-1 for a range outside it). */+function spansIn(+ registry: ReadonlyMap<string, Set<AbstractRange>>,+ element: HTMLElement,+): Span[] {+ const starts = new Map<Node, number>();+ let offset = 0;+ const walker = document.createTreeWalker(element, NodeFilter.SHOW_TEXT);+ for (let node = walker.nextNode(); node != null; node = walker.nextNode()) {+ starts.set(node, offset);+ offset += (node as Text).length;+ }+ const at = (node: Node, nodeOffset: number) =>+ starts.has(node) ? starts.get(node)! + nodeOffset : -1;+ const spans: Span[] = [];+ for (const [name, highlight] of registry) {+ for (const range of highlight) {+ spans.push({+ type: name.replace(/^astryx-/, ''),+ start: at(range.startContainer, range.startOffset),+ end: at(range.endContainer, range.endOffset),+ });+ }+ }+ return spans.sort(+ (a, b) =>+ a.start - b.start ||+ a.end - b.end ||+ (a.type < b.type ? -1 : a.type > b.type ? 1 : 0),+ );+}++/**+ * What coloring the block from scratch paints: core CodeBlock's tokens for+ * the whole code, laid over the rendered text line by line when it keeps the+ * line breaks or as one line of offsets when it drops them (nothing when it+ * is neither), and written by core CodeBlock's own range writer.+ */+function fullRepaint(+ element: HTMLElement,+ code: string,+ language: string | CodeLanguage,+): Span[] {+ const rendered = element.textContent ?? '';+ const lines = tokenize(code, language);+ let laid: TokenLine[] | null = null;+ if (rendered === code) {+ laid = lines;+ } else if (rendered === code.replaceAll('\n', '')) {+ const flat: TokenLine = [];+ let offset = 0;+ code.split('\n').forEach((line, i) => {+ for (const token of lines[i] ?? []) {+ flat.push({+ ...token,+ start: offset + token.start,+ end: offset + token.end,+ });+ }+ offset += line.length;+ });+ laid = [flat];+ }+ if (laid == null) {+ return [];+ }+ const live = CSS.highlights;+ const registry = new Map<string, TestHighlight>();+ Object.assign(CSS, {highlights: registry});+ try {+ const remove = applyHighlightRangesFlat(element, laid);+ const spans = spansIn(registry, element);+ remove();+ return spans;+ } finally {+ Object.assign(CSS, {highlights: live});+ }+}++function painted(+ registry: ReadonlyMap<string, Set<AbstractRange>>,+ type: string,+): string[] {+ return [...(registry.get(`astryx-${type}`) ?? [])].map(range =>+ (range as Range).toString(),+ );+}+describe('code block syntax colors (spec:AST-061 FR1)', () => {const code = 'let a = 1;\nreturn a;';- it("uses core CodeBlock's tokens line by line when the text keeps its line breaks", () => {- expect(renderedTokens(code, 'ts', code)).toEqual(tokenize(code, 'ts'));+ it("colors with core CodeBlock's tokens, line by line where the text keeps its line breaks", async () => {+ const registry = stubHighlights();+ const editor = await mountEditor('```ts\n' + code + '\n```\n');+ expect(codeElement(editor).textContent).toBe(code);+ expect(painted(registry, 'keyword')).toEqual(['let', 'return']);+ expect(spansIn(registry, codeElement(editor))).toEqual(+ fullRepaint(codeElement(editor), code, 'ts'),+ );});- it('paints a listed language value, as Markdown does (CodeBlock FR13)', () => {- const idl = 'struct User {}';- expect(renderedTokens(idl, 'thrift', idl)?.flat()).toEqual([]);- expect(- renderedTokens(idl, listedLanguage([thrift], 'thrift') ?? 'thrift', idl)- ?.flat()- .map(t => t.type),- ).toContain('keyword');+ it('lays the tokens over the text as one line where line breaks render without text', async () => {+ const registry = stubHighlights();+ const editor = await mountEditor('```ts\nx\n```\n');+ update(editor, () => $writeLines(code.split('\n')));+ expect(codeElement(editor).textContent).toBe(code.replace('\n', ''));+ expect(painted(registry, 'keyword')).toEqual(['let', 'return']);+ expect(spansIn(registry, codeElement(editor))).toEqual(+ fullRepaint(codeElement(editor), code, 'ts'),+ );+ });++ it('paints a listed language value, as Markdown does (CodeBlock FR13)', async () => {+ const registry = stubHighlights();+ await mountEditor('```thrift\nstruct User {}\n```\n');+ expect(painted(registry, 'keyword')).toEqual([]);+ cleanup();+ await mountEditor('```thrift\nstruct User {}\n```\n', [thrift]);+ expect(painted(registry, 'keyword')).toEqual(['struct']);+ expect(listedLanguage([thrift], 'thrift')).toBe(thrift);expect(listedLanguage([thrift], 'graphql')).toBeUndefined();});@@ -34,19 +297,24 @@ describe('code block syntax colors (spec:AST-061 FR1)', () => {expect(sameLanguages([thrift], undefined)).toBe(false);});- it('lays the tokens over the text as one line when line breaks render without text', () => {- const rendered = code.replace('\n', '');- const [line] = renderedTokens(code, 'ts', rendered) ?? [];- const texts = (line ?? []).map(- token => `${token.type}:${rendered.slice(token.start, token.end)}`,+ it('colors nothing for a language core does not know, or while the rendered text is not the code', async () => {+ const registry = stubHighlights();+ const editor = await mountEditor('```notalanguage\n' + code + '\n```\n');+ expect([...registry.values()].flatMap(highlight => [...highlight])).toEqual(+ [],);- expect(texts).toContain('keyword:let');- expect(texts).toContain('keyword:return');- });-- it('colors nothing while the rendered text is not the code, or for a language core does not know', () => {- expect(renderedTokens(code, 'ts', 'let a')).toBeNull();- expect(renderedTokens(code, 'notalanguage', code)).toEqual([]);+ update(editor, () => $block().setLanguage('ts'));+ expect(painted(registry, 'keyword')).toEqual(['let', 'return']);+ // A pasted line break node beside the imported text's own line breaks+ // leaves the rendered text neither the code nor the code without breaks.+ update(editor, () => {+ $selectCode(code.length);+ const selection = $getSelection();+ if ($isRangeSelection(selection)) {+ selection.insertRawText('\nlet b;');+ }+ });+ expect(spansIn(registry, codeElement(editor))).toEqual([]);});it('renders code blocks where the Highlight API is missing', async () => {@@ -58,3 +326,369 @@ describe('code block syntax colors (spec:AST-061 FR1)', () => {expect(await screen.findByText('let plain = 1;')).toBeInTheDocument();});});++type Edit = [find: (code: string) => number, removed: number, inserted: string];++/** Applies each edit, then checks the colors against a full repaint. */+async function expectEditsMatchFullRepaint(+ language: string,+ lines: ReadonlyArray<string>,+ layout: 'text node per line' | 'one text node',+ edits: ReadonlyArray<Edit>,+) {+ const registry = stubHighlights();+ const editor = await mountEditor(+ '```' + language + '\n' + lines.join('\n') + '\n```\n',+ );+ if (layout === 'text node per line') {+ update(editor, () => $writeLines(lines));+ }+ const check = (step: string) => {+ const {code, language: current} = codeOf(editor);+ expect(+ spansIn(registry, codeElement(editor)),+ `${layout}, ${step}: ${JSON.stringify(code)}`,+ ).toEqual(fullRepaint(codeElement(editor), code, current));+ };+ check('as loaded');+ for (const [find, removed, inserted] of edits) {+ const at = find(codeOf(editor).code);+ update(editor, () => {+ $selectCode(at, removed);+ $type(inserted);+ });+ check(`replace ${removed} at ${at} with ${JSON.stringify(inserted)}`);+ }+ cleanup();+}++function indexIn(code: string, text: string): number {+ const at = code.indexOf(text);+ if (at < 0) {+ throw new Error(`missing ${JSON.stringify(text)}`);+ }+ return at;+}+const before = (text: string) => (code: string) => indexIn(code, text);+const after = (text: string) => (code: string) =>+ indexIn(code, text) + text.length;++const TS_LINES = [+ 'const a = call(1, "row"); // note',+ 'let b = `two`;',+ '/* first */ const c = 3;',+ 'function run() {',+ ' return a + b;',+ '}',+];++describe('edited code blocks color exactly as a full repaint does', () => {+ const layouts = ['text node per line', 'one text node'] as const;++ it.each(layouts)(+ 'TypeScript, opening and closing strings, comments, and template literals (%s)',+ async layout => {+ await expectEditsMatchFullRepaint('ts', TS_LINES, layout, [+ [after('// note'), 0, 'x'],+ [after('let b = '), 0, '"'],+ [after('let b = "'), 0, 'q'],+ [before('"q'), 2, ''],+ [before('function'), 0, '/*'],+ [after('return a + b;'), 0, ' */'],+ [before('/*function'), 2, ''],+ [after('let b = `two`'), 0, '`'],+ [before('`;'), 1, ''],+ [after('const c = 3;'), 0, ' call'],+ [before('function'), 0, '('],+ [before('(function'), 1, ''],+ [after('row'), 0, 'x'],+ [after('rowx'), 0, '\\'],+ [before('1, "row'), 1, '42'],+ [after('call(42'), 0, '\n'],+ [after('return a'), 0, 'x'],+ ]);+ },+ );++ it.each(layouts)(+ 'Python, opening and closing a triple-quoted string (%s)',+ async layout => {+ await expectEditsMatchFullRepaint(+ 'python',+ ['def run():', ' x = call(1)', ' return x', 'y = 2'],+ layout,+ [+ [after('x = '), 0, '"""'],+ [after('return x'), 0, '"""'],+ [before('"""call'), 3, ''],+ [after('y = 2'), 0, ' # done'],+ [after('def run'), 0, 'ning'],+ ],+ );+ },+ );++ it.each(layouts)('shell, opening and closing ${ (%s)', async layout => {+ await expectEditsMatchFullRepaint(+ 'sh',+ ['echo "row"', 'printf 1', 'exit 0'],+ layout,+ [+ [after('echo '), 0, '${value'],+ [after('printf 1'), 0, '}'],+ [before('${value'), 7, ''],+ [after('exit 0'), 0, ' # ok'],+ ],+ );+ });++ it.each(layouts)(+ 'a language without the line cache re-tokenizes each edit (%s)',+ async layout => {+ await expectEditsMatchFullRepaint(+ 'css',+ ['.a {', ' color: red;', '}', '/* end */'],+ layout,+ [+ [after('color: red'), 0, ' !important'],+ [before('}'), 0, '/*'],+ [after('/* end */'), 0, 'x'],+ ],+ );+ },+ );++ it('follows line breaks added and removed, and keeps later lines exact', async () => {+ const registry = stubHighlights();+ const editor = await mountEditor('```ts\nx\n```\n');+ update(editor, () => $writeLines(TS_LINES));+ const check = () => {+ const {code} = codeOf(editor);+ expect(spansIn(registry, codeElement(editor))).toEqual(+ fullRepaint(codeElement(editor), code, 'ts'),+ );+ };+ update(editor, () => {+ $selectCode(after('call(1, ')(codeOf(editor).code));+ const selection = $getSelection();+ if ($isRangeSelection(selection)) {+ selection.insertLineBreak();+ }+ });+ check();+ update(editor, () => {+ $selectCode(after('let b = ')(codeOf(editor).code));+ $type('x + ');+ });+ check();+ update(editor, () => {+ $selectCode(before('\n"row"')(codeOf(editor).code), 1);+ $type('');+ });+ check();+ update(editor, () => {+ $selectCode(after('return a')(codeOf(editor).code));+ const selection = $getSelection();+ if ($isRangeSelection(selection)) {+ selection.insertRawText(' +\n\t`more`');+ }+ });+ check();+ update(editor, () => {+ $selectCode(after('`more')(codeOf(editor).code));+ $type('!');+ });+ check();+ });++ it('recolors a block whose text nodes are replaced by others with the same text', async () => {+ const registry = stubHighlights();+ const editor = await mountEditor(+ '```ts\n' + TS_LINES.join('\n') + '\n```\n',+ );+ update(editor, () => $writeLines(TS_LINES));+ expect(painted(registry, 'keyword')).toContain('const');+ expect(spansIn(registry, codeElement(editor))).toEqual(+ fullRepaint(codeElement(editor), TS_LINES.join('\n'), 'ts'),+ );+ });++ it('follows the language and restored editor states', async () => {+ const registry = stubHighlights();+ const editor = await mountEditor('```ts\nx\n```\n');+ update(editor, () => $writeLines(TS_LINES));+ const check = () => {+ const {code, language} = codeOf(editor);+ expect(spansIn(registry, codeElement(editor))).toEqual(+ fullRepaint(codeElement(editor), code, language),+ );+ };+ const loaded = editor.getEditorState();+ update(editor, () => {+ $selectCode(after('const c = ')(codeOf(editor).code), 1);+ $type('"three"');+ });+ check();+ const edited = editor.getEditorState();+ act(() => editor.setEditorState(loaded));+ check();+ expect(painted(registry, 'number')).toContain('3');+ act(() => editor.setEditorState(edited));+ check();+ expect(painted(registry, 'string')).toContain('"three"');+ update(editor, () => $block().setLanguage('python'));+ check();+ expect(painted(registry, 'keyword')).not.toContain('const');+ });+});++interface Work {+ tokenizeCalls: number;+ linesTokenized: number;+ charactersTokenized: number;+ textNodesScanned: number;+ blockTextReads: number;+ rangesAdded: number;+}++/** Counts what `run` makes the plugin do: core tokenizer calls and the lines+ * and characters they read, the DOM text nodes the plugin reads (through+ * any getter or tree walk), whole-block text reads from the editor state,+ * and ranges added. */+function measure(run: () => void): Work {+ // The plugin's own modules, not this test file's frames below them.+ const byPlugin = () =>+ /CodeSyntaxPlugin\.tsx|codeLineTokens\.ts/.test(new Error().stack ?? '');+ const scanned = new Set<Node>();+ let blockTextReads = 0;+ const visit = (node: Node) => {+ if (node.nodeType === Node.TEXT_NODE) {+ scanned.add(node);+ } else {+ node.childNodes.forEach(visit);+ }+ };+ const restores: Array<() => void> = [];+ const watch = (proto: object, name: string) => {+ const descriptor = Object.getOwnPropertyDescriptor(proto, name)!;+ Object.defineProperty(proto, name, {+ ...descriptor,+ get(this: Node) {+ if (byPlugin()) {+ visit(this);+ }+ return descriptor.get!.call(this);+ },+ });+ restores.push(() => Object.defineProperty(proto, name, descriptor));+ };+ watch(CharacterData.prototype, 'data');+ watch(CharacterData.prototype, 'length');+ watch(Node.prototype, 'nodeValue');+ watch(Node.prototype, 'textContent');+ const createTreeWalker = document.createTreeWalker;+ document.createTreeWalker = function (+ this: Document,+ ...args: Parameters<Document['createTreeWalker']>+ ) {+ const walker = createTreeWalker.apply(this, args);+ const nextNode = walker.nextNode.bind(walker);+ walker.nextNode = () => {+ const node = nextNode();+ if (node != null && byPlugin()) {+ visit(node);+ }+ return node;+ };+ return walker;+ };+ restores.push(() => {+ document.createTreeWalker = createTreeWalker;+ });+ const getTextContent = ElementNode.prototype.getTextContent;+ ElementNode.prototype.getTextContent = function (this: ElementNode) {+ if ($isCodeNode(this) && byPlugin()) {+ blockTextReads++;+ }+ return getTextContent.call(this);+ };+ restores.push(() => {+ ElementNode.prototype.getTextContent = getTextContent;+ });+ const stackTraceLimit = Error.stackTraceLimit;+ Error.stackTraceLimit = 100;+ restores.push(() => {+ Error.stackTraceLimit = stackTraceLimit;+ });+ const tokenizer = vi.mocked(tokenize);+ tokenizer.mockClear();+ rangesAdded = 0;+ try {+ run();+ } finally {+ restores.reverse().forEach(restore => restore());+ }+ const inputs = tokenizer.mock.calls.map(([text]) => text);+ return {+ tokenizeCalls: inputs.length,+ linesTokenized: inputs.reduce(+ (sum, text) => sum + text.split('\n').length,+ 0,+ ),+ charactersTokenized: inputs.reduce((sum, text) => sum + text.length, 0),+ textNodesScanned: scanned.size,+ blockTextReads,+ rangesAdded,+ };+}++const LINE = 'const a = 1;';++/** The work one character typed at the end of the middle line costs in a+ * TypeScript block of `lines` copies of LINE. */+async function typingWork(+ lines: number,+ layout: 'text node per line' | 'one text node',+): Promise<Work> {+ stubHighlights();+ const source = Array.from({length: lines}, () => LINE);+ const editor = await mountEditor('```ts\n' + source.join('\n') + '\n```\n');+ if (layout === 'text node per line') {+ update(editor, () => $writeLines(source));+ }+ const middle = Math.floor(lines / 2);+ update(editor, () => $selectCode(middle * (LINE.length + 1) + LINE.length));+ const work = measure(() => update(editor, () => $type('x')));+ cleanup();+ return work;+}++describe('typing in a long code block re-tokenizes only what it changed', () => {+ // The typed line now ends in a name, which is a function name if the next+ // visible character is `(`, so the next line's first character is+ // tokenized with it: two lines, 15 characters.+ const ONE_LINE = {+ tokenizeCalls: 1,+ linesTokenized: 2,+ charactersTokenized: LINE.length + 3,+ textNodesScanned: 1,+ blockTextReads: 0,+ };++ it('costs the same in 200 and 2,000 lines when each line is a text node', async () => {+ const small = await typingWork(200, 'text node per line');+ const large = await typingWork(2000, 'text node per line');+ expect(large).toEqual(small);+ // The typed line's four tokens: `const`, `=`, `1`, `;`.+ expect(large).toEqual({...ONE_LINE, rangesAdded: 4});+ }, 60000);++ it('tokenizes and reads the same in 200 and 2,000 lines imported as one text node', async () => {+ const small = await typingWork(200, 'one text node');+ const large = await typingWork(2000, 'one text node');+ // Lexical rewrites that one text node whole, so every range in it is+ // replaced; nothing is tokenized or read again.+ expect(small).toEqual({...ONE_LINE, rangesAdded: 200 * 4});+ expect(large).toEqual({...ONE_LINE, rangesAdded: 2000 * 4});+ }, 60000);+});diff --git a/packages/richtext/src/CodeSyntaxPlugin.tsx b/packages/richtext/src/CodeSyntaxPlugin.tsxindex 14fa7a3b8c..b7d6609990 100644--- a/packages/richtext/src/CodeSyntaxPlugin.tsx+++ b/packages/richtext/src/CodeSyntaxPlugin.tsx@@ -5,34 +5,43 @@/*** @file CodeSyntaxPlugin.tsx* @input Uses @lexical/react (composer context), @lexical/code (CodeNode),- * lexical, and core CodeBlock's tokenizer, highlight ranges, and token- * types.+ * lexical, core CodeBlock's highlight styles and token types, and+ * CodeLineTokens (codeLineTokens.ts), core CodeBlock's tokens kept line by+ * line.* @output Exports CodeSyntaxPlugin, which colors the code in every fenced code- * block with core CodeBlock's tokenizer and syntax color tokens;- * CODE_SYNTAX_CLASS, the class the theme gives code blocks for it; and- * renderedTokens, which lays tokens out over a block's rendered text.+ * block with core CodeBlock's tokenizer and syntax color tokens; and+ * CODE_SYNTAX_CLASS, the class the theme gives code blocks for it.* @position Rendered by RichTextEditor and RichTextView. A fenced code block* with a language core CodeBlock knows is colored as core Markdown colors* it: the same tokenizer finds the same tokens, and each token type takes* its `--color-syntax-*` token (spec:AST-061 FR1) — no palette of its own.* The colors are CSS Custom Highlight ranges over the text Lexical renders,* the way core CodeBlock colors editable code, so the document and its DOM- * are untouched; the ranges are rebuilt when a block's code or language- * changes. Browsers without the Highlight API show plain code.+ * are untouched. A block is colored in full when it appears, or when its+ * language, its element, or its set of text nodes changes. An edit that+ * only rewrites one text node's text re-tokenizes just the lines+ * CodeLineTokens says it can change, and replaces only the ranges in that+ * text node, reading no other. Browsers without the Highlight API show+ * plain code.*/import {useEffect, useState} from 'react';import {useLexicalComposerContext} from '@lexical/react/LexicalComposerContext';import {$isCodeNode, CodeNode} from '@lexical/code';-import {$getNodeByKey, type NodeKey} from 'lexical';import {- applyHighlightRangesFlat,+ $getNodeByKey,+ $isTextNode,+ type LexicalEditor,+ type NodeKey,+ type NodeMutation,+} from 'lexical';+import {+ ensureHighlightStyles,TOKEN_TYPES,- tokenize,type CodeLanguage,- type TokenLine,} from '@astryxdesign/core/CodeBlock';import {listedLanguage, sameLanguages} from './codeLanguages';+import {CodeLineTokens} from './codeLineTokens';/** The class the theme adds to every code block (editorTheme.ts). */export const CODE_SYNTAX_CLASS = 'astryx-richtext-code';@@ -65,44 +74,166 @@ function supportsHighlights(): boolean {);}+/** A colored token: its range, and the highlight that holds it. */+interface Painted {+ readonly range: Range;+ readonly highlight: Highlight;+}+/**- * The tokens of `code`, laid out for the rendered text of its block: line by- * line when the text keeps the line breaks, or as one line of offsets when- * each line break is an element with no text. Null when the rendered text is- * not the code (it is mid-update), so nothing is colored out of place.+ * A colored code block, with what an edit needs to recolor part of it.+ * Offsets in the rendered text are offsets in the block's DOM text nodes+ * laid end to end, which is the code itself when its line breaks render as+ * text, and the code without them when each renders as an element (`flat`).*/-export function renderedTokens(- code: string,- language: string | CodeLanguage,- rendered: string,-): TokenLine[] | null {- const lines = tokenize(code, language);- if (rendered === code) {- return lines;+interface Colored {+ readonly element: HTMLElement;+ readonly language: string;+ readonly lines: CodeLineTokens;+ readonly flat: boolean;+ /** The block's DOM text nodes in order. */+ readonly texts: Text[];+ /** Where each DOM text node starts in the rendered text. */+ readonly starts: number[];+ /** Where each leaf's DOM text node starts in the code (NaN for others). */+ readonly codeStarts: number[];+ /** The rendered text's length. */+ length: number;+ /** The Lexical text nodes that render as one DOM text node each, by key:+ * that node's index in `texts`, and the text last colored. */+ readonly leaves: Map<NodeKey, {readonly index: number; text: string}>;+ /** Each line's colored tokens. */+ ranges: Painted[][];+}++function highlightFor(type: string): Highlight {+ const name = `astryx-${type}`;+ let highlight = CSS.highlights.get(name);+ if (highlight == null) {+ highlight = new Highlight();+ CSS.highlights.set(name, highlight);}- if (rendered !== code.replaceAll('\n', '')) {+ return highlight;+}++/**+ * The DOM position of `offset` in a block's rendered text: in the text node+ * that holds the character there when `inside`, else in the first text node+ * that reaches it. Null past the end of the text.+ */+function pointAt(+ block: Colored,+ offset: number,+ inside: boolean,+): [Text, number] | null {+ const {texts, starts, length} = block;+ if (texts.length === 0 || offset > length) {return null;}- const flat: TokenLine = [];- let offset = 0;- code.split('\n').forEach((line, index) => {- for (const token of lines[index] ?? []) {- flat.push({- type: token.type,- start: offset + token.start,- end: offset + token.end,- });+ let low = 0;+ let high = texts.length - 1;+ while (low < high) {+ const mid = (low + high) >>> 1;+ const end = mid + 1 < texts.length ? starts[mid + 1] : length;+ if (inside ? end > offset : end >= offset) {+ high = mid;+ } else {+ low = mid + 1;}- offset += line.length;- });- return [flat];+ }+ return [texts[low], offset - starts[low]];}-interface Colored {- readonly element: HTMLElement;- readonly code: string;- readonly language: string;- readonly cleanup: () => void;+/** Colors one line's tokens. */+function paintLine(block: Colored, line: number): void {+ // A flat block's rendered text has no line breaks before this line.+ const base = block.lines.starts[line] - (block.flat ? line : 0);+ const painted: Painted[] = [];+ for (const token of block.lines.tokens[line] ?? []) {+ const start = base + token.start;+ const end = base + token.end;+ const from = pointAt(block, start, start < end);+ const to = pointAt(block, end, false);+ if (from == null || to == null) {+ continue;+ }+ const highlight = highlightFor(token.type);+ try {+ const range = new Range();+ range.setStart(...from);+ range.setEnd(...to);+ highlight.add(range);+ painted.push({range, highlight});+ } catch {+ // A range the DOM refuses colors nothing.+ }+ }+ block.ranges[line] = painted;+}++function unpaintLine(block: Colored, line: number): void {+ for (const {range, highlight} of block.ranges[line] ?? []) {+ highlight.delete(range);+ }+ block.ranges[line] = [];+}++function unpaintAll(block: Colored): void {+ for (let line = 0; line < block.ranges.length; line++) {+ unpaintLine(block, line);+ }+ block.ranges = [];+}++function repaintAll(block: Colored): void {+ unpaintAll(block);+ for (let line = 0; line < block.lines.tokens.length; line++) {+ paintLine(block, line);+ }+}++/**+ * Lists a block's DOM text nodes in order with their offsets, and the+ * Lexical text nodes that render as exactly one of them.+ */+function $indexText(+ editor: LexicalEditor,+ node: CodeNode,+ block: Colored,+): void {+ const walker = document.createTreeWalker(block.element, NodeFilter.SHOW_TEXT);+ while (walker.nextNode() != null) {+ const text = walker.currentNode as Text;+ block.texts.push(text);+ block.starts.push(block.length);+ block.codeStarts.push(NaN);+ block.length += text.length;+ }+ const indexes = new Map(block.texts.map((text, index) => [text, index]));+ let codeStart = 0;+ for (const child of node.getChildren()) {+ const dom = $isTextNode(child)+ ? editor.getElementByKey(child.getKey())+ : null;+ const index =+ dom?.childNodes.length === 1+ ? indexes.get(dom.firstChild as Text)+ : undefined;+ if (index != null) {+ block.leaves.set(child.getKey(), {index, text: child.getTextContent()});+ block.codeStarts[index] = codeStart;+ }+ codeStart += child.getTextContentSize();+ }+}++/** The key of the code block that holds the node `key`, if any. */+function $codeBlockOf(key: NodeKey): NodeKey | null {+ let node = $getNodeByKey(key)?.getParent() ?? null;+ while (node != null && !$isCodeNode(node)) {+ node = node.getParent();+ }+ return node?.getKey() ?? null;}export function CodeSyntaxPlugin({@@ -121,81 +252,233 @@ export function CodeSyntaxPlugin({}// Synchronizes the editor's code blocks with the CSS highlight registry:- // Lexical update and mutation listeners colour each block, and the cleanup+ // Lexical mutation and update listeners colour each block, and the cleanup// removes the listeners and every highlight range they added.useEffect(() => {if (!supportsHighlights()) {return undefined;}ensureRules();- // Every code block in the editor, and the ones colored, by node key.+ // Every code block in the editor, the ones colored, and the ones colored+ // in full during the current update.const blocks = new Set<NodeKey>();const colored = new Map<NodeKey, Colored>();+ const repainted = new Set<NodeKey>();const uncolor = (key: NodeKey) => {- colored.get(key)?.cleanup();- colored.delete(key);+ const block = colored.get(key);+ if (block != null) {+ unpaintAll(block);+ colored.delete(key);+ }};- const color = (key: NodeKey) => {- const block = editor.getEditorState().read(() => {- const node = $getNodeByKey(key);- return $isCodeNode(node)- ? {code: node.getTextContent(), language: node.getLanguage() ?? ''}- : null;- });+ // Colors a block in full, unless `force` is false and its element,+ // language, and code are as last colored.+ const $color = (key: NodeKey, force: boolean) => {+ const node = $getNodeByKey(key);const element = editor.getElementByKey(key);- if (block == null || element == null) {+ if (!$isCodeNode(node) || element == null) {uncolor(key);return;}+ const code = node.getTextContent();+ const language = node.getLanguage() ?? '';const previous = colored.get(key);if (+ !force &&previous != null &&previous.element === element &&- previous.code === block.code &&- previous.language === block.language+ previous.language === language &&+ previous.lines.text === code) {return;}- previous?.cleanup();- const tokens = renderedTokens(- block.code,- listedLanguage(languages, block.language) ?? block.language,- element.textContent ?? '',- );- if (tokens == null) {- // Not rendered yet; the next update tries again.- colored.delete(key);+ uncolor(key);+ const rendered = element.textContent ?? '';+ if (rendered !== code && rendered !== code.replaceAll('\n', '')) {+ // Not rendered yet; the block's next change tries again.return;}- colored.set(key, {+ ensureHighlightStyles();+ const block: Colored = {element,- ...block,- cleanup: applyHighlightRangesFlat(element, tokens),- });+ language,+ lines: new CodeLineTokens(+ code,+ listedLanguage(languages, language) ?? language,+ ),+ flat: rendered !== code,+ texts: [],+ starts: [],+ codeStarts: [],+ length: 0,+ leaves: new Map(),+ ranges: [],+ };+ $indexText(editor, node, block);+ repaintAll(block);+ colored.set(key, block);+ repainted.add(key);+ };+ // Recolors a block after an update that rewrote the text of its leaf+ // `leafKey` and changed nothing else in it. False when the block must+ // be colored in full instead.+ const $recolorLeaf = (key: NodeKey, leafKey: NodeKey): boolean => {+ const block = colored.get(key);+ const leaf = block?.leaves.get(leafKey);+ const node = $getNodeByKey(leafKey);+ const parent = node?.getParent();+ if (+ block == null ||+ leaf == null ||+ !$isTextNode(node) ||+ !$isCodeNode(parent) ||+ parent.getKey() !== key ||+ (parent.getLanguage() ?? '') !== block.language ||+ editor.getElementByKey(key) !== block.element+ ) {+ return false;+ }+ const text = block.texts[leaf.index];+ const dom = editor.getElementByKey(leafKey);+ const after = node.getTextContent();+ if (+ dom?.childNodes.length !== 1 ||+ dom.firstChild !== text ||+ text.data !== after+ ) {+ return false;+ }+ const before = leaf.text;+ if (after === before) {+ return true;+ }+ let first = 0;+ while (+ first < before.length &&+ first < after.length &&+ before[first] === after[first]+ ) {+ first++;+ }+ let beforeEnd = before.length;+ let afterEnd = after.length;+ while (+ beforeEnd > first &&+ afterEnd > first &&+ before[beforeEnd - 1] === after[afterEnd - 1]+ ) {+ beforeEnd--;+ afterEnd--;+ }+ const inserted = after.slice(first, afterEnd);+ // A flat block's text holds no line breaks; one typed in would no+ // longer lay the code over its rendered text.+ if (block.flat && inserted.includes('\n')) {+ return false;+ }+ const leafStart = block.codeStarts[leaf.index];+ const lines = block.lines.replace(+ leafStart + first,+ beforeEnd - first,+ inserted,+ );+ const delta = after.length - before.length;+ for (let i = leaf.index + 1; i < block.texts.length; i++) {+ block.starts[i] += delta;+ block.codeStarts[i] += delta;+ }+ block.length += delta;+ leaf.text = after;+ if (lines == null) {+ repaintAll(block);+ return true;+ }+ // Lexical rewrites a changed DOM text node whole, which leaves every+ // range in it stale: recolor each line the node holds, and any other+ // line the edit re-tokenized. Every other range sits in a text node of+ // its own line and is still right.+ const recolor = new Set(lines);+ const last = block.lines.lineAt(+ leafStart + Math.max(after.length - 1, 0),+ );+ for (let i = block.lines.lineAt(leafStart); i <= last; i++) {+ recolor.add(i);+ }+ for (const i of recolor) {+ unpaintLine(block, i);+ paintLine(block, i);+ }+ return true;};const unregisterMutations = editor.registerMutationListener(CodeNode,mutations => {- for (const [key, mutation] of mutations) {- if (mutation === 'destroyed') {- blocks.delete(key);- uncolor(key);- } else {- blocks.add(key);- color(key);+ editor.getEditorState().read(() => {+ for (const [key, mutation] of mutations) {+ if (mutation === 'destroyed') {+ blocks.delete(key);+ uncolor(key);+ } else {+ blocks.add(key);+ $color(key, false);+ }}- }+ });},{skipInitialization: false},);- // Typing in a block changes its text nodes, not always the block itself:- // recheck every block after each update (unchanged ones stop at the- // comparison above).- const unregisterUpdates = editor.registerUpdateListener(() => {- for (const key of blocks) {- color(key);- }- });+ repainted.clear();+ // Typing in a block changes its text nodes, not the block itself: after+ // each update, recolor the blocks whose text nodes changed. A block whose+ // only change is one leaf's text takes the partial path; any other+ // change colors it in full, unless the block listener just did.+ const unregisterUpdates = editor.registerUpdateListener(+ ({editorState, mutatedNodes, prevEditorState}) => {+ if (mutatedNodes != null && blocks.size > 0) {+ const mutations: Array<[NodeKey, NodeMutation]> = [];+ for (const nodes of mutatedNodes.values()) {+ for (const mutation of nodes) {+ mutations.push(mutation);+ }+ }+ const before = prevEditorState.read(() =>+ mutations.map(([key, mutation]) =>+ mutation === 'created' ? null : $codeBlockOf(key),+ ),+ );+ editorState.read(() => {+ // Each changed block, with the one leaf whose text an update+ // rewrote, or null when anything else in it changed.+ const changed = new Map<NodeKey, NodeKey | null>();+ const note = (key: NodeKey | null, leafKey: NodeKey | null) => {+ if (key != null && blocks.has(key)) {+ changed.set(key, changed.has(key) ? null : leafKey);+ }+ };+ mutations.forEach(([key, mutation], i) => {+ const after = mutation === 'destroyed' ? null : $codeBlockOf(key);+ const rewritten =+ mutation === 'updated' &&+ after === before[i] &&+ $isTextNode($getNodeByKey(key));+ note(after, rewritten ? key : null);+ if (before[i] !== after) {+ note(before[i], null);+ }+ });+ for (const [key, leafKey] of changed) {+ if (+ !repainted.has(key) &&+ (leafKey == null || !$recolorLeaf(key, leafKey))+ ) {+ $color(key, true);+ }+ }+ });+ }+ repainted.clear();+ },+ );return () => {unregisterMutations();unregisterUpdates();diff --git a/packages/richtext/src/codeLineTokens.test.ts b/packages/richtext/src/codeLineTokens.test.tsnew file mode 100644index 0000000000..897e416b59--- /dev/null+++ b/packages/richtext/src/codeLineTokens.test.ts@@ -0,0 +1,234 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++import {describe, it, expect, vi, beforeEach} from 'vitest';+import type * as codeBlock from '@astryxdesign/core/CodeBlock';+import {tokenize} from '@astryxdesign/core/CodeBlock';+import {CodeLineTokens} from './codeLineTokens';++vi.mock('@astryxdesign/core/CodeBlock', async importOriginal => {+ const actual = await importOriginal<typeof codeBlock>();+ return {...actual, tokenize: vi.fn(actual.tokenize)};+});++const tokenizer = vi.mocked(tokenize);++beforeEach(() => {+ tokenizer.mockClear();+});++/** What the cache must always hold: a full `tokenize` of its code. */+function expectWhole(lines: CodeLineTokens, language: string) {+ const text = lines.text;+ let start = 0;+ expect({tokens: lines.tokens, starts: lines.starts}).toEqual({+ tokens: tokenize(text, language),+ starts: text.split('\n').map(line => {+ const lineStart = start;+ start += line.length + 1;+ return lineStart;+ }),+ });+}++/** The characters the tokenizer read since the last check. */+function charactersTokenized(): number {+ const characters = tokenizer.mock.calls.reduce(+ (sum, [text]) => sum + text.length,+ 0,+ );+ tokenizer.mockClear();+ return characters;+}++const DELIMITERS = [+ '"',+ "'",+ '`',+ '"""',+ '/*',+ '*/',+ '//',+ '#',+ '${',+ '}',+ '<<-',+ '\r\n',+ '\\',+ '\n',+ '(',+ ')',+ ' ',+ 'x',+];++describe('CodeLineTokens', () => {+ // An edit can recolor lines above it as well as below: removing a closing+ // delimiter reopens a string or comment that started earlier.+ it.each([+ [+ 'typescript',+ 'function run() {\n const a = call(1, "row"); // line\n return a;\n}',+ ],+ ['ts', 'const a = call(1); // line\nresult'],+ ['ts', '/* first\n\nlast */\nconst a = `one\ntwo`;\ncall\n (a);'],+ ['js', 'const a = 1;\n+// "open\nclose";\nconst b = 2;'],+ ['ts', 'const a = 1; /* note\nstill note */\nconst b = 2;'],+ ['tsx', 'const view = list\n\n .map(item => item);\nrender\n\n(view)'],+ ['hack', '<?hh\n$x->call();\n/* first\nlast */'],+ ['python', 'def run():\n x = """first\nlast"""\n return x'],+ ['python', 'def run():\n x = call(1, "row") # note\n return 1\nresult'],+ ['py', 'x = 1\npass\n@decorator\ncall\n (1)'],+ ['python', 'x = "a\nb"\nreturn 1'],+ ['bash', 'echo "row"; # note\nprintf 1\nresult'],+ ['sh', 'echo ${value\n}\necho "row";'],+ ['shell', 'cat <<EOF\necho "body"\nEOF\necho "after";'],+ ['zsh', 'echo $(printf "row")\necho `printf row`;'],+ ['markdown', '# Heading\n```ts\nconst a = 1;\n```\n**bold**'],+ ['yaml', 'name: first\nvalue: |\n next line\nend: true'],+ ['json', '{\n "name"\n : "first",\n "value": 1\n}'],+ ])(+ 'matches a whole %s tokenize through delimiters typed and deleted anywhere',+ (language, text) => {+ const lines = new CodeLineTokens(text, language);+ expectWhole(lines, language);+ for (let i = 0; i <= text.length; i++) {+ for (const inserted of DELIMITERS) {+ lines.replace(i, 0, inserted);+ expectWhole(lines, language);+ lines.replace(i, inserted.length, '');+ expect(lines.text).toBe(text);+ expectWhole(lines, language);+ }+ if (i < text.length) {+ lines.replace(i, 1, '');+ expectWhole(lines, language);+ lines.replace(i, 0, text[i]);+ expectWhole(lines, language);+ }+ }+ },+ );++ it.each(['ts', 'python', 'bash'])(+ 'matches a whole %s tokenize through edits at the end of the code',+ language => {+ const lines = new CodeLineTokens('call(1);\nresult', language);+ for (const suffix of [+ 'Value',+ '(',+ ' ',+ '\n(1);',+ '\n\n (1);',+ '\n/* next\nline */',+ ]) {+ const at = lines.text.length;+ lines.replace(at, 0, suffix);+ expectWhole(lines, language);+ lines.replace(at, suffix.length, '');+ expectWhole(lines, language);+ }+ },+ );++ it.each(['', 'r', 'b', 'u', 'f', 'fr', 'rf'])(+ 'keeps Python triple quotes first behind a "%s" prefix',+ prefix => {+ for (const spelling of new Set([prefix, prefix.toUpperCase()])) {+ for (const quote of ['"', "'"]) {+ const text = `x = ${spelling}${quote}${quote}\ny = ${quote.repeat(3)}ok${quote.repeat(3)}`;+ const lines = new CodeLineTokens(text, 'python');+ const at = text.indexOf(quote);+ for (const inserted of [quote, quote.repeat(2), quote.repeat(3)]) {+ lines.replace(at, 0, inserted);+ expectWhole(lines, 'python');+ lines.replace(at, inserted.length, '');+ expectWhole(lines, 'python');+ }+ }+ }+ },+ );++ it('recolors a name ending an earlier line when the next visible character becomes "("', () => {+ const text = 'const a = call\n\n \n value;\nnext();';+ const lines = new CodeLineTokens(text, 'ts');+ const at = text.indexOf('value');+ const types = () => lines.tokens[0].map(token => token.type);+ expect(types()).not.toContain('function');+ charactersTokenized();+ expect(lines.replace(at, 0, '(')).toEqual([0, 3]);+ expectWhole(lines, 'ts');+ expect(types()).toContain('function');+ expect(lines.replace(at, 1, '')).toEqual([0, 3]);+ expectWhole(lines, 'ts');+ expect(types()).not.toContain('function');+ // Typing elsewhere on the line leaves the name above alone.+ expect(lines.replace(at + 2, 0, 'x')).toEqual([3]);+ });++ it.each([+ ['python', 'value = """closed"""'],+ ['bash', 'echo ${value};'],+ ['sh', 'cat <<-EOF'],+ ['shell', 'echo `date`;'],+ ['zsh', 'echo $(date);'],+ ['ts', 'const a = /* open'],+ ['notalanguage', 'value'],+ ])(+ 'tokenizes all of a %s block holding `%s`, whose tokens may reach other lines',+ (language, unsafe) => {+ const ordinary =+ language === 'python' ? 'value = call(1)' : 'echo "row";';+ const text = `${unsafe}\n${ordinary}\nresult`;+ const lines = new CodeLineTokens(text, language);+ charactersTokenized();+ expect(lines.replace(text.indexOf(ordinary), 0, 'x')).toBeNull();+ expect(charactersTokenized()).toBe(text.length + 1);+ expectWhole(lines, language);+ },+ );++ it.each(+ [200, 2000].flatMap(count =>+ [+ ['ts', 'const value = call(1, "row"); // note'],+ ['python', 'value = call(1) # note'],+ ['bash', 'echo "row" $value # note'],+ ].map(([language, line]) => ({count, language, line})),+ ),+ )(+ 'tokenizes only the edited $language line in a $count-line block',+ ({count, language, line}) => {+ const text = Array.from({length: count}, () => line).join('\n');+ const lines = new CodeLineTokens(text, language);+ const middle = lines.starts[count / 2];+ charactersTokenized();+ for (const at of [middle, middle + line.length]) {+ expect(lines.replace(at, 0, 'x')).toEqual([count / 2]);+ expect(charactersTokenized()).toBeLessThan(2 * line.length);+ expectWhole(lines, language);+ lines.replace(at, 1, '');+ expectWhole(lines, language);+ charactersTokenized();+ }+ },+ );++ it('keeps offsets exact through edits in a 2,000-line block', () => {+ const text = Array.from(+ {length: 2000},+ (_, i) => `const value_${i} = call(${i}, "row"); // line ${i}`,+ ).join('\n');+ const lines = new CodeLineTokens(text, 'ts');+ const offset = text.indexOf('value_1000');+ for (const inserted of ['x', '']) {+ lines.replace(offset, inserted ? 0 : 1, inserted);+ expectWhole(lines, 'ts');+ }+ lines.replace(offset, 0, '"');+ expectWhole(lines, 'ts');+ lines.replace(offset, 1, '');+ expectWhole(lines, 'ts');+ expect(lines.text).toBe(text);+ });+});diff --git a/packages/richtext/src/codeLineTokens.ts b/packages/richtext/src/codeLineTokens.tsnew file mode 100644index 0000000000..f3ba853c9f--- /dev/null+++ b/packages/richtext/src/codeLineTokens.ts@@ -0,0 +1,241 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file codeLineTokens.ts+ * @input Core CodeBlock's tokenizer (`tokenize`) and token line type.+ * @output Exports CodeLineTokens, which keeps core CodeBlock's tokens for a+ * block of code line by line and re-tokenizes only the lines an edit can+ * change.+ * @position Used by CodeSyntaxPlugin. Its tokens always equal what `tokenize`+ * returns for the whole code. An edit re-tokenizes only its own line, and+ * at most the line above it, for the built-in TypeScript, JavaScript,+ * Python, and shell grammars while every line passes `standsAlone`;+ * anything else re-tokenizes all of it. The checks follow those grammars'+ * patterns in core CodeBlock's tokenizer.ts, and codeLineTokens.test.ts+ * compares edits against a full `tokenize`, so a grammar change that+ * breaks them fails there.+ */++import {+ tokenize,+ type CodeLanguage,+ type TokenLine,+} from '@astryxdesign/core/CodeBlock';++type Family = 'javascript' | 'python' | 'shell';++const FAMILIES: ReadonlyMap<string, Family> = new Map([+ ['typescript', 'javascript'],+ ['javascript', 'javascript'],+ ['tsx', 'javascript'],+ ['jsx', 'javascript'],+ ['ts', 'javascript'],+ ['js', 'javascript'],+ ['python', 'python'],+ ['py', 'python'],+ ['bash', 'shell'],+ ['sh', 'shell'],+ ['zsh', 'shell'],+ ['shell', 'shell'],+]);++/** What can open a string or comment that runs on past its line. */+const OPENERS: Readonly<Record<Family, RegExp>> = {+ javascript: /['"`]|\/\*/,+ python: /['"]/,+ shell: /['"`]|\$\{|\$\(|<</,+};++/** A line whose last visible character can end a function name. */+const NAME_END = /[\w$]\s*$/;++/**+ * Whether the name that ends a line can change color with the code after+ * it: its last visible character is a name character outside every string,+ * comment, number, keyword, and constant, whose patterns come before+ * function names.+ */+function endsInOpenName(text: string, tokens: TokenLine): boolean {+ const last = text.trimEnd().length - 1;+ return (+ NAME_END.test(text) &&+ !tokens.some(+ token =>+ token.start <= last &&+ token.end > last &&+ token.type !== 'function' &&+ token.type !== 'type',+ )+ );+}++/** A line's first visible character, or '' when it has none. */+function firstVisible(line: string): string {+ return /\S/.exec(line)?.[0] ?? '';+}++/**+ * Whether a line's strings and comments all close on it. Their patterns+ * match line breaks, so an opener left open can run on to a later line.+ * Only a string or comment the tokenizer emitted shields an opener: one that+ * another token swallowed, such as `/*` in the operator `+/*`, still counts+ * as open. Python tries triple quotes first, so `"""` can pair with another+ * line even where its quotes close as single strings on this one.+ */+function standsAlone(text: string, tokens: TokenLine, family: Family): boolean {+ if (family === 'python' && /'''|"""/.test(text)) {+ return false;+ }+ const opener = OPENERS[family];+ let position = 0;+ for (const token of tokens) {+ if (token.end > text.length) {+ return false;+ }+ if (token.type === 'string' || token.type === 'comment') {+ if (opener.test(text.slice(position, token.start))) {+ return false;+ }+ position = token.end;+ }+ }+ return !opener.test(text.slice(position));+}++/**+ * Core CodeBlock's tokens for a block of code, kept line by line with+ * line-relative offsets, exactly as `tokenize` returns them for the whole+ * code.+ */+export class CodeLineTokens {+ readonly language: string | CodeLanguage;+ /** Where each line starts in the code. */+ starts: number[] = [];+ /** Each line's tokens. */+ tokens: TokenLine[] = [];+ private lines: string[] = [];+ private readonly family: Family | null;+ /** Whether every line stands alone, so an edit re-tokenizes lines alone. */+ private linesStandAlone = false;++ constructor(text: string, language: string | CodeLanguage) {+ this.language = language;+ this.family =+ typeof language === 'string' ? (FAMILIES.get(language) ?? null) : null;+ this.tokenizeAll(text);+ }++ /** The code. */+ get text(): string {+ return this.lines.join('\n');+ }++ /** The line that holds `offset` in the code. */+ lineAt(offset: number): number {+ let low = 0;+ let high = this.starts.length - 1;+ while (low < high) {+ const mid = (low + high + 1) >>> 1;+ if (this.starts[mid] <= offset) {+ low = mid;+ } else {+ high = mid - 1;+ }+ }+ return low;+ }++ /**+ * Replaces `removed` characters at `offset` in the code with `inserted`.+ * Returns the lines it re-tokenized when the edit stays inside one line+ * and every line still stands alone; otherwise it re-tokenizes all the+ * code and returns null.+ */+ replace(offset: number, removed: number, inserted: string): number[] | null {+ const line = this.lineAt(offset);+ const before = this.lines[line];+ const column = offset - this.starts[line];+ if (+ this.family == null ||+ !this.linesStandAlone ||+ column + removed > before.length ||+ inserted.includes('\n')+ ) {+ const text = this.text;+ this.tokenizeAll(+ text.slice(0, offset) + inserted + text.slice(offset + removed),+ );+ return null;+ }+ const after =+ before.slice(0, column) + inserted + before.slice(column + removed);+ this.lines[line] = after;+ const tokens = this.tokenizeLine(line);+ if (!standsAlone(after, tokens, this.family)) {+ this.tokenizeAll(this.text);+ return null;+ }+ this.tokens[line] = tokens;+ for (let i = line + 1; i < this.starts.length; i++) {+ this.starts[i] += inserted.length - removed;+ }+ const changed = [line];+ // A name that ends an earlier line looks past blank lines for `(`, so a+ // new first visible character here can change that name's color.+ if (firstVisible(before) !== firstVisible(after)) {+ let above = line - 1;+ while (above >= 0 && firstVisible(this.lines[above]) === '') {+ above--;+ }+ if (+ above >= 0 &&+ endsInOpenName(this.lines[above], this.tokens[above] ?? [])+ ) {+ this.tokens[above] = this.tokenizeLine(above);+ changed.unshift(above);+ }+ }+ return changed;+ }++ /**+ * A line's tokens as `tokenize` finds them in the whole code. A function+ * name is a name followed by `(` past any whitespace, line breaks+ * included, so when the line ends in a name, the code up to the next+ * visible character is tokenized with it.+ */+ private tokenizeLine(line: number): TokenLine {+ let text = this.lines[line];+ if (NAME_END.test(text)) {+ for (let next = line + 1; next < this.lines.length; next++) {+ const visible = /\S/.exec(this.lines[next]);+ text +=+ '\n' ++ (visible == null+ ? this.lines[next]+ : this.lines[next].slice(0, visible.index + 1));+ if (visible != null) {+ break;+ }+ }+ }+ return tokenize(text, this.language)[0] ?? [];+ }++ private tokenizeAll(text: string): void {+ this.lines = text.split('\n');+ let start = 0;+ this.starts = this.lines.map(line => {+ const lineStart = start;+ start += line.length + 1;+ return lineStart;+ });+ this.tokens = tokenize(text, this.language);+ const family = this.family;+ this.linesStandAlone =+ family != null &&+ this.lines.every((line, i) =>+ standsAlone(line, this.tokens[i] ?? [], family),+ );+ }+}