Skip to content

An open source design system that's fully customizable and agent ready

TypeScript
21.9 KB
Raw Blame
// Copyright (c) Meta Platforms, Inc. and affiliates.

import {describe, it, expect, vi, afterEach, beforeAll} from 'vitest';
import {act, cleanup, render, screen, waitFor} from '@testing-library/react';
import {useEffect} from 'react';
import {useLexicalComposerContext} from '@lexical/react/LexicalComposerContext';
import {$isCodeNode} from '@lexical/code';
import {
  $createLineBreakNode,
  $createRangeSelection,
  $createTextNode,
  $getRoot,
  $getSelection,
  $isRangeSelection,
  $isTextNode,
  $setSelection,
  ElementNode,
  type LexicalEditor,
} from 'lexical';
import type * as codeBlock from '@astryxdesign/core/CodeBlock';
import {
  applyHighlightRangesFlat,
  tokenize,
  type CodeLanguage,
  type TokenLine,
} from '@astryxdesign/core/CodeBlock';
import {thrift} from '@astryxdesign/core/CodeBlock/languages/thrift';
import {listedLanguage, sameLanguages} from './codeLanguages';
import {RichTextEditor} from './RichTextEditor';
import {RichTextView} from './RichTextView';
import {markdownToEditorStateJSON} from './markdownSerializers';

vi.mock('@astryxdesign/core/CodeBlock', async importOriginal => {
  const actual = await importOriginal<typeof codeBlock>();
  return {...actual, tokenize: vi.fn(actual.tokenize)};
});

let rangesAdded = 0;

/** jsdom has no CSS Custom Highlight API: named sets of ranges stand in. */
class TestHighlight extends Set<AbstractRange> {
  override add(range: AbstractRange): this {
    rangesAdded++;
    return super.add(range);
  }
}

function stubHighlights(): Map<string, TestHighlight> {
  const registry = new Map<string, TestHighlight>();
  vi.stubGlobal('CSS', {...globalThis.CSS, highlights: registry});
  vi.stubGlobal('Highlight', TestHighlight);
  return registry;
}

afterEach(() => {
  cleanup();
  vi.unstubAllGlobals();
});

// jsdom's Range has no getBoundingClientRect, which Lexical calls to scroll
// a reconciled selection into view while the editor has focus.
beforeAll(() => {
  if (typeof Range.prototype.getBoundingClientRect !== 'function') {
    Range.prototype.getBoundingClientRect = () => new DOMRect();
  }
});

function CaptureEditor({onReady}: {onReady: (editor: LexicalEditor) => void}) {
  const [editor] = useLexicalComposerContext();
  useEffect(() => {
    onReady(editor);
  }, [editor, onReady]);
  return null;
}

async function mountEditor(
  markdown: string,
  codeLanguages?: ReadonlyArray<CodeLanguage>,
): Promise<LexicalEditor> {
  let editor: LexicalEditor | undefined;
  render(
    <RichTextEditor
      label="Code"
      codeLanguages={codeLanguages}
      defaultValue={markdownToEditorStateJSON(markdown)}
      plugins={<CaptureEditor onReady={live => (editor = live)} />}
    />,
  );
  await waitFor(() => expect(editor).toBeDefined());
  return editor!;
}

function update(editor: LexicalEditor, fn: () => void) {
  act(() => editor.update(fn, {discrete: true}));
}

function $block() {
  const block = $getRoot().getFirstChild();
  if (!$isCodeNode(block)) {
    throw new Error('missing code block');
  }
  return block;
}

/** Rewrites the code block the way typing and pasting build one: a text node
 *  per line, with a line break node between lines. */
function $writeLines(lines: ReadonlyArray<string>) {
  const block = $block();
  block.clear();
  lines.forEach((line, i) => {
    if (i > 0) {
      block.append($createLineBreakNode());
    }
    if (line !== '') {
      block.append($createTextNode(line));
    }
  });
}

/** The text node and offset at `offset` in the code block's text. */
function $pointAt(offset: number): [string, number] {
  let start = 0;
  for (const child of $block().getChildren()) {
    const size = child.getTextContentSize();
    if ($isTextNode(child) && offset <= start + size) {
      return [child.getKey(), offset - start];
    }
    start += size;
  }
  throw new Error(`no text node at ${offset}`);
}

/** Selects `length` characters at `offset` in the code block's text. */
function $selectCode(offset: number, length = 0) {
  const [anchorKey, anchorOffset] = $pointAt(offset);
  const [focusKey, focusOffset] = $pointAt(offset + length);
  const selection = $createRangeSelection();
  selection.anchor.set(anchorKey, anchorOffset, 'text');
  selection.focus.set(focusKey, focusOffset, 'text');
  $setSelection(selection);
}

function $type(text: string) {
  const selection = $getSelection();
  if (!$isRangeSelection(selection)) {
    throw new Error('missing selection');
  }
  selection.insertText(text);
}

function codeOf(editor: LexicalEditor): {code: string; language: string} {
  return editor.getEditorState().read(() => ({
    code: $block().getTextContent(),
    language: $block().getLanguage() ?? '',
  }));
}

function codeElement(editor: LexicalEditor): HTMLElement {
  const key = editor.getEditorState().read(() => $block().getKey());
  return editor.getElementByKey(key)!;
}

interface Span {
  type: string;
  start: number;
  end: number;
}

/** Every range in `registry`, as a token type over offsets in the code
 *  element's rendered text (-1 for a range outside it). */
function spansIn(
  registry: ReadonlyMap<string, Set<AbstractRange>>,
  element: HTMLElement,
): Span[] {
  const starts = new Map<Node, number>();
  let offset = 0;
  const walker = document.createTreeWalker(element, NodeFilter.SHOW_TEXT);
  for (let node = walker.nextNode(); node != null; node = walker.nextNode()) {
    starts.set(node, offset);
    offset += (node as Text).length;
  }
  const at = (node: Node, nodeOffset: number) =>
    starts.has(node) ? starts.get(node)! + nodeOffset : -1;
  const spans: Span[] = [];
  for (const [name, highlight] of registry) {
    for (const range of highlight) {
      spans.push({
        type: name.replace(/^astryx-/, ''),
        start: at(range.startContainer, range.startOffset),
        end: at(range.endContainer, range.endOffset),
      });
    }
  }
  return spans.sort(
    (a, b) =>
      a.start - b.start ||
      a.end - b.end ||
      (a.type < b.type ? -1 : a.type > b.type ? 1 : 0),
  );
}

/**
 * What coloring the block from scratch paints: core CodeBlock's tokens for
 * the whole code, laid over the rendered text line by line when it keeps the
 * line breaks or as one line of offsets when it drops them (nothing when it
 * is neither), and written by core CodeBlock's own range writer.
 */
function fullRepaint(
  element: HTMLElement,
  code: string,
  language: string | CodeLanguage,
): Span[] {
  const rendered = element.textContent ?? '';
  const lines = tokenize(code, language);
  let laid: TokenLine[] | null = null;
  if (rendered === code) {
    laid = lines;
  } else if (rendered === code.replaceAll('\n', '')) {
    const flat: TokenLine = [];
    let offset = 0;
    code.split('\n').forEach((line, i) => {
      for (const token of lines[i] ?? []) {
        flat.push({
          ...token,
          start: offset + token.start,
          end: offset + token.end,
        });
      }
      offset += line.length;
    });
    laid = [flat];
  }
  if (laid == null) {
    return [];
  }
  const live = CSS.highlights;
  const registry = new Map<string, TestHighlight>();
  Object.assign(CSS, {highlights: registry});
  try {
    const remove = applyHighlightRangesFlat(element, laid);
    const spans = spansIn(registry, element);
    remove();
    return spans;
  } finally {
    Object.assign(CSS, {highlights: live});
  }
}

function painted(
  registry: ReadonlyMap<string, Set<AbstractRange>>,
  type: string,
): string[] {
  return [...(registry.get(`astryx-${type}`) ?? [])].map(range =>
    (range as Range).toString(),
  );
}

describe('code block syntax colors (spec:AST-061 FR1)', () => {
  const code = 'let a = 1;\nreturn a;';

  it("colors with core CodeBlock's tokens, line by line where the text keeps its line breaks", async () => {
    const registry = stubHighlights();
    const editor = await mountEditor('```ts\n' + code + '\n```\n');
    expect(codeElement(editor).textContent).toBe(code);
    expect(painted(registry, 'keyword')).toEqual(['let', 'return']);
    expect(spansIn(registry, codeElement(editor))).toEqual(
      fullRepaint(codeElement(editor), code, 'ts'),
    );
  });

  it('lays the tokens over the text as one line where line breaks render without text', async () => {
    const registry = stubHighlights();
    const editor = await mountEditor('```ts\nx\n```\n');
    update(editor, () => $writeLines(code.split('\n')));
    expect(codeElement(editor).textContent).toBe(code.replace('\n', ''));
    expect(painted(registry, 'keyword')).toEqual(['let', 'return']);
    expect(spansIn(registry, codeElement(editor))).toEqual(
      fullRepaint(codeElement(editor), code, 'ts'),
    );
  });

  it('paints a listed language value, as Markdown does (CodeBlock FR13)', async () => {
    const registry = stubHighlights();
    await mountEditor('```thrift\nstruct User {}\n```\n');
    expect(painted(registry, 'keyword')).toEqual([]);
    cleanup();
    await mountEditor('```thrift\nstruct User {}\n```\n', [thrift]);
    expect(painted(registry, 'keyword')).toEqual(['struct']);
    expect(listedLanguage([thrift], 'thrift')).toBe(thrift);
    expect(listedLanguage([thrift], 'graphql')).toBeUndefined();
  });

  it('treats lists naming the same values as the same', () => {
    expect(sameLanguages([thrift], [thrift])).toBe(true);
    expect(sameLanguages(undefined, undefined)).toBe(true);
    expect(sameLanguages([thrift], [])).toBe(false);
    expect(sameLanguages([thrift], undefined)).toBe(false);
  });

  it('colors nothing for a language core does not know, or while the rendered text is not the code', async () => {
    const registry = stubHighlights();
    const editor = await mountEditor('```notalanguage\n' + code + '\n```\n');
    expect([...registry.values()].flatMap(highlight => [...highlight])).toEqual(
      [],
    );
    update(editor, () => $block().setLanguage('ts'));
    expect(painted(registry, 'keyword')).toEqual(['let', 'return']);
    // A pasted line break node beside the imported text's own line breaks
    // leaves the rendered text neither the code nor the code without breaks.
    update(editor, () => {
      $selectCode(code.length);
      const selection = $getSelection();
      if ($isRangeSelection(selection)) {
        selection.insertRawText('\nlet b;');
      }
    });
    expect(spansIn(registry, codeElement(editor))).toEqual([]);
  });

  it('renders code blocks where the Highlight API is missing', async () => {
    render(
      <RichTextView
        value={markdownToEditorStateJSON('```ts\nlet plain = 1;\n```\n')}
      />,
    );
    expect(await screen.findByText('let plain = 1;')).toBeInTheDocument();
  });
});

type Edit = [find: (code: string) => number, removed: number, inserted: string];

/** Applies each edit, then checks the colors against a full repaint. */
async function expectEditsMatchFullRepaint(
  language: string,
  lines: ReadonlyArray<string>,
  layout: 'text node per line' | 'one text node',
  edits: ReadonlyArray<Edit>,
) {
  const registry = stubHighlights();
  const editor = await mountEditor(
    '```' + language + '\n' + lines.join('\n') + '\n```\n',
  );
  if (layout === 'text node per line') {
    update(editor, () => $writeLines(lines));
  }
  const check = (step: string) => {
    const {code, language: current} = codeOf(editor);
    expect(
      spansIn(registry, codeElement(editor)),
      `${layout}, ${step}: ${JSON.stringify(code)}`,
    ).toEqual(fullRepaint(codeElement(editor), code, current));
  };
  check('as loaded');
  for (const [find, removed, inserted] of edits) {
    const at = find(codeOf(editor).code);
    update(editor, () => {
      $selectCode(at, removed);
      $type(inserted);
    });
    check(`replace ${removed} at ${at} with ${JSON.stringify(inserted)}`);
  }
  cleanup();
}

function indexIn(code: string, text: string): number {
  const at = code.indexOf(text);
  if (at < 0) {
    throw new Error(`missing ${JSON.stringify(text)}`);
  }
  return at;
}
const before = (text: string) => (code: string) => indexIn(code, text);
const after = (text: string) => (code: string) =>
  indexIn(code, text) + text.length;

const TS_LINES = [
  'const a = call(1, "row"); // note',
  'let b = `two`;',
  '/* first */ const c = 3;',
  'function run() {',
  '  return a + b;',
  '}',
];

describe('edited code blocks color exactly as a full repaint does', () => {
  const layouts = ['text node per line', 'one text node'] as const;

  it.each(layouts)(
    'TypeScript, opening and closing strings, comments, and template literals (%s)',
    async layout => {
      await expectEditsMatchFullRepaint('ts', TS_LINES, layout, [
        [after('// note'), 0, 'x'],
        [after('let b = '), 0, '"'],
        [after('let b = "'), 0, 'q'],
        [before('"q'), 2, ''],
        [before('function'), 0, '/*'],
        [after('return a + b;'), 0, ' */'],
        [before('/*function'), 2, ''],
        [after('let b = `two`'), 0, '`'],
        [before('`;'), 1, ''],
        [after('const c = 3;'), 0, ' call'],
        [before('function'), 0, '('],
        [before('(function'), 1, ''],
        [after('row'), 0, 'x'],
        [after('rowx'), 0, '\\'],
        [before('1, "row'), 1, '42'],
        [after('call(42'), 0, '\n'],
        [after('return a'), 0, 'x'],
      ]);
    },
  );

  it.each(layouts)(
    'Python, opening and closing a triple-quoted string (%s)',
    async layout => {
      await expectEditsMatchFullRepaint(
        'python',
        ['def run():', '  x = call(1)', '  return x', 'y = 2'],
        layout,
        [
          [after('x = '), 0, '"""'],
          [after('return x'), 0, '"""'],
          [before('"""call'), 3, ''],
          [after('y = 2'), 0, ' # done'],
          [after('def run'), 0, 'ning'],
        ],
      );
    },
  );

  it.each(layouts)('shell, opening and closing ${ (%s)', async layout => {
    await expectEditsMatchFullRepaint(
      'sh',
      ['echo "row"', 'printf 1', 'exit 0'],
      layout,
      [
        [after('echo '), 0, '${value'],
        [after('printf 1'), 0, '}'],
        [before('${value'), 7, ''],
        [after('exit 0'), 0, ' # ok'],
      ],
    );
  });

  it.each(layouts)(
    'a language without the line cache re-tokenizes each edit (%s)',
    async layout => {
      await expectEditsMatchFullRepaint(
        'css',
        ['.a {', '  color: red;', '}', '/* end */'],
        layout,
        [
          [after('color: red'), 0, ' !important'],
          [before('}'), 0, '/*'],
          [after('/* end */'), 0, 'x'],
        ],
      );
    },
  );

  it('follows line breaks added and removed, and keeps later lines exact', async () => {
    const registry = stubHighlights();
    const editor = await mountEditor('```ts\nx\n```\n');
    update(editor, () => $writeLines(TS_LINES));
    const check = () => {
      const {code} = codeOf(editor);
      expect(spansIn(registry, codeElement(editor))).toEqual(
        fullRepaint(codeElement(editor), code, 'ts'),
      );
    };
    update(editor, () => {
      $selectCode(after('call(1, ')(codeOf(editor).code));
      const selection = $getSelection();
      if ($isRangeSelection(selection)) {
        selection.insertLineBreak();
      }
    });
    check();
    update(editor, () => {
      $selectCode(after('let b = ')(codeOf(editor).code));
      $type('x + ');
    });
    check();
    update(editor, () => {
      $selectCode(before('\n"row"')(codeOf(editor).code), 1);
      $type('');
    });
    check();
    update(editor, () => {
      $selectCode(after('return a')(codeOf(editor).code));
      const selection = $getSelection();
      if ($isRangeSelection(selection)) {
        selection.insertRawText(' +\n\t`more`');
      }
    });
    check();
    update(editor, () => {
      $selectCode(after('`more')(codeOf(editor).code));
      $type('!');
    });
    check();
  });

  it('recolors a block whose text nodes are replaced by others with the same text', async () => {
    const registry = stubHighlights();
    const editor = await mountEditor(
      '```ts\n' + TS_LINES.join('\n') + '\n```\n',
    );
    update(editor, () => $writeLines(TS_LINES));
    expect(painted(registry, 'keyword')).toContain('const');
    expect(spansIn(registry, codeElement(editor))).toEqual(
      fullRepaint(codeElement(editor), TS_LINES.join('\n'), 'ts'),
    );
  });

  it('follows the language and restored editor states', async () => {
    const registry = stubHighlights();
    const editor = await mountEditor('```ts\nx\n```\n');
    update(editor, () => $writeLines(TS_LINES));
    const check = () => {
      const {code, language} = codeOf(editor);
      expect(spansIn(registry, codeElement(editor))).toEqual(
        fullRepaint(codeElement(editor), code, language),
      );
    };
    const loaded = editor.getEditorState();
    update(editor, () => {
      $selectCode(after('const c = ')(codeOf(editor).code), 1);
      $type('"three"');
    });
    check();
    const edited = editor.getEditorState();
    act(() => editor.setEditorState(loaded));
    check();
    expect(painted(registry, 'number')).toContain('3');
    act(() => editor.setEditorState(edited));
    check();
    expect(painted(registry, 'string')).toContain('"three"');
    update(editor, () => $block().setLanguage('python'));
    check();
    expect(painted(registry, 'keyword')).not.toContain('const');
  });
});

interface Work {
  tokenizeCalls: number;
  linesTokenized: number;
  charactersTokenized: number;
  textNodesScanned: number;
  blockTextReads: number;
  rangesAdded: number;
}

/** Counts what `run` makes the plugin do: core tokenizer calls and the lines
 *  and characters they read, the DOM text nodes the plugin reads (through
 *  any getter or tree walk), whole-block text reads from the editor state,
 *  and ranges added. */
function measure(run: () => void): Work {
  // The plugin's own modules, not this test file's frames below them.
  const byPlugin = () =>
    /CodeSyntaxPlugin\.tsx|codeLineTokens\.ts/.test(new Error().stack ?? '');
  const scanned = new Set<Node>();
  let blockTextReads = 0;
  const visit = (node: Node) => {
    if (node.nodeType === Node.TEXT_NODE) {
      scanned.add(node);
    } else {
      node.childNodes.forEach(visit);
    }
  };
  const restores: Array<() => void> = [];
  const watch = (proto: object, name: string) => {
    const descriptor = Object.getOwnPropertyDescriptor(proto, name)!;
    Object.defineProperty(proto, name, {
      ...descriptor,
      get(this: Node) {
        if (byPlugin()) {
          visit(this);
        }
        return descriptor.get!.call(this);
      },
    });
    restores.push(() => Object.defineProperty(proto, name, descriptor));
  };
  watch(CharacterData.prototype, 'data');
  watch(CharacterData.prototype, 'length');
  watch(Node.prototype, 'nodeValue');
  watch(Node.prototype, 'textContent');
  const createTreeWalker = document.createTreeWalker;
  document.createTreeWalker = function (
    this: Document,
    ...args: Parameters<Document['createTreeWalker']>
  ) {
    const walker = createTreeWalker.apply(this, args);
    const nextNode = walker.nextNode.bind(walker);
    walker.nextNode = () => {
      const node = nextNode();
      if (node != null && byPlugin()) {
        visit(node);
      }
      return node;
    };
    return walker;
  };
  restores.push(() => {
    document.createTreeWalker = createTreeWalker;
  });
  const getTextContent = ElementNode.prototype.getTextContent;
  ElementNode.prototype.getTextContent = function (this: ElementNode) {
    if ($isCodeNode(this) && byPlugin()) {
      blockTextReads++;
    }
    return getTextContent.call(this);
  };
  restores.push(() => {
    ElementNode.prototype.getTextContent = getTextContent;
  });
  const stackTraceLimit = Error.stackTraceLimit;
  Error.stackTraceLimit = 100;
  restores.push(() => {
    Error.stackTraceLimit = stackTraceLimit;
  });
  const tokenizer = vi.mocked(tokenize);
  tokenizer.mockClear();
  rangesAdded = 0;
  try {
    run();
  } finally {
    restores.reverse().forEach(restore => restore());
  }
  const inputs = tokenizer.mock.calls.map(([text]) => text);
  return {
    tokenizeCalls: inputs.length,
    linesTokenized: inputs.reduce(
      (sum, text) => sum + text.split('\n').length,
      0,
    ),
    charactersTokenized: inputs.reduce((sum, text) => sum + text.length, 0),
    textNodesScanned: scanned.size,
    blockTextReads,
    rangesAdded,
  };
}

const LINE = 'const a = 1;';

/** The work one character typed at the end of the middle line costs in a
 *  TypeScript block of `lines` copies of LINE. */
async function typingWork(
  lines: number,
  layout: 'text node per line' | 'one text node',
): Promise<Work> {
  stubHighlights();
  const source = Array.from({length: lines}, () => LINE);
  const editor = await mountEditor('```ts\n' + source.join('\n') + '\n```\n');
  if (layout === 'text node per line') {
    update(editor, () => $writeLines(source));
  }
  const middle = Math.floor(lines / 2);
  update(editor, () => $selectCode(middle * (LINE.length + 1) + LINE.length));
  const work = measure(() => update(editor, () => $type('x')));
  cleanup();
  return work;
}

describe('typing in a long code block re-tokenizes only what it changed', () => {
  // The typed line now ends in a name, which is a function name if the next
  // visible character is `(`, so the next line's first character is
  // tokenized with it: two lines, 15 characters.
  const ONE_LINE = {
    tokenizeCalls: 1,
    linesTokenized: 2,
    charactersTokenized: LINE.length + 3,
    textNodesScanned: 1,
    blockTextReads: 0,
  };

  it('costs the same in 200 and 2,000 lines when each line is a text node', async () => {
    const small = await typingWork(200, 'text node per line');
    const large = await typingWork(2000, 'text node per line');
    expect(large).toEqual(small);
    // The typed line's four tokens: `const`, `=`, `1`, `;`.
    expect(large).toEqual({...ONE_LINE, rangesAdded: 4});
  }, 60000);

  it('tokenizes and reads the same in 200 and 2,000 lines imported as one text node', async () => {
    const small = await typingWork(200, 'one text node');
    const large = await typingWork(2000, 'one text node');
    // Lexical rewrites that one text node whole, so every range in it is
    // replaced; nothing is tokenized or read again.
    expect(small).toEqual({...ONE_LINE, rangesAdded: 200 * 4});
    expect(large).toEqual({...ONE_LINE, rangesAdded: 2000 * 4});
  }, 60000);
});