Visualize Your Ideas With Code
feat(2d): simplify highlighters (#1002)
Jacob8656336ea992
diff --git a/packages/2d/src/lib/code/CodeHighlighter.ts b/packages/2d/src/lib/code/CodeHighlighter.tsindex 5ec1fbda..7c9f9ed6 100644--- a/packages/2d/src/lib/code/CodeHighlighter.ts+++ b/packages/2d/src/lib/code/CodeHighlighter.ts@@ -53,9 +53,8 @@ export interface CodeHighlighter<T = unknown> {* highlighted.** @param code - The code to prepare.- * @param dialect - The language in which the code is written.*/- prepare(code: string, dialect: string): T;+ prepare(code: string): T;/*** Highlights the code at the given index.@@ -69,7 +68,6 @@ export interface CodeHighlighter<T = unknown> {* Tokenize the code.** @param code - The code to tokenize.- * @param dialect - The language in which the code is written.*/- tokenize(code: string, dialect: string): string[];+ tokenize(code: string): string[];}diff --git a/packages/2d/src/lib/code/CodeSignal.ts b/packages/2d/src/lib/code/CodeSignal.tsindex 1b7f4238..f0d77edc 100644--- a/packages/2d/src/lib/code/CodeSignal.ts+++ b/packages/2d/src/lib/code/CodeSignal.ts@@ -80,13 +80,11 @@ export class CodeSignalContext<TOwner>public constructor(initial: SignalValue<PossibleCodeScope>,owner: TOwner,- private readonly highlighter?: SignalValue<CodeHighlighter>,- private readonly dialect?: SignalValue<string>,+ private readonly highlighter?: SignalValue<CodeHighlighter | null>,) {super(initial, deepLerp, owner);if (owner instanceof Code) {this.highlighter ??= owner.highlighter;- this.dialect ??= owner.dialect;}Object.defineProperty(this.invokable, 'edit', {value: this.edit.bind(this),@@ -115,15 +113,14 @@ export class CodeSignalContext<TOwner>): ThreadGenerator {let tokenize = defaultTokenize;const highlighter = unwrap(this.highlighter);- const dialect = unwrap(this.dialect);- if (highlighter && dialect) {+ if (highlighter) {yield (async () => {do {await DependencyContext.consumePromises();highlighter.initialize();} while (DependencyContext.hasPromises());})();- tokenize = (input: string) => highlighter.tokenize(input, dialect);+ tokenize = (input: string) => highlighter.tokenize(input);}this.progress(0);diff --git a/packages/2d/src/lib/code/LezerHighlighter.ts b/packages/2d/src/lib/code/LezerHighlighter.tsindex 92f6f133..a98db5bf 100644--- a/packages/2d/src/lib/code/LezerHighlighter.ts+++ b/packages/2d/src/lib/code/LezerHighlighter.ts@@ -1,9 +1,8 @@import {HighlightStyle} from '@codemirror/language';import {Parser, SyntaxNode, Tree} from '@lezer/common';import {highlightTree} from '@lezer/highlight';-import {useLogger} from '@motion-canvas/core';import {CodeHighlighter, HighlightResult} from './CodeHighlighter';-import {defaultTokenize} from './CodeTokenizer';+import {DefaultHighlightStyle} from './DefaultHighlightStyle';interface LezerCache {tree: Tree;@@ -12,15 +11,13 @@ interface LezerCache {}export class LezerHighlighter implements CodeHighlighter<LezerCache | null> {- private static readonly parserMap = new Map<string, Parser>();private static classRegex = /\.(\S+).*color:([^;]+)/;private readonly classLookup = new Map<string, string>();- public static registerParser(parser: Parser, dialect = ''): void {- this.parserMap.set(dialect, parser);- }-- public constructor(private readonly style: HighlightStyle) {+ public constructor(+ private readonly parser: Parser,+ private readonly style: HighlightStyle = DefaultHighlightStyle,+ ) {for (const rule of this.style.module?.getRules().split('\n') ?? []) {const match = rule.match(LezerHighlighter.classRegex);if (!match) {@@ -37,17 +34,9 @@ export class LezerHighlighter implements CodeHighlighter<LezerCache | null> {return true;}- public prepare(code: string, dialect: string): LezerCache | null {- const parser = LezerHighlighter.parserMap.get(dialect);- if (!parser) {- if (dialect !== '') {- useLogger().warn(`No parser found for dialect: ${dialect}`);- }- return null;- }-+ public prepare(code: string): LezerCache | null {const colorLookup = new Map<string, string>();- const tree = parser.parse(code);+ const tree = this.parser.parse(code);highlightTree(tree, this.style, (from, to, classes) => {const color = this.classLookup.get(classes);if (!color) {@@ -97,16 +86,8 @@ export class LezerHighlighter implements CodeHighlighter<LezerCache | null> {};}- public tokenize(code: string, dialect: string): string[] {- const parser = LezerHighlighter.parserMap.get(dialect);- if (!parser) {- if (dialect !== '') {- useLogger().warn(`No parser found for dialect: ${dialect}`);- }- return defaultTokenize(code);- }-- const tree = parser.parse(code);+ public tokenize(code: string): string[] {+ const tree = this.parser.parse(code);const cursor = tree.cursor();const tokens: string[] = [];let current = 0;diff --git a/packages/2d/src/lib/components/Code.ts b/packages/2d/src/lib/components/Code.tsindex a625c518..f08ce1f1 100644--- a/packages/2d/src/lib/components/Code.ts+++ b/packages/2d/src/lib/components/Code.ts@@ -2,7 +2,6 @@ import {BBox,createSignal,experimentalLog,- lazy,map,SerializedVector2,Signal,@@ -25,10 +24,8 @@ import {CodeSignal,codeSignal,CodeSignalContext,- DefaultHighlightStyle,findAllCodeRanges,isPointInCodeSelection,- LezerHighlighter,lines,parseCodeSelection,PossibleCodeScope,@@ -81,10 +78,6 @@ export interface CodeProps extends ShapeProps {* {@inheritDoc Code.highlighter}*/highlighter?: SignalValue<CodeHighlighter | null>;- /**- * {@inheritDoc Code.dialect}- */- dialect?: SignalValue<string>;/*** {@inheritDoc Code.code}*/@@ -145,58 +138,19 @@ export class Code extends Shape {** @param initial - The initial code.* @param highlighter - Custom highlighter to use.- * @param dialect - Custom dialect to use.*/public static createSignal(initial: SignalValue<PossibleCodeScope>,highlighter?: SignalValue<CodeHighlighter>,- dialect?: SignalValue<string>,): CodeSignal<void> {return new CodeSignalContext<void>(initial,undefined,highlighter,- dialect,).toSignal();}- @lazy(() => new LezerHighlighter(DefaultHighlightStyle))- public static readonly defaultHighlighter: LezerHighlighter;-- /**- * The dialect to use for highlighting the code.- *- * @remarks- * This value will be passed to the {@link code.CodeHighlighter}- * defined by the {@link highlighter} property. Different highlighters may use- * it differently.- *- * The default {@link code.LezerHighlighter} uses it to select- * the language parser to use. The parser for the given dialect can be- * registered as follows:- * ```tsx- * // Import the lezer parser:- * import {parser} from '@lezer/javascript';- *- * // Register it in the highlighter:- * LezerHighlighter.registerParser(parser, 'js');- *- * // Use the dialect in a code node:- * <Code dialect="js" code="const a = 7;" />- * ```- * When no dialect is provided, the highlighter will use the default- * parser:- * ```tsx- * // Register the default parser by omitting the dialect:- * LezerHighlighter.registerParser(parser);- *- * // Code nodes with no dialect will now use the default parser:- * <Code code="const a = 7;" />- * ```- */- @initial('')- @signal()- public declare readonly dialect: SimpleSignal<string, this>;+ public static defaultHighlighter: CodeHighlighter | null = null;/*** The code highlighter to use for this code node.@@ -206,7 +160,10 @@ export class Code extends Shape {*/@initial(() => Code.defaultHighlighter)@signal()- public declare readonly highlighter: SimpleSignal<CodeHighlighter, this>;+ public declare readonly highlighter: SimpleSignal<+ CodeHighlighter | null,+ this+ >;/*** The code to display.@@ -327,11 +284,10 @@ export class Code extends Shape {const code = this.code();const before = resolveScope(code, false);const after = resolveScope(code, true);- const dialect = this.dialect();return {- before: highlighter.prepare(before, dialect),- after: highlighter.prepare(after, dialect),+ before: highlighter.prepare(before),+ after: highlighter.prepare(after),};}@@ -360,7 +316,6 @@ export class Code extends Shape {initial,this,this.highlighter,- this.dialect,).toSignal();}diff --git a/packages/docs/docs/components/code/index.mdx b/packages/docs/docs/components/code/index.mdxindex b351bdb6..81899bb9 100644--- a/packages/docs/docs/components/code/index.mdx+++ b/packages/docs/docs/components/code/index.mdx@@ -61,18 +61,18 @@ you should install the `javascript` grammar:npm i @lezer/javascript```-Then, in your project configuration, import the grammar and register it via-`LezerHighlighter`:+Then, in your project configuration, instantiate a new `LezerHighlighter` using+the imported grammar, and set it as the default highlighter:```ts title="src/project.ts"import {makeProject} from '@motion-canvas/core';import example from './scenes/example?scene';// highlight-start-import {LezerHighlighter} from '@motion-canvas/2d';+import {Code, LezerHighlighter} from '@motion-canvas/2d';import {parser} from '@lezer/javascript';-LezerHighlighter.registerParser(parser);+Code.defaultHighlighter = new LezerHighlighter(parser);// highlight-endexport default makeProject({@@ -80,10 +80,28 @@ export default makeProject({});```-This sets `@lezer/javascript` up as the default parser for all your code-snippets. If you want to use more than one language, check out the+Now all `Code` nodes in your project will use `@lezer/javascript` to parse and+highlight the snippets. If you want to use more than one language, check out the[Multiple Languages](#multiple-languages) section.+:::info++Note that, by default, the JavaScript parser doesn't support JSX or TypeScript.+You can enable support for these via [dialects][dialects]. The dialects+available for a given parser are usually listed in the documentation of the+grammar package.++```ts+Code.defaultHighlighter = new LezerHighlighter(+ parser.configure({+ // Provide a space-separated list of dialects to enable:+ dialect: 'jsx ts',+ }),+);+```++:::+## Defining CodeThe code to display is set via the [`code`](/api/2d/components/Code#code)@@ -130,9 +148,9 @@ Notice two things here:or tabs at the beginning of each line would be included in the snippet.- The backslash character (`\`) at the very beginning is used to escape the- first newline character. This lets us start the snippet on a new line without- actually including an empty line at the start of the snippet. Without the- slash, the equivalent code would have to be written as:+ first newline character. This lets the snippet start on a new line without+ actually including an empty line at the beginning. Without the slash, the+ equivalent code would have to be written as:```tsxview.add(<Code@@ -149,8 +167,8 @@ Notice two things here:```Template strings allow you to easily include variables in your code snippets-with `${}` syntax. In the example below, `${name}` is replaced with the value of-the `name` variable (which is `number` in this case):+with the `${}` syntax. In the example below, `${name}` is replaced with the+value of the `name` variable (which is `number` in this case):```tsx// highlight-next-line@@ -724,61 +742,105 @@ function hello() {});```-## Multiple Languages+## Custom Themes-You can register parsers for multiple different languages by giving them a-unique name during registration:+`LezerHighlighter` uses CodeMirror's+[`HighlightStyle`](https://codemirror.net/examples/styling/) to assign colors to+specific code tokens. By default, the+[`DefaultHighlightStyle`](/api/2d/code#DefaultHighlightStyle) is used. You can+specify your own style by passing it as the second argument to the+`LezerHighlighter` constructor:```ts-import {LezerHighlighter} from '@motion-canvas/2d';-import {parser as jsParser} from '@lezer/javascript';-// highlight-next-line-import {parser as rsParser} from '@lezer/rust';+import {Code, LezerHighlighter} from '@motion-canvas/2d';+import {HighlightStyle} from '@codemirror/language';+import {tags} from '@lezer/highlight';+import {parser} from '@lezer/javascript';-LezerHighlighter.registerParser(jsParser);-// highlight-next-line-LezerHighlighter.registerParser(rsParser, 'rs');+const MyStyle = HighlightStyle.define([+ {tag: tags.keyword, color: 'red'},+ {tag: tags.function(tags.variableName), color: 'yellow'},+ {tag: tags.number, color: 'blue'},+ {tag: tags.string, color: 'green'},+ // ...+]);++Code.defaultHighlighter = new LezerHighlighter(parser, MyStyle);```-This name can then be passed to the [`dialect`](/api/2d/components/Code#dialect)-property of a specific `Code` node to use a different parser:+## Multiple Languages-```tsx editor mode=code-import {makeScene2D, Code} from '@motion-canvas/2d';+You can configure highlighters on a per-node basis using the+[`highlighter`](/api/2d/components/Code#highlighter) property. This will+override the default highlighter set in the project configuration file:-export default makeScene2D(function* (view) {- // Font properties configured for the view are- // inherited by all text nodes, including Code:- view.fontSize(28).fontFamily('JetBrains Mono');+```tsx+import {Code, LezerHighlighter} from '@motion-canvas/2d';+import {parser} from '@lezer/rust';- view.add(- <>- <Code- // this node uses the default parser (javascript)- offsetX={-1}- x={-400}- code={`+const RustHighlighter = new LezerHighlighter(parser);++// ...++view.add(+ <Code+ // this node uses the default parser+ offsetX={-1}+ x={-400}+ code={`function hello() {console.log('Hello!');}`}- />- <Code- // this node uses the "rs" parser (rust)- dialect={'rs'}- offsetX={1}- x={400}- code={`+ />,+);++view.add(+ <Code+ // this node uses the Rust parser+ highlighter={RustHighlighter}+ offsetX={1}+ x={400}+ code={`fn hello() {println!("Hello!");}`}- />- </>,- );+ />,+);+```++It can be useful to create a custom component for the languages you often use.+You can use the [`withDefaults`](/api/2d/utils#withDefaults) helper function to+quickly extend any node with your own defaults:++```ts title="src/nodes/RustCode.ts"+import {Code, LezerHighlighter, withDefaults} from '@motion-canvas/2d';+import {parser} from '@lezer/rust';++const RustHighlighter = new LezerHighlighter(parser);++export const RustCode = withDefaults(Code, {+ highlighter: RustHighlighter,});```+```tsx title="src/scenes/example.tsx"+import {RustCode} from '../nodes/RustCode';++// ...++view.add(+ <RustCode+ code={`+fn hello() {+ println!("Hello!");+}+`}+ />,+);+```+## RenderingIf you're using a Chromium-based browser, you might notice that the code@@ -788,3 +850,4 @@ to fix this, we recommend you use Firefox for rendering.[tag-function]:https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Template_literals#tagged_templates+[dialects]: https://lezer.codemirror.net/docs/guide/#dialectsdiff --git a/packages/docs/src/components/Fiddle/SharedPlayer.ts b/packages/docs/src/components/Fiddle/SharedPlayer.tsindex 4cc02a43..04b4220a 100644--- a/packages/docs/src/components/Fiddle/SharedPlayer.ts+++ b/packages/docs/src/components/Fiddle/SharedPlayer.ts@@ -1,6 +1,5 @@import ExecutionEnvironment from '@docusaurus/ExecutionEnvironment';import {parser as javascript} from '@lezer/javascript';-import {parser as rust} from '@lezer/rust';import type {View2D} from '@motion-canvas/2d';import type {FullSceneDescription,@@ -120,11 +119,14 @@ export async function borrowPlayer(ValueDispatcher,DefaultPlugin,} = await import(/* webpackIgnore: true */ '@motion-canvas/core');- const {makeScene2D, LezerHighlighter} = await import(+ const {makeScene2D, Code, LezerHighlighter} = await import(/* webpackIgnore: true */ '@motion-canvas/2d');- LezerHighlighter.registerParser(javascript);- LezerHighlighter.registerParser(rust, 'rs');+ Code.defaultHighlighter = new LezerHighlighter(+ javascript.configure({+ dialect: 'jsx ts',+ }),+ );Description = makeScene2D(function* () {yield;