Skip to content

Visualize Your Ideas With Code

feat(2d): simplify highlighters (#1002)

Jacob8656336ea992
diff --git a/packages/2d/src/lib/code/CodeHighlighter.ts b/packages/2d/src/lib/code/CodeHighlighter.ts
index 5ec1fbda..7c9f9ed6 100644
--- a/packages/2d/src/lib/code/CodeHighlighter.ts
+++ b/packages/2d/src/lib/code/CodeHighlighter.ts
@@ -53,9 +53,8 @@ export interface CodeHighlighter<T = unknown> {
* highlighted.
*
* @param code - The code to prepare.
- * @param dialect - The language in which the code is written.
*/
- prepare(code: string, dialect: string): T;
+ prepare(code: string): T;
/**
* Highlights the code at the given index.
@@ -69,7 +68,6 @@ export interface CodeHighlighter<T = unknown> {
* Tokenize the code.
*
* @param code - The code to tokenize.
- * @param dialect - The language in which the code is written.
*/
- tokenize(code: string, dialect: string): string[];
+ tokenize(code: string): string[];
}
diff --git a/packages/2d/src/lib/code/CodeSignal.ts b/packages/2d/src/lib/code/CodeSignal.ts
index 1b7f4238..f0d77edc 100644
--- a/packages/2d/src/lib/code/CodeSignal.ts
+++ b/packages/2d/src/lib/code/CodeSignal.ts
@@ -80,13 +80,11 @@ export class CodeSignalContext<TOwner>
public constructor(
initial: SignalValue<PossibleCodeScope>,
owner: TOwner,
- private readonly highlighter?: SignalValue<CodeHighlighter>,
- private readonly dialect?: SignalValue<string>,
+ private readonly highlighter?: SignalValue<CodeHighlighter | null>,
) {
super(initial, deepLerp, owner);
if (owner instanceof Code) {
this.highlighter ??= owner.highlighter;
- this.dialect ??= owner.dialect;
}
Object.defineProperty(this.invokable, 'edit', {
value: this.edit.bind(this),
@@ -115,15 +113,14 @@ export class CodeSignalContext<TOwner>
): ThreadGenerator {
let tokenize = defaultTokenize;
const highlighter = unwrap(this.highlighter);
- const dialect = unwrap(this.dialect);
- if (highlighter && dialect) {
+ if (highlighter) {
yield (async () => {
do {
await DependencyContext.consumePromises();
highlighter.initialize();
} while (DependencyContext.hasPromises());
})();
- tokenize = (input: string) => highlighter.tokenize(input, dialect);
+ tokenize = (input: string) => highlighter.tokenize(input);
}
this.progress(0);
diff --git a/packages/2d/src/lib/code/LezerHighlighter.ts b/packages/2d/src/lib/code/LezerHighlighter.ts
index 92f6f133..a98db5bf 100644
--- a/packages/2d/src/lib/code/LezerHighlighter.ts
+++ b/packages/2d/src/lib/code/LezerHighlighter.ts
@@ -1,9 +1,8 @@
import {HighlightStyle} from '@codemirror/language';
import {Parser, SyntaxNode, Tree} from '@lezer/common';
import {highlightTree} from '@lezer/highlight';
-import {useLogger} from '@motion-canvas/core';
import {CodeHighlighter, HighlightResult} from './CodeHighlighter';
-import {defaultTokenize} from './CodeTokenizer';
+import {DefaultHighlightStyle} from './DefaultHighlightStyle';
interface LezerCache {
tree: Tree;
@@ -12,15 +11,13 @@ interface LezerCache {
}
export class LezerHighlighter implements CodeHighlighter<LezerCache | null> {
- private static readonly parserMap = new Map<string, Parser>();
private static classRegex = /\.(\S+).*color:([^;]+)/;
private readonly classLookup = new Map<string, string>();
- public static registerParser(parser: Parser, dialect = ''): void {
- this.parserMap.set(dialect, parser);
- }
-
- public constructor(private readonly style: HighlightStyle) {
+ public constructor(
+ private readonly parser: Parser,
+ private readonly style: HighlightStyle = DefaultHighlightStyle,
+ ) {
for (const rule of this.style.module?.getRules().split('\n') ?? []) {
const match = rule.match(LezerHighlighter.classRegex);
if (!match) {
@@ -37,17 +34,9 @@ export class LezerHighlighter implements CodeHighlighter<LezerCache | null> {
return true;
}
- public prepare(code: string, dialect: string): LezerCache | null {
- const parser = LezerHighlighter.parserMap.get(dialect);
- if (!parser) {
- if (dialect !== '') {
- useLogger().warn(`No parser found for dialect: ${dialect}`);
- }
- return null;
- }
-
+ public prepare(code: string): LezerCache | null {
const colorLookup = new Map<string, string>();
- const tree = parser.parse(code);
+ const tree = this.parser.parse(code);
highlightTree(tree, this.style, (from, to, classes) => {
const color = this.classLookup.get(classes);
if (!color) {
@@ -97,16 +86,8 @@ export class LezerHighlighter implements CodeHighlighter<LezerCache | null> {
};
}
- public tokenize(code: string, dialect: string): string[] {
- const parser = LezerHighlighter.parserMap.get(dialect);
- if (!parser) {
- if (dialect !== '') {
- useLogger().warn(`No parser found for dialect: ${dialect}`);
- }
- return defaultTokenize(code);
- }
-
- const tree = parser.parse(code);
+ public tokenize(code: string): string[] {
+ const tree = this.parser.parse(code);
const cursor = tree.cursor();
const tokens: string[] = [];
let current = 0;
diff --git a/packages/2d/src/lib/components/Code.ts b/packages/2d/src/lib/components/Code.ts
index a625c518..f08ce1f1 100644
--- a/packages/2d/src/lib/components/Code.ts
+++ b/packages/2d/src/lib/components/Code.ts
@@ -2,7 +2,6 @@ import {
BBox,
createSignal,
experimentalLog,
- lazy,
map,
SerializedVector2,
Signal,
@@ -25,10 +24,8 @@ import {
CodeSignal,
codeSignal,
CodeSignalContext,
- DefaultHighlightStyle,
findAllCodeRanges,
isPointInCodeSelection,
- LezerHighlighter,
lines,
parseCodeSelection,
PossibleCodeScope,
@@ -81,10 +78,6 @@ export interface CodeProps extends ShapeProps {
* {@inheritDoc Code.highlighter}
*/
highlighter?: SignalValue<CodeHighlighter | null>;
- /**
- * {@inheritDoc Code.dialect}
- */
- dialect?: SignalValue<string>;
/**
* {@inheritDoc Code.code}
*/
@@ -145,58 +138,19 @@ export class Code extends Shape {
*
* @param initial - The initial code.
* @param highlighter - Custom highlighter to use.
- * @param dialect - Custom dialect to use.
*/
public static createSignal(
initial: SignalValue<PossibleCodeScope>,
highlighter?: SignalValue<CodeHighlighter>,
- dialect?: SignalValue<string>,
): CodeSignal<void> {
return new CodeSignalContext<void>(
initial,
undefined,
highlighter,
- dialect,
).toSignal();
}
- @lazy(() => new LezerHighlighter(DefaultHighlightStyle))
- public static readonly defaultHighlighter: LezerHighlighter;
-
- /**
- * The dialect to use for highlighting the code.
- *
- * @remarks
- * This value will be passed to the {@link code.CodeHighlighter}
- * defined by the {@link highlighter} property. Different highlighters may use
- * it differently.
- *
- * The default {@link code.LezerHighlighter} uses it to select
- * the language parser to use. The parser for the given dialect can be
- * registered as follows:
- * ```tsx
- * // Import the lezer parser:
- * import {parser} from '@lezer/javascript';
- *
- * // Register it in the highlighter:
- * LezerHighlighter.registerParser(parser, 'js');
- *
- * // Use the dialect in a code node:
- * <Code dialect="js" code="const a = 7;" />
- * ```
- * When no dialect is provided, the highlighter will use the default
- * parser:
- * ```tsx
- * // Register the default parser by omitting the dialect:
- * LezerHighlighter.registerParser(parser);
- *
- * // Code nodes with no dialect will now use the default parser:
- * <Code code="const a = 7;" />
- * ```
- */
- @initial('')
- @signal()
- public declare readonly dialect: SimpleSignal<string, this>;
+ public static defaultHighlighter: CodeHighlighter | null = null;
/**
* The code highlighter to use for this code node.
@@ -206,7 +160,10 @@ export class Code extends Shape {
*/
@initial(() => Code.defaultHighlighter)
@signal()
- public declare readonly highlighter: SimpleSignal<CodeHighlighter, this>;
+ public declare readonly highlighter: SimpleSignal<
+ CodeHighlighter | null,
+ this
+ >;
/**
* The code to display.
@@ -327,11 +284,10 @@ export class Code extends Shape {
const code = this.code();
const before = resolveScope(code, false);
const after = resolveScope(code, true);
- const dialect = this.dialect();
return {
- before: highlighter.prepare(before, dialect),
- after: highlighter.prepare(after, dialect),
+ before: highlighter.prepare(before),
+ after: highlighter.prepare(after),
};
}
@@ -360,7 +316,6 @@ export class Code extends Shape {
initial,
this,
this.highlighter,
- this.dialect,
).toSignal();
}
diff --git a/packages/docs/docs/components/code/index.mdx b/packages/docs/docs/components/code/index.mdx
index b351bdb6..81899bb9 100644
--- a/packages/docs/docs/components/code/index.mdx
+++ b/packages/docs/docs/components/code/index.mdx
@@ -61,18 +61,18 @@ you should install the `javascript` grammar:
npm i @lezer/javascript
```
-Then, in your project configuration, import the grammar and register it via
-`LezerHighlighter`:
+Then, in your project configuration, instantiate a new `LezerHighlighter` using
+the imported grammar, and set it as the default highlighter:
```ts title="src/project.ts"
import {makeProject} from '@motion-canvas/core';
import example from './scenes/example?scene';
// highlight-start
-import {LezerHighlighter} from '@motion-canvas/2d';
+import {Code, LezerHighlighter} from '@motion-canvas/2d';
import {parser} from '@lezer/javascript';
-LezerHighlighter.registerParser(parser);
+Code.defaultHighlighter = new LezerHighlighter(parser);
// highlight-end
export default makeProject({
@@ -80,10 +80,28 @@ export default makeProject({
});
```
-This sets `@lezer/javascript` up as the default parser for all your code
-snippets. If you want to use more than one language, check out the
+Now all `Code` nodes in your project will use `@lezer/javascript` to parse and
+highlight the snippets. If you want to use more than one language, check out the
[Multiple Languages](#multiple-languages) section.
+:::info
+
+Note that, by default, the JavaScript parser doesn't support JSX or TypeScript.
+You can enable support for these via [dialects][dialects]. The dialects
+available for a given parser are usually listed in the documentation of the
+grammar package.
+
+```ts
+Code.defaultHighlighter = new LezerHighlighter(
+ parser.configure({
+ // Provide a space-separated list of dialects to enable:
+ dialect: 'jsx ts',
+ }),
+);
+```
+
+:::
+
## Defining Code
The code to display is set via the [`code`](/api/2d/components/Code#code)
@@ -130,9 +148,9 @@ Notice two things here:
or tabs at the beginning of each line would be included in the snippet.
- The backslash character (`\`) at the very beginning is used to escape the
- first newline character. This lets us start the snippet on a new line without
- actually including an empty line at the start of the snippet. Without the
- slash, the equivalent code would have to be written as:
+ first newline character. This lets the snippet start on a new line without
+ actually including an empty line at the beginning. Without the slash, the
+ equivalent code would have to be written as:
```tsx
view.add(
<Code
@@ -149,8 +167,8 @@ Notice two things here:
```
Template strings allow you to easily include variables in your code snippets
-with `${}` syntax. In the example below, `${name}` is replaced with the value of
-the `name` variable (which is `number` in this case):
+with the `${}` syntax. In the example below, `${name}` is replaced with the
+value of the `name` variable (which is `number` in this case):
```tsx
// highlight-next-line
@@ -724,61 +742,105 @@ function hello() {
});
```
-## Multiple Languages
+## Custom Themes
-You can register parsers for multiple different languages by giving them a
-unique name during registration:
+`LezerHighlighter` uses CodeMirror's
+[`HighlightStyle`](https://codemirror.net/examples/styling/) to assign colors to
+specific code tokens. By default, the
+[`DefaultHighlightStyle`](/api/2d/code#DefaultHighlightStyle) is used. You can
+specify your own style by passing it as the second argument to the
+`LezerHighlighter` constructor:
```ts
-import {LezerHighlighter} from '@motion-canvas/2d';
-import {parser as jsParser} from '@lezer/javascript';
-// highlight-next-line
-import {parser as rsParser} from '@lezer/rust';
+import {Code, LezerHighlighter} from '@motion-canvas/2d';
+import {HighlightStyle} from '@codemirror/language';
+import {tags} from '@lezer/highlight';
+import {parser} from '@lezer/javascript';
-LezerHighlighter.registerParser(jsParser);
-// highlight-next-line
-LezerHighlighter.registerParser(rsParser, 'rs');
+const MyStyle = HighlightStyle.define([
+ {tag: tags.keyword, color: 'red'},
+ {tag: tags.function(tags.variableName), color: 'yellow'},
+ {tag: tags.number, color: 'blue'},
+ {tag: tags.string, color: 'green'},
+ // ...
+]);
+
+Code.defaultHighlighter = new LezerHighlighter(parser, MyStyle);
```
-This name can then be passed to the [`dialect`](/api/2d/components/Code#dialect)
-property of a specific `Code` node to use a different parser:
+## Multiple Languages
-```tsx editor mode=code
-import {makeScene2D, Code} from '@motion-canvas/2d';
+You can configure highlighters on a per-node basis using the
+[`highlighter`](/api/2d/components/Code#highlighter) property. This will
+override the default highlighter set in the project configuration file:
-export default makeScene2D(function* (view) {
- // Font properties configured for the view are
- // inherited by all text nodes, including Code:
- view.fontSize(28).fontFamily('JetBrains Mono');
+```tsx
+import {Code, LezerHighlighter} from '@motion-canvas/2d';
+import {parser} from '@lezer/rust';
- view.add(
- <>
- <Code
- // this node uses the default parser (javascript)
- offsetX={-1}
- x={-400}
- code={`
+const RustHighlighter = new LezerHighlighter(parser);
+
+// ...
+
+view.add(
+ <Code
+ // this node uses the default parser
+ offsetX={-1}
+ x={-400}
+ code={`
function hello() {
console.log('Hello!');
}
`}
- />
- <Code
- // this node uses the "rs" parser (rust)
- dialect={'rs'}
- offsetX={1}
- x={400}
- code={`
+ />,
+);
+
+view.add(
+ <Code
+ // this node uses the Rust parser
+ highlighter={RustHighlighter}
+ offsetX={1}
+ x={400}
+ code={`
fn hello() {
println!("Hello!");
}
`}
- />
- </>,
- );
+ />,
+);
+```
+
+It can be useful to create a custom component for the languages you often use.
+You can use the [`withDefaults`](/api/2d/utils#withDefaults) helper function to
+quickly extend any node with your own defaults:
+
+```ts title="src/nodes/RustCode.ts"
+import {Code, LezerHighlighter, withDefaults} from '@motion-canvas/2d';
+import {parser} from '@lezer/rust';
+
+const RustHighlighter = new LezerHighlighter(parser);
+
+export const RustCode = withDefaults(Code, {
+ highlighter: RustHighlighter,
});
```
+```tsx title="src/scenes/example.tsx"
+import {RustCode} from '../nodes/RustCode';
+
+// ...
+
+view.add(
+ <RustCode
+ code={`
+fn hello() {
+ println!("Hello!");
+}
+`}
+ />,
+);
+```
+
## Rendering
If you're using a Chromium-based browser, you might notice that the code
@@ -788,3 +850,4 @@ to fix this, we recommend you use Firefox for rendering.
[tag-function]:
https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Template_literals#tagged_templates
+[dialects]: https://lezer.codemirror.net/docs/guide/#dialects
diff --git a/packages/docs/src/components/Fiddle/SharedPlayer.ts b/packages/docs/src/components/Fiddle/SharedPlayer.ts
index 4cc02a43..04b4220a 100644
--- a/packages/docs/src/components/Fiddle/SharedPlayer.ts
+++ b/packages/docs/src/components/Fiddle/SharedPlayer.ts
@@ -1,6 +1,5 @@
import ExecutionEnvironment from '@docusaurus/ExecutionEnvironment';
import {parser as javascript} from '@lezer/javascript';
-import {parser as rust} from '@lezer/rust';
import type {View2D} from '@motion-canvas/2d';
import type {
FullSceneDescription,
@@ -120,11 +119,14 @@ export async function borrowPlayer(
ValueDispatcher,
DefaultPlugin,
} = await import(/* webpackIgnore: true */ '@motion-canvas/core');
- const {makeScene2D, LezerHighlighter} = await import(
+ const {makeScene2D, Code, LezerHighlighter} = await import(
/* webpackIgnore: true */ '@motion-canvas/2d'
);
- LezerHighlighter.registerParser(javascript);
- LezerHighlighter.registerParser(rust, 'rs');
+ Code.defaultHighlighter = new LezerHighlighter(
+ javascript.configure({
+ dialect: 'jsx ts',
+ }),
+ );
Description = makeScene2D(function* () {
yield;