Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

feat(CodeBlock, cli): deprecate tokenizer in favour of a custom language value; astryx upgrade rewrites it (#8490)

Cindy Zhang79a7a902b8b4
diff --git a/.changeset/codeblock-tokenizer-codemod.md b/.changeset/codeblock-tokenizer-codemod.md
new file mode 100644
index 0000000000..5e35aa96e8
--- /dev/null
+++ b/.changeset/codeblock-tokenizer-codemod.md
@@ -0,0 +1,8 @@
+---
+'@astryxdesign/cli': patch
+'@astryxdesign/core': patch
+---
+
+[feat] CodeBlock's `tokenizer` prop is deprecated in favour of a custom language value: pass `language={createCodeLanguage({name, identifiers, tokenize})}` with the same function. The prop still works exactly as released. `astryx upgrade` rewrites it when it sits beside a static `language`. A function literal becomes `language={createCodeLanguage({name: 'zig', identifiers: ['zig'], tokenize: (code) => …})}`. A name or call-free property becomes `language={fn ? createCodeLanguage({…, tokenize: fn}) : 'zig'}`, since it may be undefined. Uses it cannot prove are left unchanged, with a TODO comment: a computed, missing, or `plaintext` language, any other tokenizer value, or a block that spreads props.
+
+@cixzhang
diff --git a/packages/cli/assets/codemods/transforms/next/__tests__/migrate-codeblock-tokenizer-to-language.test.mjs b/packages/cli/assets/codemods/transforms/next/__tests__/migrate-codeblock-tokenizer-to-language.test.mjs
new file mode 100644
index 0000000000..ee751d22b4
--- /dev/null
+++ b/packages/cli/assets/codemods/transforms/next/__tests__/migrate-codeblock-tokenizer-to-language.test.mjs
@@ -0,0 +1,130 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+import {describe, expect, it} from 'vitest';
+import transform from '../migrate-codeblock-tokenizer-to-language.mjs';
+
+async function applyTransform(source) {
+ const jscodeshift = (await import('jscodeshift')).default;
+ const j = jscodeshift.withParser('tsx');
+ const api = {jscodeshift: j, stats: () => {}, report: () => {}};
+ return transform({source, path: 'test.tsx'}, api) ?? source;
+}
+
+/** Collapse whitespace so assertions ignore the printer's line breaks. */
+const flat = text => text.replace(/\s+/g, ' ');
+
+describe('migrate-codeblock-tokenizer-to-language', () => {
+ it('rewrites tokenizer beside a static language to a custom language', async () => {
+ const output = flat(
+ await applyTransform(`import {CodeBlock} from '@astryxdesign/core/CodeBlock';
+const a = <CodeBlock code={src} language="zig" tokenizer={zigTokens} />;
+const b = <CodeBlock code={src} language={'nim'} tokenizer={(c) => tokens(c)} />;`),
+ );
+
+ expect(output).not.toContain('tokenizer=');
+ expect(output).toContain(
+ "import { CodeBlock, createCodeLanguage } from '@astryxdesign/core/CodeBlock';",
+ );
+ expect(output).toContain(
+ "language={zigTokens ? createCodeLanguage({ name: 'zig', identifiers: ['zig'], tokenize: zigTokens }) : 'zig'}",
+ );
+ expect(output).toContain(
+ "language={createCodeLanguage({ name: 'nim', identifiers: ['nim'], tokenize: (c) => tokens(c) })}",
+ );
+ });
+
+ it('adds the import beside a root-package import and follows aliases', async () => {
+ const output = flat(
+ await applyTransform(`import {CodeBlock as Code} from '@astryxdesign/core';
+const a = <Code code={src} language="zig" tokenizer={zigTokens} />;`),
+ );
+
+ expect(output).toContain(
+ "import { createCodeLanguage } from '@astryxdesign/core/CodeBlock';",
+ );
+ expect(output).toContain("createCodeLanguage({ name: 'zig'");
+ });
+
+ it('adds a separate import beside a namespace import', async () => {
+ const output =
+ await applyTransform(`import * as CB from '@astryxdesign/core/CodeBlock';
+const a = <CB.CodeBlock code={src} language="zig" tokenizer={zigTokens} />;`);
+ expect(output).toContain(
+ "import * as CB from '@astryxdesign/core/CodeBlock';",
+ );
+ expect(flat(output)).toContain(
+ "import { createCodeLanguage } from '@astryxdesign/core/CodeBlock';",
+ );
+ const jscodeshift = (await import('jscodeshift')).default;
+ expect(() => jscodeshift.withParser('tsx')(output)).not.toThrow();
+ });
+
+ it('guards a name or property that may be undefined', async () => {
+ const output = flat(
+ await applyTransform(`import {CodeBlock} from '@astryxdesign/core/CodeBlock';
+const a = <CodeBlock code={src} language="zig" tokenizer={props.tokenizer} />;`),
+ );
+ expect(output).toContain(
+ "language={props.tokenizer ? createCodeLanguage({ name: 'zig', identifiers: ['zig'], tokenize: props.tokenizer }) : 'zig'}",
+ );
+ });
+
+ it('imports under a free alias when the file binds the name already', async () => {
+ const fromLocal = flat(
+ await applyTransform(`import {CodeBlock} from '@astryxdesign/core/CodeBlock';
+import {createCodeLanguage} from './my-languages';
+const a = <CodeBlock code={src} language="zig" tokenizer={(c) => t(c)} />;`),
+ );
+ expect(fromLocal).toContain(
+ "import { CodeBlock, createCodeLanguage as createCodeLanguage2 } from '@astryxdesign/core/CodeBlock';",
+ );
+ expect(fromLocal).toContain(
+ "import {createCodeLanguage} from './my-languages';",
+ );
+ expect(fromLocal).toContain("language={createCodeLanguage2({ name: 'zig'");
+
+ const fromFunction = flat(
+ await applyTransform(`import {CodeBlock} from '@astryxdesign/core/CodeBlock';
+function createCodeLanguage() {}
+const a = <CodeBlock code={src} language="zig" tokenizer={(c) => t(c)} />;`),
+ );
+ expect(fromFunction).toContain('createCodeLanguage as createCodeLanguage2');
+ const jscodeshift = (await import('jscodeshift')).default;
+ const parse = jscodeshift.withParser('tsx');
+ expect(() => parse(fromLocal)).not.toThrow();
+ });
+
+ it('reports a tokenizer value it cannot read twice, naming the tokenizer', async () => {
+ const output =
+ await applyTransform(`import {CodeBlock} from '@astryxdesign/core/CodeBlock';
+const a = <CodeBlock code={src} language="zig" tokenizer={registry.get('zig').tokenize} />;
+const b = <CodeBlock code={src} language="zig" tokenizer={enabled ? t : undefined} />;`);
+ expect(output.match(/this tokenizer value is computed/g)).toHaveLength(2);
+ expect(output).not.toContain('the language here is computed');
+ expect(output).toContain("tokenizer={registry.get('zig').tokenize}");
+ });
+
+ it('reports what it cannot prove and leaves it unchanged', async () => {
+ const source = `import {CodeBlock} from '@astryxdesign/core/CodeBlock';
+const a = <CodeBlock code={src} language={lang} tokenizer={t} />;
+const b = <CodeBlock code={src} tokenizer={t} />;
+const c = <CodeBlock code={src} language="plaintext" tokenizer={t} />;
+const d = <CodeBlock {...rest} code={src} language="zig" tokenizer={t} />;
+const e = <CodeBlock code={src} language="zig" tokenizer={enabled ? t : undefined} />;
+const f = <CodeBlock code={src} language="zig" tokenizer={enabled && t} />;`;
+ const output = await applyTransform(source);
+
+ expect(output.match(/TODO\(astryx upgrade\)/g)).toHaveLength(6);
+ expect(output.match(/tokenizer=\{t\}/g)).toHaveLength(4);
+ expect(output).toContain('tokenizer={enabled ? t : undefined}');
+ expect(output).toContain('tokenizer={enabled && t}');
+ expect(output).not.toMatch(/import \{[^}]*createCodeLanguage/);
+ expect(await applyTransform(output)).toBe(output);
+ });
+
+ it('leaves files without CodeBlock tokenizer alone', async () => {
+ const source = `import {CodeBlock} from '@astryxdesign/core/CodeBlock';
+const a = <CodeBlock code={src} language="ts" />;`;
+ expect(await applyTransform(source)).toBe(source);
+ });
+});
diff --git a/packages/cli/assets/codemods/transforms/next/index.mjs b/packages/cli/assets/codemods/transforms/next/index.mjs
index 2af3bdfcf0..45dead1111 100644
--- a/packages/cli/assets/codemods/transforms/next/index.mjs
+++ b/packages/cli/assets/codemods/transforms/next/index.mjs
@@ -7,4 +7,14 @@
* this file into the resolved version folder.
*/
-export default [];
+import migrateCodeBlockTokenizerToLanguage, {
+ meta as migrateCodeBlockTokenizerToLanguageMeta,
+} from './migrate-codeblock-tokenizer-to-language.mjs';
+
+export default [
+ {
+ name: 'migrate-codeblock-tokenizer-to-language',
+ transform: migrateCodeBlockTokenizerToLanguage,
+ meta: migrateCodeBlockTokenizerToLanguageMeta,
+ },
+];
diff --git a/packages/cli/assets/codemods/transforms/next/migrate-codeblock-tokenizer-to-language.mjs b/packages/cli/assets/codemods/transforms/next/migrate-codeblock-tokenizer-to-language.mjs
new file mode 100644
index 0000000000..b76de67313
--- /dev/null
+++ b/packages/cli/assets/codemods/transforms/next/migrate-codeblock-tokenizer-to-language.mjs
@@ -0,0 +1,315 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file Codemod: migrate CodeBlock tokenizer to a custom language value
+ *
+ * `tokenizer={fn}` beside a static `language="x"` other than `plaintext`
+ * becomes `language={createCodeLanguage({name: 'x', identifiers: ['x'],
+ * tokenize: fn})}` (`component:CodeBlock` FR13). The custom language paints
+ * with the same function, labels `x`, and reflects `data-language="x"`, so the
+ * block looks the same. `createCodeLanguage` is imported from
+ * `@astryxdesign/core/CodeBlock` when the file does not import it yet, under
+ * a free alias when the file already binds the name to something else. A
+ * `tokenizer` that is a name or a call-free property may be undefined at run
+ * time, so it is guarded: `fn ? createCodeLanguage({...}) : 'x'`.
+ *
+ * Built on the shared `transformProp` helper, so it does not follow
+ * variables, helpers, wrappers, or types. Where a CodeBlock writes
+ * `tokenizer` but the codemod cannot prove the result, it keeps the old prop
+ * and leaves a TODO instead of guessing: a computed or missing `language`, a
+ * `tokenizer` value that is neither a function literal nor a name or property
+ * (a name or property is guarded with `fn ? createCodeLanguage(...) : 'x'`,
+ * since it may be undefined), a
+ * `plaintext` language (which shows no label today, while a language value
+ * does), and any CodeBlock that also spreads props. The deprecated prop keeps
+ * working there. Comments are the codemod's only report channel, and each is
+ * added once, so a second run changes nothing.
+ */
+
+import {transformProp} from '../../transform-prop.mjs';
+
+export const meta = {
+ title: 'Migrate CodeBlock tokenizer to a custom language',
+ description:
+ 'Replaces `tokenizer={fn}` beside a static `language="x"` with ' +
+ "`language={createCodeLanguage({name: 'x', identifiers: ['x'], tokenize: fn})}`. " +
+ 'Keeps the old prop with a TODO where the language is computed, missing, ' +
+ 'or `plaintext`, and where the CodeBlock also spreads props; the ' +
+ 'deprecated prop still works there.',
+};
+
+const OLD_NAME = 'tokenizer';
+const CORE_IMPORT_RE = /^@(astryxdesign|xds)\/core(?:\/|$)/;
+const CODEBLOCK_SOURCE = '@astryxdesign/core/CodeBlock';
+const COMPONENT_PROPS = new Map([['CodeBlock', new Set([OLD_NAME])]]);
+
+const UNPROVEN_COMMENT =
+ ' TODO(astryx upgrade): tokenizer is deprecated. Pass ' +
+ 'language={createCodeLanguage({name, identifiers, tokenize})} instead, ' +
+ 'with this function as tokenize; the language here is computed, missing, ' +
+ 'or plaintext, so it was left unchanged. ';
+const VALUE_COMMENT =
+ ' TODO(astryx upgrade): tokenizer is deprecated. Pass ' +
+ 'language={createCodeLanguage({name, identifiers, tokenize})} instead; ' +
+ 'this tokenizer value is computed (not a function, a name, or a ' +
+ 'call-free property), so it was left unchanged. ';
+const SPREAD_COMMENT =
+ ' TODO(astryx upgrade): tokenizer is deprecated, but this CodeBlock also ' +
+ 'spreads props that may set language or tokenizer, so it was left ' +
+ 'unchanged. Pass language={createCodeLanguage({name, identifiers, ' +
+ 'tokenize})} by hand. ';
+
+const TYPE_WRAPPERS = new Set([
+ 'TSAsExpression',
+ 'TSSatisfiesExpression',
+ 'TSTypeAssertion',
+ 'TSNonNullExpression',
+ 'TypeCastExpression',
+ 'ParenthesizedExpression',
+]);
+
+/** @param {any} node */
+function unwrap(node) {
+ let current = node;
+ while (current && TYPE_WRAPPERS.has(current.type)) {
+ current = current.expression;
+ }
+ return current;
+}
+
+/** @param {any} attribute */
+function attributeValue(attribute) {
+ return attribute.value?.type === 'JSXExpressionContainer'
+ ? attribute.value.expression
+ : attribute.value;
+}
+
+/** The static string of a node, or null. @param {any} node */
+function staticString(node) {
+ const value = unwrap(node);
+ if (
+ value != null &&
+ (value.type === 'StringLiteral' || value.type === 'Literal') &&
+ typeof value.value === 'string'
+ ) {
+ return value.value;
+ }
+ if (
+ value?.type === 'TemplateLiteral' &&
+ value.expressions.length === 0 &&
+ value.quasis.length === 1
+ ) {
+ return value.quasis[0].value.cooked;
+ }
+ return null;
+}
+
+/**
+ * A name, or a property chain on one with no call and no computed key other
+ * than a literal: safe to read twice, and narrowed by a `?:` guard.
+ * @param {any} node
+ * @returns {boolean}
+ */
+function isCallFreeChain(node) {
+ const value = unwrap(node);
+ if (value == null) {
+ return false;
+ }
+ if (value.type === 'Identifier' || value.type === 'ThisExpression') {
+ return true;
+ }
+ if (value.type === 'MemberExpression') {
+ const key = value.property;
+ const literalKey =
+ !value.computed ||
+ key.type === 'StringLiteral' ||
+ key.type === 'NumericLiteral' ||
+ key.type === 'Literal';
+ return (
+ literalKey && value.optional !== true && isCallFreeChain(value.object)
+ );
+ }
+ return false;
+}
+
+/** @param {any} j @param {any} node @param {string} text */
+function addComment(j, node, text) {
+ node.comments ??= [];
+ if (node.comments.some((/** @type {any} */ c) => c.value === text)) {
+ return false;
+ }
+ node.comments.push(j.commentBlock(text, true, false));
+ return true;
+}
+
+/**
+ * The local name `createCodeLanguage` is imported under, adding the import
+ * when the file has none.
+ * @param {any} j @param {any} root
+ */
+function ensureCreateImport(j, root) {
+ let local = null;
+ root
+ .find(j.ImportDeclaration)
+ .filter((/** @type {any} */ p) => CORE_IMPORT_RE.test(p.node.source.value))
+ .forEach((/** @type {any} */ p) => {
+ for (const specifier of p.node.specifiers ?? []) {
+ if (
+ specifier.type === 'ImportSpecifier' &&
+ specifier.imported.name === 'createCodeLanguage'
+ ) {
+ local = specifier.local?.name ?? 'createCodeLanguage';
+ }
+ }
+ });
+ if (local != null) {
+ return local;
+ }
+ // Any other binding of the name (a local import or function) would clash,
+ // so the import takes the first free alias instead.
+ const name = freeName(j, root, 'createCodeLanguage');
+ const specifier =
+ name === 'createCodeLanguage'
+ ? j.importSpecifier(j.identifier('createCodeLanguage'))
+ : j.importSpecifier(
+ j.identifier('createCodeLanguage'),
+ j.identifier(name),
+ );
+ const existing = root.find(j.ImportDeclaration).filter(
+ (/** @type {any} */ p) =>
+ p.node.source.value === CODEBLOCK_SOURCE &&
+ p.node.importKind !== 'type' &&
+ // A namespace import cannot take a named specifier beside it.
+ !(p.node.specifiers ?? []).some(
+ (/** @type {any} */ s) => s.type === 'ImportNamespaceSpecifier',
+ ),
+ );
+ if (existing.size() > 0) {
+ existing.get().node.specifiers.push(specifier);
+ } else {
+ const declaration = j.importDeclaration(
+ [specifier],
+ j.stringLiteral(CODEBLOCK_SOURCE),
+ );
+ const imports = root.find(j.ImportDeclaration);
+ if (imports.size() > 0) {
+ imports.at(imports.size() - 1).insertAfter(declaration);
+ } else {
+ root.get().node.program.body.unshift(declaration);
+ }
+ }
+ return name;
+}
+
+/**
+ * `preferred`, or `preferred` with a numeric suffix, whichever no binding or
+ * reference in the file already uses.
+ * @param {any} j @param {any} root @param {string} preferred
+ */
+function freeName(j, root, preferred) {
+ const used = new Set();
+ root.find(j.Identifier).forEach((/** @type {any} */ p) => {
+ used.add(p.node.name);
+ });
+ if (!used.has(preferred)) {
+ return preferred;
+ }
+ let n = 2;
+ while (used.has(`${preferred}${n}`)) {
+ n++;
+ }
+ return `${preferred}${n}`;
+}
+
+/**
+ * @param {import('../../../../authoring/codemod/type').AstryxCodemodFile} file
+ * @param {import('../../../../authoring/codemod/type').CodemodTransformApi} api
+ * @returns {string | null | undefined}
+ */
+export default function transformer(file, api) {
+ const j = api.jscodeshift;
+ const root = j(file.source);
+ let hasChanges = false;
+ /** @type {Array<() => void>} */
+ const rewrites = [];
+
+ transformProp(
+ root,
+ j,
+ {
+ matchesImport: source => CORE_IMPORT_RE.test(source),
+ components: COMPONENT_PROPS,
+ },
+ propPath => {
+ const opening = propPath.parent?.node;
+ const attributes = opening?.attributes ?? [];
+ const prop = propPath.node;
+ if (
+ attributes.some(
+ (/** @type {any} */ a) => a.type === 'JSXSpreadAttribute',
+ )
+ ) {
+ hasChanges = addComment(j, prop, SPREAD_COMMENT) || hasChanges;
+ return;
+ }
+ const languageAttribute = attributes.find(
+ (/** @type {any} */ a) =>
+ a.type === 'JSXAttribute' && a.name?.name === 'language',
+ );
+ const language =
+ languageAttribute == null
+ ? null
+ : staticString(attributeValue(languageAttribute));
+ const tokenize = attributeValue(prop);
+ const tokenizeNode = unwrap(tokenize);
+ // A function literal is always a function. A name or property may be
+ // undefined at run time (a forwarded optional prop), so it is guarded:
+ // `fn ? createCodeLanguage({...}) : 'x'` keeps the plain block the
+ // released prop drew without one. Anything else is reported.
+ const isFunction =
+ tokenizeNode?.type === 'ArrowFunctionExpression' ||
+ tokenizeNode?.type === 'FunctionExpression';
+ const isGuardable = isCallFreeChain(tokenizeNode);
+ if (language == null || language === '' || language === 'plaintext') {
+ hasChanges = addComment(j, prop, UNPROVEN_COMMENT) || hasChanges;
+ return;
+ }
+ if (!(isFunction || isGuardable)) {
+ hasChanges = addComment(j, prop, VALUE_COMMENT) || hasChanges;
+ return;
+ }
+ rewrites.push(() => {
+ const create = ensureCreateImport(j, root);
+ const value = j.callExpression(j.identifier(create), [
+ j.objectExpression([
+ j.objectProperty(j.identifier('name'), j.stringLiteral(language)),
+ j.objectProperty(
+ j.identifier('identifiers'),
+ j.arrayExpression([j.stringLiteral(language)]),
+ ),
+ j.objectProperty(j.identifier('tokenize'), tokenize),
+ ]),
+ ]);
+ languageAttribute.value = j.jsxExpressionContainer(
+ isFunction
+ ? value
+ : j.conditionalExpression(
+ tokenize,
+ value,
+ j.stringLiteral(language),
+ ),
+ );
+ opening.attributes = opening.attributes.filter(
+ (/** @type {any} */ a) => a !== prop,
+ );
+ });
+ },
+ );
+
+ for (const rewrite of rewrites) {
+ rewrite();
+ hasChanges = true;
+ }
+ if (!hasChanges) return undefined;
+ return root.toSource({quote: 'single'});
+}
diff --git a/packages/core/src/CodeBlock/CodeBlock.doc.mjs b/packages/core/src/CodeBlock/CodeBlock.doc.mjs
index 9bf7a740d7..60f20047f5 100644
--- a/packages/core/src/CodeBlock/CodeBlock.doc.mjs
+++ b/packages/core/src/CodeBlock/CodeBlock.doc.mjs
@@ -125,7 +125,7 @@ export const docs = {
name: 'tokenizer',
type: '(code: string, language: string) => Array<{type: string; start: number; end: number}>',
description:
- 'Your own tokenizer; it wins over the language\'s grammar. The same function also makes a custom language: createCodeLanguage({name, identifiers, tokenize}).',
+ 'Deprecated: pass language={createCodeLanguage({name, identifiers, tokenize})} with this same function instead. Still works exactly as released and wins over the language; astryx upgrade rewrites static uses.',
},
{
name: 'syntaxTheme',
diff --git a/packages/core/src/CodeBlock/CodeBlock.tsx b/packages/core/src/CodeBlock/CodeBlock.tsx
index add98b7698..96d665dad6 100644
--- a/packages/core/src/CodeBlock/CodeBlock.tsx
+++ b/packages/core/src/CodeBlock/CodeBlock.tsx
@@ -51,6 +51,7 @@ import {
isCustomCodeLanguage,
languageKey,
languageLabel as languageWord,
+ sameCodeLanguage,
} from './grammar';
import {ensureHighlightStyles} from './highlightStyles';
import {applyHighlightRangesChunked} from './highlightRanges';
@@ -603,9 +604,10 @@ export interface CodeBlockProps extends BaseProps<HTMLPreElement> {
*/
container?: 'card' | 'section';
/**
- * Your own tokenizer; it wins over `language`'s grammar. The same function
- * also makes a custom language: `createCodeLanguage({name, identifiers,
- * tokenize})`.
+ * @deprecated Pass a custom language as `language` instead:
+ * `language={createCodeLanguage({name, identifiers, tokenize})}` takes this
+ * same function as `tokenize`. Still works exactly as released and wins
+ * over `language`'s grammar; `astryx upgrade` rewrites static uses.
*/
tokenizer?: (
code: string,
@@ -652,9 +654,16 @@ function isSafari(): boolean {
*/
function useTokenLines(
code: string,
- language: CodeLanguageInput,
+ languageProp: CodeLanguageInput,
customTokenizer?: CodeBlockProps['tokenizer'],
): TokenLine[] {
+ // Keep one language while it paints the same, so an inline
+ // `createCodeLanguage({...})` — the form the `tokenizer` codemod writes —
+ // tokenizes once, as `tokenizer` did (adjusting state during render).
+ const [language, setLanguage] = useState(languageProp);
+ if (!sameCodeLanguage(language, languageProp)) {
+ setLanguage(languageProp);
+ }
const [asyncTokenResult, setAsyncTokenResult] = useState<{
code: string;
language: CodeLanguageInput;
diff --git a/packages/core/src/CodeBlock/CodeLanguage.test.tsx b/packages/core/src/CodeBlock/CodeLanguage.test.tsx
index c19f41ed2d..0d74998d85 100644
--- a/packages/core/src/CodeBlock/CodeLanguage.test.tsx
+++ b/packages/core/src/CodeBlock/CodeLanguage.test.tsx
@@ -132,6 +132,38 @@ describe('a language value as `language` (FR13)', () => {
expect(keywordTexts(value.container)).toEqual([]);
});
+ it('tokenizes an inline custom language once across re-renders', () => {
+ let calls = 0;
+ const tokenize = () => {
+ calls++;
+ return [{type: 'keyword', start: 0, end: 3}];
+ };
+ const make = () =>
+ createCodeLanguage({name: 'zig', identifiers: ['zig'], tokenize});
+ const {rerender} = render(
+ <CodeBlock code="pub fn" language={make()} highlightMode="spans" />,
+ );
+ for (let i = 0; i < 5; i++) {
+ rerender(
+ <CodeBlock code="pub fn" language={make()} highlightMode="spans" />,
+ );
+ }
+ expect(calls).toBe(1);
+ });
+
+ it('renders plain, not throwing, when a custom language has no function', () => {
+ const missing = createCodeLanguage({
+ name: 'zig',
+ identifiers: ['zig'],
+ tokenize: undefined as unknown as () => [],
+ });
+ const {container} = render(
+ <CodeBlock code="pub fn" language={missing} highlightMode="spans" />,
+ );
+ expect(keywordTexts(container)).toEqual([]);
+ expect(container.textContent).toContain('pub fn');
+ });
+
it('paints a language value in the server render', () => {
const html = renderToString(
<CodeBlock code={THRIFT} language={thrift} highlightMode="spans" />,
diff --git a/packages/core/src/CodeBlock/grammar.ts b/packages/core/src/CodeBlock/grammar.ts
index c9acde5d51..7f283f3f4e 100644
--- a/packages/core/src/CodeBlock/grammar.ts
+++ b/packages/core/src/CodeBlock/grammar.ts
@@ -149,6 +149,33 @@ export function isCustomCodeLanguage(language: string | CodeLanguage): boolean {
);
}
+/**
+ * Whether two languages paint and label the same: the same value, or custom
+ * languages with the same tokenize function, name, and identifiers — so an
+ * inline `createCodeLanguage({...})` does not re-tokenize on every render.
+ */
+export function sameCodeLanguage(
+ a: string | CodeLanguage,
+ b: string | CodeLanguage,
+): boolean {
+ if (a === b) {
+ return true;
+ }
+ if (typeof a === 'string' || typeof b === 'string') {
+ return false;
+ }
+ const painterA = codeLanguagePainter(a);
+ const painterB = codeLanguagePainter(b);
+ return (
+ painterA?.kind === 'tokenize' &&
+ painterB?.kind === 'tokenize' &&
+ painterA.tokenize === painterB.tokenize &&
+ a.name === b.name &&
+ a.identifiers.length === b.identifiers.length &&
+ a.identifiers.every((identifier, i) => identifier === b.identifiers[i])
+ );
+}
+
/** The word a language reflects and labels: a string itself, or a value's name. */
export function languageLabel(language: string | CodeLanguage): string {
return typeof language === 'string' ? language : language.name;
diff --git a/packages/core/src/CodeBlock/tokenizer.ts b/packages/core/src/CodeBlock/tokenizer.ts
index 94e06dcd6a..1b31715f7c 100644
--- a/packages/core/src/CodeBlock/tokenizer.ts
+++ b/packages/core/src/CodeBlock/tokenizer.ts
@@ -123,7 +123,12 @@ function customTokens(
language: CodeLanguageInput,
): TokenLine[] | null {
const resolved = resolveLanguage(language);
- return resolved != null && 'tokenize' in resolved
+ if (resolved == null || !('tokenize' in resolved)) {
+ return null;
+ }
+ // A missing function paints nothing rather than throwing, as an absent
+ // `tokenizer` prop does.
+ return typeof resolved.tokenize === 'function'
? flatTokensToLines(resolved.tokenize(code, resolved.name), code)
: null;
}