An open source design system that's fully customizable and agent ready
TypeScript
feat(CodeBlock, cli): deprecate tokenizer in favour of a custom language value; astryx upgrade rewrites it (#8490)
Cindy Zhang79a7a902b8b4
diff --git a/.changeset/codeblock-tokenizer-codemod.md b/.changeset/codeblock-tokenizer-codemod.mdnew file mode 100644index 0000000000..5e35aa96e8--- /dev/null+++ b/.changeset/codeblock-tokenizer-codemod.md@@ -0,0 +1,8 @@+---+'@astryxdesign/cli': patch+'@astryxdesign/core': patch+---++[feat] CodeBlock's `tokenizer` prop is deprecated in favour of a custom language value: pass `language={createCodeLanguage({name, identifiers, tokenize})}` with the same function. The prop still works exactly as released. `astryx upgrade` rewrites it when it sits beside a static `language`. A function literal becomes `language={createCodeLanguage({name: 'zig', identifiers: ['zig'], tokenize: (code) => …})}`. A name or call-free property becomes `language={fn ? createCodeLanguage({…, tokenize: fn}) : 'zig'}`, since it may be undefined. Uses it cannot prove are left unchanged, with a TODO comment: a computed, missing, or `plaintext` language, any other tokenizer value, or a block that spreads props.++@cixzhangdiff --git a/packages/cli/assets/codemods/transforms/next/__tests__/migrate-codeblock-tokenizer-to-language.test.mjs b/packages/cli/assets/codemods/transforms/next/__tests__/migrate-codeblock-tokenizer-to-language.test.mjsnew file mode 100644index 0000000000..ee751d22b4--- /dev/null+++ b/packages/cli/assets/codemods/transforms/next/__tests__/migrate-codeblock-tokenizer-to-language.test.mjs@@ -0,0 +1,130 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++import {describe, expect, it} from 'vitest';+import transform from '../migrate-codeblock-tokenizer-to-language.mjs';++async function applyTransform(source) {+ const jscodeshift = (await import('jscodeshift')).default;+ const j = jscodeshift.withParser('tsx');+ const api = {jscodeshift: j, stats: () => {}, report: () => {}};+ return transform({source, path: 'test.tsx'}, api) ?? source;+}++/** Collapse whitespace so assertions ignore the printer's line breaks. */+const flat = text => text.replace(/\s+/g, ' ');++describe('migrate-codeblock-tokenizer-to-language', () => {+ it('rewrites tokenizer beside a static language to a custom language', async () => {+ const output = flat(+ await applyTransform(`import {CodeBlock} from '@astryxdesign/core/CodeBlock';+const a = <CodeBlock code={src} language="zig" tokenizer={zigTokens} />;+const b = <CodeBlock code={src} language={'nim'} tokenizer={(c) => tokens(c)} />;`),+ );++ expect(output).not.toContain('tokenizer=');+ expect(output).toContain(+ "import { CodeBlock, createCodeLanguage } from '@astryxdesign/core/CodeBlock';",+ );+ expect(output).toContain(+ "language={zigTokens ? createCodeLanguage({ name: 'zig', identifiers: ['zig'], tokenize: zigTokens }) : 'zig'}",+ );+ expect(output).toContain(+ "language={createCodeLanguage({ name: 'nim', identifiers: ['nim'], tokenize: (c) => tokens(c) })}",+ );+ });++ it('adds the import beside a root-package import and follows aliases', async () => {+ const output = flat(+ await applyTransform(`import {CodeBlock as Code} from '@astryxdesign/core';+const a = <Code code={src} language="zig" tokenizer={zigTokens} />;`),+ );++ expect(output).toContain(+ "import { createCodeLanguage } from '@astryxdesign/core/CodeBlock';",+ );+ expect(output).toContain("createCodeLanguage({ name: 'zig'");+ });++ it('adds a separate import beside a namespace import', async () => {+ const output =+ await applyTransform(`import * as CB from '@astryxdesign/core/CodeBlock';+const a = <CB.CodeBlock code={src} language="zig" tokenizer={zigTokens} />;`);+ expect(output).toContain(+ "import * as CB from '@astryxdesign/core/CodeBlock';",+ );+ expect(flat(output)).toContain(+ "import { createCodeLanguage } from '@astryxdesign/core/CodeBlock';",+ );+ const jscodeshift = (await import('jscodeshift')).default;+ expect(() => jscodeshift.withParser('tsx')(output)).not.toThrow();+ });++ it('guards a name or property that may be undefined', async () => {+ const output = flat(+ await applyTransform(`import {CodeBlock} from '@astryxdesign/core/CodeBlock';+const a = <CodeBlock code={src} language="zig" tokenizer={props.tokenizer} />;`),+ );+ expect(output).toContain(+ "language={props.tokenizer ? createCodeLanguage({ name: 'zig', identifiers: ['zig'], tokenize: props.tokenizer }) : 'zig'}",+ );+ });++ it('imports under a free alias when the file binds the name already', async () => {+ const fromLocal = flat(+ await applyTransform(`import {CodeBlock} from '@astryxdesign/core/CodeBlock';+import {createCodeLanguage} from './my-languages';+const a = <CodeBlock code={src} language="zig" tokenizer={(c) => t(c)} />;`),+ );+ expect(fromLocal).toContain(+ "import { CodeBlock, createCodeLanguage as createCodeLanguage2 } from '@astryxdesign/core/CodeBlock';",+ );+ expect(fromLocal).toContain(+ "import {createCodeLanguage} from './my-languages';",+ );+ expect(fromLocal).toContain("language={createCodeLanguage2({ name: 'zig'");++ const fromFunction = flat(+ await applyTransform(`import {CodeBlock} from '@astryxdesign/core/CodeBlock';+function createCodeLanguage() {}+const a = <CodeBlock code={src} language="zig" tokenizer={(c) => t(c)} />;`),+ );+ expect(fromFunction).toContain('createCodeLanguage as createCodeLanguage2');+ const jscodeshift = (await import('jscodeshift')).default;+ const parse = jscodeshift.withParser('tsx');+ expect(() => parse(fromLocal)).not.toThrow();+ });++ it('reports a tokenizer value it cannot read twice, naming the tokenizer', async () => {+ const output =+ await applyTransform(`import {CodeBlock} from '@astryxdesign/core/CodeBlock';+const a = <CodeBlock code={src} language="zig" tokenizer={registry.get('zig').tokenize} />;+const b = <CodeBlock code={src} language="zig" tokenizer={enabled ? t : undefined} />;`);+ expect(output.match(/this tokenizer value is computed/g)).toHaveLength(2);+ expect(output).not.toContain('the language here is computed');+ expect(output).toContain("tokenizer={registry.get('zig').tokenize}");+ });++ it('reports what it cannot prove and leaves it unchanged', async () => {+ const source = `import {CodeBlock} from '@astryxdesign/core/CodeBlock';+const a = <CodeBlock code={src} language={lang} tokenizer={t} />;+const b = <CodeBlock code={src} tokenizer={t} />;+const c = <CodeBlock code={src} language="plaintext" tokenizer={t} />;+const d = <CodeBlock {...rest} code={src} language="zig" tokenizer={t} />;+const e = <CodeBlock code={src} language="zig" tokenizer={enabled ? t : undefined} />;+const f = <CodeBlock code={src} language="zig" tokenizer={enabled && t} />;`;+ const output = await applyTransform(source);++ expect(output.match(/TODO\(astryx upgrade\)/g)).toHaveLength(6);+ expect(output.match(/tokenizer=\{t\}/g)).toHaveLength(4);+ expect(output).toContain('tokenizer={enabled ? t : undefined}');+ expect(output).toContain('tokenizer={enabled && t}');+ expect(output).not.toMatch(/import \{[^}]*createCodeLanguage/);+ expect(await applyTransform(output)).toBe(output);+ });++ it('leaves files without CodeBlock tokenizer alone', async () => {+ const source = `import {CodeBlock} from '@astryxdesign/core/CodeBlock';+const a = <CodeBlock code={src} language="ts" />;`;+ expect(await applyTransform(source)).toBe(source);+ });+});diff --git a/packages/cli/assets/codemods/transforms/next/index.mjs b/packages/cli/assets/codemods/transforms/next/index.mjsindex 2af3bdfcf0..45dead1111 100644--- a/packages/cli/assets/codemods/transforms/next/index.mjs+++ b/packages/cli/assets/codemods/transforms/next/index.mjs@@ -7,4 +7,14 @@* this file into the resolved version folder.*/-export default [];+import migrateCodeBlockTokenizerToLanguage, {+ meta as migrateCodeBlockTokenizerToLanguageMeta,+} from './migrate-codeblock-tokenizer-to-language.mjs';++export default [+ {+ name: 'migrate-codeblock-tokenizer-to-language',+ transform: migrateCodeBlockTokenizerToLanguage,+ meta: migrateCodeBlockTokenizerToLanguageMeta,+ },+];diff --git a/packages/cli/assets/codemods/transforms/next/migrate-codeblock-tokenizer-to-language.mjs b/packages/cli/assets/codemods/transforms/next/migrate-codeblock-tokenizer-to-language.mjsnew file mode 100644index 0000000000..b76de67313--- /dev/null+++ b/packages/cli/assets/codemods/transforms/next/migrate-codeblock-tokenizer-to-language.mjs@@ -0,0 +1,315 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file Codemod: migrate CodeBlock tokenizer to a custom language value+ *+ * `tokenizer={fn}` beside a static `language="x"` other than `plaintext`+ * becomes `language={createCodeLanguage({name: 'x', identifiers: ['x'],+ * tokenize: fn})}` (`component:CodeBlock` FR13). The custom language paints+ * with the same function, labels `x`, and reflects `data-language="x"`, so the+ * block looks the same. `createCodeLanguage` is imported from+ * `@astryxdesign/core/CodeBlock` when the file does not import it yet, under+ * a free alias when the file already binds the name to something else. A+ * `tokenizer` that is a name or a call-free property may be undefined at run+ * time, so it is guarded: `fn ? createCodeLanguage({...}) : 'x'`.+ *+ * Built on the shared `transformProp` helper, so it does not follow+ * variables, helpers, wrappers, or types. Where a CodeBlock writes+ * `tokenizer` but the codemod cannot prove the result, it keeps the old prop+ * and leaves a TODO instead of guessing: a computed or missing `language`, a+ * `tokenizer` value that is neither a function literal nor a name or property+ * (a name or property is guarded with `fn ? createCodeLanguage(...) : 'x'`,+ * since it may be undefined), a+ * `plaintext` language (which shows no label today, while a language value+ * does), and any CodeBlock that also spreads props. The deprecated prop keeps+ * working there. Comments are the codemod's only report channel, and each is+ * added once, so a second run changes nothing.+ */++import {transformProp} from '../../transform-prop.mjs';++export const meta = {+ title: 'Migrate CodeBlock tokenizer to a custom language',+ description:+ 'Replaces `tokenizer={fn}` beside a static `language="x"` with ' ++ "`language={createCodeLanguage({name: 'x', identifiers: ['x'], tokenize: fn})}`. " ++ 'Keeps the old prop with a TODO where the language is computed, missing, ' ++ 'or `plaintext`, and where the CodeBlock also spreads props; the ' ++ 'deprecated prop still works there.',+};++const OLD_NAME = 'tokenizer';+const CORE_IMPORT_RE = /^@(astryxdesign|xds)\/core(?:\/|$)/;+const CODEBLOCK_SOURCE = '@astryxdesign/core/CodeBlock';+const COMPONENT_PROPS = new Map([['CodeBlock', new Set([OLD_NAME])]]);++const UNPROVEN_COMMENT =+ ' TODO(astryx upgrade): tokenizer is deprecated. Pass ' ++ 'language={createCodeLanguage({name, identifiers, tokenize})} instead, ' ++ 'with this function as tokenize; the language here is computed, missing, ' ++ 'or plaintext, so it was left unchanged. ';+const VALUE_COMMENT =+ ' TODO(astryx upgrade): tokenizer is deprecated. Pass ' ++ 'language={createCodeLanguage({name, identifiers, tokenize})} instead; ' ++ 'this tokenizer value is computed (not a function, a name, or a ' ++ 'call-free property), so it was left unchanged. ';+const SPREAD_COMMENT =+ ' TODO(astryx upgrade): tokenizer is deprecated, but this CodeBlock also ' ++ 'spreads props that may set language or tokenizer, so it was left ' ++ 'unchanged. Pass language={createCodeLanguage({name, identifiers, ' ++ 'tokenize})} by hand. ';++const TYPE_WRAPPERS = new Set([+ 'TSAsExpression',+ 'TSSatisfiesExpression',+ 'TSTypeAssertion',+ 'TSNonNullExpression',+ 'TypeCastExpression',+ 'ParenthesizedExpression',+]);++/** @param {any} node */+function unwrap(node) {+ let current = node;+ while (current && TYPE_WRAPPERS.has(current.type)) {+ current = current.expression;+ }+ return current;+}++/** @param {any} attribute */+function attributeValue(attribute) {+ return attribute.value?.type === 'JSXExpressionContainer'+ ? attribute.value.expression+ : attribute.value;+}++/** The static string of a node, or null. @param {any} node */+function staticString(node) {+ const value = unwrap(node);+ if (+ value != null &&+ (value.type === 'StringLiteral' || value.type === 'Literal') &&+ typeof value.value === 'string'+ ) {+ return value.value;+ }+ if (+ value?.type === 'TemplateLiteral' &&+ value.expressions.length === 0 &&+ value.quasis.length === 1+ ) {+ return value.quasis[0].value.cooked;+ }+ return null;+}++/**+ * A name, or a property chain on one with no call and no computed key other+ * than a literal: safe to read twice, and narrowed by a `?:` guard.+ * @param {any} node+ * @returns {boolean}+ */+function isCallFreeChain(node) {+ const value = unwrap(node);+ if (value == null) {+ return false;+ }+ if (value.type === 'Identifier' || value.type === 'ThisExpression') {+ return true;+ }+ if (value.type === 'MemberExpression') {+ const key = value.property;+ const literalKey =+ !value.computed ||+ key.type === 'StringLiteral' ||+ key.type === 'NumericLiteral' ||+ key.type === 'Literal';+ return (+ literalKey && value.optional !== true && isCallFreeChain(value.object)+ );+ }+ return false;+}++/** @param {any} j @param {any} node @param {string} text */+function addComment(j, node, text) {+ node.comments ??= [];+ if (node.comments.some((/** @type {any} */ c) => c.value === text)) {+ return false;+ }+ node.comments.push(j.commentBlock(text, true, false));+ return true;+}++/**+ * The local name `createCodeLanguage` is imported under, adding the import+ * when the file has none.+ * @param {any} j @param {any} root+ */+function ensureCreateImport(j, root) {+ let local = null;+ root+ .find(j.ImportDeclaration)+ .filter((/** @type {any} */ p) => CORE_IMPORT_RE.test(p.node.source.value))+ .forEach((/** @type {any} */ p) => {+ for (const specifier of p.node.specifiers ?? []) {+ if (+ specifier.type === 'ImportSpecifier' &&+ specifier.imported.name === 'createCodeLanguage'+ ) {+ local = specifier.local?.name ?? 'createCodeLanguage';+ }+ }+ });+ if (local != null) {+ return local;+ }+ // Any other binding of the name (a local import or function) would clash,+ // so the import takes the first free alias instead.+ const name = freeName(j, root, 'createCodeLanguage');+ const specifier =+ name === 'createCodeLanguage'+ ? j.importSpecifier(j.identifier('createCodeLanguage'))+ : j.importSpecifier(+ j.identifier('createCodeLanguage'),+ j.identifier(name),+ );+ const existing = root.find(j.ImportDeclaration).filter(+ (/** @type {any} */ p) =>+ p.node.source.value === CODEBLOCK_SOURCE &&+ p.node.importKind !== 'type' &&+ // A namespace import cannot take a named specifier beside it.+ !(p.node.specifiers ?? []).some(+ (/** @type {any} */ s) => s.type === 'ImportNamespaceSpecifier',+ ),+ );+ if (existing.size() > 0) {+ existing.get().node.specifiers.push(specifier);+ } else {+ const declaration = j.importDeclaration(+ [specifier],+ j.stringLiteral(CODEBLOCK_SOURCE),+ );+ const imports = root.find(j.ImportDeclaration);+ if (imports.size() > 0) {+ imports.at(imports.size() - 1).insertAfter(declaration);+ } else {+ root.get().node.program.body.unshift(declaration);+ }+ }+ return name;+}++/**+ * `preferred`, or `preferred` with a numeric suffix, whichever no binding or+ * reference in the file already uses.+ * @param {any} j @param {any} root @param {string} preferred+ */+function freeName(j, root, preferred) {+ const used = new Set();+ root.find(j.Identifier).forEach((/** @type {any} */ p) => {+ used.add(p.node.name);+ });+ if (!used.has(preferred)) {+ return preferred;+ }+ let n = 2;+ while (used.has(`${preferred}${n}`)) {+ n++;+ }+ return `${preferred}${n}`;+}++/**+ * @param {import('../../../../authoring/codemod/type').AstryxCodemodFile} file+ * @param {import('../../../../authoring/codemod/type').CodemodTransformApi} api+ * @returns {string | null | undefined}+ */+export default function transformer(file, api) {+ const j = api.jscodeshift;+ const root = j(file.source);+ let hasChanges = false;+ /** @type {Array<() => void>} */+ const rewrites = [];++ transformProp(+ root,+ j,+ {+ matchesImport: source => CORE_IMPORT_RE.test(source),+ components: COMPONENT_PROPS,+ },+ propPath => {+ const opening = propPath.parent?.node;+ const attributes = opening?.attributes ?? [];+ const prop = propPath.node;+ if (+ attributes.some(+ (/** @type {any} */ a) => a.type === 'JSXSpreadAttribute',+ )+ ) {+ hasChanges = addComment(j, prop, SPREAD_COMMENT) || hasChanges;+ return;+ }+ const languageAttribute = attributes.find(+ (/** @type {any} */ a) =>+ a.type === 'JSXAttribute' && a.name?.name === 'language',+ );+ const language =+ languageAttribute == null+ ? null+ : staticString(attributeValue(languageAttribute));+ const tokenize = attributeValue(prop);+ const tokenizeNode = unwrap(tokenize);+ // A function literal is always a function. A name or property may be+ // undefined at run time (a forwarded optional prop), so it is guarded:+ // `fn ? createCodeLanguage({...}) : 'x'` keeps the plain block the+ // released prop drew without one. Anything else is reported.+ const isFunction =+ tokenizeNode?.type === 'ArrowFunctionExpression' ||+ tokenizeNode?.type === 'FunctionExpression';+ const isGuardable = isCallFreeChain(tokenizeNode);+ if (language == null || language === '' || language === 'plaintext') {+ hasChanges = addComment(j, prop, UNPROVEN_COMMENT) || hasChanges;+ return;+ }+ if (!(isFunction || isGuardable)) {+ hasChanges = addComment(j, prop, VALUE_COMMENT) || hasChanges;+ return;+ }+ rewrites.push(() => {+ const create = ensureCreateImport(j, root);+ const value = j.callExpression(j.identifier(create), [+ j.objectExpression([+ j.objectProperty(j.identifier('name'), j.stringLiteral(language)),+ j.objectProperty(+ j.identifier('identifiers'),+ j.arrayExpression([j.stringLiteral(language)]),+ ),+ j.objectProperty(j.identifier('tokenize'), tokenize),+ ]),+ ]);+ languageAttribute.value = j.jsxExpressionContainer(+ isFunction+ ? value+ : j.conditionalExpression(+ tokenize,+ value,+ j.stringLiteral(language),+ ),+ );+ opening.attributes = opening.attributes.filter(+ (/** @type {any} */ a) => a !== prop,+ );+ });+ },+ );++ for (const rewrite of rewrites) {+ rewrite();+ hasChanges = true;+ }+ if (!hasChanges) return undefined;+ return root.toSource({quote: 'single'});+}diff --git a/packages/core/src/CodeBlock/CodeBlock.doc.mjs b/packages/core/src/CodeBlock/CodeBlock.doc.mjsindex 9bf7a740d7..60f20047f5 100644--- a/packages/core/src/CodeBlock/CodeBlock.doc.mjs+++ b/packages/core/src/CodeBlock/CodeBlock.doc.mjs@@ -125,7 +125,7 @@ export const docs = {name: 'tokenizer',type: '(code: string, language: string) => Array<{type: string; start: number; end: number}>',description:- 'Your own tokenizer; it wins over the language\'s grammar. The same function also makes a custom language: createCodeLanguage({name, identifiers, tokenize}).',+ 'Deprecated: pass language={createCodeLanguage({name, identifiers, tokenize})} with this same function instead. Still works exactly as released and wins over the language; astryx upgrade rewrites static uses.',},{name: 'syntaxTheme',diff --git a/packages/core/src/CodeBlock/CodeBlock.tsx b/packages/core/src/CodeBlock/CodeBlock.tsxindex add98b7698..96d665dad6 100644--- a/packages/core/src/CodeBlock/CodeBlock.tsx+++ b/packages/core/src/CodeBlock/CodeBlock.tsx@@ -51,6 +51,7 @@ import {isCustomCodeLanguage,languageKey,languageLabel as languageWord,+ sameCodeLanguage,} from './grammar';import {ensureHighlightStyles} from './highlightStyles';import {applyHighlightRangesChunked} from './highlightRanges';@@ -603,9 +604,10 @@ export interface CodeBlockProps extends BaseProps<HTMLPreElement> {*/container?: 'card' | 'section';/**- * Your own tokenizer; it wins over `language`'s grammar. The same function- * also makes a custom language: `createCodeLanguage({name, identifiers,- * tokenize})`.+ * @deprecated Pass a custom language as `language` instead:+ * `language={createCodeLanguage({name, identifiers, tokenize})}` takes this+ * same function as `tokenize`. Still works exactly as released and wins+ * over `language`'s grammar; `astryx upgrade` rewrites static uses.*/tokenizer?: (code: string,@@ -652,9 +654,16 @@ function isSafari(): boolean {*/function useTokenLines(code: string,- language: CodeLanguageInput,+ languageProp: CodeLanguageInput,customTokenizer?: CodeBlockProps['tokenizer'],): TokenLine[] {+ // Keep one language while it paints the same, so an inline+ // `createCodeLanguage({...})` — the form the `tokenizer` codemod writes —+ // tokenizes once, as `tokenizer` did (adjusting state during render).+ const [language, setLanguage] = useState(languageProp);+ if (!sameCodeLanguage(language, languageProp)) {+ setLanguage(languageProp);+ }const [asyncTokenResult, setAsyncTokenResult] = useState<{code: string;language: CodeLanguageInput;diff --git a/packages/core/src/CodeBlock/CodeLanguage.test.tsx b/packages/core/src/CodeBlock/CodeLanguage.test.tsxindex c19f41ed2d..0d74998d85 100644--- a/packages/core/src/CodeBlock/CodeLanguage.test.tsx+++ b/packages/core/src/CodeBlock/CodeLanguage.test.tsx@@ -132,6 +132,38 @@ describe('a language value as `language` (FR13)', () => {expect(keywordTexts(value.container)).toEqual([]);});+ it('tokenizes an inline custom language once across re-renders', () => {+ let calls = 0;+ const tokenize = () => {+ calls++;+ return [{type: 'keyword', start: 0, end: 3}];+ };+ const make = () =>+ createCodeLanguage({name: 'zig', identifiers: ['zig'], tokenize});+ const {rerender} = render(+ <CodeBlock code="pub fn" language={make()} highlightMode="spans" />,+ );+ for (let i = 0; i < 5; i++) {+ rerender(+ <CodeBlock code="pub fn" language={make()} highlightMode="spans" />,+ );+ }+ expect(calls).toBe(1);+ });++ it('renders plain, not throwing, when a custom language has no function', () => {+ const missing = createCodeLanguage({+ name: 'zig',+ identifiers: ['zig'],+ tokenize: undefined as unknown as () => [],+ });+ const {container} = render(+ <CodeBlock code="pub fn" language={missing} highlightMode="spans" />,+ );+ expect(keywordTexts(container)).toEqual([]);+ expect(container.textContent).toContain('pub fn');+ });+it('paints a language value in the server render', () => {const html = renderToString(<CodeBlock code={THRIFT} language={thrift} highlightMode="spans" />,diff --git a/packages/core/src/CodeBlock/grammar.ts b/packages/core/src/CodeBlock/grammar.tsindex c9acde5d51..7f283f3f4e 100644--- a/packages/core/src/CodeBlock/grammar.ts+++ b/packages/core/src/CodeBlock/grammar.ts@@ -149,6 +149,33 @@ export function isCustomCodeLanguage(language: string | CodeLanguage): boolean {);}+/**+ * Whether two languages paint and label the same: the same value, or custom+ * languages with the same tokenize function, name, and identifiers — so an+ * inline `createCodeLanguage({...})` does not re-tokenize on every render.+ */+export function sameCodeLanguage(+ a: string | CodeLanguage,+ b: string | CodeLanguage,+): boolean {+ if (a === b) {+ return true;+ }+ if (typeof a === 'string' || typeof b === 'string') {+ return false;+ }+ const painterA = codeLanguagePainter(a);+ const painterB = codeLanguagePainter(b);+ return (+ painterA?.kind === 'tokenize' &&+ painterB?.kind === 'tokenize' &&+ painterA.tokenize === painterB.tokenize &&+ a.name === b.name &&+ a.identifiers.length === b.identifiers.length &&+ a.identifiers.every((identifier, i) => identifier === b.identifiers[i])+ );+}+/** The word a language reflects and labels: a string itself, or a value's name. */export function languageLabel(language: string | CodeLanguage): string {return typeof language === 'string' ? language : language.name;diff --git a/packages/core/src/CodeBlock/tokenizer.ts b/packages/core/src/CodeBlock/tokenizer.tsindex 94e06dcd6a..1b31715f7c 100644--- a/packages/core/src/CodeBlock/tokenizer.ts+++ b/packages/core/src/CodeBlock/tokenizer.ts@@ -123,7 +123,12 @@ function customTokens(language: CodeLanguageInput,): TokenLine[] | null {const resolved = resolveLanguage(language);- return resolved != null && 'tokenize' in resolved+ if (resolved == null || !('tokenize' in resolved)) {+ return null;+ }+ // A missing function paints nothing rather than throwing, as an absent+ // `tokenizer` prop does.+ return typeof resolved.tokenize === 'function'? flatTokensToLines(resolved.tokenize(code, resolved.name), code): null;}