An open source design system that's fully customizable and agent ready
TypeScript
feat(CodeBlock): built-in C, C++, SQL, Rust, Kotlin, Swift, Go; per-language values passed as language; createCodeLanguage (#8333)
Cindy Zhang4adaeda939c2
diff --git a/.changeset/codeblock-languages.md b/.changeset/codeblock-languages.mdnew file mode 100644index 0000000000..d30525ec1e--- /dev/null+++ b/.changeset/codeblock-languages.md@@ -0,0 +1,17 @@+---+'@astryxdesign/core': patch+---++[feat] `CodeBlock` highlights C, C++, SQL, Rust, Kotlin, Swift, and Go with no setup.++Every other language is opt in, one at a time:++- Import that language on its own and pass it as `language`, for example `graphql` from `@astryxdesign/core/CodeBlock/languages/graphql`.+- Modules ship for `graphql`, `java`, `thrift`, `objectiveC` (path `languages/objective-c`), `toml`, `powershell`, `ruby`, `lua`, and `dockerfile`.+- A string such as `language="graphql"` still selects only a built-in language.++Your own languages work the same way. `createCodeLanguage({name, identifiers, fileExtensions, fileNames, tokenize})` makes a language value from your own tokenize function.++The built-in tokenizer stays within 4,608 bytes gzipped. Each module adds at most 512 bytes, and only when you import it.++@cixzhangdiff --git a/apps/storybook/stories/CodeBlock.stories.tsx b/apps/storybook/stories/CodeBlock.stories.tsxindex 7b7afd0c89..81e9cdb67a 100644--- a/apps/storybook/stories/CodeBlock.stories.tsx+++ b/apps/storybook/stories/CodeBlock.stories.tsx@@ -2,6 +2,8 @@import type {Meta, StoryObj} from '@storybook/react';import {CodeBlock} from '@astryxdesign/core/CodeBlock';+import {java} from '@astryxdesign/core/CodeBlock/languages/java';+import {thrift} from '@astryxdesign/core/CodeBlock/languages/thrift';import {IconButton} from '@astryxdesign/core/IconButton';const meta: Meta<typeof CodeBlock> = {@@ -226,6 +228,52 @@ export const EndContentTall: Story = {},};+// Built-in languages colour with no setup.+export const BuiltInLanguages: Story = {+ render: () => (+ <div style={{display: 'grid', gap: 16, maxWidth: 520}}>+ <CodeBlock+ language="cpp"+ code={+ '#include <vector>\n// Sum a vector\nint sum(const std::vector<int>& v) {\n int total = 0;\n for (int x : v) total += x;\n return total;\n}'+ }+ />+ <CodeBlock+ language="rust"+ code={+ '// Parse a port\nfn port(s: &str) -> Result<u16, String> {\n s.parse::<u16>().map_err(|e| e.to_string())\n}'+ }+ />+ <CodeBlock+ language="sql"+ code={+ "-- Active users this week\nSELECT id, name FROM users\nWHERE last_seen > NOW() - INTERVAL '7 days'\nORDER BY name;"+ }+ />+ </div>+ ),+};++// A less common language is one import, passed as the language.+export const LanguageModules: Story = {+ render: () => (+ <div style={{display: 'grid', gap: 16, maxWidth: 520}}>+ <CodeBlock+ language={thrift}+ code={+ '# A user record\nstruct User {\n 1: required i64 id;\n 2: optional string name;\n}'+ }+ />+ <CodeBlock+ language={java}+ code={+ '// Greet a user\npublic final class Greeter {\n String greet(String name) { return "hello, " + name; }\n}'+ }+ />+ </div>+ ),+};+export const JSON: Story = {args: {code: `{diff --git a/packages/core/package.json b/packages/core/package.jsonindex 6e9a4db109..462f6f61ad 100644--- a/packages/core/package.json+++ b/packages/core/package.json@@ -769,6 +769,51 @@"types": "./dist/Typeahead/utils.d.ts","default": "./dist/Typeahead/utils.js"},+ "./CodeBlock/languages/dockerfile": {+ "source": "./src/CodeBlock/languages/dockerfile.ts",+ "types": "./dist/CodeBlock/languages/dockerfile.d.ts",+ "default": "./dist/CodeBlock/languages/dockerfile.js"+ },+ "./CodeBlock/languages/graphql": {+ "source": "./src/CodeBlock/languages/graphql.ts",+ "types": "./dist/CodeBlock/languages/graphql.d.ts",+ "default": "./dist/CodeBlock/languages/graphql.js"+ },+ "./CodeBlock/languages/java": {+ "source": "./src/CodeBlock/languages/java.ts",+ "types": "./dist/CodeBlock/languages/java.d.ts",+ "default": "./dist/CodeBlock/languages/java.js"+ },+ "./CodeBlock/languages/lua": {+ "source": "./src/CodeBlock/languages/lua.ts",+ "types": "./dist/CodeBlock/languages/lua.d.ts",+ "default": "./dist/CodeBlock/languages/lua.js"+ },+ "./CodeBlock/languages/objective-c": {+ "source": "./src/CodeBlock/languages/objective-c.ts",+ "types": "./dist/CodeBlock/languages/objective-c.d.ts",+ "default": "./dist/CodeBlock/languages/objective-c.js"+ },+ "./CodeBlock/languages/powershell": {+ "source": "./src/CodeBlock/languages/powershell.ts",+ "types": "./dist/CodeBlock/languages/powershell.d.ts",+ "default": "./dist/CodeBlock/languages/powershell.js"+ },+ "./CodeBlock/languages/ruby": {+ "source": "./src/CodeBlock/languages/ruby.ts",+ "types": "./dist/CodeBlock/languages/ruby.d.ts",+ "default": "./dist/CodeBlock/languages/ruby.js"+ },+ "./CodeBlock/languages/thrift": {+ "source": "./src/CodeBlock/languages/thrift.ts",+ "types": "./dist/CodeBlock/languages/thrift.d.ts",+ "default": "./dist/CodeBlock/languages/thrift.js"+ },+ "./CodeBlock/languages/toml": {+ "source": "./src/CodeBlock/languages/toml.ts",+ "types": "./dist/CodeBlock/languages/toml.d.ts",+ "default": "./dist/CodeBlock/languages/toml.js"+ },"./Markdown/remark": {"source": "./src/Markdown/remark.ts","types": "./dist/Markdown/remark.d.ts",diff --git a/packages/core/src/CodeBlock/CodeBlock.doc.mjs b/packages/core/src/CodeBlock/CodeBlock.doc.mjsindex fd17c2990f..9bf7a740d7 100644--- a/packages/core/src/CodeBlock/CodeBlock.doc.mjs+++ b/packages/core/src/CodeBlock/CodeBlock.doc.mjs@@ -28,9 +28,9 @@ export const docs = {},{name: 'language',- type: 'string',+ type: 'string | CodeLanguage',description:- 'Language for syntax highlighting. Use "plaintext" to disable.',+ 'Language for syntax highlighting. A string selects a built-in: ts, js, tsx, jsx, json, html, xml, svg, css, scss, less, python, bash/sh/zsh/shell, php, hack, yaml, markdown, c, cpp (c++, cc, cxx, hpp, hh), sql, rust, kotlin, swift, go. Any other language is a value you import on its own and pass here: graphql, java, thrift, objectiveC, toml, powershell, ruby, lua, dockerfile from @astryxdesign/core/CodeBlock/languages/<path> (e.g. languages/graphql, languages/objective-c), or your own made with createCodeLanguage({name, identifiers, tokenize}). A value labels with its name and reflects its first identifier as data-language. Use "plaintext" to disable.',default: "'plaintext'",},{@@ -78,7 +78,7 @@ export const docs = {name: 'endContent',type: 'ReactNode',description:- 'Your own controls, shown just before Copy: in the header when the block has one, otherwise beside the code in the top inline-end corner, never over it. It never creates a header and never collapses the block. Use ghost sm IconButtons to match Copy.',+ 'Your own controls, shown just before Copy: in the header when the block has one, otherwise beside the code in the top inline-end corner, never over it. It never creates a header and never collapses the block. Ghost sm IconButtons match Copy’s size and variant.',slotElements: [{__element: 'IconButton',@@ -124,7 +124,8 @@ export const docs = {{name: 'tokenizer',type: '(code: string, language: string) => Array<{type: string; start: number; end: number}>',- description: 'Custom tokenizer override for unsupported languages.',+ description:+ 'Your own tokenizer; it wins over the language\'s grammar. The same function also makes a custom language: createCodeLanguage({name, identifiers, tokenize}).',},{name: 'syntaxTheme',diff --git a/packages/core/src/CodeBlock/CodeBlock.tsx b/packages/core/src/CodeBlock/CodeBlock.tsxindex 95ef4d32a1..add98b7698 100644--- a/packages/core/src/CodeBlock/CodeBlock.tsx+++ b/packages/core/src/CodeBlock/CodeBlock.tsx@@ -45,7 +45,13 @@ import {flatTokensToLines,SYNC_TOKENIZE_THRESHOLD,} from './tokenizer';-import type {SyntaxToken, TokenLine} from './tokenizer';+import type {CodeLanguageInput, SyntaxToken, TokenLine} from './tokenizer';+import type {CodeLanguage} from './grammar';+import {+ isCustomCodeLanguage,+ languageKey,+ languageLabel as languageWord,+} from './grammar';import {ensureHighlightStyles} from './highlightStyles';import {applyHighlightRangesChunked} from './highlightRanges';import {themeProps} from '../utils/themeProps';@@ -541,7 +547,15 @@ function renderLines(export interface CodeBlockProps extends BaseProps<HTMLPreElement> {ref?: React.Ref<HTMLPreElement>;code: string;- language?: string;+ /**+ * The code's language: a built-in identifier such as `'rust'`, or a+ * language value — a module imported on its own, such as `graphql` from+ * `@astryxdesign/core/CodeBlock/languages/graphql`, or one made with+ * `createCodeLanguage`. A string selects only a built-in grammar; the label+ * shows the string, or the value's `name`.+ * @default 'plaintext'+ */+ language?: string | CodeLanguage;title?: string;hasLanguageLabel?: boolean;hasLineNumbers?: boolean;@@ -562,7 +576,7 @@ export interface CodeBlockProps extends BaseProps<HTMLPreElement> {* has one, otherwise beside the code in the top inline-end corner, never* over it. It never creates a header, and activating it never collapses the* block. You own each control's label, state, and action; ghost `sm`- * IconButtons match Copy.+ * IconButtons match Copy's size and variant.*/endContent?: React.ReactNode;isWrapped?: boolean;@@ -588,6 +602,11 @@ export interface CodeBlockProps extends BaseProps<HTMLPreElement> {* @default 'card'*/container?: 'card' | 'section';+ /**+ * Your own tokenizer; it wins over `language`'s grammar. The same function+ * also makes a custom language: `createCodeLanguage({name, identifiers,+ * tokenize})`.+ */tokenizer?: (code: string,language: string,@@ -633,27 +652,42 @@ function isSafari(): boolean {*/function useTokenLines(code: string,- language: string,+ language: CodeLanguageInput,customTokenizer?: CodeBlockProps['tokenizer'],): TokenLine[] {const [asyncTokenResult, setAsyncTokenResult] = useState<{code: string;- language: string;+ language: CodeLanguageInput;tokens: TokenLine[];} | null>(null);const syncTokens = useMemo(() => {if (customTokenizer) {- return flatTokensToLines(customTokenizer(code, language), code);+ return flatTokensToLines(+ customTokenizer(code, languageWord(language)),+ code,+ );}- if (code.length >= SYNC_TOKENIZE_THRESHOLD) {+ // A custom language paints synchronously at any length, as `tokenizer`+ // does, so the codemod's rewrite keeps the same first paint (FR13).+ if (+ code.length >= SYNC_TOKENIZE_THRESHOLD &&+ !isCustomCodeLanguage(language)+ ) {return null;}return tokenize(code, language);}, [code, language, customTokenizer]);+ // Synchronizes long code with a scheduler-yielding tokenize task: it starts+ // when the code, language, or tokenizer changes, and the+ // cleanup aborts the previous one.useEffect(() => {- if (code.length < SYNC_TOKENIZE_THRESHOLD || customTokenizer) {+ if (+ code.length < SYNC_TOKENIZE_THRESHOLD ||+ customTokenizer ||+ isCustomCodeLanguage(language)+ ) {return;}@@ -988,14 +1022,18 @@ export function CodeBlock({// With the diff layer on, the label and viewport name say so: `diff` over// plaintext, `tsx diff` over a language — so a diff keeps its header,// collapse control, and a name that still matches a `diff` fence.+ // A language value labels with its `name` and reflects its first+ // identifier as `data-language`, so theme selectors keep matching (FR13).+ const languageName = languageWord(language);+ const languageAttr = languageKey(language);const languageLabel = !hasLanguageLabel? null: isDiff- ? language === 'plaintext'+ ? languageName === 'plaintext'? 'diff'- : `${language} diff`- : language !== 'plaintext'- ? language+ : `${languageName} diff`+ : languageName !== 'plaintext'+ ? languageName: null;const showHeader = title != null || languageLabel != null;@@ -1071,7 +1109,11 @@ export function CodeBlock({const headerEl = showHeader ? (<div{...mergeProps(- themeProps('code-block-header', {size, language, container}),+ themeProps('code-block-header', {+ size,+ language: languageAttr,+ container,+ }),stylex.props(styles.headerRow,hasLineNumbers ? styles.headerWithDivider : styles.headerCompact,@@ -1104,7 +1146,7 @@ export function CodeBlock({)}><span{...mergeProps(- themeProps('code-block-title', {size, language}),+ themeProps('code-block-title', {size, language: languageAttr}),stylex.props(styles.headerTitle),)}>{canCollapse && (@@ -1179,7 +1221,7 @@ export function CodeBlock({<preref={ref}{...mergeProps(- themeProps('code-block', {size, language, container}),+ themeProps('code-block', {size, language: languageAttr, container}),stylex.props(styles.root,hasControls && !showHeader && styles.rootCorner,diff --git a/packages/core/src/CodeBlock/CodeLanguage.test.tsx b/packages/core/src/CodeBlock/CodeLanguage.test.tsxnew file mode 100644index 0000000000..c19f41ed2d--- /dev/null+++ b/packages/core/src/CodeBlock/CodeLanguage.test.tsx@@ -0,0 +1,141 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file CodeLanguage.test.tsx+ * @input CodeBlock, language modules, createCodeLanguage+ * @output CodeBlock FR13 evidence: a language value passed as `language`+ * paints, labels with its name, and reflects its first identifier; a string+ * never reaches a module; the `tokenizer` prop still wins; server rendering+ * paints in the first render+ * @position CodeBlock FR13 evidence+ */++import {render, screen} from '@testing-library/react';+import {renderToString} from 'react-dom/server';+import {describe, expect, it} from 'vitest';+import {CodeBlock} from './CodeBlock';+import {createCodeLanguage} from './grammar';+import {objectiveC} from './languages/objective-c';+import {thrift} from './languages/thrift';++const THRIFT = 'struct User {\n 1: required string name;\n}';++function keywordTexts(root: ParentNode): string[] {+ return Array.from(root.querySelectorAll('.astryx-token-keyword')).map(+ node => node.textContent ?? '',+ );+}++describe('a language value as `language` (FR13)', () => {+ it('paints and labels with the value; a string never reaches a module', () => {+ const plain = render(+ <CodeBlock code={THRIFT} language="thrift" highlightMode="spans" />,+ );+ expect(keywordTexts(plain.container)).toEqual([]);++ const {container} = render(+ <CodeBlock code={THRIFT} language={thrift} highlightMode="spans" />,+ );+ expect(keywordTexts(container)).toEqual(+ expect.arrayContaining(['struct', 'required', 'string']),+ );+ expect(screen.getAllByText('thrift').length).toBeGreaterThan(0);+ });++ it('reflects the first identifier as data-language', () => {+ const {container} = render(+ <CodeBlock code="@interface A\n@end" language={objectiveC} />,+ );+ expect(container.querySelector('pre')?.getAttribute('data-language')).toBe(+ 'objective-c',+ );+ });++ it('paints a custom language made with createCodeLanguage', () => {+ const zig = createCodeLanguage({+ name: 'zig',+ identifiers: ['zig'],+ tokenize: () => [{type: 'keyword', start: 0, end: 3}],+ });+ const {container} = render(+ <CodeBlock+ code="pub fn main() void {}"+ language={zig}+ highlightMode="spans"+ />,+ );+ expect(keywordTexts(container)).toEqual(['pub']);+ expect(container.querySelector('pre')?.getAttribute('data-language')).toBe(+ 'zig',+ );+ });++ it('labels with the value name and reflects its first identifier', () => {+ const zig = createCodeLanguage({+ name: 'Zig Lang',+ identifiers: ['zig', 'ziglang'],+ tokenize: () => [],+ });+ const {container} = render(+ <CodeBlock code="const a = 1;" language={zig} />,+ );+ expect(screen.getByText('Zig Lang')).toBeInTheDocument();+ expect(container.querySelector('pre')?.getAttribute('data-language')).toBe(+ 'zig',+ );+ });++ it('paints a custom language synchronously at any length, as tokenizer does', () => {+ const long = 'pub '.repeat(600);+ const zig = createCodeLanguage({+ name: 'zig',+ identifiers: ['zig'],+ tokenize: code =>+ Array.from(code.matchAll(/pub/g), match => ({+ type: 'keyword',+ start: match.index ?? 0,+ end: (match.index ?? 0) + 3,+ })),+ });+ const html = renderToString(+ <CodeBlock code={long} language={zig} highlightMode="spans" />,+ );+ expect(html.match(/astryx-token-keyword/g)?.length).toBe(600);+ const {container} = render(+ <CodeBlock code={long} language={zig} highlightMode="spans" />,+ );+ expect(keywordTexts(container)).toHaveLength(600);+ });++ it('keeps the tokenizer prop winning, as released', () => {+ const tokenizer = () => [{type: 'string', start: 0, end: 6}];+ const builtIn = render(+ <CodeBlock+ code="const x = 1;"+ language="ts"+ tokenizer={tokenizer}+ highlightMode="spans"+ />,+ );+ expect(keywordTexts(builtIn.container)).toEqual([]);+ expect(+ builtIn.container.querySelector('.astryx-token-string')?.textContent,+ ).toBe('const ');+ const value = render(+ <CodeBlock+ code={THRIFT}+ language={thrift}+ tokenizer={tokenizer}+ highlightMode="spans"+ />,+ );+ expect(keywordTexts(value.container)).toEqual([]);+ });++ it('paints a language value in the server render', () => {+ const html = renderToString(+ <CodeBlock code={THRIFT} language={thrift} highlightMode="spans" />,+ );+ expect(html).toContain('astryx-token-keyword');+ });+});diff --git a/packages/core/src/CodeBlock/__tests__/CodeBlockLanguages.a11y.chromium.spec.ts b/packages/core/src/CodeBlock/__tests__/CodeBlockLanguages.a11y.chromium.spec.tsnew file mode 100644index 0000000000..f140c6c687--- /dev/null+++ b/packages/core/src/CodeBlock/__tests__/CodeBlockLanguages.a11y.chromium.spec.ts@@ -0,0 +1,64 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file CodeBlockLanguages.a11y.chromium.spec.ts+ * @input Storybook stories core-codeblock--built-in-languages and+ * core-codeblock--language-modules+ * @output CodeBlock FR12 and FR13 in a real browser: built-in languages and+ * language values passed as `language` register syntax highlights over the+ * right text+ * @position CodeBlock FR12, FR13 browser evidence+ */++import {expect, test, type Page} from '@playwright/test';+import {+ DEFAULT_STORYBOOK_DIR,+ serveStorybook,+ type StaticServer,+} from '@astryxdesign/a11y-spec/storybook';++let storybook: StaticServer;+test.beforeAll(async () => {+ storybook = await serveStorybook(+ process.env.ASTRYX_STORYBOOK_DIR ?? DEFAULT_STORYBOOK_DIR,+ );+});+test.afterAll(async () => {+ await storybook?.close();+});++/** The text each range of the `astryx-<type>` syntax highlight covers. */+async function highlighted(page: Page, type: string) {+ return page.evaluate(name => {+ const highlight = CSS.highlights.get(`astryx-${name}`);+ return highlight+ ? Array.from(highlight as unknown as Iterable<Range>).map(range =>+ range.toString(),+ )+ : [];+ }, type);+}++test('built-in C++, Rust, and SQL colour with no setup', async ({page}) => {+ await page.goto(+ `${storybook.origin}/iframe.html?id=core-codeblock--built-in-languages&viewMode=story`,+ );+ await expect(page.locator('pre.astryx-code-block')).toHaveCount(3);+ await expect+ .poll(async () => highlighted(page, 'keyword'))+ .toEqual(expect.arrayContaining(['int', 'return', 'fn', 'SELECT']));+ expect(await highlighted(page, 'comment')).toEqual(+ expect.arrayContaining(['// Sum a vector', '-- Active users this week']),+ );+});++test('Thrift and Java colour when passed as the language', async ({page}) => {+ await page.goto(+ `${storybook.origin}/iframe.html?id=core-codeblock--language-modules&viewMode=story`,+ );+ await expect(page.locator('pre.astryx-code-block')).toHaveCount(2);+ await expect+ .poll(async () => highlighted(page, 'keyword'))+ .toEqual(expect.arrayContaining(['struct', 'required', 'public', 'class']));+ expect(await highlighted(page, 'string')).toContain('"hello, "');+});diff --git a/packages/core/src/CodeBlock/grammar.ts b/packages/core/src/CodeBlock/grammar.tsnew file mode 100644index 0000000000..89d885dc67--- /dev/null+++ b/packages/core/src/CodeBlock/grammar.ts@@ -0,0 +1,248 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file grammar.ts+ * @input Nothing+ * @output Shared pattern builders for CodeBlock grammars, the CodeLanguage+ * value, createCodeLanguage, and lookups by identifier and file name+ * @position Shared utility; consumed by tokenizer.ts and the modules under+ * languages/+ *+ * SYNC: When modified, update:+ * - /packages/core/src/CodeBlock/tokenizer.ts+ * - /packages/core/src/CodeBlock/languages/+ */++/** One grammar rule: text matching `regex` paints as `type`. */+export type GrammarPattern = {type: string; regex: RegExp};++/** A language's rules, tried in order at each position. */+export type Grammar = {+ patterns: GrammarPattern[];+ /** SyntaxToken type that matches the element's default text color (skipped in output). */+ defaultType: string;+};++const LANGUAGE = Symbol.for('astryx.codeLanguage');++/** A flat token from a custom tokenize function: absolute offsets in `code`. */+export type CodeLanguageToken = {type: string; start: number; end: number};++/** A custom language's tokenize function, the shape `tokenizer` takes. */+export type CodeLanguageTokenize = (+ code: string,+ language: string,+) => CodeLanguageToken[];++/**+ * A language value: an Astryx module from+ * `@astryxdesign/core/CodeBlock/languages/<path>`, or a custom language from+ * `createCodeLanguage`. Pass it as `language`.+ */+export type CodeLanguage = {+ /** The identifier the label shows, such as `graphql`; shown verbatim. */+ readonly name: string;+ /** The words that name this language in `language` strings and fences. */+ readonly identifiers: ReadonlyArray<string>;+ /** File extensions (no dot) that infer this language, such as `graphql`. */+ readonly fileExtensions: ReadonlyArray<string>;+ /** Whole file names that infer this language, such as `Dockerfile`. */+ readonly fileNames: ReadonlyArray<string>;+ readonly [LANGUAGE]: true;+};++/** How a language value paints: Astryx grammar data or a custom function. */+export type CodeLanguagePainter =+ | {kind: 'grammar'; grammar: Grammar}+ | {kind: 'tokenize'; tokenize: CodeLanguageTokenize};++type InternalCodeLanguage = CodeLanguage & {+ readonly painter: () => CodeLanguagePainter;+};++type CodeLanguageMeta = {+ name: string;+ identifiers: ReadonlyArray<string>;+ fileExtensions?: ReadonlyArray<string>;+ fileNames?: ReadonlyArray<string>;+};++function makeLanguage(+ meta: CodeLanguageMeta,+ painter: () => CodeLanguagePainter,+): CodeLanguage {+ const language: InternalCodeLanguage = {+ name: meta.name,+ identifiers: meta.identifiers,+ fileExtensions: meta.fileExtensions ?? [],+ fileNames: meta.fileNames ?? [],+ [LANGUAGE]: true,+ painter,+ };+ return language;+}++/** Make an Astryx language module from grammar data. Internal. */+export function defineCodeLanguage(+ meta: CodeLanguageMeta,+ grammar: () => Grammar,+): CodeLanguage {+ return makeLanguage(meta, () => ({kind: 'grammar', grammar: grammar()}));+}++export type CreateCodeLanguageOptions = CodeLanguageMeta & {+ /** Returns the tokens of `code`; runs synchronously. */+ tokenize: CodeLanguageTokenize;+};++/**+ * Make a custom language value from your own tokenize function. It works+ * everywhere an Astryx language module does, starting with `language`.+ *+ * @example+ * ```+ * const zig = createCodeLanguage({+ * name: 'zig',+ * identifiers: ['zig'],+ * fileExtensions: ['zig'],+ * tokenize: zigTokens,+ * });+ * <CodeBlock language={zig} code={source} />+ * ```+ */+export function createCodeLanguage({+ tokenize,+ ...meta+}: CreateCodeLanguageOptions): CodeLanguage {+ return makeLanguage(meta, () => ({kind: 'tokenize', tokenize}));+}++/** Whether `value` is a language value. */+export function isCodeLanguage(value: unknown): value is CodeLanguage {+ return (+ typeof value === 'object' &&+ value !== null &&+ (value as Partial<CodeLanguage>)[LANGUAGE] === true+ );+}++/** How a language value paints, or null for any other value. */+export function codeLanguagePainter(+ language: CodeLanguage,+): CodeLanguagePainter | null {+ const internal = language as Partial<InternalCodeLanguage>;+ return isCodeLanguage(language) && typeof internal.painter === 'function'+ ? internal.painter()+ : null;+}++/**+ * Whether `language` is a custom language (made with `createCodeLanguage`),+ * whose function runs synchronously at any length, as `tokenizer` does.+ */+export function isCustomCodeLanguage(language: string | CodeLanguage): boolean {+ return (+ typeof language !== 'string' &&+ codeLanguagePainter(language)?.kind === 'tokenize'+ );+}++/** The word a language reflects and labels: a string itself, or a value's name. */+export function languageLabel(language: string | CodeLanguage): string {+ return typeof language === 'string' ? language : language.name;+}++/** The word a language reflects as `data-language`: its first identifier. */+export function languageKey(language: string | CodeLanguage): string {+ return typeof language === 'string'+ ? language+ : (language.identifiers[0] ?? language.name);+}++/** The listed value that claims `word`; a later entry wins. */+export function listedLanguage(+ languages: ReadonlyArray<CodeLanguage> | undefined,+ word: string,+): CodeLanguage | undefined {+ if (languages == null) {+ return undefined;+ }+ for (let i = languages.length - 1; i >= 0; i--) {+ if (languages[i].identifiers.includes(word)) {+ return languages[i];+ }+ }+ return undefined;+}++/** The listed value a file's name or extension infers; a later entry wins. */+export function listedLanguageForFile(+ languages: ReadonlyArray<CodeLanguage> | undefined,+ fileName: string,+ extension: string | null,+): CodeLanguage | undefined {+ if (languages == null) {+ return undefined;+ }+ for (let i = languages.length - 1; i >= 0; i--) {+ const language = languages[i];+ if (+ language.fileNames.includes(fileName) ||+ (extension != null && language.fileExtensions.includes(extension))+ ) {+ return language;+ }+ }+ return undefined;+}++/** A keyword alternation from a space-separated word list. */+export const keywords = (words: string): RegExp =>+ new RegExp(`\\b(?:${words.trim().split(/\s+/).join('|')})\\b`);++export const NUMBER =+ /\b(?:0[xX][\da-fA-F_]+|0[bB][01_]+|0[oO][0-7_]+|\d[\d_]*\.?[\d_]*(?:[eE][+-]?\d+)?)[a-zA-Z]*\b/;+export const DOUBLE_QUOTED = /"(?:[^"\\\n]|\\.)*"/;+export const SINGLE_QUOTED = /'(?:[^'\\\n]|\\.)*'/;+export const BLOCK_COMMENT = /\/\*[\s\S]*?\*\//;+export const LINE_COMMENT = /\/\/[^\n]*/;+export const HASH_COMMENT = /#[^\n]*/;+export const CALL = /\b[a-zA-Z_$][\w$]*(?=\s*\()/;+export const TYPE_NAME = /\b[A-Z][a-zA-Z0-9_]*\b/;+export const OPERATOR = /[+\-*/%=!<>&|^~?:]+/;+// eslint-disable-next-line no-useless-escape+export const PUNCTUATION = /[{}()\[\];,.@]/;++/**+ * A C-like grammar: comments, any extra rules, strings, numbers, keywords,+ * calls, type names, operators, and punctuation, in that order.+ */+export function cLike(+ words: string,+ extra: GrammarPattern[] = [],+ comments: RegExp[] = [BLOCK_COMMENT, LINE_COMMENT],+): Grammar {+ return {+ defaultType: 'variable',+ patterns: [+ ...comments.map(regex => ({type: 'comment', regex})),+ ...extra,+ {type: 'string', regex: DOUBLE_QUOTED},+ {type: 'string', regex: SINGLE_QUOTED},+ {type: 'number', regex: NUMBER},+ {type: 'keyword', regex: keywords(words)},+ {type: 'function', regex: CALL},+ {type: 'type', regex: TYPE_NAME},+ {type: 'operator', regex: OPERATOR},+ {type: 'punctuation', regex: PUNCTUATION},+ ],+ };+}++/** A preprocessor directive at the start of a line (`#include`). */+export const DIRECTIVE = /^[ \t]*#\s*\w+/m;++export const C_WORDS =+ 'auto break case char const continue default do double else enum extern float for goto if inline int long register restrict return short signed sizeof static struct switch typedef union unsigned void volatile while bool true false NULL nullptr';++export const CPP_WORDS = `${C_WORDS} alignas alignof catch class concept constexpr consteval co_await co_return co_yield decltype delete explicit export friend mutable namespace new noexcept operator override private protected public requires static_assert template this throw try typename using virtual final`;diff --git a/packages/core/src/CodeBlock/index.ts b/packages/core/src/CodeBlock/index.tsindex 15dcac75e4..2cfd8debb6 100644--- a/packages/core/src/CodeBlock/index.ts+++ b/packages/core/src/CodeBlock/index.ts@@ -13,6 +13,13 @@export {CodeBlock} from './CodeBlock';export type {CodeBlockProps} from './CodeBlock';+export {createCodeLanguage} from './grammar';+export type {+ CodeLanguage,+ CodeLanguageToken,+ CodeLanguageTokenize,+ CreateCodeLanguageOptions,+} from './grammar';export {Code} from '../Code';export type {CodeProps, CodeColor, CodeSize} from '../Code';diff --git a/packages/core/src/CodeBlock/languages.test.ts b/packages/core/src/CodeBlock/languages.test.tsnew file mode 100644index 0000000000..09ab157898--- /dev/null+++ b/packages/core/src/CodeBlock/languages.test.ts@@ -0,0 +1,402 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file languages.test.ts+ * @input tokenizer.ts and the language modules+ * @output CodeBlock FR12 (built-in identifiers) and FR13 (language modules)+ * tokenizer evidence+ * @position CodeBlock FR12, FR13 evidence+ */++import {describe, expect, it} from 'vitest';+import {tokenize} from './tokenizer';+import {dockerfile} from './languages/dockerfile';+import {graphql} from './languages/graphql';+import {java} from './languages/java';+import {lua} from './languages/lua';+import {objectiveC} from './languages/objective-c';+import {powershell} from './languages/powershell';+import {ruby} from './languages/ruby';+import {thrift} from './languages/thrift';+import {toml} from './languages/toml';+import {+ createCodeLanguage,+ isCodeLanguage,+ languageKey,+ listedLanguage,+ listedLanguageForFile,+} from './grammar';+import type {CodeLanguage} from './grammar';++/** Token types painted for `code`, by the text they cover. */+function painted(+ code: string,+ language: string | CodeLanguage,+): Map<string, Set<string>> {+ const lines = code.split('\n');+ const out = new Map<string, Set<string>>();+ tokenize(code, language).forEach((tokens, index) => {+ for (const token of tokens) {+ const text = lines[index].slice(token.start, token.end);+ const types = out.get(token.type) ?? new Set<string>();+ types.add(text);+ out.set(token.type, types);+ }+ });+ return out;+}++type Case = {+ ids: string[];+ code: string;+ comment?: string;+ string?: string;+ number?: string;+ keyword: string;+};++const NATIVE: Case[] = [+ {+ ids: ['typescript', 'ts', 'javascript', 'js', 'tsx', 'jsx'],+ code: '// c\nconst n = 3; const s = "x";',+ comment: '// c',+ string: '"x"',+ number: '3',+ keyword: 'const',+ },+ {+ ids: ['json'],+ code: '{"s": "x", "n": 3}',+ string: '"x"',+ number: '3',+ keyword: '',+ },+ {+ ids: ['html', 'xml', 'svg'],+ code: '<!-- c -->\n<a href="x">t</a>',+ comment: '<!-- c -->',+ string: '"x"',+ keyword: '',+ },+ {+ ids: ['css', 'scss', 'less'],+ code: '/* c */\na { width: 3px; content: "x"; }',+ comment: '/* c */',+ string: '"x"',+ number: '3px',+ keyword: '',+ },+ {+ ids: ['python', 'py'],+ code: '# c\ndef f(): return "x", 3',+ comment: '# c',+ string: '"x"',+ number: '3',+ keyword: 'def',+ },+ {+ ids: ['bash', 'sh', 'zsh', 'shell'],+ code: '# c\nif true; then echo "x" 3; fi',+ comment: '# c',+ string: '"x"',+ number: '3',+ keyword: 'if',+ },+ {+ ids: ['php'],+ code: '<?php\n// c\nfunction f() { return "x" . 3; }',+ comment: '// c',+ string: '"x"',+ number: '3',+ keyword: 'function',+ },+ {+ ids: ['hack'],+ code: '// c\nfunction f(): vec<int> { return vec["x", 3]; }',+ comment: '// c',+ string: '"x"',+ number: '3',+ keyword: 'function',+ },+ {+ ids: ['yaml', 'yml'],+ code: '# c\nname: "x"\nn: 3',+ comment: '# c',+ string: '"x"',+ number: '3',+ keyword: '',+ },+ {+ ids: ['markdown', 'md'],+ code: '# Title\n\nSome `code` here.',+ keyword: '# Title',+ },+ {+ ids: ['c', 'h'],+ code: '#include <stdio.h>\n// c\nint n = 42; char *s = "x";',+ comment: '// c',+ string: '"x"',+ number: '42',+ keyword: 'int',+ },+ {+ ids: ['cpp', 'c++', 'cc', 'cxx', 'hpp', 'hh'],+ code: '// c\nauto v = std::vector<int>{1};\nconst char* s = "x";',+ comment: '// c',+ string: '"x"',+ number: '1',+ keyword: 'auto',+ },+ {+ ids: ['sql'],+ code: "-- c\nSELECT id FROM t WHERE n > 7 AND s = 'x';",+ comment: '-- c',+ string: "'x'",+ number: '7',+ keyword: 'SELECT',+ },+ {+ ids: ['rust', 'rs'],+ code: '// c\nfn main() { let n = 3; let s = "x"; }',+ comment: '// c',+ string: '"x"',+ number: '3',+ keyword: 'fn',+ },+ {+ ids: ['kotlin', 'kt', 'kts'],+ code: '// c\nfun main() { val n = 3; val s = "x" }',+ comment: '// c',+ string: '"x"',+ number: '3',+ keyword: 'fun',+ },+ {+ ids: ['swift'],+ code: '// c\nfunc f() { let n = 3; let s = "x" }',+ comment: '// c',+ string: '"x"',+ number: '3',+ keyword: 'func',+ },+ {+ ids: ['go', 'golang'],+ code: '// c\nfunc main() { n := 3; s := "x" }',+ comment: '// c',+ string: '"x"',+ number: '3',+ keyword: 'func',+ },+];++const MODULES: (Case & {module: CodeLanguage})[] = [+ {+ module: graphql,+ ids: ['graphql', 'gql'],+ code: '# c\nquery Q { user(id: "x", n: 3) { name } }',+ comment: '# c',+ string: '"x"',+ number: '3',+ keyword: 'query',+ },+ {+ module: java,+ ids: ['java'],+ code: '// c\npublic class A { int n = 3; String s = "x"; }',+ comment: '// c',+ string: '"x"',+ number: '3',+ keyword: 'public',+ },+ {+ module: thrift,+ ids: ['thrift'],+ code: '# c\nstruct U { 1: required string s = "x"; }',+ comment: '# c',+ string: '"x"',+ number: '1',+ keyword: 'struct',+ },+ {+ module: objectiveC,+ ids: ['objective-c', 'objc', 'm', 'mm'],+ code: '// c\n@interface A : NSObject\nint n = 3; NSString *s = @"x";',+ comment: '// c',+ string: '@"x"',+ number: '3',+ keyword: '@interface',+ },+ {+ module: toml,+ ids: ['toml'],+ code: '# c\n[pkg]\nname = "x"\nn = 3',+ comment: '# c',+ string: '"x"',+ number: '3',+ keyword: '',+ },+ {+ module: powershell,+ ids: ['powershell', 'ps1', 'pwsh'],+ code: '# c\nfunction F { $n = 3; $s = "x" }',+ comment: '# c',+ string: '"x"',+ number: '3',+ keyword: 'function',+ },+ {+ module: ruby,+ ids: ['ruby', 'rb'],+ code: '# c\ndef f\n n = 3; s = "x"\nend',+ comment: '# c',+ string: '"x"',+ number: '3',+ keyword: 'def',+ },+ {+ module: lua,+ ids: ['lua'],+ code: '-- c\nlocal function f() local n = 3; local s = "x" end',+ comment: '-- c',+ string: '"x"',+ number: '3',+ keyword: 'local',+ },+ {+ module: dockerfile,+ ids: ['dockerfile', 'docker'],+ code: '# c\nFROM node:20 AS build\nRUN echo "x" 3',+ comment: '# c',+ string: '"x"',+ number: '3',+ keyword: 'FROM',+ },+];++function expectPaints(map: Map<string, Set<string>>, c: Case) {+ for (const type of ['comment', 'string', 'number', 'keyword'] as const) {+ const text = c[type];+ if (text) {+ expect(map.get(type)).toContain(text);+ }+ }+}++describe('built-in languages (FR12)', () => {+ for (const c of NATIVE) {+ for (const id of c.ids) {+ it(`paints ${id} with no setup`, () => {+ expectPaints(painted(c.code, id), c);+ });+ }+ }++ it('paints a call, not the operand before `!=`, and Rust macros', () => {+ const go = painted('if err != nil { f(1) }', 'go');+ expect(go.get('function')).toEqual(new Set(['f']));+ expect(go.get('operator')).toContain('!=');+ expect(painted('println!("x"); a != b', 'rust').get('function')).toEqual(+ new Set(['println!']),+ );+ });++ it('reads a raw string with inner quotes as one string', () => {+ const rust = 'let j = r#"{"id": 1, "type": "user"}"#; let u = user;';+ expect(painted(rust, 'rust').get('string')).toEqual(+ new Set(['r#"{"id": 1, "type": "user"}"#']),+ );+ const cpp = 'auto j = R"json({"id": ")"})json"; int n;';+ expect(painted(cpp, 'cpp').get('string')).toEqual(+ new Set(['R"json({"id": ")"})json"']),+ );+ });++ it('reads prefixed C++ and extended-delimiter Swift raw strings as one string', () => {+ expect(painted('auto s = u8R"(a "b" c)";', 'cpp').get('string')).toEqual(+ new Set(['u8R"(a "b" c)"']),+ );+ expect(painted('auto s = LR"x(say "y")x";', 'cpp').get('string')).toEqual(+ new Set(['LR"x(say "y")x"']),+ );+ expect(painted('let s = #"He said "hi""#', 'swift').get('string')).toEqual(+ new Set(['#"He said "hi""#']),+ );+ });++ it('renders an identifier with no grammar as plain code', () => {+ expect(tokenize('const x = 1;', 'zig').flat()).toEqual([]);+ expect(tokenize('', 'rust')).toEqual([[]]);+ });+});++describe('language values (FR13)', () => {+ for (const c of MODULES) {+ it(`paints ${c.module.name} only when its value is passed`, () => {+ for (const id of c.ids) {+ // A string never resolves to a module.+ expect(tokenize(c.code, id).flat()).toEqual([]);+ }+ expectPaints(painted(c.code, c.module), c);+ });+ }++ it('carries its name, identifiers, file extensions, and file names', () => {+ expect(thrift.name).toBe('thrift');+ expect(objectiveC.identifiers).toContain('objc');+ expect(graphql.fileExtensions).toEqual(['graphql', 'gql']);+ expect(dockerfile.fileNames).toContain('Dockerfile');+ expect(java.fileNames).toEqual([]);+ expect(languageKey(objectiveC)).toBe('objective-c');+ });++ it('makes a custom language from a tokenize function', () => {+ const seen: string[] = [];+ const zig = createCodeLanguage({+ name: 'zig',+ identifiers: ['zig'],+ tokenize: (code, language) => {+ seen.push(language);+ return [{type: 'keyword', start: 0, end: 3}];+ },+ });+ expect(zig.fileExtensions).toEqual([]);+ expect(zig.fileNames).toEqual([]);+ expect(painted('pub fn\nx', zig).get('keyword')).toEqual(new Set(['pub']));+ expect(seen).toEqual(['zig']);+ expect(isCodeLanguage(zig)).toBe(true);+ });++ it('finds a listed value by identifier, later entries first', () => {+ const gqlToo = createCodeLanguage({+ name: 'gql2',+ identifiers: ['gql'],+ tokenize: () => [],+ });+ expect(listedLanguage([graphql], 'gql')).toBe(graphql);+ expect(listedLanguage([graphql, gqlToo], 'gql')).toBe(gqlToo);+ expect(listedLanguage([graphql], 'thrift')).toBeUndefined();+ expect(listedLanguage(undefined, 'gql')).toBeUndefined();+ });++ it('finds a listed value by file name or extension', () => {+ const list = [graphql, dockerfile, ruby];+ expect(listedLanguageForFile(list, 'schema.graphql', 'graphql')).toBe(+ graphql,+ );+ expect(listedLanguageForFile(list, 'Dockerfile', null)).toBe(dockerfile);+ expect(listedLanguageForFile(list, 'Gemfile', null)).toBe(ruby);+ expect(listedLanguageForFile(list, 'a.rs', 'rs')).toBeUndefined();+ });++ it('ignores values that are not language values', () => {+ const fake = {+ name: 'thrift',+ identifiers: ['thrift'],+ } as unknown as CodeLanguage;+ expect(isCodeLanguage(fake)).toBe(false);+ expect(tokenize('struct U {}', fake).flat()).toEqual([]);+ expect(powershell.name).toBe('powershell');+ expect(lua.name).toBe('lua');+ expect(toml.name).toBe('toml');+ });+});diff --git a/packages/core/src/CodeBlock/languages/dockerfile.ts b/packages/core/src/CodeBlock/languages/dockerfile.tsnew file mode 100644index 0000000000..962cc94fe2--- /dev/null+++ b/packages/core/src/CodeBlock/languages/dockerfile.ts@@ -0,0 +1,43 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file languages/dockerfile.ts+ * @input Shared grammar builders from ../grammar+ * @output The `dockerfile` language value for CodeBlock: Dockerfile: `dockerfile`, `docker`+ * @position Subpath entry: @astryxdesign/core/CodeBlock/languages/dockerfile+ */++import {+ DOUBLE_QUOTED,+ HASH_COMMENT,+ NUMBER,+ SINGLE_QUOTED,+ defineCodeLanguage,+} from '../grammar';++/** Dockerfile: `dockerfile`, `docker`. */+export const dockerfile = /* @__PURE__ */ defineCodeLanguage(+ {+ name: 'dockerfile',+ identifiers: ['dockerfile', 'docker'],+ fileExtensions: ['dockerfile'],+ fileNames: ['Dockerfile', 'Containerfile'],+ },+ () => ({+ defaultType: 'variable',+ patterns: [+ {type: 'comment', regex: HASH_COMMENT},+ {+ type: 'keyword',+ regex:+ /^[ \t]*(?:FROM|RUN|CMD|LABEL|EXPOSE|ENV|ADD|COPY|ENTRYPOINT|VOLUME|USER|WORKDIR|ARG|ONBUILD|STOPSIGNAL|HEALTHCHECK|SHELL)\b/im,+ },+ {type: 'keyword', regex: /\bAS\b/},+ {type: 'string', regex: DOUBLE_QUOTED},+ {type: 'string', regex: SINGLE_QUOTED},+ {type: 'variable', regex: /\$\{?\w+\}?/},+ {type: 'number', regex: NUMBER},+ {type: 'operator', regex: /[&|\\=]+/},+ ],+ }),+);diff --git a/packages/core/src/CodeBlock/languages/graphql.ts b/packages/core/src/CodeBlock/languages/graphql.tsnew file mode 100644index 0000000000..71e634eb7e--- /dev/null+++ b/packages/core/src/CodeBlock/languages/graphql.ts@@ -0,0 +1,30 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file languages/graphql.ts+ * @input Shared grammar builders from ../grammar+ * @output The `graphql` language value for CodeBlock: GraphQL: `graphql`, `gql`+ * @position Subpath entry: @astryxdesign/core/CodeBlock/languages/graphql+ */++import {HASH_COMMENT, cLike, defineCodeLanguage} from '../grammar';++/** GraphQL: `graphql`, `gql`. */+export const graphql = /* @__PURE__ */ defineCodeLanguage(+ {+ name: 'graphql',+ identifiers: ['graphql', 'gql'],+ fileExtensions: ['graphql', 'gql'],+ fileNames: [],+ },+ () =>+ cLike(+ 'query mutation subscription fragment on type input enum interface union scalar schema extend directive implements repeatable true false null',+ [+ {type: 'variable', regex: /\$\w+/},+ {type: 'constant', regex: /@\w+/},+ {type: 'string', regex: /"""[\s\S]*?"""/},+ ],+ [HASH_COMMENT],+ ),+);diff --git a/packages/core/src/CodeBlock/languages/java.ts b/packages/core/src/CodeBlock/languages/java.tsnew file mode 100644index 0000000000..0516e999f3--- /dev/null+++ b/packages/core/src/CodeBlock/languages/java.ts@@ -0,0 +1,28 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file languages/java.ts+ * @input Shared grammar builders from ../grammar+ * @output The `java` language value for CodeBlock: Java: `java`+ * @position Subpath entry: @astryxdesign/core/CodeBlock/languages/java+ */++import {cLike, defineCodeLanguage} from '../grammar';++/** Java: `java`. */+export const java = /* @__PURE__ */ defineCodeLanguage(+ {+ name: 'java',+ identifiers: ['java'],+ fileExtensions: ['java'],+ fileNames: [],+ },+ () =>+ cLike(+ 'abstract assert boolean break byte case catch char class const continue default do double else enum extends final finally float for if implements import instanceof int interface long native new package private protected public return short static super switch synchronized this throw throws transient try var void volatile while record sealed permits yield true false null',+ [+ {type: 'constant', regex: /@\w+/},+ {type: 'string', regex: /"""[\s\S]*?"""/},+ ],+ ),+);diff --git a/packages/core/src/CodeBlock/languages/lua.ts b/packages/core/src/CodeBlock/languages/lua.tsnew file mode 100644index 0000000000..c57b45bcc4--- /dev/null+++ b/packages/core/src/CodeBlock/languages/lua.ts@@ -0,0 +1,21 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file languages/lua.ts+ * @input Shared grammar builders from ../grammar+ * @output The `lua` language value for CodeBlock: Lua: `lua`+ * @position Subpath entry: @astryxdesign/core/CodeBlock/languages/lua+ */++import {cLike, defineCodeLanguage} from '../grammar';++/** Lua: `lua`. */+export const lua = /* @__PURE__ */ defineCodeLanguage(+ {name: 'lua', identifiers: ['lua'], fileExtensions: ['lua'], fileNames: []},+ () =>+ cLike(+ 'and break do else elseif end false for function goto if in local nil not or repeat return then true until while',+ [{type: 'string', regex: /\[(=*)\[[\s\S]*?\]\1\]/}],+ [/--\[(=*)\[[\s\S]*?\]\1\]/, /--[^\n]*/],+ ),+);diff --git a/packages/core/src/CodeBlock/languages/objective-c.ts b/packages/core/src/CodeBlock/languages/objective-c.tsnew file mode 100644index 0000000000..a811c61a2b--- /dev/null+++ b/packages/core/src/CodeBlock/languages/objective-c.ts@@ -0,0 +1,26 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file languages/objective-c.ts+ * @input Shared grammar builders from ../grammar+ * @output The `objectiveC` language value for CodeBlock: Objective-C: `objective-c`, `objc`, `m`, `mm`+ * @position Subpath entry: @astryxdesign/core/CodeBlock/languages/objective-c+ */++import {CPP_WORDS, DIRECTIVE, cLike, defineCodeLanguage} from '../grammar';++/** Objective-C: `objective-c`, `objc`, `m`, `mm`. */+export const objectiveC = /* @__PURE__ */ defineCodeLanguage(+ {+ name: 'objective-c',+ identifiers: ['objective-c', 'objc', 'm', 'mm'],+ fileExtensions: ['m', 'mm'],+ fileNames: [],+ },+ () =>+ cLike(`${CPP_WORDS} id self super nil Nil YES NO SEL BOOL instancetype`, [+ {type: 'keyword', regex: DIRECTIVE},+ {type: 'string', regex: /@"(?:[^"\\\n]|\\.)*"/},+ {type: 'keyword', regex: /@\w+/},+ ]),+);diff --git a/packages/core/src/CodeBlock/languages/powershell.ts b/packages/core/src/CodeBlock/languages/powershell.tsnew file mode 100644index 0000000000..a1a1a383d3--- /dev/null+++ b/packages/core/src/CodeBlock/languages/powershell.ts@@ -0,0 +1,31 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file languages/powershell.ts+ * @input Shared grammar builders from ../grammar+ * @output The `powershell` language value for CodeBlock: PowerShell+ * @position Subpath entry: @astryxdesign/core/CodeBlock/languages/powershell+ */++import {HASH_COMMENT, cLike, defineCodeLanguage} from '../grammar';++/** PowerShell: `powershell`, `ps1`, `pwsh`. */+export const powershell = /* @__PURE__ */ defineCodeLanguage(+ {+ name: 'powershell',+ identifiers: ['powershell', 'ps1', 'pwsh'],+ fileExtensions: ['ps1', 'psm1', 'psd1'],+ fileNames: [],+ },+ () =>+ cLike(+ 'begin break catch class continue data do dynamicparam else elseif end exit filter finally for foreach function if in param process return switch throw trap try until using while',+ [+ {type: 'comment', regex: /<#[\s\S]*?#>/},+ {type: 'variable', regex: /\$[\w:]+/},+ {type: 'function', regex: /\b[A-Z][a-z]+-[A-Z]\w*/},+ {type: 'operator', regex: /-[a-z]+\b/},+ ],+ [HASH_COMMENT],+ ),+);diff --git a/packages/core/src/CodeBlock/languages/ruby.ts b/packages/core/src/CodeBlock/languages/ruby.tsnew file mode 100644index 0000000000..175021ed2c--- /dev/null+++ b/packages/core/src/CodeBlock/languages/ruby.ts@@ -0,0 +1,29 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file languages/ruby.ts+ * @input Shared grammar builders from ../grammar+ * @output The `ruby` language value for CodeBlock: Ruby: `ruby`, `rb`+ * @position Subpath entry: @astryxdesign/core/CodeBlock/languages/ruby+ */++import {HASH_COMMENT, cLike, defineCodeLanguage} from '../grammar';++/** Ruby: `ruby`, `rb`. */+export const ruby = /* @__PURE__ */ defineCodeLanguage(+ {+ name: 'ruby',+ identifiers: ['ruby', 'rb'],+ fileExtensions: ['rb', 'rake', 'gemspec'],+ fileNames: ['Gemfile', 'Rakefile'],+ },+ () =>+ cLike(+ 'alias and begin break case class def defined do else elsif end ensure false for if in module next nil not or redo rescue retry return self super then true undef unless until when while yield require attr_accessor',+ [+ {type: 'constant', regex: /:\w+/},+ {type: 'variable', regex: /@{1,2}\w+|\$\w+/},+ ],+ [HASH_COMMENT],+ ),+);diff --git a/packages/core/src/CodeBlock/languages/thrift.ts b/packages/core/src/CodeBlock/languages/thrift.tsnew file mode 100644index 0000000000..1bb49f6a5a--- /dev/null+++ b/packages/core/src/CodeBlock/languages/thrift.ts@@ -0,0 +1,32 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file languages/thrift.ts+ * @input Shared grammar builders from ../grammar+ * @output The `thrift` language value for CodeBlock: Thrift IDL: `thrift`+ * @position Subpath entry: @astryxdesign/core/CodeBlock/languages/thrift+ */++import {+ BLOCK_COMMENT,+ HASH_COMMENT,+ LINE_COMMENT,+ cLike,+ defineCodeLanguage,+} from '../grammar';++/** Thrift IDL: `thrift`. */+export const thrift = /* @__PURE__ */ defineCodeLanguage(+ {+ name: 'thrift',+ identifiers: ['thrift'],+ fileExtensions: ['thrift'],+ fileNames: [],+ },+ () =>+ cLike(+ 'namespace include cpp_include typedef enum struct union exception service extends oneway void throws const required optional bool byte i8 i16 i32 i64 double string binary list set map true false',+ [],+ [BLOCK_COMMENT, LINE_COMMENT, HASH_COMMENT],+ ),+);diff --git a/packages/core/src/CodeBlock/languages/toml.ts b/packages/core/src/CodeBlock/languages/toml.tsnew file mode 100644index 0000000000..0506f77ba5--- /dev/null+++ b/packages/core/src/CodeBlock/languages/toml.ts@@ -0,0 +1,36 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file languages/toml.ts+ * @input Shared grammar builders from ../grammar+ * @output The `toml` language value for CodeBlock: TOML: `toml`+ * @position Subpath entry: @astryxdesign/core/CodeBlock/languages/toml+ */++import {DOUBLE_QUOTED, HASH_COMMENT, defineCodeLanguage} from '../grammar';++/** TOML: `toml`. */+export const toml = /* @__PURE__ */ defineCodeLanguage(+ {+ name: 'toml',+ identifiers: ['toml'],+ fileExtensions: ['toml'],+ fileNames: [],+ },+ () => ({+ defaultType: 'variable',+ patterns: [+ {type: 'comment', regex: HASH_COMMENT},+ {type: 'tag', regex: /^[ \t]*\[\[?[^\]\n]+\]\]?/m},+ {type: 'property', regex: /^[ \t]*[\w."'-]+(?=\s*=)/m},+ {type: 'string', regex: /"""[\s\S]*?"""|'''[\s\S]*?'''/},+ {type: 'string', regex: DOUBLE_QUOTED},+ {type: 'string', regex: /'[^'\n]*'/},+ {type: 'constant', regex: /\b(?:true|false)\b/},+ {type: 'number', regex: /[+-]?\b\d[\d_:.TZ+-]*\b/},+ {type: 'operator', regex: /=/},+ // eslint-disable-next-line no-useless-escape+ {type: 'punctuation', regex: /[\[\]{},.]/},+ ],+ }),+);diff --git a/packages/core/src/CodeBlock/tokenizer.ts b/packages/core/src/CodeBlock/tokenizer.tsindex 10732c7c13..94e06dcd6a 100644--- a/packages/core/src/CodeBlock/tokenizer.ts+++ b/packages/core/src/CodeBlock/tokenizer.ts@@ -17,6 +17,24 @@*/declare const scheduler: {yield?: () => Promise<void>} | undefined;+import {+ BLOCK_COMMENT,+ CALL,+ CPP_WORDS,+ C_WORDS,+ DIRECTIVE,+ NUMBER,+ PUNCTUATION,+ cLike,+ codeLanguagePainter,+} from './grammar';+import type {CodeLanguage, CodeLanguageTokenize, Grammar} from './grammar';++export type {CodeLanguage} from './grammar';++/** A language identifier string or a language value (FR13). */+export type CodeLanguageInput = string | CodeLanguage;+export type SyntaxToken = {type: string; start: number; end: number};/**@@ -55,22 +73,67 @@ const PHP_KEYWORDS =const HACK_KEYWORDS =/\b(function|class|if|else|for|foreach|while|return|echo|public|private|protected|static|new|try|catch|throw|namespace|use|require|include|extends|implements|interface|abstract|final|const|shape|vec|dict|keyset|async|await|concurrent|enum|type|newtype|tuple|inout)\b/;-function buildLanguage(lang: string): LangDef | null {- const cached = langCache.get(lang);+/** How each language value paints, compiled once per value. */+const valueCache = new WeakMap<+ CodeLanguage,+ LangDef | {tokenize: CodeLanguageTokenize; name: string} | null+>();++/**+ * How `language` paints: a compiled grammar (a built-in identifier, or an+ * Astryx module), a custom tokenize function, or null (plain code). An+ * identifier string selects only a built-in grammar (FR12).+ */+function resolveLanguage(+ language: CodeLanguageInput,+): LangDef | {tokenize: CodeLanguageTokenize; name: string} | null {+ if (typeof language !== 'string') {+ const cachedValue = valueCache.get(language);+ if (cachedValue !== undefined) {+ return cachedValue;+ }+ const painter = codeLanguagePainter(language);+ const resolved =+ painter == null+ ? null+ : painter.kind === 'grammar'+ ? compile(painter.grammar)+ : {tokenize: painter.tokenize, name: language.name};+ valueCache.set(language, resolved);+ return resolved;+ }+ const cached = langCache.get(language);if (cached !== undefined) {return cached;}-- const def = buildLanguageUncached(lang);- langCache.set(lang, def);+ const def = buildLanguageUncached(language);+ langCache.set(language, def);return def;}+/** A compiled grammar for `language`, or null for a custom or plain one. */+function buildLanguage(language: CodeLanguageInput): LangDef | null {+ const resolved = resolveLanguage(language);+ return resolved != null && 'patterns' in resolved ? resolved : null;+}++/** The tokens of a custom language, or null when `language` is not one. */+function customTokens(+ code: string,+ language: CodeLanguageInput,+): TokenLine[] | null {+ const resolved = resolveLanguage(language);+ return resolved != null && 'tokenize' in resolved+ ? flatTokensToLines(resolved.tokenize(code, resolved.name), code)+ : null;+}+function buildLanguageUncached(lang: string): LangDef | null {const raw = buildLanguagePatterns(lang);- if (!raw) {- return null;- }+ return raw ? compile(raw) : null;+}++function compile(raw: Grammar): LangDef {return {defaultType: raw.defaultType,patterns: raw.patterns.map(p => {@@ -80,10 +143,7 @@ function buildLanguageUncached(lang: string): LangDef | null {};}-function buildLanguagePatterns(lang: string): {- patterns: {type: string; regex: RegExp}[];- defaultType: string;-} | null {+function buildLanguagePatterns(lang: string): Grammar | null {switch (lang) {case 'typescript':case 'javascript':@@ -308,6 +368,84 @@ function buildLanguagePatterns(lang: string): {],};+ case 'c':+ case 'h':+ return cLike(C_WORDS, [{type: 'keyword', regex: DIRECTIVE}]);++ case 'cpp':+ case 'c++':+ case 'cc':+ case 'cxx':+ case 'hpp':+ case 'hh':+ return cLike(CPP_WORDS, [+ {type: 'keyword', regex: DIRECTIVE},+ {+ type: 'string',+ regex: /\b(?:u8|[uUL])?R"([^()\\\s]{0,16})\([\s\S]*?\)\1"/,+ },+ ]);++ case 'sql':+ return {+ defaultType: 'variable',+ patterns: [+ {type: 'comment', regex: /--[^\n]*/},+ {type: 'comment', regex: BLOCK_COMMENT},+ {type: 'string', regex: /'(?:[^']|'')*'/},+ {type: 'property', regex: /"[^"\n]*"|`[^`\n]*`/},+ {type: 'number', regex: NUMBER},+ {+ type: 'keyword',+ regex:+ /\b(?:select|from|where|and|or|not|in|is|null|as|join|inner|left|right|full|outer|cross|on|group|by|order|having|limit|offset|union|all|distinct|insert|into|values|update|set|delete|create|table|view|index|drop|alter|add|primary|key|foreign|references|with|case|when|then|else|end|exists|between|like|asc|desc|true|false|default|returning|interval)\b/i,+ },+ {type: 'function', regex: CALL},+ {type: 'operator', regex: /[+\-*/%=<>!|]+/},+ {type: 'punctuation', regex: PUNCTUATION},+ ],+ };++ case 'rust':+ case 'rs':+ return cLike(+ 'as async await break const continue crate dyn else enum extern false fn for if impl in let loop match mod move mut pub ref return self Self static struct super trait true type unsafe use where while',+ [+ {type: 'constant', regex: /#!?\[[^\]\n]*\]/},+ {type: 'function', regex: /\b[a-zA-Z_]\w*!(?=\s*[([{])/},+ {type: 'string', regex: /\bb?r(#*)"[\s\S]*?"\1/},+ {type: 'type', regex: /'[a-z_]\w*\b(?!')/},+ ],+ );++ case 'kotlin':+ case 'kt':+ case 'kts':+ return cLike(+ 'as break class continue do else false for fun if in interface is null object package return super this throw true try typealias val var when while by constructor init companion data sealed enum open override private protected public internal suspend inline lateinit import',+ [+ {type: 'constant', regex: /@\w+/},+ {type: 'string', regex: /"""[\s\S]*?"""/},+ ],+ );++ case 'swift':+ return cLike(+ 'associatedtype class deinit enum extension func import init inout internal let open operator private protocol public static struct subscript typealias var break case continue default defer do else fallthrough for guard if in repeat return switch where while as catch false is nil rethrows super self Self throw throws true try async await actor some any',+ [+ {type: 'string', regex: /(#+)"[\s\S]*?"\1/},+ {type: 'constant', regex: /[@#]\w+/},+ {type: 'string', regex: /"""[\s\S]*?"""/},+ ],+ );++ case 'go':+ case 'golang':+ return cLike(+ 'break case chan const continue default defer else fallthrough for func go goto if import interface map package range return select struct switch type var true false nil iota',+ [{type: 'string', regex: /`[^`]*`/}],+ );+default:return null;}@@ -494,10 +632,17 @@ function tokenizeLine(* language's default color type are omitted (~30% reduction).** @param code - The source code string to tokenize- * @param language - Language identifier (e.g. 'typescript', 'python')+ * @param language - A built-in identifier (e.g. 'typescript') or a language value* @returns Array of token arrays, one per line*/-export function tokenize(code: string, language: string): TokenLine[] {+export function tokenize(+ code: string,+ language: CodeLanguageInput,+): TokenLine[] {+ const custom = customTokens(code, language);+ if (custom != null) {+ return custom;+ }const langDef = buildLanguage(language);if (!langDef) {return [];@@ -522,9 +667,14 @@ export function tokenize(code: string, language: string): TokenLine[] {*/export async function tokenizeAsync(code: string,- language: string,+ language: CodeLanguageInput,signal?: AbortSignal,): Promise<TokenLine[]> {+ // A custom language runs synchronously, as the `tokenizer` prop does.+ const custom = customTokens(code, language);+ if (custom != null) {+ return custom;+ }const langDef = buildLanguage(language);if (!langDef) {return [];@@ -560,10 +710,15 @@ export async function tokenizeAsync(*/export async function tokenizeStreaming(code: string,- language: string,+ language: CodeLanguageInput,onBatch: (lines: TokenLine[], startLine: number) => void,signal?: AbortSignal,): Promise<void> {+ const custom = customTokens(code, language);+ if (custom != null) {+ onBatch(custom, 0);+ return;+ }const langDef = buildLanguage(language);if (!langDef) {return;diff --git a/packages/core/src/CodeBlock/tokenizerBudget.test.ts b/packages/core/src/CodeBlock/tokenizerBudget.test.tsnew file mode 100644index 0000000000..685e69c20a--- /dev/null+++ b/packages/core/src/CodeBlock/tokenizerBudget.test.ts@@ -0,0 +1,112 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file tokenizerBudget.test.ts+ * @input tokenizer.ts and each module under languages/, bundled and minified+ * by esbuild+ * @output The CodeBlock language budget (PR4): the built-in tokenizer within+ * 4,608 bytes and each language module within 512 bytes, minified and gzipped+ * @position CodeBlock PR4 evidence+ */++import {execFileSync} from 'node:child_process';+import {mkdtempSync, rmSync, writeFileSync} from 'node:fs';+import path from 'node:path';+import {fileURLToPath} from 'node:url';+import {afterAll, describe, expect, it} from 'vitest';++const here = path.dirname(fileURLToPath(import.meta.url));+const repoRoot = path.resolve(here, '../../../..');+// Inside the repo so the script resolves `esbuild` by bare specifier, and+// under node_modules so it is ignored and swept with the install.+const workspace = mkdtempSync(+ path.join(repoRoot, 'node_modules', '.astryx-tokenizer-budget-'),+);+afterAll(() => {+ rmSync(workspace, {recursive: true, force: true});+});++/** Each module's export name and subpath under ./languages. */+const MODULES = [+ 'graphql:graphql',+ 'java:java',+ 'thrift:thrift',+ 'objectiveC:objective-c',+ 'toml:toml',+ 'powershell:powershell',+ 'ruby:ruby',+ 'lua:lua',+ 'dockerfile:dockerfile',+];++/**+ * Gzipped, minified size of each entry. A module is measured as its marginal+ * cost: the built-in tokenizer plus that module, less the tokenizer alone.+ */+const script = path.join(workspace, 'measure.mjs');+writeFileSync(+ script,+ [+ "import {build} from 'esbuild';",+ "import {gzipSync} from 'node:zlib';",+ 'const [dir, ...modules] = process.argv.slice(2);',+ 'async function bundle(contents) {',+ ' const out = await build({',+ " stdin: {contents, resolveDir: dir, loader: 'ts'},",+ " bundle: true, minify: true, format: 'esm', write: false, logLevel: 'silent',",+ ' });',+ ' return out.outputFiles[0].text;',+ '}',+ 'async function size(contents) {',+ ' const out = await build({',+ " stdin: {contents, resolveDir: dir, loader: 'ts'},",+ " bundle: true, minify: true, format: 'esm', write: false, logLevel: 'silent',",+ ' });',+ ' return gzipSync(out.outputFiles[0].contents, {level: 9}).length;',+ '}',+ 'const base = await size("import {tokenize} from \'./tokenizer\'; console.log(tokenize);");',+ 'const core = await bundle("import {CodeBlock, createCodeLanguage, tokenize} from \'./index\'; console.log(CodeBlock, createCodeLanguage, tokenize);");',+ "const result = {tokenizer: base, modules: {}, leaked: ['cpp_include', 'dynamicparam', 'HEALTHCHECK', 'instancetype', 'subscription'].filter(word => core.includes(word))};",+ 'for (const entry of modules) {',+ " const [name, path] = entry.split(':');",+ " const both = await size(`import {tokenize} from './tokenizer'; import {${name}} from './languages/${path}'; console.log(tokenize, ${name});`);",+ ' result.modules[name] = both - base;',+ '}',+ 'console.log(JSON.stringify(result));',+ ].join('\n'),+);++describe('CodeBlock language budget (PR4)', () => {+ const result = JSON.parse(+ execFileSync(process.execPath, [script, here, ...MODULES], {+ encoding: 'utf8',+ timeout: 120_000,+ })+ .trim()+ .split('\n')+ .pop() ?? '{}',+ ) as {+ tokenizer: number;+ modules: Record<string, number>;+ leaked: string[];+ };++ it('keeps the built-in tokenizer within 4,608 bytes minified and gzipped', () => {+ expect(result.tokenizer).toBeGreaterThan(0);+ expect(result.tokenizer).toBeLessThanOrEqual(4608);+ });++ it('adds no module a bundle does not import', () => {+ // Words only the Thrift, PowerShell, Dockerfile, Objective-C, and GraphQL+ // grammars contain never reach a bundle of the CodeBlock entry.+ expect(result.leaked).toEqual([]);+ });++ it.each(MODULES.map(entry => entry.split(':')[0]))(+ 'keeps the %s module within 512 bytes',+ name => {+ expect(result.modules[name]).toBeGreaterThan(0);+ expect(result.modules[name]).toBeLessThanOrEqual(512);+ },+ );+});diff --git a/scripts/sync-exports.js b/scripts/sync-exports.jsindex ddc98d5f03..0bbe258f11 100644--- a/scripts/sync-exports.js+++ b/scripts/sync-exports.js@@ -222,7 +222,19 @@ const UTIL_SUBPATH_DIRS = [* `Markdown/remark` is the limited Remark compatibility adapter* (`module:Markdown/remark`, `spec:AST-036` FR24).*/-const FILE_MODULE_SUBPATH_EXPORTS = ['Markdown/remark'];+const FILE_MODULE_SUBPATH_EXPORTS = [+ // One opt-in module per CodeBlock language (`component:CodeBlock` FR13).+ 'CodeBlock/languages/dockerfile',+ 'CodeBlock/languages/graphql',+ 'CodeBlock/languages/java',+ 'CodeBlock/languages/lua',+ 'CodeBlock/languages/objective-c',+ 'CodeBlock/languages/powershell',+ 'CodeBlock/languages/ruby',+ 'CodeBlock/languages/thrift',+ 'CodeBlock/languages/toml',+ 'Markdown/remark',+];/*** Discover all exportable directories under src/.