Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.
import { createHighlighterCore, type HighlighterCore } from "@shikijs/core";
import { createJavaScriptRegexEngine } from "@shikijs/engine-javascript";
import bashLanguage from "@shikijs/langs/bash";
import javascriptLanguage from "@shikijs/langs/javascript";
import jsonLanguage from "@shikijs/langs/json";
import jsxLanguage from "@shikijs/langs/jsx";
import tsxLanguage from "@shikijs/langs/tsx";
import typescriptLanguage from "@shikijs/langs/typescript";
import yamlLanguage from "@shikijs/langs/yaml";
import githubDarkDefault from "@shikijs/themes/github-dark-default";
import githubLightDefault from "@shikijs/themes/github-light-default";
export type CodeHighlightTheme = "light" | "dark";
export interface HighlightedCodeToken {
readonly content: string;
readonly color: string | null;
readonly fontStyle: number | null;
}
const SHIKI_THEME_NAME_BY_SCHEME = {
light: "github-light-default",
dark: "github-dark-default",
} as const;
// Tests always use the JavaScript regex engine; apps prefer the native one when present.
const PREFER_NATIVE_ENGINE =
(process.env.EXPO_PUBLIC_CODE_HIGHLIGHTER_ENGINE ??
(process.env.NODE_ENV === "test" ? "javascript" : "native")) === "native";
const HIGHLIGHT_CHUNK_LINE_THRESHOLD = 8;
const HIGHLIGHT_CHUNK_SIZE = 200;
const TOKENIZE_MAX_LINE_LENGTH = 1_000;
const INITIAL_LANGUAGE_MODULES = [
bashLanguage,
javascriptLanguage,
jsonLanguage,
jsxLanguage,
tsxLanguage,
typescriptLanguage,
yamlLanguage,
] satisfies Parameters<typeof createHighlighterCore>[0]["langs"];
const loadedLanguages = new Set<string>([
"text",
"bash",
"javascript",
"json",
"jsx",
"tsx",
"typescript",
"yaml",
]);
const languageLoadingPromises = new Map<string, Promise<boolean>>();
const languageImports = new Map(
Object.entries({
javascript: () => import("@shikijs/langs/javascript"),
typescript: () => import("@shikijs/langs/typescript"),
jsx: () => import("@shikijs/langs/jsx"),
tsx: () => import("@shikijs/langs/tsx"),
python: () => import("@shikijs/langs/python"),
rust: () => import("@shikijs/langs/rust"),
go: () => import("@shikijs/langs/go"),
java: () => import("@shikijs/langs/java"),
kotlin: () => import("@shikijs/langs/kotlin"),
swift: () => import("@shikijs/langs/swift"),
"objective-c": () => import("@shikijs/langs/objective-c"),
c: () => import("@shikijs/langs/c"),
cpp: () => import("@shikijs/langs/cpp"),
csharp: () => import("@shikijs/langs/csharp"),
php: () => import("@shikijs/langs/php"),
ruby: () => import("@shikijs/langs/ruby"),
lua: () => import("@shikijs/langs/lua"),
perl: () => import("@shikijs/langs/perl"),
r: () => import("@shikijs/langs/r"),
dart: () => import("@shikijs/langs/dart"),
scala: () => import("@shikijs/langs/scala"),
elixir: () => import("@shikijs/langs/elixir"),
haskell: () => import("@shikijs/langs/haskell"),
clojure: () => import("@shikijs/langs/clojure"),
ocaml: () => import("@shikijs/langs/ocaml"),
fsharp: () => import("@shikijs/langs/fsharp"),
erlang: () => import("@shikijs/langs/erlang"),
zig: () => import("@shikijs/langs/zig"),
nim: () => import("@shikijs/langs/nim"),
html: () => import("@shikijs/langs/html"),
css: () => import("@shikijs/langs/css"),
scss: () => import("@shikijs/langs/scss"),
less: () => import("@shikijs/langs/less"),
xml: () => import("@shikijs/langs/xml"),
svg: () => import("@shikijs/langs/xml"),
vue: () => import("@shikijs/langs/vue"),
svelte: () => import("@shikijs/langs/svelte"),
astro: () => import("@shikijs/langs/astro"),
json: () => import("@shikijs/langs/json"),
jsonc: () => import("@shikijs/langs/jsonc"),
yaml: () => import("@shikijs/langs/yaml"),
toml: () => import("@shikijs/langs/toml"),
ini: () => import("@shikijs/langs/ini"),
bash: () => import("@shikijs/langs/bash"),
shellscript: () => import("@shikijs/langs/shellscript"),
powershell: () => import("@shikijs/langs/powershell"),
fish: () => import("@shikijs/langs/fish"),
sql: () => import("@shikijs/langs/sql"),
graphql: () => import("@shikijs/langs/graphql"),
prisma: () => import("@shikijs/langs/prisma"),
docker: () => import("@shikijs/langs/docker"),
hcl: () => import("@shikijs/langs/hcl"),
nix: () => import("@shikijs/langs/nix"),
markdown: () => import("@shikijs/langs/markdown"),
mdx: () => import("@shikijs/langs/mdx"),
tex: () => import("@shikijs/langs/tex"),
diff: () => import("@shikijs/langs/diff"),
regex: () => import("@shikijs/langs/regex"),
viml: () => import("@shikijs/langs/viml"),
makefile: () => import("@shikijs/langs/makefile"),
cmake: () => import("@shikijs/langs/cmake"),
groovy: () => import("@shikijs/langs/groovy"),
} as const),
);
const languageAliases = new Map(
Object.entries({
js: "javascript",
mjs: "javascript",
cjs: "javascript",
ts: "typescript",
mts: "typescript",
cts: "typescript",
py: "python",
rb: "ruby",
rs: "rust",
sh: "bash",
zsh: "bash",
shell: "shellscript",
yml: "yaml",
md: "markdown",
"c++": "cpp",
"c#": "csharp",
cs: "csharp",
dockerfile: "docker",
vim: "viml",
objc: "objective-c",
objectivec: "objective-c",
"obj-c": "objective-c",
ps1: "powershell",
pwsh: "powershell",
hs: "haskell",
ex: "elixir",
exs: "elixir",
erl: "erlang",
clj: "clojure",
ml: "ocaml",
fs: "fsharp",
tf: "hcl",
make: "makefile",
plain: "text",
plaintext: "text",
txt: "text",
} as const),
);
let highlighterPromise: Promise<HighlighterCore> | null = null;
function waitForNextFrame(): Promise<void> {
return new Promise((resolve) => {
setTimeout(resolve, 0);
});
}
async function createHighlighter(): Promise<HighlighterCore> {
const themes = [githubLightDefault, githubDarkDefault];
if (PREFER_NATIVE_ENGINE) {
try {
const nativeEngineModule = await import("react-native-shiki-engine");
if (nativeEngineModule.isNativeEngineAvailable()) {
return await createHighlighterCore({
themes,
langs: INITIAL_LANGUAGE_MODULES,
engine: nativeEngineModule.createNativeEngine(),
});
}
} catch {
// Fall back to the JavaScript regex engine below.
}
}
return createHighlighterCore({
themes,
langs: INITIAL_LANGUAGE_MODULES,
engine: createJavaScriptRegexEngine(),
});
}
function getHighlighter(): Promise<HighlighterCore> {
if (!highlighterPromise) {
highlighterPromise = createHighlighter().catch((error) => {
highlighterPromise = null;
throw error;
});
}
return highlighterPromise;
}
function resolveLanguageAlias(language: string): string {
const normalized = language.toLowerCase();
return languageAliases.get(normalized) ?? normalized;
}
async function loadSingleLanguage(
highlighter: HighlighterCore,
language: string,
): Promise<boolean> {
if (loadedLanguages.has(language)) {
return true;
}
const existingPromise = languageLoadingPromises.get(language);
if (existingPromise) {
return existingPromise;
}
const importer = languageImports.get(language);
if (!importer) {
return false;
}
const loadingPromise = (async () => {
try {
const languageModule = await importer();
await highlighter.loadLanguage(languageModule.default);
loadedLanguages.add(language);
return true;
} catch {
return false;
} finally {
languageLoadingPromises.delete(language);
}
})();
languageLoadingPromises.set(language, loadingPromise);
return loadingPromise;
}
async function resolveLanguage(languageHint: string): Promise<string> {
const candidate = resolveLanguageAlias(languageHint);
if (candidate === "text" || candidate === "ansi" || !languageImports.has(candidate)) {
return "text";
}
if (loadedLanguages.has(candidate)) {
return candidate;
}
const loaded = await loadSingleLanguage(await getHighlighter(), candidate);
return loaded ? candidate : "text";
}
function normalizeHighlightedLines(
tokenLines: ReadonlyArray<ReadonlyArray<{ content: string; color?: string; fontStyle?: number }>>,
): ReadonlyArray<ReadonlyArray<HighlightedCodeToken>> {
return tokenLines.map((line) =>
line.map((token) => ({
content: token.content,
color: token.color ?? null,
fontStyle: token.fontStyle ?? null,
})),
);
}
async function highlightLines(
code: string,
language: string,
theme: string,
): Promise<ReadonlyArray<ReadonlyArray<HighlightedCodeToken>>> {
if (code.length === 0) {
return [];
}
const highlighter = await getHighlighter();
const sourceLines = code.split("\n");
const highlightedLines: Array<ReadonlyArray<HighlightedCodeToken>> = [];
const shortLineBatch: string[] = [];
const flushShortLineBatch = async (): Promise<void> => {
if (shortLineBatch.length === 0) {
return;
}
const tokenLines = highlighter.codeToTokensBase(shortLineBatch.join("\n"), {
lang: language,
theme,
});
highlightedLines.push(...normalizeHighlightedLines(tokenLines));
shortLineBatch.length = 0;
};
for (let lineIndex = 0; lineIndex < sourceLines.length; lineIndex += 1) {
const line = sourceLines[lineIndex] ?? "";
if (line.length > TOKENIZE_MAX_LINE_LENGTH) {
await flushShortLineBatch();
highlightedLines.push([{ content: line, color: null, fontStyle: null }]);
} else {
shortLineBatch.push(line);
}
if (shortLineBatch.length >= HIGHLIGHT_CHUNK_SIZE) {
await flushShortLineBatch();
}
if (
sourceLines.length > HIGHLIGHT_CHUNK_LINE_THRESHOLD &&
lineIndex + 1 < sourceLines.length &&
(shortLineBatch.length === 0 || line.length > TOKENIZE_MAX_LINE_LENGTH)
) {
await waitForNextFrame();
}
}
await flushShortLineBatch();
return highlightedLines;
}
/** Tokenizes a fenced code block from chat markdown. Unknown languages render as plain text. */
export async function highlightCodeSnippet(input: {
readonly code: string;
readonly language?: string | null;
readonly theme: CodeHighlightTheme;
}): Promise<ReadonlyArray<ReadonlyArray<HighlightedCodeToken>>> {
const language = await resolveLanguage(input.language?.trim() || "text");
return highlightLines(input.code, language, SHIKI_THEME_NAME_BY_SCHEME[input.theme]);
}