Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Reduce Studio code highlighter bundle (#15740)

Daniel Han8f5ed05b15b4
diff --git a/studio/frontend/src/components/assistant-ui/code-plugin.ts b/studio/frontend/src/components/assistant-ui/code-plugin.ts
index 0004c9fba73..ecb0622731b 100644
--- a/studio/frontend/src/components/assistant-ui/code-plugin.ts
+++ b/studio/frontend/src/components/assistant-ui/code-plugin.ts
@@ -8,20 +8,34 @@ import type {
HighlightResult,
ThemeInput,
} from "@streamdown/code";
-import {
- type BundledLanguage,
- bundledLanguages,
- bundledLanguagesInfo,
- createHighlighter,
- type GrammarState,
- type ThemedToken,
-} from "shiki";
+import type { GrammarState, ThemedToken } from "shiki";
+import { createBundledHighlighter } from "shiki/core";
import {
createJavaScriptRegexEngine,
defaultJavaScriptRegexConstructor,
} from "shiki/engine/javascript";
+import {
+ type BundledLanguage,
+ bundledLanguages,
+ bundledLanguagesInfo,
+} from "shiki/langs";
+
+// Keep named themes limited to Studio and Streamdown defaults; other callers pass objects.
+type NamedTheme = "github-light" | "github-dark";
+const NAMED_THEMES = {
+ "github-light": () => import("@shikijs/themes/github-light"),
+ "github-dark": () => import("@shikijs/themes/github-dark"),
+};
+
+const createHighlighter = createBundledHighlighter<
+ BundledLanguage,
+ NamedTheme
+>({
+ langs: bundledLanguages,
+ themes: NAMED_THEMES,
+ engine: () => createJavaScriptRegexEngine({ forgiving: true }),
+});
-// Fence tags shiki does not alias: lower-cased input to canonical shiki id.
const LANGUAGE_ALIAS_OVERRIDES: Record<string, BundledLanguage> = {
objectivec: "objective-c",
"obj-c": "objective-c",
@@ -63,32 +77,20 @@ const SUPPORTED_LANGUAGE_LIST = Array.from(
) as BundledLanguage[];
const PLAIN_TEXT = "text" as BundledLanguage;
-/** Exported so per-grammar caches key on the same identity as `highlight`. */
export const normalizeLanguage = (language: string): BundledLanguage => {
const key = language.trim().toLowerCase();
const alias = LANGUAGE_ALIAS_OVERRIDES[key] ?? SHIKI_LANGUAGE_ALIASES[key];
return alias ?? (key as BundledLanguage);
};
-// Past this size, cache completed lines and grammar state so a streaming fence tokenizes only new text.
export const MIN_INCREMENTAL_CHARS = 2000;
const REFRESH_MS = 250;
const REFRESH_SLICE_MS = 8;
-// Shiki's own tokenizer guard is a wall clock: a line that takes longer than `tokenizeTimeLimit`
-// (default 500 ms) is abandoned part-way and the rest of it is emitted as one uncoloured token.
-// Dual themes are two passes with two separate budgets, and only the first pays to compile the
-// grammar's regexes, so a loaded machine drops the light theme to plain and keeps the dark one
-// correct -- for the same line, in the same result. Committed lines are never re-tokenized, so that
-// half-plain line is then cached for the life of the fence. Bound the work by line length instead:
-// same protection against a minified blob (~120 ms at this length, growing quadratically past it,
-// which is VS Code's `editor.maxTokenizationLineLength` default for the same reason), but the
-// output depends on the source rather than on the host.
+// A wall-clock limit can cache a half-plain theme; bound line length for deterministic dual-theme output.
export const TOKENIZE_LIMITS = {
tokenizeTimeLimit: 0,
tokenizeMaxLineLength: 20_000,
} as const;
-// An unvirtualized thread mounts every fence. Bound both their count and source
-// size; token data measured roughly 30 bytes per source character.
export const MAX_FENCES = 512;
export const MAX_CACHED_CHARACTERS = 512_000;
@@ -157,11 +159,9 @@ const shiftLine = (line: TokenLine, offset: number): TokenLine =>
? line
: line.map((token) => ({ ...token, offset: token.offset + offset }));
-// Adjacent-token coalescing measured zero (dual tokens 537013 -> merged 537013); Shiki already
-// coalesces. Reproduce with scripts/coal-span-census.mjs.
-
// Markdown reports a closing fence as body until it recognizes it, so that line (possibly
// a partial run) is all a fence can lose.
+// Shiki already coalesces (537013 -> merged 537013); reproduce with scripts/coal-span-census.mjs.
const CLOSING_FENCE = /^ {0,3}(?:`+|~+)[ \t]*$/;
const shedsClosingRun = (shorter: string, longer: string): boolean =>
@@ -222,7 +222,6 @@ export function createCodePlugin(
: { forgiving: true },
);
const highlighters = new Map<string, { highlighter: Highlighter | null }>();
- // Most recently used first.
const fences: Fence[] = [];
const fencesByCode = new Map<string, Fence>();
let cachedCharacters = 0;
@@ -231,7 +230,6 @@ export function createCodePlugin(
const codeKey = (key: string, code: string): string =>
`${key}\n${code.length}\n${code.slice(0, 32)}\n${code.slice(-32)}`;
- // Identical fences can share an index key.
const dropCodeIndex = (fence: Fence): void => {
const indexKey = codeKey(fence.key, fence.code);
if (fencesByCode.get(indexKey) === fence) fencesByCode.delete(indexKey);
@@ -275,7 +273,6 @@ export function createCodePlugin(
fences.length > MAX_FENCES ||
(cachedCharacters > MAX_CACHED_CHARACTERS && fences.length > 1)
) {
- // Keep pending callbacks until their grammar resolves.
let dropIndex = fences.length - 1;
while (dropIndex >= 0 && fences[dropIndex].pending !== null) {
dropIndex -= 1;
@@ -318,7 +315,6 @@ export function createCodePlugin(
return anchor && reaches ? Math.min(anchor.length, code.length) : -1;
};
- /** The fence whose cached code reaches furthest into `code`. */
const findFence = (key: string, code: string): Fence => {
const exact = fencesByCode.get(codeKey(key, code));
if (exact && exact.code === code) return promote(exact);
diff --git a/studio/frontend/tests/code-plugin-engine-and-cache.test.ts b/studio/frontend/tests/code-plugin-engine-and-cache.test.ts
index f4eb684d345..4a8797513b5 100644
--- a/studio/frontend/tests/code-plugin-engine-and-cache.test.ts
+++ b/studio/frontend/tests/code-plugin-engine-and-cache.test.ts
@@ -1,11 +1,6 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-/**
- * The desktop CSP forbids WASM, so shiki must use the JS regex engine; also exercises the
- * character-budget eviction branch, unreachable from other fixtures.
- */
-
import assert from "node:assert/strict";
import test from "node:test";
import type {
@@ -38,7 +33,6 @@ const highlightOnce = (
});
test("the highlighter uses the JavaScript regex engine, not the WASM one", () => {
- // assert.match prints the whole subject, burying the message, so assert a boolean.
assert.ok(
/createJavaScriptRegexEngine/.test(PLUGIN_SOURCE),
"code-plugin must build its highlighter with shiki's JavaScript regex engine",
@@ -51,6 +45,16 @@ test("the highlighter uses the JavaScript regex engine, not the WASM one", () =>
);
});
+const RUNTIME_SHIKI_IMPORT = /^import\s+(?!type\b)[^;]*?\bfrom\s+"shiki";/m;
+
+test("the highlighter factory is built without shiki's default WASM engine", () => {
+ assert.ok(
+ !RUNTIME_SHIKI_IMPORT.test(PLUGIN_SOURCE),
+ 'code-plugin must take only types from "shiki"; build the factory from shiki/core',
+ );
+ assert.ok(PLUGIN_SOURCE.includes("= createBundledHighlighter<"));
+});
+
const sourceOfSize = (id: number, chars: number): string => {
const lines: string[] = [];
let length = 0;
@@ -71,7 +75,6 @@ const stillCached = async (
(await highlightOnce(plugin, { code, language, themes: THEMES })) === previous;
test("the character budget evicts even while the fence count is under its limit", async () => {
- // ~600 KB over 100 fences exceeds MAX_CACHED_CHARACTERS but not MAX_FENCES.
const plugin = createCodePlugin({ themes: THEMES });
const first = sourceOfSize(0, 6_000);
const held = await highlightOnce(plugin, {
diff --git a/studio/frontend/tests/code-plugin-named-themes.test.ts b/studio/frontend/tests/code-plugin-named-themes.test.ts
new file mode 100644
index 00000000000..4a72b41c486
--- /dev/null
+++ b/studio/frontend/tests/code-plugin-named-themes.test.ts
@@ -0,0 +1,88 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import assert from "node:assert/strict";
+import { readdirSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import test from "node:test";
+import { fileURLToPath } from "node:url";
+import type {
+ HighlightOptions,
+ HighlightResult,
+ ThemeInput,
+} from "@streamdown/code";
+import { bundledThemesInfo } from "shiki/themes";
+
+import { createCodePlugin } from "../src/components/assistant-ui/code-plugin.ts";
+
+import { readSrc } from "./helpers/kit.ts";
+
+const PLUGIN_SOURCE = readSrc("components/assistant-ui/code-plugin.ts");
+const SRC_ROOT = fileURLToPath(new URL("../src/", import.meta.url));
+
+const sourceFiles = (dir: string): string[] =>
+ readdirSync(dir, { withFileTypes: true }).flatMap((entry) => {
+ const path = join(dir, entry.name);
+ if (entry.isDirectory()) return sourceFiles(path);
+ return /\.(ts|tsx)$/.test(entry.name) ? [path] : [];
+ });
+
+const registeredNames = (): Set<string> => {
+ const map = /const NAMED_THEMES = \{([\s\S]*?)\n\};/.exec(PLUGIN_SOURCE);
+ assert.ok(map, "code-plugin must declare its NAMED_THEMES map");
+ return new Set(
+ Array.from(map[1].matchAll(/^\s*"([a-z0-9-]+)":/gm), (m) => m[1]),
+ );
+};
+
+test("the highlighter does not register shiki's full themes map", () => {
+ assert.ok(
+ !/^import\s+(?!type\b)[^;]*?\bfrom\s+"shiki\/themes";/m.test(PLUGIN_SOURCE),
+ "bundledThemes emits a lazy chunk per bundled theme; list only the names Studio passes",
+ );
+ assert.ok(PLUGIN_SOURCE.includes("themes: NAMED_THEMES,"));
+});
+
+test("every bundled theme name a highlighting call site passes is registered", () => {
+ const ids = bundledThemesInfo.map((theme) => theme.id);
+ const literal = new RegExp(`["'\`](${ids.join("|")})["'\`]`, "g");
+ const registered = registeredNames();
+ const used = new Set<string>();
+ for (const file of sourceFiles(SRC_ROOT)) {
+ const text = readFileSync(file, "utf8");
+ if (!/shikiTheme|createCodePlugin|ThemeInput/.test(text)) continue;
+ for (const match of text.matchAll(literal)) used.add(match[1]);
+ }
+ // Streamdown's default shikiTheme, used by every Streamdown that passes none.
+ used.add("github-light");
+ used.add("github-dark");
+ assert.deepEqual(
+ [...used].filter((name) => !registered.has(name)),
+ [],
+ );
+ assert.deepEqual([...registered].sort(), [...used].sort());
+});
+
+const highlightOnce = (
+ plugin: ReturnType<typeof createCodePlugin>,
+ options: HighlightOptions,
+): Promise<HighlightResult> =>
+ new Promise((resolve) => {
+ const immediate = plugin.highlight(options, resolve);
+ if (immediate) resolve(immediate);
+ });
+
+test("the registered names still highlight", async () => {
+ const themes: [ThemeInput, ThemeInput] = ["github-light", "github-dark"];
+ const result = await highlightOnce(createCodePlugin({ themes }), {
+ code: "def f(x):\n return x + 1\n",
+ language: "python" as HighlightOptions["language"],
+ themes,
+ });
+ const colours = new Set(
+ result.tokens
+ .flat()
+ .flatMap((token) => Object.values(token.htmlStyle ?? {})),
+ );
+ assert.ok(colours.size > 2, "expected real token colours for both themes");
+});
diff --git a/studio/frontend/tests/shiki-tokenization-counter.mts b/studio/frontend/tests/shiki-tokenization-counter.mts
index 58a68892283..fe92fa108fd 100644
--- a/studio/frontend/tests/shiki-tokenization-counter.mts
+++ b/studio/frontend/tests/shiki-tokenization-counter.mts
@@ -1,23 +1,23 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-// `shiki` with one change: every highlighter it hands out counts the source it
-// is asked to tokenize. The explicit `createHighlighter` below shadows the one
-// the star export would provide.
-export * from "shiki";
-import { createHighlighter as createRealHighlighter } from "shiki";
+export * from "shiki/core";
+import { createBundledHighlighter as createRealBundledHighlighter } from "shiki/core";
export const tokenized = { characters: 0, calls: 0 };
-export const createHighlighter = async (
- ...args: Parameters<typeof createRealHighlighter>
+export const createBundledHighlighter: typeof createRealBundledHighlighter = (
+ bundle,
) => {
- const highlighter = await createRealHighlighter(...args);
- const codeToTokens = highlighter.codeToTokens.bind(highlighter);
- highlighter.codeToTokens = (code, options) => {
- tokenized.characters += code.length;
- tokenized.calls += 1;
- return codeToTokens(code, options);
+ const createRealHighlighter = createRealBundledHighlighter(bundle);
+ return async (...args) => {
+ const highlighter = await createRealHighlighter(...args);
+ const codeToTokens = highlighter.codeToTokens.bind(highlighter);
+ highlighter.codeToTokens = (code, options) => {
+ tokenized.characters += code.length;
+ tokenized.calls += 1;
+ return codeToTokens(code, options);
+ };
+ return highlighter;
};
- return highlighter;
};
diff --git a/studio/frontend/tests/shiki-tokenization-resolver.mjs b/studio/frontend/tests/shiki-tokenization-resolver.mjs
index 37bd9af6a8a..af9b51bb79a 100644
--- a/studio/frontend/tests/shiki-tokenization-resolver.mjs
+++ b/studio/frontend/tests/shiki-tokenization-resolver.mjs
@@ -1,12 +1,14 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-// code-plugin.ts imports shiki statically and ES namespaces are read-only, so redirect it.
const COUNTER = new URL("./shiki-tokenization-counter.mts", import.meta.url)
.href;
export function resolve(specifier, context, next) {
- if (specifier === "shiki" && context.parentURL?.endsWith("/code-plugin.ts")) {
+ if (
+ specifier === "shiki/core" &&
+ context.parentURL?.endsWith("/code-plugin.ts")
+ ) {
return next(COUNTER, context);
}
return next(specifier, context);