Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Reduce Studio code highlighter bundle (#15740)
Daniel Han8f5ed05b15b4
diff --git a/studio/frontend/src/components/assistant-ui/code-plugin.ts b/studio/frontend/src/components/assistant-ui/code-plugin.tsindex 0004c9fba73..ecb0622731b 100644--- a/studio/frontend/src/components/assistant-ui/code-plugin.ts+++ b/studio/frontend/src/components/assistant-ui/code-plugin.ts@@ -8,20 +8,34 @@ import type {HighlightResult,ThemeInput,} from "@streamdown/code";-import {- type BundledLanguage,- bundledLanguages,- bundledLanguagesInfo,- createHighlighter,- type GrammarState,- type ThemedToken,-} from "shiki";+import type { GrammarState, ThemedToken } from "shiki";+import { createBundledHighlighter } from "shiki/core";import {createJavaScriptRegexEngine,defaultJavaScriptRegexConstructor,} from "shiki/engine/javascript";+import {+ type BundledLanguage,+ bundledLanguages,+ bundledLanguagesInfo,+} from "shiki/langs";++// Keep named themes limited to Studio and Streamdown defaults; other callers pass objects.+type NamedTheme = "github-light" | "github-dark";+const NAMED_THEMES = {+ "github-light": () => import("@shikijs/themes/github-light"),+ "github-dark": () => import("@shikijs/themes/github-dark"),+};++const createHighlighter = createBundledHighlighter<+ BundledLanguage,+ NamedTheme+>({+ langs: bundledLanguages,+ themes: NAMED_THEMES,+ engine: () => createJavaScriptRegexEngine({ forgiving: true }),+});-// Fence tags shiki does not alias: lower-cased input to canonical shiki id.const LANGUAGE_ALIAS_OVERRIDES: Record<string, BundledLanguage> = {objectivec: "objective-c","obj-c": "objective-c",@@ -63,32 +77,20 @@ const SUPPORTED_LANGUAGE_LIST = Array.from() as BundledLanguage[];const PLAIN_TEXT = "text" as BundledLanguage;-/** Exported so per-grammar caches key on the same identity as `highlight`. */export const normalizeLanguage = (language: string): BundledLanguage => {const key = language.trim().toLowerCase();const alias = LANGUAGE_ALIAS_OVERRIDES[key] ?? SHIKI_LANGUAGE_ALIASES[key];return alias ?? (key as BundledLanguage);};-// Past this size, cache completed lines and grammar state so a streaming fence tokenizes only new text.export const MIN_INCREMENTAL_CHARS = 2000;const REFRESH_MS = 250;const REFRESH_SLICE_MS = 8;-// Shiki's own tokenizer guard is a wall clock: a line that takes longer than `tokenizeTimeLimit`-// (default 500 ms) is abandoned part-way and the rest of it is emitted as one uncoloured token.-// Dual themes are two passes with two separate budgets, and only the first pays to compile the-// grammar's regexes, so a loaded machine drops the light theme to plain and keeps the dark one-// correct -- for the same line, in the same result. Committed lines are never re-tokenized, so that-// half-plain line is then cached for the life of the fence. Bound the work by line length instead:-// same protection against a minified blob (~120 ms at this length, growing quadratically past it,-// which is VS Code's `editor.maxTokenizationLineLength` default for the same reason), but the-// output depends on the source rather than on the host.+// A wall-clock limit can cache a half-plain theme; bound line length for deterministic dual-theme output.export const TOKENIZE_LIMITS = {tokenizeTimeLimit: 0,tokenizeMaxLineLength: 20_000,} as const;-// An unvirtualized thread mounts every fence. Bound both their count and source-// size; token data measured roughly 30 bytes per source character.export const MAX_FENCES = 512;export const MAX_CACHED_CHARACTERS = 512_000;@@ -157,11 +159,9 @@ const shiftLine = (line: TokenLine, offset: number): TokenLine =>? line: line.map((token) => ({ ...token, offset: token.offset + offset }));-// Adjacent-token coalescing measured zero (dual tokens 537013 -> merged 537013); Shiki already-// coalesces. Reproduce with scripts/coal-span-census.mjs.-// Markdown reports a closing fence as body until it recognizes it, so that line (possibly// a partial run) is all a fence can lose.+// Shiki already coalesces (537013 -> merged 537013); reproduce with scripts/coal-span-census.mjs.const CLOSING_FENCE = /^ {0,3}(?:`+|~+)[ \t]*$/;const shedsClosingRun = (shorter: string, longer: string): boolean =>@@ -222,7 +222,6 @@ export function createCodePlugin(: { forgiving: true },);const highlighters = new Map<string, { highlighter: Highlighter | null }>();- // Most recently used first.const fences: Fence[] = [];const fencesByCode = new Map<string, Fence>();let cachedCharacters = 0;@@ -231,7 +230,6 @@ export function createCodePlugin(const codeKey = (key: string, code: string): string =>`${key}\n${code.length}\n${code.slice(0, 32)}\n${code.slice(-32)}`;- // Identical fences can share an index key.const dropCodeIndex = (fence: Fence): void => {const indexKey = codeKey(fence.key, fence.code);if (fencesByCode.get(indexKey) === fence) fencesByCode.delete(indexKey);@@ -275,7 +273,6 @@ export function createCodePlugin(fences.length > MAX_FENCES ||(cachedCharacters > MAX_CACHED_CHARACTERS && fences.length > 1)) {- // Keep pending callbacks until their grammar resolves.let dropIndex = fences.length - 1;while (dropIndex >= 0 && fences[dropIndex].pending !== null) {dropIndex -= 1;@@ -318,7 +315,6 @@ export function createCodePlugin(return anchor && reaches ? Math.min(anchor.length, code.length) : -1;};- /** The fence whose cached code reaches furthest into `code`. */const findFence = (key: string, code: string): Fence => {const exact = fencesByCode.get(codeKey(key, code));if (exact && exact.code === code) return promote(exact);diff --git a/studio/frontend/tests/code-plugin-engine-and-cache.test.ts b/studio/frontend/tests/code-plugin-engine-and-cache.test.tsindex f4eb684d345..4a8797513b5 100644--- a/studio/frontend/tests/code-plugin-engine-and-cache.test.ts+++ b/studio/frontend/tests/code-plugin-engine-and-cache.test.ts@@ -1,11 +1,6 @@// SPDX-License-Identifier: AGPL-3.0-only// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0-/**- * The desktop CSP forbids WASM, so shiki must use the JS regex engine; also exercises the- * character-budget eviction branch, unreachable from other fixtures.- */-import assert from "node:assert/strict";import test from "node:test";import type {@@ -38,7 +33,6 @@ const highlightOnce = (});test("the highlighter uses the JavaScript regex engine, not the WASM one", () => {- // assert.match prints the whole subject, burying the message, so assert a boolean.assert.ok(/createJavaScriptRegexEngine/.test(PLUGIN_SOURCE),"code-plugin must build its highlighter with shiki's JavaScript regex engine",@@ -51,6 +45,16 @@ test("the highlighter uses the JavaScript regex engine, not the WASM one", () =>);});+const RUNTIME_SHIKI_IMPORT = /^import\s+(?!type\b)[^;]*?\bfrom\s+"shiki";/m;++test("the highlighter factory is built without shiki's default WASM engine", () => {+ assert.ok(+ !RUNTIME_SHIKI_IMPORT.test(PLUGIN_SOURCE),+ 'code-plugin must take only types from "shiki"; build the factory from shiki/core',+ );+ assert.ok(PLUGIN_SOURCE.includes("= createBundledHighlighter<"));+});+const sourceOfSize = (id: number, chars: number): string => {const lines: string[] = [];let length = 0;@@ -71,7 +75,6 @@ const stillCached = async ((await highlightOnce(plugin, { code, language, themes: THEMES })) === previous;test("the character budget evicts even while the fence count is under its limit", async () => {- // ~600 KB over 100 fences exceeds MAX_CACHED_CHARACTERS but not MAX_FENCES.const plugin = createCodePlugin({ themes: THEMES });const first = sourceOfSize(0, 6_000);const held = await highlightOnce(plugin, {diff --git a/studio/frontend/tests/code-plugin-named-themes.test.ts b/studio/frontend/tests/code-plugin-named-themes.test.tsnew file mode 100644index 00000000000..4a72b41c486--- /dev/null+++ b/studio/frontend/tests/code-plugin-named-themes.test.ts@@ -0,0 +1,88 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import assert from "node:assert/strict";+import { readdirSync, readFileSync } from "node:fs";+import { join } from "node:path";+import test from "node:test";+import { fileURLToPath } from "node:url";+import type {+ HighlightOptions,+ HighlightResult,+ ThemeInput,+} from "@streamdown/code";+import { bundledThemesInfo } from "shiki/themes";++import { createCodePlugin } from "../src/components/assistant-ui/code-plugin.ts";++import { readSrc } from "./helpers/kit.ts";++const PLUGIN_SOURCE = readSrc("components/assistant-ui/code-plugin.ts");+const SRC_ROOT = fileURLToPath(new URL("../src/", import.meta.url));++const sourceFiles = (dir: string): string[] =>+ readdirSync(dir, { withFileTypes: true }).flatMap((entry) => {+ const path = join(dir, entry.name);+ if (entry.isDirectory()) return sourceFiles(path);+ return /\.(ts|tsx)$/.test(entry.name) ? [path] : [];+ });++const registeredNames = (): Set<string> => {+ const map = /const NAMED_THEMES = \{([\s\S]*?)\n\};/.exec(PLUGIN_SOURCE);+ assert.ok(map, "code-plugin must declare its NAMED_THEMES map");+ return new Set(+ Array.from(map[1].matchAll(/^\s*"([a-z0-9-]+)":/gm), (m) => m[1]),+ );+};++test("the highlighter does not register shiki's full themes map", () => {+ assert.ok(+ !/^import\s+(?!type\b)[^;]*?\bfrom\s+"shiki\/themes";/m.test(PLUGIN_SOURCE),+ "bundledThemes emits a lazy chunk per bundled theme; list only the names Studio passes",+ );+ assert.ok(PLUGIN_SOURCE.includes("themes: NAMED_THEMES,"));+});++test("every bundled theme name a highlighting call site passes is registered", () => {+ const ids = bundledThemesInfo.map((theme) => theme.id);+ const literal = new RegExp(`["'\`](${ids.join("|")})["'\`]`, "g");+ const registered = registeredNames();+ const used = new Set<string>();+ for (const file of sourceFiles(SRC_ROOT)) {+ const text = readFileSync(file, "utf8");+ if (!/shikiTheme|createCodePlugin|ThemeInput/.test(text)) continue;+ for (const match of text.matchAll(literal)) used.add(match[1]);+ }+ // Streamdown's default shikiTheme, used by every Streamdown that passes none.+ used.add("github-light");+ used.add("github-dark");+ assert.deepEqual(+ [...used].filter((name) => !registered.has(name)),+ [],+ );+ assert.deepEqual([...registered].sort(), [...used].sort());+});++const highlightOnce = (+ plugin: ReturnType<typeof createCodePlugin>,+ options: HighlightOptions,+): Promise<HighlightResult> =>+ new Promise((resolve) => {+ const immediate = plugin.highlight(options, resolve);+ if (immediate) resolve(immediate);+ });++test("the registered names still highlight", async () => {+ const themes: [ThemeInput, ThemeInput] = ["github-light", "github-dark"];+ const result = await highlightOnce(createCodePlugin({ themes }), {+ code: "def f(x):\n return x + 1\n",+ language: "python" as HighlightOptions["language"],+ themes,+ });+ const colours = new Set(+ result.tokens+ .flat()+ .flatMap((token) => Object.values(token.htmlStyle ?? {})),+ );+ assert.ok(colours.size > 2, "expected real token colours for both themes");+});diff --git a/studio/frontend/tests/shiki-tokenization-counter.mts b/studio/frontend/tests/shiki-tokenization-counter.mtsindex 58a68892283..fe92fa108fd 100644--- a/studio/frontend/tests/shiki-tokenization-counter.mts+++ b/studio/frontend/tests/shiki-tokenization-counter.mts@@ -1,23 +1,23 @@// SPDX-License-Identifier: AGPL-3.0-only// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0-// `shiki` with one change: every highlighter it hands out counts the source it-// is asked to tokenize. The explicit `createHighlighter` below shadows the one-// the star export would provide.-export * from "shiki";-import { createHighlighter as createRealHighlighter } from "shiki";+export * from "shiki/core";+import { createBundledHighlighter as createRealBundledHighlighter } from "shiki/core";export const tokenized = { characters: 0, calls: 0 };-export const createHighlighter = async (- ...args: Parameters<typeof createRealHighlighter>+export const createBundledHighlighter: typeof createRealBundledHighlighter = (+ bundle,) => {- const highlighter = await createRealHighlighter(...args);- const codeToTokens = highlighter.codeToTokens.bind(highlighter);- highlighter.codeToTokens = (code, options) => {- tokenized.characters += code.length;- tokenized.calls += 1;- return codeToTokens(code, options);+ const createRealHighlighter = createRealBundledHighlighter(bundle);+ return async (...args) => {+ const highlighter = await createRealHighlighter(...args);+ const codeToTokens = highlighter.codeToTokens.bind(highlighter);+ highlighter.codeToTokens = (code, options) => {+ tokenized.characters += code.length;+ tokenized.calls += 1;+ return codeToTokens(code, options);+ };+ return highlighter;};- return highlighter;};diff --git a/studio/frontend/tests/shiki-tokenization-resolver.mjs b/studio/frontend/tests/shiki-tokenization-resolver.mjsindex 37bd9af6a8a..af9b51bb79a 100644--- a/studio/frontend/tests/shiki-tokenization-resolver.mjs+++ b/studio/frontend/tests/shiki-tokenization-resolver.mjs@@ -1,12 +1,14 @@// SPDX-License-Identifier: AGPL-3.0-only// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0-// code-plugin.ts imports shiki statically and ES namespaces are read-only, so redirect it.const COUNTER = new URL("./shiki-tokenization-counter.mts", import.meta.url).href;export function resolve(specifier, context, next) {- if (specifier === "shiki" && context.parentURL?.endsWith("/code-plugin.ts")) {+ if (+ specifier === "shiki/core" &&+ context.parentURL?.endsWith("/code-plugin.ts")+ ) {return next(COUNTER, context);}return next(specifier, context);