Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

22.2 KB
Raw Blame
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

import type {
  CodeHighlighterPlugin,
  CodePluginOptions,
  HighlightOptions,
  HighlightResult,
  ThemeInput,
} from "@streamdown/code";
import type { GrammarState, ThemedToken } from "shiki";
import { createBundledHighlighter } from "shiki/core";
import {
  createJavaScriptRegexEngine,
  defaultJavaScriptRegexConstructor,
} from "shiki/engine/javascript";
import {
  type BundledLanguage,
  bundledLanguages,
  bundledLanguagesInfo,
} from "shiki/langs";

// Keep named themes limited to Studio and Streamdown defaults; other callers pass objects.
type NamedTheme = "github-light" | "github-dark";
const NAMED_THEMES = {
  "github-light": () => import("@shikijs/themes/github-light"),
  "github-dark": () => import("@shikijs/themes/github-dark"),
};

const createHighlighter = createBundledHighlighter<
  BundledLanguage,
  NamedTheme
>({
  langs: bundledLanguages,
  themes: NAMED_THEMES,
  engine: () => createJavaScriptRegexEngine({ forgiving: true }),
});

const LANGUAGE_ALIAS_OVERRIDES: Record<string, BundledLanguage> = {
  objectivec: "objective-c",
  "obj-c": "objective-c",
  objectivecpp: "objective-cpp",
  "objective-cplusplus": "objective-cpp",
  objcpp: "objective-cpp",
  "c++": "cpp",
  cplusplus: "cpp",
  "c#": "csharp",
  cs: "csharp",
  "f#": "fsharp",
  "c-sharp": "csharp",
  "f-sharp": "fsharp",
  golang: "go",
  rs: "rust",
  rb: "ruby",
  py: "python",
  sh: "shellscript",
  bash: "shellscript",
  zsh: "shellscript",
  shell: "shellscript",
  yml: "yaml",
  ts: "typescript",
  js: "javascript",
  kt: "kotlin",
  rsx: "rust",
  "vue-html": "vue",
};

const SHIKI_LANGUAGE_ALIASES: Record<string, BundledLanguage> =
  Object.fromEntries(
    bundledLanguagesInfo.flatMap((info) =>
      (info.aliases ?? []).map((alias) => [alias, info.id as BundledLanguage]),
    ),
  );
const SUPPORTED_LANGUAGES = new Set(Object.keys(bundledLanguages));
const SUPPORTED_LANGUAGE_LIST = Array.from(
  SUPPORTED_LANGUAGES,
) as BundledLanguage[];
const PLAIN_TEXT = "text" as BundledLanguage;

// Shiki names shells by grammar ("Shell" for bash and zsh); readers name them by the tag they typed.
const DISPLAY_NAME_OVERRIDES: Record<string, string> = {
  bash: "Bash",
  zsh: "Zsh",
  text: "Text",
  plaintext: "Text",
  txt: "Text",
  cuda: "CUDA",
};
const DISPLAY_NAMES = new Map(
  bundledLanguagesInfo.flatMap((info) =>
    [info.id, ...(info.aliases ?? [])].map((name) => [name.toLowerCase(), info.name] as const),
  ),
);

export const languageDisplayName = (language: string): string => {
  const key = language.trim().toLowerCase();
  return DISPLAY_NAME_OVERRIDES[key] ?? DISPLAY_NAMES.get(key) ?? key.charAt(0).toUpperCase() + key.slice(1);
};

export const normalizeLanguage = (language: string): BundledLanguage => {
  const key = language.trim().toLowerCase();
  const alias = LANGUAGE_ALIAS_OVERRIDES[key] ?? SHIKI_LANGUAGE_ALIASES[key];
  return alias ?? (key as BundledLanguage);
};

export const MIN_INCREMENTAL_CHARS = 2000;
export const LIVE_TAIL_CHARS = 1024;
const REFRESH_MS = 250;
const REFRESH_SLICE_MS = 8;
// A wall-clock limit can cache a half-plain theme; bound line length for deterministic dual-theme output.
export const TOKENIZE_LIMITS = {
  tokenizeTimeLimit: 0,
  tokenizeMaxLineLength: 20_000,
} as const;
export const MAX_FENCES = 512;
export const MAX_CACHED_CHARACTERS = 512_000;

// Date.now() can step backwards (NTP, sleep resume), so use a monotonic clock.
const monotonicNow = (): number =>
  typeof performance !== "undefined" && typeof performance.now === "function"
    ? performance.now()
    : Date.now();

type Highlighter = Awaited<ReturnType<typeof createHighlighter>>;
type TokenLine = HighlightResult["tokens"][number];
type ResultMeta = Omit<HighlightResult, "tokens">;
type ThemeNames = { light: string; dark: string };
type HighlightCallback = (result: HighlightResult) => void;
type Pending = { code: string; callbacks: Set<HighlightCallback> };

type Fence = {
  key: string;
  code: string;
  result: HighlightResult | null;
  meta: ResultMeta | null;
  lines: TokenLine[];
  committedLength: number;
  state: GrammarState | undefined;
  liveTokens: TokenLine | null;
  lastTokenizedAt: number;
  trailing: ReturnType<typeof setTimeout> | null;
  pending: Pending | null;
  seeded: boolean;
};

const plainLine = (content: string): TokenLine =>
  [{ content, offset: 0 } as unknown as ThemedToken] as TokenLine;

const themeName = (theme: ThemeInput): string =>
  typeof theme === "string" ? theme : (theme.name ?? "custom");

// Custom themes can share or lack names, so key on a short id per distinct definition.
const themeIds = new Map<string, string>();
const themeIdByInput = new WeakMap<object, string>();

const themeKey = (theme: ThemeInput): string => {
  if (typeof theme === "string") return theme;
  const known = themeIdByInput.get(theme);
  if (known !== undefined) return known;
  const definition = JSON.stringify(theme);
  let id = themeIds.get(definition);
  if (id === undefined) {
    id = `${themeName(theme)}#${themeIds.size}`;
    themeIds.set(definition, id);
  }
  themeIdByInput.set(theme, id);
  return id;
};

const stripTokens = (result: HighlightResult): ResultMeta => ({
  bg: result.bg,
  fg: result.fg,
  themeName: result.themeName,
  rootStyle: result.rootStyle,
  grammarState: result.grammarState,
});

const shiftLine = (line: TokenLine, offset: number): TokenLine =>
  offset === 0
    ? line
    : line.map((token) => ({ ...token, offset: token.offset + offset }));

// Markdown reports a closing fence as body until it recognizes it, so that line (possibly
// a partial run) is all a fence can lose.
// Shiki already coalesces (537013 -> merged 537013); reproduce with scripts/coal-span-census.mjs.
const CLOSING_FENCE = /^ {0,3}(?:`+|~+)[ \t]*$/;

const shedsClosingRun = (shorter: string, longer: string): boolean =>
  (shorter === "" || shorter.endsWith("\n")) &&
  longer.startsWith(shorter) &&
  CLOSING_FENCE.test(longer.slice(shorter.length));

export type FenceCover = {
  uncovered: number;
  result: HighlightResult | null;
};

export type UnslothCodePlugin = CodeHighlighterPlugin & {
  cached: (opts: HighlightOptions) => HighlightResult | null;
  cover: (opts: HighlightOptions) => FenceCover;
  seed: (opts: HighlightOptions, result: HighlightResult) => void;
  highlight: (
    opts: HighlightOptions,
    callback?: (result: HighlightResult) => void,
    exact?: boolean,
  ) => HighlightResult | null;
  highlightExact: (
    opts: HighlightOptions,
    callback?: (result: HighlightResult) => void,
  ) => HighlightResult | null;
};

const fenceKeyOf = (opts: HighlightOptions): string =>
  `${normalizeLanguage(opts.language)} ${themeKey(opts.themes[0])} ${themeKey(opts.themes[1])}`;

const javaScriptCore =
  typeof navigator !== "undefined"
  && navigator.userAgent.includes("AppleWebKit/")
  && !/Chrom(?:e|ium)\//.test(navigator.userAgent);

export function createCodePlugin(
  options: CodePluginOptions = {},
): UnslothCodePlugin {
  const defaultThemes: [ThemeInput, ThemeInput] = options.themes ?? [
    "github-light",
    "github-dark",
  ];
  const engine = createJavaScriptRegexEngine(
    javaScriptCore
      ? {
          forgiving: true,
          regexConstructor: (pattern) => {
            try {
              return defaultJavaScriptRegexConstructor(pattern, {
                target: "ES2018",
                accuracy: "strict",
              });
            } catch {
              return defaultJavaScriptRegexConstructor(pattern);
            }
          },
        }
      : { forgiving: true },
  );
  const highlighters = new Map<string, { highlighter: Highlighter | null }>();
  const fences: Fence[] = [];
  const fencesByCode = new Map<string, Fence>();
  let cachedCharacters = 0;

  // Avoid hashing the whole source on each update; verify compact-key hits.
  const codeKey = (key: string, code: string): string =>
    `${key}\n${code.length}\n${code.slice(0, 32)}\n${code.slice(-32)}`;

  const dropCodeIndex = (fence: Fence): void => {
    const indexKey = codeKey(fence.key, fence.code);
    if (fencesByCode.get(indexKey) === fence) fencesByCode.delete(indexKey);
  };

  const clearTrailing = (fence: Fence): void => {
    if (fence.trailing !== null) clearTimeout(fence.trailing);
    fence.trailing = null;
    fence.pending = null;
  };

  const queuePending = (
    fence: Fence,
    code: string,
    callback?: HighlightCallback,
  ): void => {
    if (!fence.pending || fence.pending.code !== code) {
      fence.pending = { code, callbacks: new Set() };
    }
    if (callback) fence.pending.callbacks.add(callback);
  };

  const notifyPending = (
    pending: Pending,
    result: HighlightResult,
  ): void => {
    for (const callback of pending.callbacks) callback(result);
  };

  const dropFence = (fence: Fence): void => {
    const index = fences.indexOf(fence);
    if (index < 0) return;
    fences.splice(index, 1);
    cachedCharacters -= fence.code.length;
    dropCodeIndex(fence);
    clearTrailing(fence);
  };

  const evict = (): void => {
    while (
      fences.length > MAX_FENCES ||
      (cachedCharacters > MAX_CACHED_CHARACTERS && fences.length > 1)
    ) {
      let dropIndex = fences.length - 1;
      while (dropIndex >= 0 && fences[dropIndex].pending !== null) {
        dropIndex -= 1;
      }
      if (dropIndex < 0) return;
      dropFence(fences[dropIndex]);
    }
  };

  const forkFence = (fence: Fence): Fence => {
    const fork: Fence = {
      ...fence,
      lines: [...fence.lines],
      liveTokens: null,
      trailing: null,
      pending: null,
      seeded: false,
    };
    fences.unshift(fork);
    cachedCharacters += fence.code.length;
    return fork;
  };

  const promote = (fence: Fence): Fence => {
    const index = fences.indexOf(fence);
    if (index > 0) {
      fences.splice(index, 1);
      fences.unshift(fence);
    }
    return fence;
  };

  const reachInto = (fence: Fence, code: string): number => {
    if (fence.result === null || fence.seeded) return -1;
    const anchor = fence.code;
    // Lost more than its closing delimiter: a different fence, so do not share the entry.
    const reaches =
      code.startsWith(anchor) ||
      (code.length >= fence.committedLength && shedsClosingRun(code, anchor));
    return anchor && reaches ? Math.min(anchor.length, code.length) : -1;
  };

  const findFence = (key: string, code: string): Fence => {
    const exact = fencesByCode.get(codeKey(key, code));
    if (exact && exact.code === code) return promote(exact);
    let match: Fence | null = null;
    let matchLength = -1;
    for (const fence of fences) {
      if (fence.key !== key) continue;
      // Prefix-related pending calls may belong to different blocks.
      if (fence.result === null) {
        if (fence.pending?.code === code) return promote(fence);
        continue;
      }
      const reach = reachInto(fence, code);
      if (reach <= matchLength) continue;
      match = fence;
      matchLength = reach;
    }
    if (match) return promote(match);
    const fence: Fence = {
      key,
      code: "",
      result: null,
      meta: null,
      lines: [],
      committedLength: 0,
      state: undefined,
      liveTokens: null,
      lastTokenizedAt: 0,
      trailing: null,
      pending: null,
      seeded: false,
    };
    fences.unshift(fence);
    return fence;
  };

  const tokenize = (
    fence: Fence,
    highlighter: Highlighter,
    code: string,
    language: BundledLanguage,
    themes: ThemeNames,
    budgetMs: number,
  ): HighlightResult => {
    const lang = highlighter.getLoadedLanguages().includes(language)
      ? language
      : PLAIN_TEXT;
    const tokenizeFrom = (text: string) =>
      highlighter.codeToTokens(text, {
        lang,
        themes,
        grammarState: fence.state,
        ...TOKENIZE_LIMITS,
      });

    const lastNewline = code.lastIndexOf("\n");
    if (lastNewline >= fence.committedLength && budgetMs < Number.POSITIVE_INFINITY) {
      const deadline = monotonicNow() + budgetMs;
      while (fence.committedLength <= lastNewline) {
        const newline = code.indexOf("\n", fence.committedLength);
        const end =
          newline > fence.committedLength && code[newline - 1] === "\r"
            ? newline - 1
            : newline;
        const completed = tokenizeFrom(code.slice(fence.committedLength, end));
        for (const line of completed.tokens) {
          fence.lines.push(shiftLine(line, fence.committedLength));
        }
        fence.state = completed.grammarState;
        fence.committedLength = newline + 1;
        fence.liveTokens = null;
        fence.meta = stripTokens(completed);
        if (monotonicNow() >= deadline) break;
      }
      if (fence.committedLength <= lastNewline) {
        fence.lastTokenizedAt = monotonicNow();
        const tail = code.slice(fence.committedLength).split("\n");
        return {
          ...(fence.meta as ResultMeta),
          tokens: [...fence.lines, ...tail.map(plainLine)],
        };
      }
    }

    if (lastNewline >= fence.committedLength) {
      // Shiki omits CR from CRLF token content.
      const completedEnd =
        lastNewline > 0 && code[lastNewline - 1] === "\r"
          ? lastNewline - 1
          : lastNewline;
      const completed = tokenizeFrom(
        code.slice(fence.committedLength, completedEnd),
      );
      for (const line of completed.tokens) {
        fence.lines.push(shiftLine(line, fence.committedLength));
      }
      fence.state = completed.grammarState;
      fence.committedLength = lastNewline + 1;
      fence.liveTokens = null;
      fence.meta = stripTokens(completed);
    }

    const live = code.slice(fence.committedLength);
    const liveResult = tokenizeFrom(live);
    fence.meta = stripTokens(liveResult);
    fence.liveTokens = shiftLine(
      liveResult.tokens[0] ?? ([] as unknown as TokenLine),
      fence.committedLength,
    );
    fence.lastTokenizedAt = monotonicNow();
    return { ...fence.meta, tokens: [...fence.lines, fence.liveTokens] };
  };

  const approximateResult = (fence: Fence, code: string): HighlightResult => {
    const exact = fence.result as HighlightResult;
    // A live line may have grown mid-token, so render the whole tail plain.
    const keptLines = exact.tokens.slice(0, fence.lines.length);
    const tail = code.slice(fence.committedLength).split("\n");
    return { ...exact, tokens: [...keptLines, ...tail.map(plainLine)] };
  };

  const resetFence = (fence: Fence): void => {
    fence.lines = [];
    fence.committedLength = 0;
    fence.state = undefined;
    fence.liveTokens = null;
    fence.meta = null;
  };

  const update = (
    fence: Fence,
    highlighter: Highlighter,
    code: string,
    language: BundledLanguage,
    themes: ThemeNames,
    budgetMs = Number.POSITIVE_INFINITY,
  ): HighlightResult | null => {
    let result: HighlightResult;
    try {
      result = tokenize(fence, highlighter, code, language, themes, budgetMs);
    } catch (error) {
      console.error("[Unsloth Code] Failed to highlight code:", error);
      resetFence(fence);
      // A fence that never produced tokens has no anchor, so a failing block would strand one per render.
      if (fence.result === null) dropFence(fence);
      return null;
    }
    cachedCharacters += code.length - fence.code.length;
    dropCodeIndex(fence);
    fence.code = code;
    fence.result = result;
    fencesByCode.set(codeKey(fence.key, code), fence);
    evict();
    return result;
  };

  const settlePending = (
    fence: Fence,
    highlighter: Highlighter,
    language: BundledLanguage,
    themes: ThemeNames,
  ): void => {
    const pending = fence.pending;
    clearTrailing(fence);
    if (!pending) return;
    const refreshed = update(
      fence,
      highlighter,
      pending.code,
      language,
      themes,
    );
    if (refreshed) notifyPending(pending, refreshed);
  };

  const loadHighlighter = (
    key: string,
    language: BundledLanguage,
    themes: [ThemeInput, ThemeInput],
    resume: (highlighter: Highlighter) => void,
  ): Highlighter | null => {
    const loading = highlighters.get(key);
    if (loading) return loading.highlighter;
    const entry: { highlighter: Highlighter | null } = { highlighter: null };
    highlighters.set(key, entry);
    createHighlighter({
      themes,
      langs: [SUPPORTED_LANGUAGES.has(language) ? language : PLAIN_TEXT],
      engine,
    })
      .then((highlighter) => {
        entry.highlighter = highlighter;
        resume(highlighter);
      })
      .catch((error) => {
        console.error("[Unsloth Code] Failed to highlight code:", error);
        highlighters.delete(key);
        // Failed callbacks must not pin fences or fire after a later retry.
        for (const waiting of fences) {
          if (waiting.key === key) clearTrailing(waiting);
        }
        evict();
      });
    return null;
  };

  const plugin: UnslothCodePlugin = {
    name: "shiki",
    type: "code-highlighter",
    cached: (opts) => {
      const exact = fencesByCode.get(codeKey(fenceKeyOf(opts), opts.code));
      return exact && exact.code === opts.code && exact.result
        ? promote(exact).result
        : null;
    },
    cover: (opts) => {
      const key = fenceKeyOf(opts);
      const exact = fencesByCode.get(codeKey(key, opts.code));
      if (exact && exact.code === opts.code && exact.result) {
        return { uncovered: 0, result: exact.result };
      }
      let match: Fence | null = null;
      let matchLength = -1;
      for (const fence of fences) {
        if (fence.key !== key) continue;
        const reach = reachInto(fence, opts.code);
        if (reach <= matchLength) continue;
        match = fence;
        matchLength = reach;
      }
      if (match === null) return { uncovered: opts.code.length, result: null };
      return {
        uncovered:
          opts.code.length - Math.min(match.committedLength, opts.code.length),
        result: approximateResult(match, opts.code),
      };
    },
    seed: (opts, result) => {
      const key = fenceKeyOf(opts);
      const exact = fencesByCode.get(codeKey(key, opts.code));
      if (exact && exact.code === opts.code && exact.result) return;
      const fence: Fence = {
        key,
        code: opts.code,
        result,
        meta: stripTokens(result),
        lines: result.tokens,
        committedLength: opts.code.length,
        state: undefined,
        liveTokens: null,
        lastTokenizedAt: monotonicNow(),
        trailing: null,
        pending: null,
        seeded: true,
      };
      fences.unshift(fence);
      cachedCharacters += opts.code.length;
      fencesByCode.set(codeKey(key, opts.code), fence);
      evict();
    },
    getSupportedLanguages: () => SUPPORTED_LANGUAGE_LIST,
    getThemes: () => defaultThemes,
    supportsLanguage: (language) =>
      SUPPORTED_LANGUAGES.has(normalizeLanguage(language)),
    highlight: (
      opts: HighlightOptions,
      callback?: (result: HighlightResult) => void,
      exact = false,
    ): HighlightResult | null => {
      const language = normalizeLanguage(opts.language);
      const themes: ThemeNames = {
        light: themeName(opts.themes[0]),
        dark: themeName(opts.themes[1]),
      };
      const key = `${language} ${themeKey(opts.themes[0])} ${themeKey(opts.themes[1])}`;
      const fence = findFence(key, opts.code);

      const partial = fence.committedLength <= fence.code.lastIndexOf("\n");
      if (fence.result && fence.code === opts.code && !(exact && partial)) {
        const pending = fence.pending;
        if (
          pending !== null &&
          shedsClosingRun(opts.code, pending.code)
        ) {
          // Settle the queued body before serving the shorter exact hit, so neither caller loses state.
          const loaded = highlighters.get(key)?.highlighter;
          if (loaded) {
            const exact = fence.result;
            settlePending(fence, loaded, language, themes);
            return exact;
          }
        }
        return fence.result;
      }

      const highlighter = loadHighlighter(key, language, opts.themes, (ready) => {
        // Oldest-first snapshot because updates can evict fences.
        for (const waiting of [...fences].reverse()) {
          const pending = waiting.pending;
          if (waiting.key !== key || !pending) continue;
          clearTrailing(waiting);
          const resumed = update(waiting, ready, pending.code, language, themes);
          if (resumed) notifyPending(pending, resumed);
        }
      });
      if (!highlighter) {
        queuePending(fence, opts.code, callback);
        evict();
        return null;
      }
      if (exact && fence.result !== null && fence.code !== opts.code) {
        return update(forkFence(fence), highlighter, opts.code, language, themes);
      }

      const pending = fence.pending;
      if (pending && shedsClosingRun(opts.code, pending.code)) {
        settlePending(fence, highlighter, language, themes);
      } else {
        clearTrailing(fence);
      }
      const elapsed = monotonicNow() - fence.lastTokenizedAt;
      const grewLargeFence =
        fence.result !== null &&
        fence.code.length >= MIN_INCREMENTAL_CHARS &&
        opts.code.length > fence.code.length &&
        opts.code.length - fence.committedLength > LIVE_TAIL_CHARS;
      if (!exact && grewLargeFence && elapsed < REFRESH_MS) {
        const result = approximateResult(fence, opts.code);
        queuePending(fence, opts.code, callback);
        const refresh = (): void => {
          const pending = fence.pending;
          fence.trailing = null;
          fence.pending = null;
          if (!pending) return;
          const refreshed = update(
            fence,
            highlighter,
            pending.code,
            language,
            themes,
            REFRESH_SLICE_MS,
          );
          if (refreshed) notifyPending(pending, refreshed);
          if (
            refreshed
            && fence.code === pending.code
            && fence.committedLength <= pending.code.lastIndexOf("\n")
          ) {
            fence.pending = pending;
            fence.trailing = setTimeout(refresh, 0);
          }
        };
        fence.trailing = setTimeout(
          refresh,
          partial ? 0 : Math.max(0, REFRESH_MS - elapsed),
        );
        return result;
      }
      return update(fence, highlighter, opts.code, language, themes);
    },
    highlightExact: (opts, callback) => plugin.highlight(opts, callback, true),
  };
  return plugin;
}