Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

3.0 KB
Raw Blame
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

/** In lib/ to avoid a feature-boundary import cycle. Mirrors the backend's _CANONICAL_SPEC_MODES. */
export const SPECULATIVE_TYPES = [
  "auto",
  "mtp",
  "dspark",
  "dflash",
  "ngram",
  "mtp+ngram",
  "off",
] as const;

/** Values only an MLX load reads (studio/backend/core/inference/mlx_speculative.py). */
export const MLX_ONLY_SPEC_TYPES = ["eagle3"] as const;

export const MLX_SPECULATIVE_TYPES = [
  "auto",
  "mtp",
  "dflash",
  "dspark",
  "eagle3",
  "ngram",
  "off",
] as const;

export interface MlxDrafter {
  repo: string;
  kind: string;
  /** False when the drafter only fits the model's architecture and is not named for it. */
  named: boolean;
}

/** Drafters the Drafter picker offers in `mode`, as `[repo, label]`: every cached kind under Auto,
 *  else that kind, plus a saved choice that is not cached (or is a local folder) so it stays visible. */
export function mlxDrafterChoices(
  drafters: readonly MlxDrafter[],
  mode: string,
  selected: string | null,
): [string, string][] {
  const choices = drafters
    .filter((drafter) => mode === "auto" || drafter.kind === mode)
    .map((drafter): [string, string] => [
      drafter.repo,
      drafter.named ? drafter.repo : `${drafter.repo} (same architecture)`,
    ]);
  return selected != null && !choices.some(([repo]) => repo === selected)
    ? [[selected, selected], ...choices]
    : choices;
}

/** The mode an MLX load runs: every drafter kind also copies repeated text, so llama.cpp's `mtp+ngram` is `mtp`. */
export function mlxSpeculativeMode(mode: string): string {
  return mode === "mtp+ngram" ? "mtp" : mode;
}

/**
 * Modes that read spec_draft_n_max, named for the setting rather than MTP so DSpark and DFlash
 * count too; mirrors DRAFT_N_MAX_SPEC_TYPES in studio/backend/utils/openai_auto_switch_settings.py.
 */
export const DRAFT_N_MAX_SPEC_TYPES: ReadonlySet<string> = new Set([
  "mtp",
  "mtp+ngram",
  "dspark",
  "dflash",
  ...MLX_ONLY_SPEC_TYPES,
]);

export const DRAFTER_MODEL_SPEC_TYPES: ReadonlySet<string> = new Set([
  ...DRAFT_N_MAX_SPEC_TYPES,
  "auto",
]);

/** The mode a load sends: the model's own choice, else the standing preference, which GGUF loads write.
 *  On MLX its ngram reads as auto: n-gram copying alone would move a text model onto the vision runtime. */
export function resolveSpeculativeType(
  chosen: string | null,
  standing: string,
  isMlx: boolean,
): string {
  return chosen ?? (isMlx && standing === "ngram" ? "auto" : standing);
}

/**
 * Modes that always launch a separate draft model with its own KV cache; MTP is left out since a
 * drafter file or baked-in heads is known only after metadata loads, but draft cache flags go
 * wherever --model-draft does, so a stored setting still reaches MTP loads that attach a drafter.
 */
export const SEPARATE_DRAFT_MODEL_SPEC_TYPES: ReadonlySet<string> = new Set([
  "dspark",
  "dflash",
]);