Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import type { ModelType } from "@/types/training";
import type { PipelineType } from "@huggingface/hub";
export const CONTEXT_LENGTHS = [
512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072, 262144,
];
export const TARGET_MODULES = [
"q_proj",
"k_proj",
"v_proj",
"o_proj",
"gate_proj",
"up_proj",
"down_proj",
];
export const CPT_EMBEDDING_MODULES = ["embed_tokens", "lm_head"] as const;
export const CPT_TARGET_MODULES = [...TARGET_MODULES, ...CPT_EMBEDDING_MODULES];
const CPT_UI_TARGET_MODULES = ["all-linear", ...CPT_TARGET_MODULES] as const;
export function isCptAllLinearTargetModules(
targetModules: readonly string[],
): boolean {
const loraTargetModules = targetModules.filter(
(module) =>
!CPT_EMBEDDING_MODULES.some(
(embeddingModule) => embeddingModule === module,
),
);
return (
loraTargetModules.length === 1 && loraTargetModules[0] === "all-linear"
);
}
export function resolveCptTargetModules(
currentTargetModules: readonly string[],
): string[] {
if (isCptAllLinearTargetModules(currentTargetModules)) {
return ["all-linear", ...CPT_EMBEDDING_MODULES];
}
return [...CPT_TARGET_MODULES];
}
export function getCptUiTargetModules(): readonly string[] {
return CPT_UI_TARGET_MODULES;
}
/** all-linear is a chip outside CPT too: model defaults (Qwen3-VL, Qwen3.5, LFM2, ...) ship it. */
const UI_TARGET_MODULES = ["all-linear", ...TARGET_MODULES] as const;
export function getUiTargetModules(isCpt: boolean): readonly string[] {
return isCpt ? CPT_UI_TARGET_MODULES : UI_TARGET_MODULES;
}
export function isCptTargetModuleActive(
targetModules: readonly string[],
module: string,
): boolean {
return module === "all-linear"
? isCptAllLinearTargetModules(targetModules)
: targetModules.includes(module);
}
export function toggleCptTargetModule(
targetModules: readonly string[],
module: string,
): string[] {
if (
CPT_EMBEDDING_MODULES.some((embeddingModule) => embeddingModule === module)
) {
return targetModules.includes(module)
? targetModules.filter((candidate) => candidate !== module)
: [...targetModules, module];
}
if (module === "all-linear") {
const embeddingModules = targetModules.filter((candidate) =>
CPT_EMBEDDING_MODULES.some(
(embeddingModule) => embeddingModule === candidate,
),
);
return isCptAllLinearTargetModules(targetModules)
? [...TARGET_MODULES, ...embeddingModules]
: ["all-linear", ...embeddingModules];
}
const namedTargetModules = targetModules.filter(
(candidate) => candidate !== "all-linear",
);
return targetModules.includes(module)
? namedTargetModules.filter((candidate) => candidate !== module)
: [...namedTargetModules, module];
}
export const OPTIMIZER_OPTIONS: ReadonlyArray<{
value: string;
label: string;
}> = [
{ value: "adamw_8bit", label: "AdamW 8-bit" },
{ value: "paged_adamw_8bit", label: "Paged AdamW 8-bit" },
{ value: "adamw_bnb_8bit", label: "AdamW BNB 8-bit" },
{ value: "paged_adamw_32bit", label: "Paged AdamW 32-bit" },
{ value: "adamw_torch", label: "AdamW (PyTorch)" },
{ value: "adamw_torch_fused", label: "AdamW (PyTorch Fused)" },
];
// Must match SUPPORTED_MLX_OPTIMIZERS in unsloth-zoo's mlx/trainer.py.
export const MLX_OPTIMIZER_OPTIONS: ReadonlyArray<{
value: string;
label: string;
}> = [
{ value: "adamw", label: "AdamW" },
{ value: "adam", label: "Adam" },
{ value: "lion", label: "Lion" },
{ value: "muon", label: "Muon" },
{ value: "sgd", label: "SGD" },
{ value: "adafactor", label: "Adafactor" },
];
export const LR_SCHEDULER_OPTIONS: ReadonlyArray<{
value: string;
label: string;
}> = [
{ value: "linear", label: "Linear" },
{ value: "cosine", label: "Cosine" },
];
/** Method-aware LR defaults; mirrored in studio/backend/assets/configs/. */
export const LR_DEFAULT_LORA = 2e-4;
export const LR_DEFAULT_FULL = 2e-5;
export const LR_DEFAULT_CPT = 5e-5;
export const LR_DEFAULT_DECISION_FULL = 2.5e-5;
// Also the advanced-settings summary's "unchanged" reference while CPT is selected.
export const CPT_LORA_HYPERPARAMS = {
loraRank: 128,
loraAlpha: 32,
loraVariant: "rslora",
} as const;
export const DEFAULT_HYPERPARAMS = {
epochs: 3,
contextLength: 2048,
visionImageSize: null as number | null,
learningRate: LR_DEFAULT_LORA,
// null = let backend auto-compute (lr/10 per Unsloth CPT recipe). Only used by CPT.
embeddingLearningRate: null as number | null,
optimizerType: "adamw_8bit",
lrSchedulerType: "linear",
loraRank: 16,
loraAlpha: 32,
loraDropout: 0.05,
loraVariant: "lora" as const,
batchSize: 4,
gradientAccumulation: 8,
weightDecay: 0.001,
warmupSteps: 5,
maxSteps: 60,
saveSteps: 0,
evalSteps: 0.0,
earlyStoppingPatience: 0,
drafterEnabled: false,
drafterStrategy: "auto" as const,
drafterDataSource: "dataset" as const,
drafterMaxExamples: 2000,
drafterEpochs: 1,
drafterLearningRate: null as number | null,
drafterDraftSteps: 3,
packing: true,
trainOnCompletions: true,
gradientCheckpointing: "unsloth" as const,
offloadLayers: 0 as number | "auto",
offloadVramGb: null as number | null,
offloadVramGbPerDevice: {} as Record<string, number | null>,
prefetchDepth: 2 as number | "auto",
randomSeed: 3407,
enableWandb: false,
wandbToken: "",
wandbProject: "llm-finetuning",
enableTensorboard: false,
tensorboardDir: "runs",
logFrequency: 10,
trustRemoteCode: false,
finetuneVisionLayers: true,
finetuneLanguageLayers: true,
finetuneAttentionModules: true,
finetuneMLPModules: true,
targetModules: TARGET_MODULES,
s3Config: null as import("@/types/training").S3Config | null,
};
export const MODEL_TYPE_TO_HF_TASKS: Record<
ModelType,
readonly PipelineType[]
> = {
text: ["text-generation"],
vision: [
"image-text-to-text",
"visual-question-answering",
"document-question-answering",
"image-to-text",
"any-to-any",
],
audio: [
"text-to-speech",
"automatic-speech-recognition",
"audio-text-to-text",
"text-to-audio",
],
embeddings: ["feature-extraction"],
decision: ["text-classification"],
};
export const PRIORITY_TRAINING_MODELS: readonly string[] = [
"unsloth/gemma-4-E2B-it",
"unsloth/gemma-4-E4B-it",
"unsloth/gemma-4-31B-it",
"unsloth/gemma-4-26B-A4B-it",
"unsloth/Qwen3.5-2B",
"unsloth/Qwen3.5-9B",
"unsloth/gpt-oss-20b",
"unsloth/NVIDIA-Nemotron-3-Nano-4B",
"unsloth/Qwen3-0.6B",
"unsloth/gemma-3-4b-it",
"unsloth/embeddinggemma-300m",
"convaiinnovations/laya",
"unsloth/orpheus-3b-0.1-ft",
"unsloth/Llama-3.1-8B-Instruct",
"unsloth/Llama-3.2-3B-Instruct",
];
const LAYA_TRAINING_MODEL = "convaiinnovations/laya";
const CLEF_OMNI_MLX_TRAINING_MODEL = "mlx-community/clef-omni-8bit";
/** The pinned models for a device: Clef Omni trains only through MLX, so only a Mac lists it. */
export function priorityTrainingModels(
deviceType?: string | null,
): readonly string[] {
return deviceType === "mac"
? PRIORITY_TRAINING_MODELS.flatMap((id) =>
id === LAYA_TRAINING_MODEL ? [id, CLEF_OMNI_MLX_TRAINING_MODEL] : [id],
)
: PRIORITY_TRAINING_MODELS;
}
/** Pin priority models to the top of a list of model IDs, preserving their defined order. */
export function applyPriorityOrdering(
ids: string[],
deviceType?: string | null,
): string[] {
const idSet = new Set(ids);
const pinned = priorityTrainingModels(deviceType).filter((id) =>
idSet.has(id),
);
const pinnedSet = new Set(pinned);
const rest = ids.filter((id) => !pinnedSet.has(id));
return [...pinned, ...rest];
}