Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Linear / Embedding weights stay at stored half dtype and upcast exactly per call. UNSLOTH_STUDIO_MANUAL_CAST=0
disables. LoRA adapters on them stay at compute dtype, unfused (UNSLOTH_STUDIO_MANUAL_CAST_LORA=0 disables)."""
from __future__ import annotations
import os
from typing import Any, Optional
MANUAL_CAST_ENV = "UNSLOTH_STUDIO_MANUAL_CAST"
MANUAL_CAST_ATTR = "_unsloth_manual_cast"
MANUAL_CAST_LORA_ENV = "UNSLOTH_STUDIO_MANUAL_CAST_LORA"
_PEFT_PATCHED = "_unsloth_manual_cast_lora"
_CLASSES: dict[str, Any] = {}
def manual_cast_disabled() -> bool:
return (os.environ.get(MANUAL_CAST_ENV) or "").strip().lower() in ("0", "off", "false", "no")
def manual_cast_lora_disabled() -> bool:
return (os.environ.get(MANUAL_CAST_LORA_ENV) or "").strip().lower() in (
"0",
"off",
"false",
"no",
)
def lora_compute_dtype(base: Any) -> Any:
if not _CLASSES or type(base) not in (_CLASSES["linear"], _CLASSES["embedding"]):
return None
dtype = getattr(base, "_unsloth_compute_dtype", None)
weight = getattr(base, "weight", None)
if dtype is None or weight is None or weight.dtype == dtype or manual_cast_lora_disabled():
return None
return dtype
def _patch_peft_adapter_dtype() -> None:
"""PEFT casts adapters to the base weight dtype (bf16 GEMMs, rounded fp32 files); use compute dtype instead."""
try:
from peft.tuners.tuners_utils import BaseTunerLayer
except Exception: # noqa: BLE001 -- no PEFT, no adapters to place
return
original = BaseTunerLayer.__dict__.get("_move_adapter_to_device_of_base_layer")
if original is None or getattr(original, _PEFT_PATCHED, False):
return
import torch
def _move_adapter_to_device_of_base_layer(
self,
adapter_name,
device = None,
):
base = self.get_base_layer()
dtype = lora_compute_dtype(base)
if dtype is None:
return original(self, adapter_name, device)
if device is None:
device = base.weight.device
meta = torch.device("meta")
for name in (*self.adapter_layer_names, *self.other_param_names):
layer = getattr(self, name, None)
if not hasattr(layer, "keys") or adapter_name not in layer.keys():
continue
item = layer[adapter_name]
if not isinstance(item, (torch.nn.Module, torch.Tensor)):
continue
tensors = item.parameters() if isinstance(item, torch.nn.Module) else (item,)
if any(t.device == meta for t in tensors):
continue
if isinstance(item, torch.nn.Module) or item.is_floating_point():
layer[adapter_name] = item.to(device, dtype = dtype)
else:
layer[adapter_name] = item.to(device)
setattr(_move_adapter_to_device_of_base_layer, _PEFT_PATCHED, True)
_move_adapter_to_device_of_base_layer.__wrapped__ = original
BaseTunerLayer._move_adapter_to_device_of_base_layer = _move_adapter_to_device_of_base_layer
def _classes() -> tuple[Any, Any]:
if not _CLASSES:
import torch
import torch.nn.functional as F
class ManualCastLinear(torch.nn.Linear):
def forward(self, x):
dtype = self._unsloth_compute_dtype
weight = self.weight if self.weight.dtype == dtype else self.weight.to(dtype)
bias = self.bias
if bias is not None and bias.dtype != dtype:
bias = bias.to(dtype)
return F.linear(x if x.dtype == dtype else x.to(dtype), weight, bias)
class ManualCastEmbedding(torch.nn.Embedding):
def forward(self, ids):
out = F.embedding(
ids,
self.weight,
self.padding_idx,
self.max_norm,
self.norm_type,
self.scale_grad_by_freq,
self.sparse,
)
dtype = self._unsloth_compute_dtype
return out if out.dtype == dtype else out.to(dtype)
_CLASSES["linear"] = ManualCastLinear
_CLASSES["embedding"] = ManualCastEmbedding
return _CLASSES["linear"], _CLASSES["embedding"]
def _keep_fp32_patterns(module: Any) -> tuple[str, ...]:
pats = getattr(module, "_keep_in_fp32_modules", None) or ()
strict = getattr(module, "_keep_in_fp32_modules_strict", None) or ()
return tuple(str(p) for p in (*pats, *strict))
def _first_float_owner(module: Any, keep: tuple[str, ...]) -> Optional[str]:
"""Owner of the param ``module.dtype`` reads; converted whole so pipelines cast latents to the compute dtype."""
for name, param in module.named_parameters():
if keep and any(k in name for k in keep):
continue
if param.is_floating_point():
return name.rpartition(".")[0]
return None
def apply_manual_cast_(module: Any, compute_dtype: Any) -> int:
"""Swap plain half Linear / Embedding for manual-cast twins, convert other half tensors; returns bytes kept.
Subclasses (quantised, GGUF, LoRA) are left alone: they do their own cast."""
import torch
linear_cls, embedding_cls = _classes()
_patch_peft_adapter_dtype()
half = (torch.bfloat16, torch.float16)
keep = _keep_fp32_patterns(module)
first_owner = _first_float_owner(module, keep)
kept_bytes = 0
with torch.no_grad():
for mname, sub in module.named_modules():
swap = None
if type(sub) is torch.nn.Linear:
swap = linear_cls
elif type(sub) is torch.nn.Embedding:
swap = embedding_cls
weight = getattr(sub, "weight", None)
if (
swap is not None
and mname != first_owner
and isinstance(weight, torch.Tensor)
and weight.dtype in half
and weight.device.type != "meta"
):
sub.__class__ = swap
sub._unsloth_compute_dtype = compute_dtype
kept_bytes += weight.numel() * weight.element_size()
bias = getattr(sub, "bias", None)
if isinstance(bias, torch.Tensor):
kept_bytes += bias.numel() * bias.element_size()
continue
for _pname, param in list(sub.named_parameters(recurse = False)):
if param.dtype in half:
param.data = param.data.to(compute_dtype)
for bname, buf in list(sub.named_buffers(recurse = False)):
if buf is not None and buf.dtype in half:
sub._buffers[bname] = buf.to(compute_dtype)
setattr(
module, MANUAL_CAST_ATTR, {"compute_dtype": str(compute_dtype), "kept_bytes": kept_bytes}
)
return kept_bytes
def manual_cast_info(module: Any) -> Optional[dict]:
info = getattr(module, MANUAL_CAST_ATTR, None)
return info if isinstance(info, dict) else None