Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Prevent FP16 overflow in grouped multi-LoRA expert projections (#16414)
Daniel Han9a8b22971465
diff --git a/tests/test_multi_lora_scale_overflow_cpu.py b/tests/test_multi_lora_scale_overflow_cpu.pynew file mode 100644index 00000000000..e7f95c2e6cf--- /dev/null+++ b/tests/test_multi_lora_scale_overflow_cpu.py@@ -0,0 +1,124 @@+# SPDX-License-Identifier: AGPL-3.0-only++from __future__ import annotations+import ast+import functools+import math+from pathlib import Path+import sys+import types+from typing import Iterable, Optional++import pytest+import torch+from torch import nn+from torch.nn import functional as F+++def _load_experts():+ path = Path(__file__).resolve().parents[1] / "unsloth/kernels/multi_lora.py"+ tree = ast.parse(path.read_text(), filename = str(path))+ names = {"MultiLoraExpertParam", "MultiLoraExperts", "_expert_stack_shape"}+ tree.body = [+ n for n in tree.body if isinstance(n, (ast.ClassDef, ast.FunctionDef)) and n.name in names+ ]+ mod = types.ModuleType("_multi_lora_overflow_source")+ mod.__dict__.update(+ torch = torch,+ nn = nn,+ F = F,+ math = math,+ Optional = Optional,+ Iterable = Iterable,+ functools = functools,+ _COMPILE_STACKS_ENABLED = False,+ _TRITON_ENABLED = False,+ _peft_expert_in_axis = lambda transposed: 1 if transposed else 2,+ )+ sys.modules[mod.__name__] = mod+ exec(compile(tree, str(path), "exec"), mod.__dict__)+ return mod+++def _grouped_mm(x, weight, offsets):+ bounds = [0, *offsets.tolist()]+ return torch.cat([x[start:end] @ w for w, start, end in zip(weight, bounds, bounds[1:])])++[email protected]("reversed_io", [False, True])+ "scale,small_factors",+ [(s, False) for s in [-0.25, 0.25, 2.0, 16.0, 100000.0]] + [(100000.0, True)],+)+def test_grouped_projection_scales_output_without_fp16_factor_overflow(+ reversed_io, scale, small_factors+):+ ml = _load_experts()+ base = nn.Module()+ base.gate_up_proj = nn.Parameter(torch.zeros(1, 16, 8, dtype = torch.float16))+ base.down_proj = nn.Parameter(torch.zeros(1, 8, 8, dtype = torch.float16))+ experts = ml.MultiLoraExperts(base, ["gate_up_proj"], 2, r = 1, lora_alpha = [scale, 1.0])+ experts.reversed_io = reversed_io+ adapter = experts.adapters["gate_up_proj"]+ first_value, second_value = (+ (0.000001, 40000.0)+ if scale == 100000+ else (0.01, 40000.0 if scale == 2 or abs(scale) < 1 else 5000.0)+ )+ if small_factors:+ first_value, second_value = 0.001, 0.001+ with torch.no_grad():+ for u, (A, B) in enumerate(zip(adapter.lora_A, adapter.lora_B)):+ value = 1.0 if abs(scale) < 1 and u == 0 else first_value+ A.fill_(second_value if reversed_io else value)+ B.fill_(value if reversed_io else second_value)+ in_dim, out_dim = (16, 8) if reversed_io else (8, 16)+ x = torch.full(+ (3, in_dim),+ 1.0 if small_factors else 4 / in_dim if abs(scale) < 1 else 0.01,+ dtype = torch.float16,+ requires_grad = True,+ )+ offsets = torch.tensor([1, 2, 3], dtype = torch.int32)+ slots = torch.tensor([0, 1, 2])+ backend = types.SimpleNamespace(+ _get_base_weight = lambda *a: torch.zeros(1, in_dim, out_dim, dtype = torch.float16),+ preprocess_weight = lambda w, *a, **kw: w,+ _moe_recompute_enabled = lambda *a, **kw: False,+ _base_grouped_mm = lambda x, offs, provider, *a: _grouped_mm(x, provider(), offs),+ _grouped_mm_with_backward_fix = _grouped_mm,+ )+ args = (+ backend,+ "gate_up_proj",+ x,+ torch.tensor([3]),+ offsets,+ torch.tensor([3]),+ None,+ torch.float16,+ )+ if "pair_slots" in ml.MultiLoraExperts._project.__code__.co_varnames:+ got = experts._project(*args, slots)+ else:+ got = experts._project(*args)+ refs = []+ for u in range(2):+ a, b, scaling, _ = adapter.grouped_mm_factors(u, reversed_io)+ h = x[u : u + 1] @ a[0].half()+ if abs(scaling) <= 1 or small_factors:+ refs.append(h @ (b[0].half() * scaling))+ else:+ refs.append((h @ b[0].half()) * scaling)+ ref = torch.cat([*refs, x[2:3] @ torch.zeros(in_dim, out_dim, dtype = x.dtype)])+ grad = torch.full_like(got, 1.0 if small_factors else 1e-6)+ leaves = [x, *adapter.lora_A, *adapter.lora_B]+ got_grads = torch.autograd.grad(got, leaves, grad, retain_graph = True)+ ref_grads = torch.autograd.grad(ref, leaves, grad)+ assert (+ got.isfinite().all()+ ), "finite LoRA projection overflowed while folding scaling into its factor"+ torch.testing.assert_close(got, ref, rtol = 0, atol = 0)+ for actual, expected in zip(got_grads, ref_grads):+ assert actual.isfinite().all()+ torch.testing.assert_close(actual, expected, rtol = 0, atol = 0)diff --git a/unsloth/kernels/multi_lora.py b/unsloth/kernels/multi_lora.pyindex 8c5eccfd9b8..373ab20c5d6 100644--- a/unsloth/kernels/multi_lora.py+++ b/unsloth/kernels/multi_lora.py@@ -59,6 +59,13 @@ __all__ = [# Read once: an environment lookup inside a compiled forward would be a guard on every call._TRITON_ENABLED = HAS_TRITON and os.environ.get("UNSLOTH_MULTI_LORA_TRITON", "1") != "0"+_COMPILE_STACKS_ENABLED = (+ HAS_TRITON+ and os.name != "nt"+ and torch.version.hip is None+ and os.environ.get("UNSLOTH_MULTI_LORA_COMPILE_STACKS", "1") != "0"+ and os.environ.get("UNSLOTH_COMPILE_DISABLE", "0") not in ("1", "true", "True")+)def set_multi_lora_triton(enabled: bool) -> None:@@ -743,18 +750,47 @@ class MultiLoraExpertParam(nn.Module):)def grouped_stacks(self, reversed_io: bool, dtype: torch.dtype) -> tuple:+ if (+ not torch.compiler.is_compiling()+ and self.lora_A[0].device.type == "cuda"+ and _COMPILE_STACKS_ENABLED+ and _TRITON_ENABLED+ ):+ return _compiled_expert_stacks()(self, reversed_io, dtype)[:3]+ return self._grouped_stacks(reversed_io, dtype)[:3]++ def _grouped_stacks(self, reversed_io: bool, dtype: torch.dtype) -> tuple:"""Every adapter's factors as (E*(U+1), in, R) and (E*(U+1), R, out), group e*(U+1) + u, R the largest rank- rounded up to 8 (zero padding is exact), scale folded into the second; group e*(U+1) + U is zero."""+ rounded up to 8, plus per-group output scales; group e*(U+1) + U is zero."""R = -(-max(self.ranks) // 8) * 8- firsts, seconds = [], []+ firsts, seconds, scales, masks = [], [], [], []+ scale_dtype = torch.float64 if dtype == torch.float64 else torch.float32for u in range(self.num_adapters):first, second, scale, _ = self.grouped_mm_factors(u, reversed_io)r = first.shape[-1]firsts.append(F.pad(first.to(dtype), (0, R - r)))- seconds.append(F.pad(second.to(dtype) * scale, (0, 0, 0, R - r)))+ second = second.to(dtype)+ scaled = (second.to(scale_dtype) * scale).to(dtype)+ if abs(scale) > 1:+ # Keep the original gradient path unless folding actually overflows this expert.+ finite = torch.isfinite(scaled).all(dim = (-2, -1), keepdim = True)+ masks.append(finite)+ second = torch.where(finite, scaled, second)+ scales.append(torch.where(finite.flatten(), 1.0, scale).to(scale_dtype))+ else:+ second = scaled+ scales.append(torch.ones(second.shape[0], device = second.device, dtype = scale_dtype))+ seconds.append(F.pad(second, (0, 0, 0, R - r)))firsts.append(torch.zeros_like(firsts[0]))seconds.append(torch.zeros_like(seconds[0]))- return torch.stack(firsts, 1).flatten(0, 1), torch.stack(seconds, 1).flatten(0, 1)+ scales.append(torch.zeros_like(scales[0]))+ return (+ torch.stack(firsts, 1).flatten(0, 1),+ torch.stack(seconds, 1).flatten(0, 1),+ torch.stack(scales, 1).flatten(),+ # Escaping masks cannot be donated by AOT before a retained backward.+ tuple(masks),+ )def folded(self, W: torch.Tensor, adapter: int, transposed: bool) -> torch.Tensor:"""W + delta in W's dtype, as PEFT's ParamWrapper folds it (one baddbmm, autocast off)."""@@ -775,6 +811,16 @@ class MultiLoraExpertParam(nn.Module):return flat_A.detach().clone(), flat_B.detach().clone()[email protected]_cache(maxsize = 1)+def _compiled_expert_stacks():+ return torch.compile(+ MultiLoraExpertParam._grouped_stacks,+ fullgraph = True,+ dynamic = True,+ options = {"emulate_precision_casts": True},+ )++@functools.lru_cache(maxsize = None)def _peft_expert_in_axis(is_transposed: bool) -> int:"""Which axis of an (E, d1, d2) stack the installed PEFT's ParamWrapper calls in_features (it moved in 0.19)."""@@ -1162,7 +1208,7 @@ class MultiLoraExperts(nn.Module):return f"the grouped probe differs by {err:.3g}"def _project(- self, moe_utils, name: str, x, offs_expert, offs_pair, counts, plan, dtype+ self, moe_utils, name: str, x, offs_expert, offs_pair, counts, plan, dtype, pair_slots) -> torch.Tensor:"""Frozen base over E expert groups through Zoo's base GEMM (its weight provider, recompute and selectivedequant, as the single-job forward), plus every adapter's LoRA over the E * (U + 1) pair groups."""@@ -1184,9 +1230,11 @@ class MultiLoraExperts(nn.Module):x, offs_expert, provider, recompute, plan if recompute else None)if name in self.adapters:- first, second = self.adapters[name].grouped_stacks(self.reversed_io, dtype)+ adapter = self.adapters[name]+ first, second, scales = adapter.grouped_stacks(self.reversed_io, dtype)h = moe_utils._grouped_mm_with_backward_fix(x, first, offs_pair).contiguous()- y = y + moe_utils._grouped_mm_with_backward_fix(h, second, offs_pair)+ delta = moe_utils._grouped_mm_with_backward_fix(h, second, offs_pair)+ y.addcmul_(delta, scales[pair_slots].unsqueeze(-1))return ydef _grouped(@@ -1218,6 +1266,7 @@ class MultiLoraExperts(nn.Module):slot = torch.where(adapter_ids < 0, U, adapter_ids).to(index.device).unsqueeze(1)key = (torch.where(valid, index, 0) * Ug + torch.where(valid, slot, U)).reshape(-1)order = torch.argsort(key, stable = True)+ pair_slots = key[order]# Fixed-size counts: bincount's output size depends on the data, so it syncs on CUDA and breaks a compiled graph.counts = torch.zeros(E * Ug, dtype = torch.int32, device = key.device)counts.scatter_add_(0, key, torch.ones_like(key, dtype = torch.int32))@@ -1241,7 +1290,15 @@ class MultiLoraExperts(nn.Module):)x = x[rows]h = self._project(- moe_utils, "gate_up_proj", x, offs_expert, offs_pair, counts, plan, dtype+ moe_utils,+ "gate_up_proj",+ x,+ offs_expert,+ offs_pair,+ counts,+ plan,+ dtype,+ pair_slots,)gate, up = h.chunk(2, dim = -1)inter = base.act_fn(gate) * up@@ -1258,7 +1315,15 @@ class MultiLoraExperts(nn.Module):inter = moe_utils._MoEGateGradIdentity.apply(inter, weights)weights = weights.detach()y = self._project(- moe_utils, "down_proj", inter, offs_expert, offs_pair, counts, plan, dtype+ moe_utils,+ "down_proj",+ inter,+ offs_expert,+ offs_pair,+ counts,+ plan,+ dtype,+ pair_slots,)out = moe_utils.weighted_unpermute(y, order, weights, T, k, out_dtype = hidden_states.dtype