Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Prevent FP16 overflow in grouped multi-LoRA expert projections (#16414)

Daniel Han9a8b22971465
diff --git a/tests/test_multi_lora_scale_overflow_cpu.py b/tests/test_multi_lora_scale_overflow_cpu.py
new file mode 100644
index 00000000000..e7f95c2e6cf
--- /dev/null
+++ b/tests/test_multi_lora_scale_overflow_cpu.py
@@ -0,0 +1,124 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+
+from __future__ import annotations
+import ast
+import functools
+import math
+from pathlib import Path
+import sys
+import types
+from typing import Iterable, Optional
+
+import pytest
+import torch
+from torch import nn
+from torch.nn import functional as F
+
+
+def _load_experts():
+ path = Path(__file__).resolve().parents[1] / "unsloth/kernels/multi_lora.py"
+ tree = ast.parse(path.read_text(), filename = str(path))
+ names = {"MultiLoraExpertParam", "MultiLoraExperts", "_expert_stack_shape"}
+ tree.body = [
+ n for n in tree.body if isinstance(n, (ast.ClassDef, ast.FunctionDef)) and n.name in names
+ ]
+ mod = types.ModuleType("_multi_lora_overflow_source")
+ mod.__dict__.update(
+ torch = torch,
+ nn = nn,
+ F = F,
+ math = math,
+ Optional = Optional,
+ Iterable = Iterable,
+ functools = functools,
+ _COMPILE_STACKS_ENABLED = False,
+ _TRITON_ENABLED = False,
+ _peft_expert_in_axis = lambda transposed: 1 if transposed else 2,
+ )
+ sys.modules[mod.__name__] = mod
+ exec(compile(tree, str(path), "exec"), mod.__dict__)
+ return mod
+
+
+def _grouped_mm(x, weight, offsets):
+ bounds = [0, *offsets.tolist()]
+ return torch.cat([x[start:end] @ w for w, start, end in zip(weight, bounds, bounds[1:])])
+
+
[email protected]("reversed_io", [False, True])
+ "scale,small_factors",
+ [(s, False) for s in [-0.25, 0.25, 2.0, 16.0, 100000.0]] + [(100000.0, True)],
+)
+def test_grouped_projection_scales_output_without_fp16_factor_overflow(
+ reversed_io, scale, small_factors
+):
+ ml = _load_experts()
+ base = nn.Module()
+ base.gate_up_proj = nn.Parameter(torch.zeros(1, 16, 8, dtype = torch.float16))
+ base.down_proj = nn.Parameter(torch.zeros(1, 8, 8, dtype = torch.float16))
+ experts = ml.MultiLoraExperts(base, ["gate_up_proj"], 2, r = 1, lora_alpha = [scale, 1.0])
+ experts.reversed_io = reversed_io
+ adapter = experts.adapters["gate_up_proj"]
+ first_value, second_value = (
+ (0.000001, 40000.0)
+ if scale == 100000
+ else (0.01, 40000.0 if scale == 2 or abs(scale) < 1 else 5000.0)
+ )
+ if small_factors:
+ first_value, second_value = 0.001, 0.001
+ with torch.no_grad():
+ for u, (A, B) in enumerate(zip(adapter.lora_A, adapter.lora_B)):
+ value = 1.0 if abs(scale) < 1 and u == 0 else first_value
+ A.fill_(second_value if reversed_io else value)
+ B.fill_(value if reversed_io else second_value)
+ in_dim, out_dim = (16, 8) if reversed_io else (8, 16)
+ x = torch.full(
+ (3, in_dim),
+ 1.0 if small_factors else 4 / in_dim if abs(scale) < 1 else 0.01,
+ dtype = torch.float16,
+ requires_grad = True,
+ )
+ offsets = torch.tensor([1, 2, 3], dtype = torch.int32)
+ slots = torch.tensor([0, 1, 2])
+ backend = types.SimpleNamespace(
+ _get_base_weight = lambda *a: torch.zeros(1, in_dim, out_dim, dtype = torch.float16),
+ preprocess_weight = lambda w, *a, **kw: w,
+ _moe_recompute_enabled = lambda *a, **kw: False,
+ _base_grouped_mm = lambda x, offs, provider, *a: _grouped_mm(x, provider(), offs),
+ _grouped_mm_with_backward_fix = _grouped_mm,
+ )
+ args = (
+ backend,
+ "gate_up_proj",
+ x,
+ torch.tensor([3]),
+ offsets,
+ torch.tensor([3]),
+ None,
+ torch.float16,
+ )
+ if "pair_slots" in ml.MultiLoraExperts._project.__code__.co_varnames:
+ got = experts._project(*args, slots)
+ else:
+ got = experts._project(*args)
+ refs = []
+ for u in range(2):
+ a, b, scaling, _ = adapter.grouped_mm_factors(u, reversed_io)
+ h = x[u : u + 1] @ a[0].half()
+ if abs(scaling) <= 1 or small_factors:
+ refs.append(h @ (b[0].half() * scaling))
+ else:
+ refs.append((h @ b[0].half()) * scaling)
+ ref = torch.cat([*refs, x[2:3] @ torch.zeros(in_dim, out_dim, dtype = x.dtype)])
+ grad = torch.full_like(got, 1.0 if small_factors else 1e-6)
+ leaves = [x, *adapter.lora_A, *adapter.lora_B]
+ got_grads = torch.autograd.grad(got, leaves, grad, retain_graph = True)
+ ref_grads = torch.autograd.grad(ref, leaves, grad)
+ assert (
+ got.isfinite().all()
+ ), "finite LoRA projection overflowed while folding scaling into its factor"
+ torch.testing.assert_close(got, ref, rtol = 0, atol = 0)
+ for actual, expected in zip(got_grads, ref_grads):
+ assert actual.isfinite().all()
+ torch.testing.assert_close(actual, expected, rtol = 0, atol = 0)
diff --git a/unsloth/kernels/multi_lora.py b/unsloth/kernels/multi_lora.py
index 8c5eccfd9b8..373ab20c5d6 100644
--- a/unsloth/kernels/multi_lora.py
+++ b/unsloth/kernels/multi_lora.py
@@ -59,6 +59,13 @@ __all__ = [
# Read once: an environment lookup inside a compiled forward would be a guard on every call.
_TRITON_ENABLED = HAS_TRITON and os.environ.get("UNSLOTH_MULTI_LORA_TRITON", "1") != "0"
+_COMPILE_STACKS_ENABLED = (
+ HAS_TRITON
+ and os.name != "nt"
+ and torch.version.hip is None
+ and os.environ.get("UNSLOTH_MULTI_LORA_COMPILE_STACKS", "1") != "0"
+ and os.environ.get("UNSLOTH_COMPILE_DISABLE", "0") not in ("1", "true", "True")
+)
def set_multi_lora_triton(enabled: bool) -> None:
@@ -743,18 +750,47 @@ class MultiLoraExpertParam(nn.Module):
)
def grouped_stacks(self, reversed_io: bool, dtype: torch.dtype) -> tuple:
+ if (
+ not torch.compiler.is_compiling()
+ and self.lora_A[0].device.type == "cuda"
+ and _COMPILE_STACKS_ENABLED
+ and _TRITON_ENABLED
+ ):
+ return _compiled_expert_stacks()(self, reversed_io, dtype)[:3]
+ return self._grouped_stacks(reversed_io, dtype)[:3]
+
+ def _grouped_stacks(self, reversed_io: bool, dtype: torch.dtype) -> tuple:
"""Every adapter's factors as (E*(U+1), in, R) and (E*(U+1), R, out), group e*(U+1) + u, R the largest rank
- rounded up to 8 (zero padding is exact), scale folded into the second; group e*(U+1) + U is zero."""
+ rounded up to 8, plus per-group output scales; group e*(U+1) + U is zero."""
R = -(-max(self.ranks) // 8) * 8
- firsts, seconds = [], []
+ firsts, seconds, scales, masks = [], [], [], []
+ scale_dtype = torch.float64 if dtype == torch.float64 else torch.float32
for u in range(self.num_adapters):
first, second, scale, _ = self.grouped_mm_factors(u, reversed_io)
r = first.shape[-1]
firsts.append(F.pad(first.to(dtype), (0, R - r)))
- seconds.append(F.pad(second.to(dtype) * scale, (0, 0, 0, R - r)))
+ second = second.to(dtype)
+ scaled = (second.to(scale_dtype) * scale).to(dtype)
+ if abs(scale) > 1:
+ # Keep the original gradient path unless folding actually overflows this expert.
+ finite = torch.isfinite(scaled).all(dim = (-2, -1), keepdim = True)
+ masks.append(finite)
+ second = torch.where(finite, scaled, second)
+ scales.append(torch.where(finite.flatten(), 1.0, scale).to(scale_dtype))
+ else:
+ second = scaled
+ scales.append(torch.ones(second.shape[0], device = second.device, dtype = scale_dtype))
+ seconds.append(F.pad(second, (0, 0, 0, R - r)))
firsts.append(torch.zeros_like(firsts[0]))
seconds.append(torch.zeros_like(seconds[0]))
- return torch.stack(firsts, 1).flatten(0, 1), torch.stack(seconds, 1).flatten(0, 1)
+ scales.append(torch.zeros_like(scales[0]))
+ return (
+ torch.stack(firsts, 1).flatten(0, 1),
+ torch.stack(seconds, 1).flatten(0, 1),
+ torch.stack(scales, 1).flatten(),
+ # Escaping masks cannot be donated by AOT before a retained backward.
+ tuple(masks),
+ )
def folded(self, W: torch.Tensor, adapter: int, transposed: bool) -> torch.Tensor:
"""W + delta in W's dtype, as PEFT's ParamWrapper folds it (one baddbmm, autocast off)."""
@@ -775,6 +811,16 @@ class MultiLoraExpertParam(nn.Module):
return flat_A.detach().clone(), flat_B.detach().clone()
[email protected]_cache(maxsize = 1)
+def _compiled_expert_stacks():
+ return torch.compile(
+ MultiLoraExpertParam._grouped_stacks,
+ fullgraph = True,
+ dynamic = True,
+ options = {"emulate_precision_casts": True},
+ )
+
+
@functools.lru_cache(maxsize = None)
def _peft_expert_in_axis(is_transposed: bool) -> int:
"""Which axis of an (E, d1, d2) stack the installed PEFT's ParamWrapper calls in_features (it moved in 0.19)."""
@@ -1162,7 +1208,7 @@ class MultiLoraExperts(nn.Module):
return f"the grouped probe differs by {err:.3g}"
def _project(
- self, moe_utils, name: str, x, offs_expert, offs_pair, counts, plan, dtype
+ self, moe_utils, name: str, x, offs_expert, offs_pair, counts, plan, dtype, pair_slots
) -> torch.Tensor:
"""Frozen base over E expert groups through Zoo's base GEMM (its weight provider, recompute and selective
dequant, as the single-job forward), plus every adapter's LoRA over the E * (U + 1) pair groups."""
@@ -1184,9 +1230,11 @@ class MultiLoraExperts(nn.Module):
x, offs_expert, provider, recompute, plan if recompute else None
)
if name in self.adapters:
- first, second = self.adapters[name].grouped_stacks(self.reversed_io, dtype)
+ adapter = self.adapters[name]
+ first, second, scales = adapter.grouped_stacks(self.reversed_io, dtype)
h = moe_utils._grouped_mm_with_backward_fix(x, first, offs_pair).contiguous()
- y = y + moe_utils._grouped_mm_with_backward_fix(h, second, offs_pair)
+ delta = moe_utils._grouped_mm_with_backward_fix(h, second, offs_pair)
+ y.addcmul_(delta, scales[pair_slots].unsqueeze(-1))
return y
def _grouped(
@@ -1218,6 +1266,7 @@ class MultiLoraExperts(nn.Module):
slot = torch.where(adapter_ids < 0, U, adapter_ids).to(index.device).unsqueeze(1)
key = (torch.where(valid, index, 0) * Ug + torch.where(valid, slot, U)).reshape(-1)
order = torch.argsort(key, stable = True)
+ pair_slots = key[order]
# Fixed-size counts: bincount's output size depends on the data, so it syncs on CUDA and breaks a compiled graph.
counts = torch.zeros(E * Ug, dtype = torch.int32, device = key.device)
counts.scatter_add_(0, key, torch.ones_like(key, dtype = torch.int32))
@@ -1241,7 +1290,15 @@ class MultiLoraExperts(nn.Module):
)
x = x[rows]
h = self._project(
- moe_utils, "gate_up_proj", x, offs_expert, offs_pair, counts, plan, dtype
+ moe_utils,
+ "gate_up_proj",
+ x,
+ offs_expert,
+ offs_pair,
+ counts,
+ plan,
+ dtype,
+ pair_slots,
)
gate, up = h.chunk(2, dim = -1)
inter = base.act_fn(gate) * up
@@ -1258,7 +1315,15 @@ class MultiLoraExperts(nn.Module):
inter = moe_utils._MoEGateGradIdentity.apply(inter, weights)
weights = weights.detach()
y = self._project(
- moe_utils, "down_proj", inter, offs_expert, offs_pair, counts, plan, dtype
+ moe_utils,
+ "down_proj",
+ inter,
+ offs_expert,
+ offs_pair,
+ counts,
+ plan,
+ dtype,
+ pair_slots,
)
out = moe_utils.weighted_unpermute(
y, order, weights, T, k, out_dtype = hidden_states.dtype