Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
from __future__ import annotations
import ast
import functools
import math
from pathlib import Path
import sys
import types
from typing import Iterable, Optional
import pytest
import torch
from torch import nn
from torch.nn import functional as F
def _load_experts():
path = Path(__file__).resolve().parents[1] / "unsloth/kernels/multi_lora.py"
tree = ast.parse(path.read_text(encoding = "utf-8"), filename = str(path))
names = {"MultiLoraExpertParam", "MultiLoraExperts", "_expert_stack_shape"}
tree.body = [
n for n in tree.body if isinstance(n, (ast.ClassDef, ast.FunctionDef)) and n.name in names
]
mod = types.ModuleType("_multi_lora_overflow_source")
mod.__dict__.update(
torch = torch,
nn = nn,
F = F,
math = math,
Optional = Optional,
Iterable = Iterable,
functools = functools,
_COMPILE_STACKS_ENABLED = False,
_TRITON_ENABLED = False,
_peft_expert_in_axis = lambda transposed: 1 if transposed else 2,
)
sys.modules[mod.__name__] = mod
exec(compile(tree, str(path), "exec"), mod.__dict__)
return mod
def _grouped_mm(x, weight, offsets):
bounds = [0, *offsets.tolist()]
return torch.cat([x[start:end] @ w for w, start, end in zip(weight, bounds, bounds[1:])])
@pytest.mark.parametrize("reversed_io", [False, True])
@pytest.mark.parametrize(
"scale,small_factors",
[(s, False) for s in [-0.25, 0.25, 2.0, 16.0, 100000.0]] + [(100000.0, True)],
)
def test_grouped_projection_scales_output_without_fp16_factor_overflow(
reversed_io, scale, small_factors
):
ml = _load_experts()
base = nn.Module()
base.gate_up_proj = nn.Parameter(torch.zeros(1, 16, 8, dtype = torch.float16))
base.down_proj = nn.Parameter(torch.zeros(1, 8, 8, dtype = torch.float16))
experts = ml.MultiLoraExperts(base, ["gate_up_proj"], 2, r = 1, lora_alpha = [scale, 1.0])
experts.reversed_io = reversed_io
adapter = experts.adapters["gate_up_proj"]
first_value, second_value = (
(0.000001, 40000.0)
if scale == 100000
else (0.01, 40000.0 if scale == 2 or abs(scale) < 1 else 5000.0)
)
if small_factors:
first_value, second_value = 0.001, 0.001
with torch.no_grad():
for u, (A, B) in enumerate(zip(adapter.lora_A, adapter.lora_B)):
value = 1.0 if abs(scale) < 1 and u == 0 else first_value
A.fill_(second_value if reversed_io else value)
B.fill_(value if reversed_io else second_value)
in_dim, out_dim = (16, 8) if reversed_io else (8, 16)
x = torch.full(
(3, in_dim),
1.0 if small_factors else 4 / in_dim if abs(scale) < 1 else 0.01,
dtype = torch.float16,
requires_grad = True,
)
offsets = torch.tensor([1, 2, 3], dtype = torch.int32)
slots = torch.tensor([0, 1, 2])
backend = types.SimpleNamespace(
_get_base_weight = lambda *a: torch.zeros(1, in_dim, out_dim, dtype = torch.float16),
preprocess_weight = lambda w, *a, **kw: w,
_moe_recompute_enabled = lambda *a, **kw: False,
_base_grouped_mm = lambda x, offs, provider, *a: _grouped_mm(x, provider(), offs),
_grouped_mm_with_backward_fix = _grouped_mm,
)
args = (
backend,
"gate_up_proj",
x,
torch.tensor([3]),
offsets,
torch.tensor([3]),
None,
torch.float16,
)
if "pair_slots" in ml.MultiLoraExperts._project.__code__.co_varnames:
got = experts._project(*args, slots)
else:
got = experts._project(*args)
refs = []
for u in range(2):
a, b, scaling, _ = adapter.grouped_mm_factors(u, reversed_io)
h = x[u : u + 1] @ a[0].half()
if abs(scaling) <= 1 or small_factors:
refs.append(h @ (b[0].half() * scaling))
else:
refs.append((h @ b[0].half()) * scaling)
ref = torch.cat([*refs, x[2:3] @ torch.zeros(in_dim, out_dim, dtype = x.dtype)])
grad = torch.full_like(got, 1.0 if small_factors else 1e-6)
leaves = [x, *adapter.lora_A, *adapter.lora_B]
got_grads = torch.autograd.grad(got, leaves, grad, retain_graph = True)
ref_grads = torch.autograd.grad(ref, leaves, grad)
assert (
got.isfinite().all()
), "finite LoRA projection overflowed while folding scaling into its factor"
torch.testing.assert_close(got, ref, rtol = 0, atol = 0)
for actual, expected in zip(got_grads, ref_grads):
assert actual.isfinite().all()
torch.testing.assert_close(actual, expected, rtol = 0, atol = 0)