Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
"""Batched expert LoRA on a fused-experts MoE: each adapter's rows match a lone PEFT run of that adapter."""
from __future__ import annotations
import copy
import pytest
import torch
import unsloth # noqa: F401
from real_accelerator import has_real_cuda
from unsloth.kernels import multi_lora as ml
transformers = pytest.importorskip("transformers")
peft = pytest.importorskip("peft")
Qwen3_5Moe = getattr(transformers, "Qwen3_5MoeForConditionalGeneration", None)
pytestmark = pytest.mark.skipif(Qwen3_5Moe is None, reason = "transformers has no Qwen3.5 MoE")
REPO = "trl-internal-testing/tiny-Qwen3_5MoeForConditionalGeneration-3.6"
DENSE = ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"]
EXPERTS = ["mlp.experts.down_proj", "mlp.experts.gate_up_proj"]
RANKS, ALPHAS = [2, 4, 3], [4.0, 8.0, 3.0]
T = 6
# Zoo's MoE forward launches Triton whenever CUDA is up, so on a GPU host everything runs on the GPU.
DEVICE = "cuda" if has_real_cuda() else "cpu"
TOL = 1e-4 if DEVICE == "cuda" else 1e-5
@pytest.fixture(scope = "module")
def base():
from huggingface_hub import snapshot_download
path = snapshot_download(REPO)
return Qwen3_5Moe.from_pretrained(path, torch_dtype = torch.float32).eval().to(DEVICE)
def _batched(base):
torch.manual_seed(0)
model = copy.deepcopy(base)
lm = model.model.language_model
ml.inject_multi_lora(lm, DENSE, 3, RANKS, ALPHAS)
wrapped = ml.inject_multi_lora_experts(lm, EXPERTS, 3, RANKS, ALPHAS)
with torch.no_grad():
# Non-zero B: with PEFT's zero init a wrong adapter would still give the base output.
for _, layer in ml._layers(model):
for B in layer.lora_B:
B.normal_(0, 0.5)
return model, wrapped
def _lone(base, model, u):
"""Adapter ``u`` exported, loaded into a plain PEFT model of the base, as a lone job would run it."""
cfg = peft.LoraConfig(
r = RANKS[u],
lora_alpha = ALPHAS[u],
target_modules = DENSE,
target_parameters = EXPERTS,
lora_dropout = 0.0,
)
lone = peft.get_peft_model(copy.deepcopy(base), cfg)
state = ml.adapter_state_dict(model, u)
result = peft.set_peft_model_state_dict(lone, state)
assert not result.unexpected_keys, result.unexpected_keys[:3]
assert not [k for k in result.missing_keys if "lora_" in k]
return lone
def _ids():
return torch.randint(0, 100, (3, T), generator = torch.Generator().manual_seed(1)).to(DEVICE)
MODES = ["grouped", "stash", "fold"]
def _need_grouped(dtype = torch.float32):
"""Skip where this torch / device has no native grouped GEMM in ``dtype`` (the module then runs per adapter)."""
if not ml._grouped_supported(torch.device(DEVICE), dtype):
pytest.skip(f"no native grouped GEMM in {dtype} on {DEVICE} with torch {torch.__version__}")
def _use_mode(monkeypatch, mode):
"""grouped: every adapter in one pass; stash / fold: one adapter at a time, through Zoo's stash or folded."""
if mode == "grouped":
_need_grouped()
if mode == "grouped":
monkeypatch.delenv("UNSLOTH_MULTI_LORA_MOE_GROUPED", raising = False)
else:
monkeypatch.setenv("UNSLOTH_MULTI_LORA_MOE_GROUPED", "0")
if mode == "fold":
monkeypatch.setattr(ml, "_zoo_stash_read_store", lambda experts: None)
def _modes(model):
return {m.mode for m in model.modules() if isinstance(m, ml.MultiLoraExperts)}
@pytest.mark.parametrize("mode", MODES)
def test_each_adapter_matches_its_lone_peft_run(base, monkeypatch, mode):
_use_mode(monkeypatch, mode)
model, wrapped = _batched(base)
assert len(wrapped) == 2 * len(model.model.language_model.layers)
ids = _ids()
ml.set_multi_lora_batch(model, torch.tensor([0, 1, 2], device = DEVICE), seq_len = T)
out = model(input_ids = ids, use_cache = False).logits
assert _modes(model) == {mode}
for u in range(3):
lone = _lone(base, model, u)
ref = lone(input_ids = ids[u : u + 1], use_cache = False).logits[0]
moved = (ref - base(input_ids = ids[u : u + 1]).logits[0]).abs().max()
assert moved > 1e-3 # the adapter really changes the output
torch.testing.assert_close(out[u], ref, atol = TOL, rtol = 1e-4)
@pytest.mark.parametrize("mode", MODES)
def test_expert_grads_match_the_lone_run(base, monkeypatch, mode):
_use_mode(monkeypatch, mode)
model, _ = _batched(base)
ids = _ids()
ml.set_multi_lora_batch(model, torch.tensor([0, 1, 2], device = DEVICE), seq_len = T)
u = 1
model(input_ids = ids, use_cache = False).logits[u].pow(2).mean().backward()
assert _modes(model) == {mode}
lone = _lone(base, model, u)
lone(input_ids = ids[u : u + 1], use_cache = False).logits[0].pow(2).mean().backward()
for name, experts in model.named_modules():
if not isinstance(experts, ml.MultiLoraExperts):
continue
layout = "rank_major" if experts.peft_mode == "fold" else ml._peft_moe_lora_b_layout()
depth = len(experts.parameter_names)
for i, param in enumerate(experts.parameter_names):
holder = lone.base_model.model.get_submodule(name)
for _ in range(depth - 1 - i):
holder = holder.base_layer
mine = experts.adapters[param]
gA, gB = mine.lora_A[u].grad, mine.lora_B[u].grad
flat = ml.MultiLoraExpertParam.peft_tensors
packed = type("G", (), {"lora_A": {u: gA}, "lora_B": {u: gB}})
exp_A, exp_B = flat(packed, u, layout)
torch.testing.assert_close(
exp_A, holder.lora_A["default"].weight.grad, atol = 1e-6, rtol = 1e-4
)
torch.testing.assert_close(
exp_B, holder.lora_B["default"].weight.grad, atol = 1e-6, rtol = 1e-4
)
other = experts.adapters[param].lora_A[0].grad
assert other is None or other.abs().max() == 0
dense = [(n, m) for n, m in model.named_modules() if isinstance(m, ml.MultiLoraLinear)]
assert dense
for name, layer in dense:
holder = lone.base_model.model.get_submodule(name)
assert layer.lora_A[u].requires_grad and layer.lora_B[u].requires_grad
torch.testing.assert_close(
layer.lora_A[u].grad, holder.lora_A["default"].weight.grad, atol = 1e-6, rtol = 1e-4
)
torch.testing.assert_close(
layer.lora_B[u].grad, holder.lora_B["default"].weight.grad, atol = 1e-6, rtol = 1e-4
)
@pytest.mark.parametrize("mode", MODES)
def test_another_adapter_cannot_move_a_row(base, monkeypatch, mode):
_use_mode(monkeypatch, mode)
model, _ = _batched(base)
ids = _ids()
ml.set_multi_lora_batch(model, torch.tensor([0, 1, 2], device = DEVICE), seq_len = T)
before = model(input_ids = ids, use_cache = False).logits.detach()
with torch.no_grad():
for _, layer in ml._layers(model):
layer.lora_A[0].mul_(3.0)
layer.lora_B[0].add_(1.0)
after = model(input_ids = ids, use_cache = False).logits.detach()
assert _modes(model) == {mode}
assert (after[0] - before[0]).abs().max() > 1e-3
# Adapter 0's attention LoRA reroutes its tokens, so in grouped mode the other adapters' rows sit at other
# positions of each expert's base GEMM. GPU kernels are row-position independent (bit-exact); AVX2 / NEON CPU
# BLAS tails round a row by an ulp or two depending on where it falls, so allow that much there.
atol = 1e-6 if mode == "grouped" and DEVICE == "cpu" else 0
torch.testing.assert_close(after[1:], before[1:], atol = atol, rtol = 0)
@pytest.mark.parametrize("mode", ["grouped", "fold"])
def test_non_reentrant_checkpointing(base, monkeypatch, mode):
_use_mode(monkeypatch, mode)
# A Zoo stash store that this forward never fills: the experts fall back to folding.
monkeypatch.setattr(ml, "_zoo_stash_read_store", lambda experts: {})
model, _ = _batched(base)
ids = _ids()
ml.set_multi_lora_batch(model, torch.tensor([0, 1, 2], device = DEVICE), seq_len = T)
experts = next(m for m in model.modules() if isinstance(m, ml.MultiLoraExperts))
block = next(m for m in model.modules() if getattr(m, "experts", None) is experts)
hidden = torch.randn(
3 * T, model.config.text_config.hidden_size, device = DEVICE, requires_grad = True
)
router = block.gate
with torch.no_grad():
_, weights, indices = router(hidden)
out = torch.utils.checkpoint.checkpoint(experts, hidden, indices, weights, use_reentrant = False)
out.sum().backward()
assert experts.mode == mode and experts.peft_mode == "fold" and hidden.grad is not None
eager = hidden.grad.clone()
hidden.grad = None
experts(hidden, indices, weights).sum().backward()
torch.testing.assert_close(hidden.grad, eager)
@pytest.mark.parametrize("mode", MODES)
def test_absent_adapters_stay_in_the_graph(base, monkeypatch, mode):
_use_mode(monkeypatch, mode)
model, _ = _batched(base)
ids = _ids()
clean = None
with torch.no_grad():
ml.set_multi_lora_batch(model, torch.tensor([0, 1, 1], device = DEVICE), seq_len = T)
clean = model(input_ids = ids, use_cache = False).logits
# A diverged absent adapter must not reach the others' outputs.
for m in model.modules():
if isinstance(m, ml.MultiLoraExperts):
for adapter in m.adapters.values():
adapter.lora_B[2].fill_(float("nan"))
ml.set_multi_lora_batch(model, torch.tensor([0, 1, 1], device = DEVICE), seq_len = T)
logits = model(input_ids = ids, use_cache = False).logits
torch.testing.assert_close(logits, clean, atol = 0, rtol = 0)
logits.sum().backward()
assert _modes(model) == {mode}
for m in model.modules():
if isinstance(m, ml.MultiLoraExperts):
for adapter in m.adapters.values():
assert (
adapter.lora_B[2].grad is not None and adapter.lora_B[2].grad.abs().max() == 0
)
assert (
adapter.lora_A[2].grad is not None and adapter.lora_A[2].grad.abs().max() == 0
)
assert adapter.lora_A[1].grad is not None
@pytest.fixture(scope = "module")
def input_major(base):
"""The same experts stored input-major and flagged as Zoo's grouped-mm format (Qwen3-VL-MoE, GPT-OSS)."""
model = copy.deepcopy(base)
flagged = 0
for module in model.modules():
if (
hasattr(module, "gate_up_proj")
and hasattr(module, "down_proj")
and module.gate_up_proj.dim() == 3
):
with torch.no_grad():
for name in ("gate_up_proj", "down_proj"):
param = getattr(module, name)
param.data = param.data.transpose(1, 2).contiguous()
module._unsloth_grouped_mm_format = True
flagged += 1
assert flagged
return model
@pytest.mark.parametrize("mode", MODES)
def test_input_major_grouped_mm_experts_match_their_lone_peft_run(
base, input_major, monkeypatch, mode
):
_use_mode(monkeypatch, mode)
ids = _ids()
# Zoo's forward reads the flag: the relaid experts compute what the original ones do.
torch.testing.assert_close(
input_major(input_ids = ids).logits, base(input_ids = ids).logits, atol = TOL, rtol = 1e-4
)
model, _ = _batched(input_major)
ml.set_multi_lora_batch(model, torch.tensor([0, 1, 2], device = DEVICE), seq_len = T)
out = model(input_ids = ids, use_cache = False).logits
assert _modes(model) == {mode}
for u in range(3):
ref = _lone(input_major, model, u)(input_ids = ids[u : u + 1], use_cache = False).logits[0]
torch.testing.assert_close(out[u], ref, atol = TOL, rtol = 1e-4)
@pytest.mark.parametrize("mode", MODES)
def test_autocast_matches_the_lone_run(base, monkeypatch, mode):
if DEVICE == "cuda" and not torch.cuda.is_bf16_supported(including_emulation = False):
pytest.skip("no bfloat16 autocast on this GPU")
_use_mode(monkeypatch, mode)
model, _ = _batched(base)
ids = _ids()
ml.set_multi_lora_batch(model, torch.tensor([0, 1, 2], device = DEVICE), seq_len = T)
with torch.autocast(device_type = DEVICE, dtype = torch.bfloat16):
out = model(input_ids = ids, use_cache = False).logits
refs = [
_lone(base, model, u)(input_ids = ids[u : u + 1], use_cache = False).logits[0]
for u in range(3)
]
assert _modes(model) == {mode}
for u in range(3):
# One bf16 ulp of the logits; the bf16 fold was off by about 0.1.
torch.testing.assert_close(out[u].float(), refs[u].float(), atol = 3e-2, rtol = 0)
def test_experts_read_directly_by_their_parent_are_refused():
class Experts(torch.nn.Module):
def __init__(self):
super().__init__()
self.gate_up_proj = torch.nn.Parameter(torch.zeros(2, 4, 8))
self.down_proj = torch.nn.Parameter(torch.zeros(2, 4, 4))
class FastBlock(torch.nn.Module):
def __init__(self):
super().__init__()
self.experts = Experts()
def forward(self, x): # a grouped-GEMM fast path that never calls self.experts
return x @ self.experts.gate_up_proj[0]
with pytest.raises(ValueError, match = "reads experts.gate_up_proj itself"):
ml.inject_multi_lora_experts(FastBlock(), ["experts.gate_up_proj"], 2, 2, 2)
class _Experts(torch.nn.Module):
"""Fused experts in transformers' layout and eager forward: gate_up_proj (E, 2I, H), down_proj (E, H, I)."""
def __init__(
self,
E = 5,
H = 20,
I = 12,
bias = False,
):
super().__init__()
g = torch.Generator().manual_seed(0)
self.gate_up_proj = torch.nn.Parameter(torch.randn(E, 2 * I, H, generator = g) * 0.3)
self.down_proj = torch.nn.Parameter(torch.randn(E, H, I, generator = g) * 0.3)
if bias:
self.gate_up_proj_bias = torch.nn.Parameter(torch.randn(E, 2 * I, generator = g))
self.act_fn = torch.nn.SiLU()
def forward(self, hidden_states, top_k_index, top_k_weights):
out = torch.zeros_like(hidden_states)
for t in range(hidden_states.shape[0]):
for j, e in enumerate(top_k_index[t].tolist()):
if e >= self.gate_up_proj.shape[0]:
continue
h = hidden_states[t] @ self.gate_up_proj[e].t()
if hasattr(self, "gate_up_proj_bias"):
h = h + self.gate_up_proj_bias[e]
gate, up = h.chunk(2, dim = -1)
y = (self.act_fn(gate) * up) @ self.down_proj[e].t()
out[t] = out[t] + y * top_k_weights[t, j]
return out
def _routes(
T,
k,
E,
seed = 0,
):
"""Skewed routing: most routes on expert 0, expert E-2 never hit, one dropped route (id E)."""
g = torch.Generator().manual_seed(seed)
index = torch.randint(0, E, (T, k), generator = g)
index[torch.rand(T, k, generator = g) < 0.5] = 0
index[index == E - 2] = E - 1
index[0, -1] = E
weights = torch.rand(T, k, generator = g)
return index, weights
def _wrapped(module, ranks, alphas, rslora, ids):
experts = ml.MultiLoraExperts(
module, ["gate_up_proj", "down_proj"], len(ranks), ranks, alphas, rslora
)
torch.manual_seed(0)
with torch.no_grad():
for adapter in experts.adapters.values():
adapter.reset_parameters()
for B in adapter.lora_B:
B.normal_(0, 0.5, generator = torch.Generator().manual_seed(B.numel()))
ml.set_multi_lora_batch(experts, ids)
return experts
def _oracle(experts, hidden, index, weights, ids, rslora):
"""float64, route by route: base plus the token's own adapter delta (scale alpha / r or alpha / sqrt(r))."""
base = experts.base_layer
E = base.gate_up_proj.shape[0]
factors = {
name: [(A.double(), B.double()) for A, B in zip(adapter.lora_A, adapter.lora_B)]
for name, adapter in experts.adapters.items()
}
out = []
for t in range(hidden.shape[0]):
u = int(ids[t])
acc = 0
for j in range(index.shape[1]):
e = int(index[t, j])
if e >= E:
continue
def proj(name, x):
W = getattr(base, name)[e].double()
if u >= 0:
A, B = factors[name][u]
r = A.shape[1]
scale = ALPHAS[u] / (r**0.5 if rslora else r)
W = W + scale * B[e] @ A[e]
return x @ W.t()
gate, up = proj("gate_up_proj", hidden[t]).chunk(2, dim = -1)
acc = acc + proj("down_proj", torch.nn.functional.silu(gate) * up) * weights[t, j]
out.append(acc)
return torch.stack(out), factors
@pytest.mark.parametrize("rslora", [False, True])
@pytest.mark.parametrize("present", ["all", "no_adapter_1"])
def test_grouped_matches_a_float64_oracle(monkeypatch, rslora, present):
_need_grouped()
monkeypatch.delenv("UNSLOTH_MULTI_LORA_MOE_GROUPED", raising = False)
T, k, E = 24, 2, 5
ids = torch.tensor([0, 0, 2, -1, 0, 1, 2, 0] * 3)
if present == "no_adapter_1":
ids[ids == 1] = 2
experts = _wrapped(_Experts(E), RANKS, ALPHAS, rslora, ids)
assert not experts.transposed and not experts.reversed_io
index, weights = _routes(T, k, E)
hidden = torch.randn(T, 20, generator = torch.Generator().manual_seed(2))
hidden64, weights64 = hidden.double().requires_grad_(), weights.double().requires_grad_()
hidden, weights = hidden.requires_grad_(), weights.requires_grad_()
calls = ml.MULTI_LORA_MOE_STATS["grouped_calls"]
out = experts(hidden, index, weights)
assert experts.mode == "grouped" and ml.MULTI_LORA_MOE_STATS["grouped_calls"] == calls + 1
ref, factors = _oracle(experts, hidden64, index, weights64, ids, rslora)
torch.testing.assert_close(out.double(), ref, atol = 1e-5, rtol = 1e-4)
grad = torch.randn(out.shape, generator = torch.Generator().manual_seed(3))
(out * grad).sum().backward()
leaves = [hidden64, weights64] + [
t for pairs in factors.values() for pair in pairs for t in pair
]
ref_grads = torch.autograd.grad((ref * grad.double()).sum(), leaves, allow_unused = True)
mine = [hidden.grad, weights.grad] + [
p.grad
for adapter in experts.adapters.values()
for pair in zip(adapter.lora_A, adapter.lora_B)
for p in pair
]
for got, want in zip(mine, ref_grads):
want = torch.zeros_like(got, dtype = torch.float64) if want is None else want
assert got is not None
torch.testing.assert_close(got.double(), want, atol = 1e-5, rtol = 1e-4)
if present == "no_adapter_1":
for adapter in experts.adapters.values():
assert (
adapter.lora_A[1].grad.abs().max() == 0 and adapter.lora_B[1].grad.abs().max() == 0
)
@pytest.mark.parametrize("U", [2, 3])
def test_grouped_is_one_pass_on_zoo_gemms(monkeypatch, U):
"""Every adapter in one pass: 2 base GEMMs through Zoo's base path and 4 LoRA grouped GEMMs, whatever U is."""
_need_grouped()
from unsloth_zoo.temporary_patches import moe_utils
monkeypatch.delenv("UNSLOTH_MULTI_LORA_MOE_GROUPED", raising = False)
T, k, E = 24, 2, 5
ids = torch.tensor([u % U for u in range(T)])
experts = _wrapped(_Experts(E), RANKS[:U], ALPHAS[:U], False, ids)
index, weights = _routes(T, k, E)
hidden = torch.randn(T, 20, generator = torch.Generator().manual_seed(2))
experts(hidden, index, weights)
assert experts.mode == "grouped"
calls = {"base": 0, "lora": 0}
base_mm, lora_mm = moe_utils._base_grouped_mm, moe_utils._grouped_mm_with_backward_fix
def count(kind, fn):
def wrapper(*args, **kwargs):
# LoRA GEMMs are the ones on the stacked (E * (U + 1), ...) factors; Zoo's base path may call the same op.
if kind == "base" or args[1].shape[0] == E * (U + 1):
calls[kind] += 1
return fn(*args, **kwargs)
return wrapper
monkeypatch.setattr(moe_utils, "_base_grouped_mm", count("base", base_mm))
monkeypatch.setattr(moe_utils, "_grouped_mm_with_backward_fix", count("lora", lora_mm))
experts(hidden, index, weights)
assert calls == {"base": 2, "lora": 4}
def test_one_present_adapter_runs_its_own_pass(monkeypatch):
"""A batch with one adapter's tokens has nothing to fuse: that call runs per adapter, with the same result."""
_need_grouped()
monkeypatch.delenv("UNSLOTH_MULTI_LORA_MOE_GROUPED", raising = False)
T, k, E = 24, 2, 5
index, weights = _routes(T, k, E)
hidden = torch.randn(T, 20, generator = torch.Generator().manual_seed(2))
experts = _wrapped(_Experts(E), RANKS, ALPHAS, False, torch.tensor([0, 1, 2, -1] * 6))
experts(hidden, index, weights)
assert experts.mode == "grouped"
ml.set_multi_lora_batch(experts, torch.tensor([1, -1] * 12))
before = dict(ml.MULTI_LORA_MOE_STATS)
out = experts(hidden, index, weights)
assert ml.MULTI_LORA_MOE_STATS["per_adapter_calls"] == before["per_adapter_calls"] + 1
assert ml.MULTI_LORA_MOE_STATS["grouped_calls"] == before["grouped_calls"]
torch.testing.assert_close(
out, experts._grouped(hidden, index, weights, torch.tensor([1, -1] * 12))
)
def test_grouped_engages_and_the_switch_turns_it_off(base, monkeypatch):
_need_grouped()
ids = _ids()
counts = []
for value in ("1", "0"):
monkeypatch.setenv("UNSLOTH_MULTI_LORA_MOE_GROUPED", value)
model, wrapped = _batched(base)
ml.set_multi_lora_batch(model, torch.tensor([0, 1, 2], device = DEVICE), seq_len = T)
before = dict(ml.MULTI_LORA_MOE_STATS)
model(input_ids = ids, use_cache = False)
counts.append({key: ml.MULTI_LORA_MOE_STATS[key] - before[key] for key in before})
layers = len(wrapped) // 2
assert counts[0] == {"grouped_calls": layers, "per_adapter_calls": 0, "grouped_fallbacks": 0}
assert counts[1] == {"grouped_calls": 0, "per_adapter_calls": layers, "grouped_fallbacks": 0}
@pytest.mark.parametrize("why", ["bias", "parity"])
def test_unsupported_experts_fall_back_to_one_adapter_at_a_time(monkeypatch, why):
monkeypatch.delenv("UNSLOTH_MULTI_LORA_MOE_GROUPED", raising = False)
if why == "parity":
grouped = ml.MultiLoraExperts._grouped
monkeypatch.setattr(ml.MultiLoraExperts, "_grouped", lambda *a: grouped(*a) * 1.01)
T, k, E = 24, 2, 5
ids = torch.tensor([0, 1, 2, -1] * 6)
index, weights = _routes(T, k, E)
hidden = torch.randn(T, 20, generator = torch.Generator().manual_seed(2))
experts = _wrapped(_Experts(E, bias = why == "bias"), RANKS, ALPHAS, False, ids)
before = dict(ml.MULTI_LORA_MOE_STATS)
out = experts(hidden, index, weights)
assert experts.mode == "fold"
assert ml.MULTI_LORA_MOE_STATS["grouped_fallbacks"] == before["grouped_fallbacks"] + 1
assert ml.MULTI_LORA_MOE_STATS["grouped_calls"] == before["grouped_calls"]
monkeypatch.setenv("UNSLOTH_MULTI_LORA_MOE_GROUPED", "0")
experts.mode = experts.peft_mode = None
torch.testing.assert_close(out, experts(hidden, index, weights), atol = 0, rtol = 0)
def test_grouped_reads_bnb_4bit_stacks_through_zoo(monkeypatch):
_need_grouped()
bnb = pytest.importorskip("bitsandbytes")
from unsloth_zoo.temporary_patches import moe_utils
monkeypatch.delenv("UNSLOTH_MULTI_LORA_MOE_GROUPED", raising = False)
# The synthetic forward cannot run a 4-bit stack, so there is no per-adapter path to probe.
monkeypatch.setattr(ml.MultiLoraExperts, "_run_stash", lambda *a: None)
monkeypatch.setattr(ml.MultiLoraExperts, "_grouped_parity", lambda *a: None)
T, k, E = 24, 2, 5
module = _Experts(E, H = 32, I = 16)
for name in ("gate_up_proj", "down_proj"):
W = getattr(module, name).data.to(torch.bfloat16)
packed, state = bnb.functional.quantize_4bit(W.reshape(-1, W.shape[-1]), quant_type = "nf4")
param = torch.Tensor._make_subclass(bnb.nn.Params4bit, packed, False)
param.quant_state, param._original_shape = state, W.shape
module._parameters[name] = param
ids = torch.tensor([0, 1, 2, -1] * 6)
experts = _wrapped(module, RANKS, ALPHAS, False, ids)
index, weights = _routes(T, k, E)
hidden = torch.randn(T, 32, generator = torch.Generator().manual_seed(2), requires_grad = True)
seen = []
get_base = moe_utils._get_base_weight
monkeypatch.setattr(
moe_utils, "_get_base_weight", lambda *a, **kw: seen.append(1) or get_base(*a, **kw)
)
out = experts(hidden, index, weights)
assert experts.mode == "grouped" and out.dtype == torch.float32
# Zoo's provider dequantizes both stacks in forward and, where Zoo recomputes, again in backward.
forward_reads = len(seen)
assert forward_reads >= 2
out.sum().backward()
if moe_utils._moe_recompute_enabled(experts.base_layer.gate_up_proj, dtype = torch.bfloat16):
assert len(seen) >= 2 * forward_reads
float_module = _Experts(E, H = 32, I = 16)
with torch.no_grad():
for name in ("gate_up_proj", "down_proj"):
# The oracle reads the stack as Zoo's provider does (CPU bitsandbytes slices differ from full reads).
param = getattr(experts.base_layer, name)
getattr(float_module, name).copy_(get_base(param, torch.bfloat16).float())
reference = _wrapped(float_module, RANKS, ALPHAS, False, ids)
ref, _ = _oracle(reference, hidden.detach().double(), index, weights.double(), ids, False)
# bfloat16 compute, as Unsloth Zoo runs 4-bit experts.
torch.testing.assert_close(out.double(), ref, atol = 2e-2 * ref.abs().max().item(), rtol = 0)
def test_grouped_dropped_route_never_poisons_gradients(monkeypatch):
_need_grouped()
# A dropped route (expert id E) with a NaN weight must add exactly nothing, forward and backward.
monkeypatch.delenv("UNSLOTH_MULTI_LORA_MOE_GROUPED", raising = False)
T, k, E = 8, 2, 5
ids = torch.tensor([0, 1, 2, -1, 0, 1, 2, 0])
experts = _wrapped(_Experts(E), RANKS, ALPHAS, False, ids)
index, weights = _routes(T, k, E)
hidden = torch.randn(T, 20, generator = torch.Generator().manual_seed(2))
experts(hidden, index, weights)
assert experts.mode == "grouped"
weights[0, -1] = float("nan")
kept, clean = index.clone(), weights.clone()
kept[0, -1], clean[0, -1] = 0, 0.0
def run(index, weights):
h, w = hidden.clone().requires_grad_(), weights.clone().requires_grad_()
experts.zero_grad(set_to_none = True)
out = experts(h, index, w)
out.backward(torch.ones_like(out))
grads = [p.grad.clone() for p in experts.parameters() if p.grad is not None]
return out, h.grad, w.grad, grads
out, d_hidden, d_weights, grads = run(index, weights)
ref, ref_hidden, ref_weights, ref_grads = run(kept, clean)
torch.testing.assert_close(out, ref)
torch.testing.assert_close(d_hidden, ref_hidden)
assert d_weights[0, -1] == 0
torch.testing.assert_close(d_weights[:, :1], ref_weights[:, :1])
for g, r in zip(grads, ref_grads):
torch.testing.assert_close(g, r)
def test_grouped_empty_top_k_gives_zeros(monkeypatch):
_need_grouped()
monkeypatch.delenv("UNSLOTH_MULTI_LORA_MOE_GROUPED", raising = False)
ids = torch.tensor([0, 1, 2, -1])
experts = _wrapped(_Experts(5), RANKS, ALPHAS, False, ids)
index, weights = _routes(4, 2, 5)
experts(torch.randn(4, 20), index, weights)
assert experts.mode == "grouped"
hidden = torch.randn(4, 20, requires_grad = True)
out = experts(hidden, index[:, :0], weights[:, :0])
assert out.shape == (4, 20) and not out.any()
out.sum().backward()
assert not hidden.grad.any()
for adapter in experts.adapters.values():
for p in (*adapter.lora_A, *adapter.lora_B):
assert p.grad is not None and not p.grad.any()
class _StaticExperts(_Experts):
"""The same experts with a data-independent forward (every expert, masked), so it traces with fullgraph."""
def forward(self, hidden_states, top_k_index, top_k_weights):
out = torch.zeros_like(hidden_states)
for e in range(self.gate_up_proj.shape[0]):
w = (top_k_weights * (top_k_index == e)).sum(-1, keepdim = True)
gate, up = (hidden_states @ self.gate_up_proj[e].t()).chunk(2, dim = -1)
out = out + ((self.act_fn(gate) * up) @ self.down_proj[e].t()) * w
return out
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
def test_grouped_under_torch_compile(monkeypatch, dtype):
# Grouped GEMM only traces in bfloat16; other dtypes take the per-adapter path inside a compiled graph.
_need_grouped(dtype)
monkeypatch.delenv("UNSLOTH_MULTI_LORA_MOE_GROUPED", raising = False)
T, k, E = 16, 2, 5
ids = torch.tensor([0, 1, 2, -1] * 4)
experts = _wrapped(_StaticExperts(E, H = 24, I = 16).to(dtype), RANKS, ALPHAS, False, ids).to(
DEVICE
)
for _, layer in ml._layers(experts):
layer.seen = None # a plain attribute, so .to() left it on the CPU
ml.set_multi_lora_batch(experts, ids.to(DEVICE))
index, weights = _routes(T, k, E)
index[0, -1] = 0
index, weights = index.to(DEVICE), weights.to(DEVICE)
hidden = (
torch.randn(T, 24, generator = torch.Generator().manual_seed(2))
.to(DEVICE, dtype)
.requires_grad_()
)
weights = weights.to(dtype)
eager = experts(hidden, index, weights)
assert experts.mode == "grouped"
torch._dynamo.reset()
# fullgraph on CUDA, where Zoo's grouped GEMM traces as a custom op; on CPU it is a compile-disabled loop.
compiled = torch.compile(experts, backend = "eager", fullgraph = DEVICE == "cuda")(
hidden, index, weights
)
torch.testing.assert_close(compiled.float(), eager.float(), atol = 2e-2, rtol = 2e-2)
compiled.sum().backward()
assert torch.isfinite(hidden.grad).all()