Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Batched MoE LoRA: every job's expert LoRA in one grouped pass on Zoo's kernels (#16012)
Daniel Han69117f1bf36c
diff --git a/tests/test_multi_lora_experts.py b/tests/test_multi_lora_experts.pyindex 82493859a4c..567cde700ba 100644--- a/tests/test_multi_lora_experts.py+++ b/tests/test_multi_lora_experts.py@@ -73,17 +73,40 @@ def _ids():return torch.randint(0, 100, (3, T), generator = torch.Generator().manual_seed(1)).to(DEVICE)[email protected]("mode", ["stash", "fold"])-def test_each_adapter_matches_its_lone_peft_run(base, monkeypatch, mode):+MODES = ["grouped", "stash", "fold"]+++def _need_grouped(dtype = torch.float32):+ """Skip where this torch / device has no native grouped GEMM in ``dtype`` (the module then runs per adapter)."""+ if not ml._grouped_supported(torch.device(DEVICE), dtype):+ pytest.skip(f"no native grouped GEMM in {dtype} on {DEVICE} with torch {torch.__version__}")+++def _use_mode(monkeypatch, mode):+ """grouped: every adapter in one pass; stash / fold: one adapter at a time, through Zoo's stash or folded."""+ if mode == "grouped":+ _need_grouped()+ if mode == "grouped":+ monkeypatch.delenv("UNSLOTH_MULTI_LORA_MOE_GROUPED", raising = False)+ else:+ monkeypatch.setenv("UNSLOTH_MULTI_LORA_MOE_GROUPED", "0")if mode == "fold":monkeypatch.setattr(ml, "_zoo_stash_read_store", lambda experts: None)+++def _modes(model):+ return {m.mode for m in model.modules() if isinstance(m, ml.MultiLoraExperts)}++[email protected]("mode", MODES)+def test_each_adapter_matches_its_lone_peft_run(base, monkeypatch, mode):+ _use_mode(monkeypatch, mode)model, wrapped = _batched(base)assert len(wrapped) == 2 * len(model.model.language_model.layers)ids = _ids()ml.set_multi_lora_batch(model, torch.tensor([0, 1, 2], device = DEVICE), seq_len = T)out = model(input_ids = ids, use_cache = False).logits- modes = {m.mode for m in model.modules() if isinstance(m, ml.MultiLoraExperts)}- assert modes == {mode}+ assert _modes(model) == {mode}for u in range(3):lone = _lone(base, model, u)ref = lone(input_ids = ids[u : u + 1], use_cache = False).logits[0]@@ -92,18 +115,21 @@ def test_each_adapter_matches_its_lone_peft_run(base, monkeypatch, mode):torch.testing.assert_close(out[u], ref, atol = TOL, rtol = 1e-4)-def test_expert_grads_match_the_lone_run(base):[email protected]("mode", MODES)+def test_expert_grads_match_the_lone_run(base, monkeypatch, mode):+ _use_mode(monkeypatch, mode)model, _ = _batched(base)ids = _ids()ml.set_multi_lora_batch(model, torch.tensor([0, 1, 2], device = DEVICE), seq_len = T)u = 1model(input_ids = ids, use_cache = False).logits[u].pow(2).mean().backward()+ assert _modes(model) == {mode}lone = _lone(base, model, u)lone(input_ids = ids[u : u + 1], use_cache = False).logits[0].pow(2).mean().backward()- layout = ml._peft_moe_lora_b_layout()for name, experts in model.named_modules():if not isinstance(experts, ml.MultiLoraExperts):continue+ layout = "rank_major" if experts.peft_mode == "fold" else ml._peft_moe_lora_b_layout()depth = len(experts.parameter_names)for i, param in enumerate(experts.parameter_names):holder = lone.base_model.model.get_submodule(name)@@ -135,7 +161,9 @@ def test_expert_grads_match_the_lone_run(base):)-def test_another_adapter_cannot_move_a_row(base):[email protected]("mode", MODES)+def test_another_adapter_cannot_move_a_row(base, monkeypatch, mode):+ _use_mode(monkeypatch, mode)model, _ = _batched(base)ids = _ids()ml.set_multi_lora_batch(model, torch.tensor([0, 1, 2], device = DEVICE), seq_len = T)@@ -145,11 +173,18 @@ def test_another_adapter_cannot_move_a_row(base):layer.lora_A[0].mul_(3.0)layer.lora_B[0].add_(1.0)after = model(input_ids = ids, use_cache = False).logits.detach()+ assert _modes(model) == {mode}assert (after[0] - before[0]).abs().max() > 1e-3- torch.testing.assert_close(after[1:], before[1:], atol = 0, rtol = 0)+ # Adapter 0's attention LoRA reroutes its tokens, so in grouped mode the other adapters' rows sit at other+ # positions of each expert's base GEMM. GPU kernels are row-position independent (bit-exact); AVX2 / NEON CPU+ # BLAS tails round a row by an ulp or two depending on where it falls, so allow that much there.+ atol = 1e-6 if mode == "grouped" and DEVICE == "cpu" else 0+ torch.testing.assert_close(after[1:], before[1:], atol = atol, rtol = 0)-def test_fold_mode_survives_non_reentrant_checkpointing(base, monkeypatch):[email protected]("mode", ["grouped", "fold"])+def test_non_reentrant_checkpointing(base, monkeypatch, mode):+ _use_mode(monkeypatch, mode)# A Zoo stash store that this forward never fills: the experts fall back to folding.monkeypatch.setattr(ml, "_zoo_stash_read_store", lambda experts: {})model, _ = _batched(base)@@ -165,10 +200,16 @@ def test_fold_mode_survives_non_reentrant_checkpointing(base, monkeypatch):_, weights, indices = router(hidden)out = torch.utils.checkpoint.checkpoint(experts, hidden, indices, weights, use_reentrant = False)out.sum().backward()- assert experts.mode == "fold" and hidden.grad is not None+ assert experts.mode == mode and experts.peft_mode == "fold" and hidden.grad is not None+ eager = hidden.grad.clone()+ hidden.grad = None+ experts(hidden, indices, weights).sum().backward()+ torch.testing.assert_close(hidden.grad, eager)-def test_absent_adapters_stay_in_the_graph(base):[email protected]("mode", MODES)+def test_absent_adapters_stay_in_the_graph(base, monkeypatch, mode):+ _use_mode(monkeypatch, mode)model, _ = _batched(base)ids = _ids()clean = None@@ -184,9 +225,13 @@ def test_absent_adapters_stay_in_the_graph(base):logits = model(input_ids = ids, use_cache = False).logitstorch.testing.assert_close(logits, clean, atol = 0, rtol = 0)logits.sum().backward()+ assert _modes(model) == {mode}for m in model.modules():if isinstance(m, ml.MultiLoraExperts):for adapter in m.adapters.values():+ assert (+ adapter.lora_B[2].grad is not None and adapter.lora_B[2].grad.abs().max() == 0+ )assert (adapter.lora_A[2].grad is not None and adapter.lora_A[2].grad.abs().max() == 0)@@ -214,12 +259,11 @@ def input_major(base):return model[email protected]("mode", ["stash", "fold"])[email protected]("mode", MODES)def test_input_major_grouped_mm_experts_match_their_lone_peft_run(base, input_major, monkeypatch, mode):- if mode == "fold":- monkeypatch.setattr(ml, "_zoo_stash_read_store", lambda experts: None)+ _use_mode(monkeypatch, mode)ids = _ids()# Zoo's forward reads the flag: the relaid experts compute what the original ones do.torch.testing.assert_close(@@ -228,14 +272,17 @@ def test_input_major_grouped_mm_experts_match_their_lone_peft_run(model, _ = _batched(input_major)ml.set_multi_lora_batch(model, torch.tensor([0, 1, 2], device = DEVICE), seq_len = T)out = model(input_ids = ids, use_cache = False).logits- assert {m.mode for m in model.modules() if isinstance(m, ml.MultiLoraExperts)} == {mode}+ assert _modes(model) == {mode}for u in range(3):ref = _lone(input_major, model, u)(input_ids = ids[u : u + 1], use_cache = False).logits[0]torch.testing.assert_close(out[u], ref, atol = TOL, rtol = 1e-4)-def test_fold_mode_under_autocast_matches_the_lone_run(base, monkeypatch):- monkeypatch.setattr(ml, "_zoo_stash_read_store", lambda experts: None)[email protected]("mode", MODES)+def test_autocast_matches_the_lone_run(base, monkeypatch, mode):+ if DEVICE == "cuda" and not torch.cuda.is_bf16_supported(including_emulation = False):+ pytest.skip("no bfloat16 autocast on this GPU")+ _use_mode(monkeypatch, mode)model, _ = _batched(base)ids = _ids()ml.set_multi_lora_batch(model, torch.tensor([0, 1, 2], device = DEVICE), seq_len = T)@@ -245,6 +292,7 @@ def test_fold_mode_under_autocast_matches_the_lone_run(base, monkeypatch):_lone(base, model, u)(input_ids = ids[u : u + 1], use_cache = False).logits[0]for u in range(3)]+ assert _modes(model) == {mode}for u in range(3):# One bf16 ulp of the logits; the bf16 fold was off by about 0.1.torch.testing.assert_close(out[u].float(), refs[u].float(), atol = 3e-2, rtol = 0)@@ -267,3 +315,373 @@ def test_experts_read_directly_by_their_parent_are_refused():with pytest.raises(ValueError, match = "reads experts.gate_up_proj itself"):ml.inject_multi_lora_experts(FastBlock(), ["experts.gate_up_proj"], 2, 2, 2)+++class _Experts(torch.nn.Module):+ """Fused experts in transformers' layout and eager forward: gate_up_proj (E, 2I, H), down_proj (E, H, I)."""++ def __init__(+ self,+ E = 5,+ H = 20,+ I = 12,+ bias = False,+ ):+ super().__init__()+ g = torch.Generator().manual_seed(0)+ self.gate_up_proj = torch.nn.Parameter(torch.randn(E, 2 * I, H, generator = g) * 0.3)+ self.down_proj = torch.nn.Parameter(torch.randn(E, H, I, generator = g) * 0.3)+ if bias:+ self.gate_up_proj_bias = torch.nn.Parameter(torch.randn(E, 2 * I, generator = g))+ self.act_fn = torch.nn.SiLU()++ def forward(self, hidden_states, top_k_index, top_k_weights):+ out = torch.zeros_like(hidden_states)+ for t in range(hidden_states.shape[0]):+ for j, e in enumerate(top_k_index[t].tolist()):+ if e >= self.gate_up_proj.shape[0]:+ continue+ h = hidden_states[t] @ self.gate_up_proj[e].t()+ if hasattr(self, "gate_up_proj_bias"):+ h = h + self.gate_up_proj_bias[e]+ gate, up = h.chunk(2, dim = -1)+ y = (self.act_fn(gate) * up) @ self.down_proj[e].t()+ out[t] = out[t] + y * top_k_weights[t, j]+ return out+++def _routes(+ T,+ k,+ E,+ seed = 0,+):+ """Skewed routing: most routes on expert 0, expert E-2 never hit, one dropped route (id E)."""+ g = torch.Generator().manual_seed(seed)+ index = torch.randint(0, E, (T, k), generator = g)+ index[torch.rand(T, k, generator = g) < 0.5] = 0+ index[index == E - 2] = E - 1+ index[0, -1] = E+ weights = torch.rand(T, k, generator = g)+ return index, weights+++def _wrapped(module, ranks, alphas, rslora, ids):+ experts = ml.MultiLoraExperts(+ module, ["gate_up_proj", "down_proj"], len(ranks), ranks, alphas, rslora+ )+ torch.manual_seed(0)+ with torch.no_grad():+ for adapter in experts.adapters.values():+ adapter.reset_parameters()+ for B in adapter.lora_B:+ B.normal_(0, 0.5, generator = torch.Generator().manual_seed(B.numel()))+ ml.set_multi_lora_batch(experts, ids)+ return experts+++def _oracle(experts, hidden, index, weights, ids, rslora):+ """float64, route by route: base plus the token's own adapter delta (scale alpha / r or alpha / sqrt(r))."""+ base = experts.base_layer+ E = base.gate_up_proj.shape[0]+ factors = {+ name: [(A.double(), B.double()) for A, B in zip(adapter.lora_A, adapter.lora_B)]+ for name, adapter in experts.adapters.items()+ }+ out = []+ for t in range(hidden.shape[0]):+ u = int(ids[t])+ acc = 0+ for j in range(index.shape[1]):+ e = int(index[t, j])+ if e >= E:+ continue++ def proj(name, x):+ W = getattr(base, name)[e].double()+ if u >= 0:+ A, B = factors[name][u]+ r = A.shape[1]+ scale = ALPHAS[u] / (r**0.5 if rslora else r)+ W = W + scale * B[e] @ A[e]+ return x @ W.t()++ gate, up = proj("gate_up_proj", hidden[t]).chunk(2, dim = -1)+ acc = acc + proj("down_proj", torch.nn.functional.silu(gate) * up) * weights[t, j]+ out.append(acc)+ return torch.stack(out), factors++[email protected]("rslora", [False, True])[email protected]("present", ["all", "no_adapter_1"])+def test_grouped_matches_a_float64_oracle(monkeypatch, rslora, present):+ _need_grouped()+ monkeypatch.delenv("UNSLOTH_MULTI_LORA_MOE_GROUPED", raising = False)+ T, k, E = 24, 2, 5+ ids = torch.tensor([0, 0, 2, -1, 0, 1, 2, 0] * 3)+ if present == "no_adapter_1":+ ids[ids == 1] = 2+ experts = _wrapped(_Experts(E), RANKS, ALPHAS, rslora, ids)+ assert not experts.transposed and not experts.reversed_io+ index, weights = _routes(T, k, E)+ hidden = torch.randn(T, 20, generator = torch.Generator().manual_seed(2))+ hidden64, weights64 = hidden.double().requires_grad_(), weights.double().requires_grad_()+ hidden, weights = hidden.requires_grad_(), weights.requires_grad_()+ calls = ml.MULTI_LORA_MOE_STATS["grouped_calls"]+ out = experts(hidden, index, weights)+ assert experts.mode == "grouped" and ml.MULTI_LORA_MOE_STATS["grouped_calls"] == calls + 1+ ref, factors = _oracle(experts, hidden64, index, weights64, ids, rslora)+ torch.testing.assert_close(out.double(), ref, atol = 1e-5, rtol = 1e-4)+ grad = torch.randn(out.shape, generator = torch.Generator().manual_seed(3))+ (out * grad).sum().backward()+ leaves = [hidden64, weights64] + [+ t for pairs in factors.values() for pair in pairs for t in pair+ ]+ ref_grads = torch.autograd.grad((ref * grad.double()).sum(), leaves, allow_unused = True)+ mine = [hidden.grad, weights.grad] + [+ p.grad+ for adapter in experts.adapters.values()+ for pair in zip(adapter.lora_A, adapter.lora_B)+ for p in pair+ ]+ for got, want in zip(mine, ref_grads):+ want = torch.zeros_like(got, dtype = torch.float64) if want is None else want+ assert got is not None+ torch.testing.assert_close(got.double(), want, atol = 1e-5, rtol = 1e-4)+ if present == "no_adapter_1":+ for adapter in experts.adapters.values():+ assert (+ adapter.lora_A[1].grad.abs().max() == 0 and adapter.lora_B[1].grad.abs().max() == 0+ )++[email protected]("U", [2, 3])+def test_grouped_is_one_pass_on_zoo_gemms(monkeypatch, U):+ """Every adapter in one pass: 2 base GEMMs through Zoo's base path and 4 LoRA grouped GEMMs, whatever U is."""+ _need_grouped()+ from unsloth_zoo.temporary_patches import moe_utils++ monkeypatch.delenv("UNSLOTH_MULTI_LORA_MOE_GROUPED", raising = False)+ T, k, E = 24, 2, 5+ ids = torch.tensor([u % U for u in range(T)])+ experts = _wrapped(_Experts(E), RANKS[:U], ALPHAS[:U], False, ids)+ index, weights = _routes(T, k, E)+ hidden = torch.randn(T, 20, generator = torch.Generator().manual_seed(2))+ experts(hidden, index, weights)+ assert experts.mode == "grouped"+ calls = {"base": 0, "lora": 0}+ base_mm, lora_mm = moe_utils._base_grouped_mm, moe_utils._grouped_mm_with_backward_fix++ def count(kind, fn):+ def wrapper(*args, **kwargs):+ # LoRA GEMMs are the ones on the stacked (E * (U + 1), ...) factors; Zoo's base path may call the same op.+ if kind == "base" or args[1].shape[0] == E * (U + 1):+ calls[kind] += 1+ return fn(*args, **kwargs)++ return wrapper++ monkeypatch.setattr(moe_utils, "_base_grouped_mm", count("base", base_mm))+ monkeypatch.setattr(moe_utils, "_grouped_mm_with_backward_fix", count("lora", lora_mm))+ experts(hidden, index, weights)+ assert calls == {"base": 2, "lora": 4}+++def test_one_present_adapter_runs_its_own_pass(monkeypatch):+ """A batch with one adapter's tokens has nothing to fuse: that call runs per adapter, with the same result."""+ _need_grouped()+ monkeypatch.delenv("UNSLOTH_MULTI_LORA_MOE_GROUPED", raising = False)+ T, k, E = 24, 2, 5+ index, weights = _routes(T, k, E)+ hidden = torch.randn(T, 20, generator = torch.Generator().manual_seed(2))+ experts = _wrapped(_Experts(E), RANKS, ALPHAS, False, torch.tensor([0, 1, 2, -1] * 6))+ experts(hidden, index, weights)+ assert experts.mode == "grouped"+ ml.set_multi_lora_batch(experts, torch.tensor([1, -1] * 12))+ before = dict(ml.MULTI_LORA_MOE_STATS)+ out = experts(hidden, index, weights)+ assert ml.MULTI_LORA_MOE_STATS["per_adapter_calls"] == before["per_adapter_calls"] + 1+ assert ml.MULTI_LORA_MOE_STATS["grouped_calls"] == before["grouped_calls"]+ torch.testing.assert_close(+ out, experts._grouped(hidden, index, weights, torch.tensor([1, -1] * 12))+ )+++def test_grouped_engages_and_the_switch_turns_it_off(base, monkeypatch):+ _need_grouped()+ ids = _ids()+ counts = []+ for value in ("1", "0"):+ monkeypatch.setenv("UNSLOTH_MULTI_LORA_MOE_GROUPED", value)+ model, wrapped = _batched(base)+ ml.set_multi_lora_batch(model, torch.tensor([0, 1, 2], device = DEVICE), seq_len = T)+ before = dict(ml.MULTI_LORA_MOE_STATS)+ model(input_ids = ids, use_cache = False)+ counts.append({key: ml.MULTI_LORA_MOE_STATS[key] - before[key] for key in before})+ layers = len(wrapped) // 2+ assert counts[0] == {"grouped_calls": layers, "per_adapter_calls": 0, "grouped_fallbacks": 0}+ assert counts[1] == {"grouped_calls": 0, "per_adapter_calls": layers, "grouped_fallbacks": 0}++[email protected]("why", ["bias", "parity"])+def test_unsupported_experts_fall_back_to_one_adapter_at_a_time(monkeypatch, why):+ monkeypatch.delenv("UNSLOTH_MULTI_LORA_MOE_GROUPED", raising = False)+ if why == "parity":+ grouped = ml.MultiLoraExperts._grouped+ monkeypatch.setattr(ml.MultiLoraExperts, "_grouped", lambda *a: grouped(*a) * 1.01)+ T, k, E = 24, 2, 5+ ids = torch.tensor([0, 1, 2, -1] * 6)+ index, weights = _routes(T, k, E)+ hidden = torch.randn(T, 20, generator = torch.Generator().manual_seed(2))+ experts = _wrapped(_Experts(E, bias = why == "bias"), RANKS, ALPHAS, False, ids)+ before = dict(ml.MULTI_LORA_MOE_STATS)+ out = experts(hidden, index, weights)+ assert experts.mode == "fold"+ assert ml.MULTI_LORA_MOE_STATS["grouped_fallbacks"] == before["grouped_fallbacks"] + 1+ assert ml.MULTI_LORA_MOE_STATS["grouped_calls"] == before["grouped_calls"]+ monkeypatch.setenv("UNSLOTH_MULTI_LORA_MOE_GROUPED", "0")+ experts.mode = experts.peft_mode = None+ torch.testing.assert_close(out, experts(hidden, index, weights), atol = 0, rtol = 0)+++def test_grouped_reads_bnb_4bit_stacks_through_zoo(monkeypatch):+ _need_grouped()+ bnb = pytest.importorskip("bitsandbytes")+ from unsloth_zoo.temporary_patches import moe_utils++ monkeypatch.delenv("UNSLOTH_MULTI_LORA_MOE_GROUPED", raising = False)+ # The synthetic forward cannot run a 4-bit stack, so there is no per-adapter path to probe.+ monkeypatch.setattr(ml.MultiLoraExperts, "_run_stash", lambda *a: None)+ monkeypatch.setattr(ml.MultiLoraExperts, "_grouped_parity", lambda *a: None)+ T, k, E = 24, 2, 5+ module = _Experts(E, H = 32, I = 16)+ for name in ("gate_up_proj", "down_proj"):+ W = getattr(module, name).data.to(torch.bfloat16)+ packed, state = bnb.functional.quantize_4bit(W.reshape(-1, W.shape[-1]), quant_type = "nf4")+ param = torch.Tensor._make_subclass(bnb.nn.Params4bit, packed, False)+ param.quant_state, param._original_shape = state, W.shape+ module._parameters[name] = param+ ids = torch.tensor([0, 1, 2, -1] * 6)+ experts = _wrapped(module, RANKS, ALPHAS, False, ids)+ index, weights = _routes(T, k, E)+ hidden = torch.randn(T, 32, generator = torch.Generator().manual_seed(2), requires_grad = True)+ seen = []+ get_base = moe_utils._get_base_weight+ monkeypatch.setattr(+ moe_utils, "_get_base_weight", lambda *a, **kw: seen.append(1) or get_base(*a, **kw)+ )+ out = experts(hidden, index, weights)+ assert experts.mode == "grouped" and out.dtype == torch.float32+ # Zoo's provider dequantizes both stacks in forward and, where Zoo recomputes, again in backward.+ forward_reads = len(seen)+ assert forward_reads >= 2+ out.sum().backward()+ if moe_utils._moe_recompute_enabled(experts.base_layer.gate_up_proj, dtype = torch.bfloat16):+ assert len(seen) >= 2 * forward_reads+ float_module = _Experts(E, H = 32, I = 16)+ with torch.no_grad():+ for name in ("gate_up_proj", "down_proj"):+ # The oracle reads the stack as Zoo's provider does (CPU bitsandbytes slices differ from full reads).+ param = getattr(experts.base_layer, name)+ getattr(float_module, name).copy_(get_base(param, torch.bfloat16).float())+ reference = _wrapped(float_module, RANKS, ALPHAS, False, ids)+ ref, _ = _oracle(reference, hidden.detach().double(), index, weights.double(), ids, False)+ # bfloat16 compute, as Unsloth Zoo runs 4-bit experts.+ torch.testing.assert_close(out.double(), ref, atol = 2e-2 * ref.abs().max().item(), rtol = 0)+++def test_grouped_dropped_route_never_poisons_gradients(monkeypatch):+ _need_grouped()+ # A dropped route (expert id E) with a NaN weight must add exactly nothing, forward and backward.+ monkeypatch.delenv("UNSLOTH_MULTI_LORA_MOE_GROUPED", raising = False)+ T, k, E = 8, 2, 5+ ids = torch.tensor([0, 1, 2, -1, 0, 1, 2, 0])+ experts = _wrapped(_Experts(E), RANKS, ALPHAS, False, ids)+ index, weights = _routes(T, k, E)+ hidden = torch.randn(T, 20, generator = torch.Generator().manual_seed(2))+ experts(hidden, index, weights)+ assert experts.mode == "grouped"+ weights[0, -1] = float("nan")+ kept, clean = index.clone(), weights.clone()+ kept[0, -1], clean[0, -1] = 0, 0.0++ def run(index, weights):+ h, w = hidden.clone().requires_grad_(), weights.clone().requires_grad_()+ experts.zero_grad(set_to_none = True)+ out = experts(h, index, w)+ out.backward(torch.ones_like(out))+ grads = [p.grad.clone() for p in experts.parameters() if p.grad is not None]+ return out, h.grad, w.grad, grads++ out, d_hidden, d_weights, grads = run(index, weights)+ ref, ref_hidden, ref_weights, ref_grads = run(kept, clean)+ torch.testing.assert_close(out, ref)+ torch.testing.assert_close(d_hidden, ref_hidden)+ assert d_weights[0, -1] == 0+ torch.testing.assert_close(d_weights[:, :1], ref_weights[:, :1])+ for g, r in zip(grads, ref_grads):+ torch.testing.assert_close(g, r)+++def test_grouped_empty_top_k_gives_zeros(monkeypatch):+ _need_grouped()+ monkeypatch.delenv("UNSLOTH_MULTI_LORA_MOE_GROUPED", raising = False)+ ids = torch.tensor([0, 1, 2, -1])+ experts = _wrapped(_Experts(5), RANKS, ALPHAS, False, ids)+ index, weights = _routes(4, 2, 5)+ experts(torch.randn(4, 20), index, weights)+ assert experts.mode == "grouped"+ hidden = torch.randn(4, 20, requires_grad = True)+ out = experts(hidden, index[:, :0], weights[:, :0])+ assert out.shape == (4, 20) and not out.any()+ out.sum().backward()+ assert not hidden.grad.any()+ for adapter in experts.adapters.values():+ for p in (*adapter.lora_A, *adapter.lora_B):+ assert p.grad is not None and not p.grad.any()+++class _StaticExperts(_Experts):+ """The same experts with a data-independent forward (every expert, masked), so it traces with fullgraph."""++ def forward(self, hidden_states, top_k_index, top_k_weights):+ out = torch.zeros_like(hidden_states)+ for e in range(self.gate_up_proj.shape[0]):+ w = (top_k_weights * (top_k_index == e)).sum(-1, keepdim = True)+ gate, up = (hidden_states @ self.gate_up_proj[e].t()).chunk(2, dim = -1)+ out = out + ((self.act_fn(gate) * up) @ self.down_proj[e].t()) * w+ return out++[email protected]("dtype", [torch.float32, torch.bfloat16])+def test_grouped_under_torch_compile(monkeypatch, dtype):+ # Grouped GEMM only traces in bfloat16; other dtypes take the per-adapter path inside a compiled graph.+ _need_grouped(dtype)+ monkeypatch.delenv("UNSLOTH_MULTI_LORA_MOE_GROUPED", raising = False)+ T, k, E = 16, 2, 5+ ids = torch.tensor([0, 1, 2, -1] * 4)+ experts = _wrapped(_StaticExperts(E, H = 24, I = 16).to(dtype), RANKS, ALPHAS, False, ids).to(+ DEVICE+ )+ for _, layer in ml._layers(experts):+ layer.seen = None # a plain attribute, so .to() left it on the CPU+ ml.set_multi_lora_batch(experts, ids.to(DEVICE))+ index, weights = _routes(T, k, E)+ index[0, -1] = 0+ index, weights = index.to(DEVICE), weights.to(DEVICE)+ hidden = (+ torch.randn(T, 24, generator = torch.Generator().manual_seed(2))+ .to(DEVICE, dtype)+ .requires_grad_()+ )+ weights = weights.to(dtype)+ eager = experts(hidden, index, weights)+ assert experts.mode == "grouped"+ torch._dynamo.reset()+ # fullgraph on CUDA, where Zoo's grouped GEMM traces as a custom op; on CPU it is a compile-disabled loop.+ compiled = torch.compile(experts, backend = "eager", fullgraph = DEVICE == "cuda")(+ hidden, index, weights+ )+ torch.testing.assert_close(compiled.float(), eager.float(), atol = 2e-2, rtol = 2e-2)+ compiled.sum().backward()+ assert torch.isfinite(hidden.grad).all()diff --git a/unsloth/kernels/multi_lora.py b/unsloth/kernels/multi_lora.pyindex 7543c379cee..9e6a442df5e 100644--- a/unsloth/kernels/multi_lora.py+++ b/unsloth/kernels/multi_lora.py@@ -10,6 +10,7 @@ from __future__ import annotationsimport contextlibimport functoolsimport inspect+import loggingimport mathimport osimport re@@ -52,6 +53,7 @@ __all__ = ["drop_absent_adapter_grads_","adapter_state_dict","set_multi_lora_triton",+ "MULTI_LORA_MOE_STATS",]@@ -740,6 +742,20 @@ class MultiLoraExpertParam(nn.Module):self.num_experts,)+ def grouped_stacks(self, reversed_io: bool, dtype: torch.dtype) -> tuple:+ """Every adapter's factors as (E*(U+1), in, R) and (E*(U+1), R, out), group e*(U+1) + u, R the largest rank+ rounded up to 8 (zero padding is exact), scale folded into the second; group e*(U+1) + U is zero."""+ R = -(-max(self.ranks) // 8) * 8+ firsts, seconds = [], []+ for u in range(self.num_adapters):+ first, second, scale, _ = self.grouped_mm_factors(u, reversed_io)+ r = first.shape[-1]+ firsts.append(F.pad(first.to(dtype), (0, R - r)))+ seconds.append(F.pad(second.to(dtype) * scale, (0, 0, 0, R - r)))+ firsts.append(torch.zeros_like(firsts[0]))+ seconds.append(torch.zeros_like(seconds[0]))+ return torch.stack(firsts, 1).flatten(0, 1), torch.stack(seconds, 1).flatten(0, 1)+def folded(self, W: torch.Tensor, adapter: int, transposed: bool) -> torch.Tensor:"""W + delta in W's dtype, as PEFT's ParamWrapper folds it (one baddbmm, autocast off)."""A, B = self.lora_A[adapter].to(W.dtype), self.lora_B[adapter].to(W.dtype)@@ -819,9 +835,92 @@ def _adapter_groups(batch: MultiLoraBatch) -> list:return groups+def _token_adapters(batch: MultiLoraBatch) -> torch.Tensor:+ """Each token's adapter id (-1 = none)."""+ ids = batch.token_ids+ if ids is None:+ ids = batch.__dict__.get("_token_adapters")+ if ids is None:+ n, U = batch.num_tokens, batch.num_adapters+ seg = batch.seg_start.long()+ pos = torch.arange(n, device = seg.device)+ ids = torch.full((n,), -1, dtype = torch.int64, device = seg.device)+ ids = ids.scatter(0, batch.sorted_idx.long(), torch.searchsorted(seg[1:], pos, right = True))+ ids = torch.where(ids >= U, torch.full_like(ids, -1), ids)+ batch.__dict__["_token_adapters"] = ids+ return ids+++MULTI_LORA_MOE_STATS = {"grouped_calls": 0, "per_adapter_calls": 0, "grouped_fallbacks": 0}+_HALF = (torch.float16, torch.bfloat16)+_FLOAT = (torch.float16, torch.bfloat16, torch.float32, torch.float64)+_ZOO_GROUPED = (+ "_base_grouped_mm",+ "_get_base_weight",+ "preprocess_weight",+ "_moe_recompute_enabled",+ "_selective_dequant_plan",+ "_grouped_mm_with_backward_fix",+ "_grouped_mm_supported_on",+ "weighted_unpermute",+ "combine_permuted_moe_outputs",+)+++def _zoo_moe_utils():+ """Unsloth Zoo's MoE module when it has the grouped GEMMs its single-job forward runs on, else None."""+ try:+ from unsloth_zoo.temporary_patches import moe_utils+ except Exception: # noqa: BLE001+ return None+ return moe_utils if all(hasattr(moe_utils, n) for n in _ZOO_GROUPED) else None+++def _grouped_supported(device: torch.device, dtype: torch.dtype) -> bool:+ """Zoo's grouped GEMM runs as a kernel here: its Triton grouped GEMM (CUDA, half) or torch's grouped GEMM.+ Elsewhere it loops over groups, which at E * (U + 1) groups is slower than one adapter at a time."""+ moe_utils = _zoo_moe_utils()+ if moe_utils is None:+ return False+ if device.type != "cuda":+ return True+ try:+ if dtype in _HALF:+ from unsloth_zoo.temporary_patches import moe_grouped_fp16+ if moe_grouped_fp16.triton_grouped_available(device):+ return True+ return bool(moe_utils._grouped_mm_supported_on(device))+ except Exception: # noqa: BLE001+ return False+++def _is_quantized(param) -> bool:+ return getattr(param, "quant_state", None) is not None or param.dtype not in _FLOAT+++def _grouped_disabled() -> bool:+ return os.environ.get("UNSLOTH_MULTI_LORA_MOE_GROUPED", "1") == "0"+++_WARNED: set = set()+++def _warn_once(message: str, level: int = logging.WARNING) -> None:+ if message in _WARNED:+ return+ _WARNED.add(message)+ try:+ from unsloth_zoo.log import logger+ except Exception: # noqa: BLE001+ logger = logging.getLogger(__name__)+ logger.log(level, message)++class MultiLoraExperts(nn.Module):- """Frozen fused experts; each adapter's tokens run the experts' own forward with only its factors, as a lone- PEFT job does: through Zoo's stash when its MoE forward reads one, else with the delta folded in."""+ """Frozen fused experts. Mode "grouped": one pass for every adapter, rows sorted by (expert, adapter), the frozen+ stacks in E-group GEMMs and each adapter's LoRA in E*(U+1)-group GEMMs over the same rows. Otherwise each+ adapter's tokens run the experts' own forward with only its factors, as a lone PEFT job does: through Zoo's+ stash when its MoE forward reads one ("stash"), else with the delta folded in ("fold")."""def __init__(self,@@ -854,7 +953,10 @@ class MultiLoraExperts(nn.Module):self.adapters[name] = MultiLoraExpertParam(E, in_features, out_features, num_adapters, r, lora_alpha, use_rslora, device = device)- self.mode: Optional[str] = None # "stash" or "fold", decided on the first adapter call+ # How a lone PEFT run of these experts applies LoRA ("stash" or "fold"), and how this module runs+ # ("grouped", or the same as peft_mode); both decided on the first call with an adapter.+ self.peft_mode: Optional[str] = None+ self.mode: Optional[str] = Nonedef __getattr__(self, name: str):try:@@ -903,34 +1005,289 @@ class MultiLoraExperts(nn.Module):def _run(self, adapter: Optional[int], args) -> torch.Tensor:if adapter is None:return self.base_layer(*args)- if self.mode is None:- # Probe on one token without autograd, so a checkpointed forward and its recompute run the same ops.- with torch.no_grad():- probe = self._run_stash(adapter, tuple(a[:1] for a in args))- self.mode = "fold" if probe is None else "stash"- if self.mode == "fold":+ if self.peft_mode == "fold":return self._run_fold(adapter, args)out = self._run_stash(adapter, args)if out is None:raise RuntimeError("The experts' forward stopped reading the LoRA stash mid-run.")return out+ def _per_adapter(self, hidden_states: torch.Tensor, args, groups) -> torch.Tensor:+ if len(groups) == 1:+ return self._run(groups[0][0], (hidden_states, *args))+ out = None+ for u, rows in groups:+ y = self._run(u, (hidden_states[rows], *(a[rows] for a in args)))+ if out is None:+ out = y.new_zeros((hidden_states.shape[0], *y.shape[1:]))+ out = out.index_copy(0, rows, y)+ return out++ def _choose_mode(self, hidden_states: torch.Tensor, args, groups) -> None:+ adapter, rows = next(((u, rows) for u, rows in groups if u is not None), (None, None))+ if adapter is None:+ return+ # Probes run without autograd, so a checkpointed forward and its recompute run the same ops.+ with torch.no_grad():+ probe = self._run_stash(+ adapter, (hidden_states[rows[:1]], *(a[rows[:1]] for a in args))+ )+ self.peft_mode = "fold" if probe is None else "stash"+ self.mode = self.peft_mode+ if _grouped_disabled():+ return+ try:+ reason = self._grouped_unsupported(hidden_states, args) or self._grouped_parity(+ hidden_states, args, groups+ )+ except Exception as error: # noqa: BLE001 - anything unexpected keeps the per-adapter path+ reason = f"{type(error).__name__}: {error}"+ if reason is None:+ self.mode = "grouped"+ _warn_once(+ f"Unsloth: batched MoE LoRA on {type(self.base_layer).__name__} runs every adapter in one grouped pass.",+ logging.INFO,+ )+ return+ if not torch.compiler.is_compiling():+ MULTI_LORA_MOE_STATS["grouped_fallbacks"] += 1+ _warn_once(+ f"Unsloth: batched MoE LoRA on {type(self.base_layer).__name__} runs one adapter at a time ({reason})."+ )++ def _in_out(self, name: str) -> tuple[int, int, int]:+ """(E, in, out) of a fused stack, from its stored (E, d1, d2) and layout."""+ base = self.base_layer+ E, d1, d2 = _expert_stack_shape(getattr(base, name))+ input_major = getattr(base, "is_transposed", False) or getattr(+ base, "_unsloth_grouped_mm_format", False+ )+ return (E, d1, d2) if input_major else (E, d2, d1)++ def _compute_dtype(self, hidden_states: torch.Tensor) -> torch.dtype:+ """As Unsloth Zoo's MoE forward: a half activation runs as it is, half stacks set the dtype, quantized ones+ take Zoo's ``moe_compute_dtype``; float32 stacks stay float32 even under autocast (grouped GEMM is not+ autocast-cast)."""+ if hidden_states.dtype in _HALF:+ return hidden_states.dtype+ param = self.base_layer.gate_up_proj+ if param.dtype in _HALF:+ return param.dtype+ if _is_quantized(param):+ from unsloth_zoo.temporary_patches.moe_utils import moe_compute_dtype+ return moe_compute_dtype(hidden_states)+ return hidden_states.dtype++ def _grouped_unsupported(self, hidden_states: torch.Tensor, args) -> Optional[str]:+ """Why the grouped path cannot run these experts, or None."""+ base = self.base_layer+ if not set(self.parameter_names) <= {"gate_up_proj", "down_proj"}:+ return f"LoRA on {self.parameter_names}"+ if "GptOss" in type(base).__name__ or getattr(base, "is_concatenated", True) is False:+ return "interleaved gate_up_proj"+ if base.__dict__.get("_unsloth_own_apply_gate") or getattr(+ type(base), "_unsloth_own_apply_gate", False+ ):+ return "a custom gate"+ if not callable(getattr(base, "act_fn", None)):+ return "no act_fn"+ names = [n for n, _ in base.named_parameters()] + [n for n, _ in base.named_buffers()]+ if any("bias" in n for n in names):+ return "expert biases"+ params = [getattr(base, n, None) for n in ("gate_up_proj", "down_proj")]+ if not all(isinstance(p, torch.Tensor) for p in params):+ return "no fused gate_up_proj / down_proj"+ if len(args) != 2 or hidden_states.dim() != 2:+ return "an experts forward not called as (hidden_states, top_k_index, top_k_weights)"+ top_k_index, top_k_weights = args+ if (+ top_k_index.dtype.is_floating_point+ or not top_k_weights.dtype.is_floating_point+ or top_k_index.shape != top_k_weights.shape+ or top_k_index.dim() != 2+ ):+ return "an experts forward not called as (hidden_states, top_k_index, top_k_weights)"+ moe_utils = _zoo_moe_utils()+ if moe_utils is None:+ return "an Unsloth Zoo without its grouped MoE GEMMs"+ layout = getattr(moe_utils, "_mxfp4_expert_layout", None)+ model_type = getattr(base, "_unsloth_model_type", None)+ H = hidden_states.shape[-1]+ if layout is not None and any(+ layout(getattr(base, n), proj, H, model_type, base) is not None+ for n, proj in (("gate_up_proj", "gate_up"), ("down_proj", "down"))+ ):+ return "MXFP4 experts"+ dtype = self._compute_dtype(hidden_states)+ (E, gate_in, gate_out), (E_down, down_in, down_out) = (+ self._in_out("gate_up_proj"),+ self._in_out("down_proj"),+ )+ if E != E_down or gate_in != H or down_out != H or gate_out != 2 * down_in:+ return "unexpected expert shapes"+ # Grouped GEMM needs 16-byte rows (the LoRA rank is padded to 8).+ step = 16 // torch.empty((), dtype = dtype).element_size()+ if any(d % step for d in (H, down_in)):+ return "expert dims not a multiple of 16 bytes"+ if not _grouped_supported(hidden_states.device, dtype):+ return f"no grouped GEMM kernel in {dtype} on {hidden_states.device.type}"+ return None++ def _grouped_parity(self, hidden_states: torch.Tensor, args, groups) -> Optional[str]:+ """A few rows per adapter through both paths; None when they agree."""+ ids = _token_adapters(next(iter(self.adapters.values())).batch)+ picks = [rows[:4] for _, rows in groups]+ sel = torch.cat(picks)+ with torch.no_grad():+ got = self._grouped(hidden_states[sel], *(a[sel] for a in args), ids[sel]).float()+ ref = torch.cat(+ [+ self._run(u, (hidden_states[r], *(a[r] for a in args))).float()+ for (u, _), r in zip(groups, picks)+ ]+ )+ err = (got - ref).abs().max().item()+ # Under autocast the per-adapter path may itself run in half precision (Zoo's CPU fallback does).+ half = self._compute_dtype(hidden_states) in _HALF or torch.is_autocast_enabled(+ hidden_states.device.type+ )+ tol = 5e-2 if half else 1e-3+ if err <= tol * max(ref.abs().max().item(), 1e-6):+ return None+ return f"the grouped probe differs by {err:.3g}"++ def _project(+ self, moe_utils, name: str, x, offs_expert, offs_pair, counts, plan, dtype+ ) -> torch.Tensor:+ """Frozen base over E expert groups through Zoo's base GEMM (its weight provider, recompute and selective+ dequant, as the single-job forward), plus every adapter's LoRA over the E * (U + 1) pair groups."""+ base = self.base_layer+ src, proj = getattr(base, name), name.removesuffix("_proj")+ hidden, model_type = (+ self._in_out("gate_up_proj")[1],+ getattr(base, "_unsloth_model_type", None),+ )++ def provider(experts = None):+ weight = moe_utils._get_base_weight(src, dtype, counts, experts)+ return moe_utils.preprocess_weight(+ weight, proj, hidden, model_type, experts_module = base+ )++ recompute = moe_utils._moe_recompute_enabled(src, dtype = dtype)+ y = moe_utils._base_grouped_mm(+ x, offs_expert, provider, recompute, plan if recompute else None+ )+ if name in self.adapters:+ first, second = self.adapters[name].grouped_stacks(self.reversed_io, dtype)+ h = moe_utils._grouped_mm_with_backward_fix(x, first, offs_pair).contiguous()+ y = y + moe_utils._grouped_mm_with_backward_fix(h, second, offs_pair)+ return y++ def _grouped(+ self,+ hidden_states: torch.Tensor,+ top_k_index: torch.Tensor,+ top_k_weights: torch.Tensor,+ adapter_ids: torch.Tensor,+ ) -> torch.Tensor:+ T, k = top_k_index.shape+ if T * k == 0:+ # Nothing routed: zeros still tied to the hidden states and every adapter (empty slices, so exact 0 even+ # for an inf / NaN adapter), as backward and default DDP need.+ zero = hidden_states.reshape(-1)[:0].sum() + sum(+ p.reshape(-1)[:0].sum()+ for adapter in self.adapters.values()+ for p in (*adapter.lora_A, *adapter.lora_B)+ )+ return hidden_states.new_zeros(hidden_states.shape) + zero.to(hidden_states.dtype)+ E = _expert_stack_shape(self.base_layer.gate_up_proj)[0]+ U = self.num_adapters+ Ug = U + 1+ dtype = self._compute_dtype(hidden_states)+ # Route = expert * Ug + adapter, slot U for rows of no adapter (zero factors) and for dropped routes+ # (expert id outside [0, E): they read a zero row and get weight 0, so they add exactly 0 and nothing can+ # overflow or turn a gradient NaN); sorted, each expert's rows stay contiguous.+ index = top_k_index.long()+ valid = (index >= 0) & (index < E)+ slot = torch.where(adapter_ids < 0, U, adapter_ids).to(index.device).unsqueeze(1)+ key = (torch.where(valid, index, 0) * Ug + torch.where(valid, slot, U)).reshape(-1)+ order = torch.argsort(key, stable = True)+ # Fixed-size counts: bincount's output size depends on the data, so it syncs on CUDA and breaks a compiled graph.+ counts = torch.zeros(E * Ug, dtype = torch.int32, device = key.device)+ counts.scatter_add_(0, key, torch.ones_like(key, dtype = torch.int32))+ offs_pair = torch.cumsum(counts, 0, dtype = torch.int32)+ offs_expert = offs_pair.view(E, Ug)[:, -1].contiguous()+ counts = counts.view(E, Ug).sum(1, dtype = torch.int32)+ moe_utils = _zoo_moe_utils()+ base = self.base_layer+ plan = None+ if moe_utils._moe_recompute_enabled(base.gate_up_proj, dtype = dtype):+ plan = moe_utils._selective_dequant_plan(+ (base.gate_up_proj, base.down_proj), counts, T * k+ )+ with torch.autocast(device_type = hidden_states.device.type, enabled = False):+ rows = torch.where(valid.reshape(-1)[order], order // k, T)+ x = torch.cat(+ [+ hidden_states.to(dtype),+ hidden_states.new_zeros(1, hidden_states.shape[1], dtype = dtype),+ ]+ )+ x = x[rows]+ h = self._project(+ moe_utils, "gate_up_proj", x, offs_expert, offs_pair, counts, plan, dtype+ )+ gate, up = h.chunk(2, dim = -1)+ inter = base.act_fn(gate) * up+ # From here on as Unsloth Zoo's MoE forward combines routes: its gate-gradient identity when enabled,+ # then its fused weighted unpermute (deterministic fallback without Triton).+ weights = torch.where(valid, top_k_weights, 0).reshape(-1)[order]+ gategrad = getattr(moe_utils, "_moe_gategrad_enabled", lambda: False)()+ gategrad = gategrad and hasattr(moe_utils, "_MoEGateGradIdentity")+ gategrad = gategrad and top_k_weights.requires_grad and inter.dtype != torch.float16+ if gategrad:+ raw = weights.float() if weights.dtype == torch.float16 else weights+ floored = torch.where(raw >= 0, raw.clamp(min = 1e-12), raw.clamp(max = -1e-12))+ weights = raw + (floored - raw).detach()+ inter = moe_utils._MoEGateGradIdentity.apply(inter, weights)+ weights = weights.detach()+ y = self._project(+ moe_utils, "down_proj", inter, offs_expert, offs_pair, counts, plan, dtype+ )+ out = moe_utils.weighted_unpermute(+ y, order, weights, T, k, out_dtype = hidden_states.dtype+ )+ if out is None:+ out = moe_utils.combine_permuted_moe_outputs(+ y * weights.unsqueeze(-1), order, T, k, out_dtype = hidden_states.dtype+ )+ return out+def forward(self, hidden_states: torch.Tensor, *args):batch = next(iter(self.adapters.values())).batchif batch is None or hidden_states.shape[0] != batch.num_tokens:raise ValueError("Call set_multi_lora_batch(model, ...) with this batch's adapter ids first.")+ if self.mode is None:+ self._choose_mode(hidden_states, args, _adapter_groups(batch))+ grouped = self.mode == "grouped"+ if grouped and torch.compiler.is_compiling():+ # Grouped GEMM only traces in bfloat16 (its meta checks the dtype on every device).+ grouped = self._compute_dtype(hidden_states) == torch.bfloat16+ elif grouped:+ # With one adapter's tokens there is nothing to fuse: its own pass is the same work in fewer groups.+ grouped = sum(u is not None for u, _ in _adapter_groups(batch)) >= 2+ if grouped:+ if not torch.compiler.is_compiling():+ MULTI_LORA_MOE_STATS["grouped_calls"] += 1+ # Every adapter's factors are in the graph, so absent adapters get exact zero grads.+ return self._grouped(hidden_states, *args, _token_adapters(batch))+ if not torch.compiler.is_compiling():+ MULTI_LORA_MOE_STATS["per_adapter_calls"] += 1groups = _adapter_groups(batch)- if len(groups) == 1:- out = self._run(groups[0][0], (hidden_states, *args))- else:- out = None- for u, rows in groups:- y = self._run(u, (hidden_states[rows], *(a[rows] for a in args)))- if out is None:- out = y.new_zeros((hidden_states.shape[0], *y.shape[1:]))- out = out.index_copy(0, rows, y)+ out = self._per_adapter(hidden_states, args, groups)present = {u for u, _ in groups}absent = [u for u in range(self.num_adapters) if u not in present]if absent and torch.is_grad_enabled():@@ -1222,8 +1579,9 @@ def adapter_state_dict(depth = len(experts.parameter_names)for i, param in enumerate(experts.parameter_names):key = f"{prefix}{name}" + ".base_layer" * (depth - 1 - i)- # Folded experts load through PEFT's own ParamWrapper delta, which is rank-major whatever Zoo uses.- layout = "rank_major" if experts.mode == "fold" else b_layout+ # Experts whose forward ignores the stash load through PEFT's own ParamWrapper delta, which is+ # rank-major whatever Zoo uses; how they trained here (grouped or not) does not matter.+ layout = "rank_major" if experts.peft_mode == "fold" else b_layoutA, B = experts.adapters[param].peft_tensors(adapter, layout)out[f"{key}.lora_A.weight"] = Aout[f"{key}.lora_B.weight"] = B