Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Batched MoE LoRA: every job's expert LoRA in one grouped pass on Zoo's kernels (#16012)

Daniel Han69117f1bf36c
diff --git a/tests/test_multi_lora_experts.py b/tests/test_multi_lora_experts.py
index 82493859a4c..567cde700ba 100644
--- a/tests/test_multi_lora_experts.py
+++ b/tests/test_multi_lora_experts.py
@@ -73,17 +73,40 @@ def _ids():
return torch.randint(0, 100, (3, T), generator = torch.Generator().manual_seed(1)).to(DEVICE)
[email protected]("mode", ["stash", "fold"])
-def test_each_adapter_matches_its_lone_peft_run(base, monkeypatch, mode):
+MODES = ["grouped", "stash", "fold"]
+
+
+def _need_grouped(dtype = torch.float32):
+ """Skip where this torch / device has no native grouped GEMM in ``dtype`` (the module then runs per adapter)."""
+ if not ml._grouped_supported(torch.device(DEVICE), dtype):
+ pytest.skip(f"no native grouped GEMM in {dtype} on {DEVICE} with torch {torch.__version__}")
+
+
+def _use_mode(monkeypatch, mode):
+ """grouped: every adapter in one pass; stash / fold: one adapter at a time, through Zoo's stash or folded."""
+ if mode == "grouped":
+ _need_grouped()
+ if mode == "grouped":
+ monkeypatch.delenv("UNSLOTH_MULTI_LORA_MOE_GROUPED", raising = False)
+ else:
+ monkeypatch.setenv("UNSLOTH_MULTI_LORA_MOE_GROUPED", "0")
if mode == "fold":
monkeypatch.setattr(ml, "_zoo_stash_read_store", lambda experts: None)
+
+
+def _modes(model):
+ return {m.mode for m in model.modules() if isinstance(m, ml.MultiLoraExperts)}
+
+
[email protected]("mode", MODES)
+def test_each_adapter_matches_its_lone_peft_run(base, monkeypatch, mode):
+ _use_mode(monkeypatch, mode)
model, wrapped = _batched(base)
assert len(wrapped) == 2 * len(model.model.language_model.layers)
ids = _ids()
ml.set_multi_lora_batch(model, torch.tensor([0, 1, 2], device = DEVICE), seq_len = T)
out = model(input_ids = ids, use_cache = False).logits
- modes = {m.mode for m in model.modules() if isinstance(m, ml.MultiLoraExperts)}
- assert modes == {mode}
+ assert _modes(model) == {mode}
for u in range(3):
lone = _lone(base, model, u)
ref = lone(input_ids = ids[u : u + 1], use_cache = False).logits[0]
@@ -92,18 +115,21 @@ def test_each_adapter_matches_its_lone_peft_run(base, monkeypatch, mode):
torch.testing.assert_close(out[u], ref, atol = TOL, rtol = 1e-4)
-def test_expert_grads_match_the_lone_run(base):
[email protected]("mode", MODES)
+def test_expert_grads_match_the_lone_run(base, monkeypatch, mode):
+ _use_mode(monkeypatch, mode)
model, _ = _batched(base)
ids = _ids()
ml.set_multi_lora_batch(model, torch.tensor([0, 1, 2], device = DEVICE), seq_len = T)
u = 1
model(input_ids = ids, use_cache = False).logits[u].pow(2).mean().backward()
+ assert _modes(model) == {mode}
lone = _lone(base, model, u)
lone(input_ids = ids[u : u + 1], use_cache = False).logits[0].pow(2).mean().backward()
- layout = ml._peft_moe_lora_b_layout()
for name, experts in model.named_modules():
if not isinstance(experts, ml.MultiLoraExperts):
continue
+ layout = "rank_major" if experts.peft_mode == "fold" else ml._peft_moe_lora_b_layout()
depth = len(experts.parameter_names)
for i, param in enumerate(experts.parameter_names):
holder = lone.base_model.model.get_submodule(name)
@@ -135,7 +161,9 @@ def test_expert_grads_match_the_lone_run(base):
)
-def test_another_adapter_cannot_move_a_row(base):
[email protected]("mode", MODES)
+def test_another_adapter_cannot_move_a_row(base, monkeypatch, mode):
+ _use_mode(monkeypatch, mode)
model, _ = _batched(base)
ids = _ids()
ml.set_multi_lora_batch(model, torch.tensor([0, 1, 2], device = DEVICE), seq_len = T)
@@ -145,11 +173,18 @@ def test_another_adapter_cannot_move_a_row(base):
layer.lora_A[0].mul_(3.0)
layer.lora_B[0].add_(1.0)
after = model(input_ids = ids, use_cache = False).logits.detach()
+ assert _modes(model) == {mode}
assert (after[0] - before[0]).abs().max() > 1e-3
- torch.testing.assert_close(after[1:], before[1:], atol = 0, rtol = 0)
+ # Adapter 0's attention LoRA reroutes its tokens, so in grouped mode the other adapters' rows sit at other
+ # positions of each expert's base GEMM. GPU kernels are row-position independent (bit-exact); AVX2 / NEON CPU
+ # BLAS tails round a row by an ulp or two depending on where it falls, so allow that much there.
+ atol = 1e-6 if mode == "grouped" and DEVICE == "cpu" else 0
+ torch.testing.assert_close(after[1:], before[1:], atol = atol, rtol = 0)
-def test_fold_mode_survives_non_reentrant_checkpointing(base, monkeypatch):
[email protected]("mode", ["grouped", "fold"])
+def test_non_reentrant_checkpointing(base, monkeypatch, mode):
+ _use_mode(monkeypatch, mode)
# A Zoo stash store that this forward never fills: the experts fall back to folding.
monkeypatch.setattr(ml, "_zoo_stash_read_store", lambda experts: {})
model, _ = _batched(base)
@@ -165,10 +200,16 @@ def test_fold_mode_survives_non_reentrant_checkpointing(base, monkeypatch):
_, weights, indices = router(hidden)
out = torch.utils.checkpoint.checkpoint(experts, hidden, indices, weights, use_reentrant = False)
out.sum().backward()
- assert experts.mode == "fold" and hidden.grad is not None
+ assert experts.mode == mode and experts.peft_mode == "fold" and hidden.grad is not None
+ eager = hidden.grad.clone()
+ hidden.grad = None
+ experts(hidden, indices, weights).sum().backward()
+ torch.testing.assert_close(hidden.grad, eager)
-def test_absent_adapters_stay_in_the_graph(base):
[email protected]("mode", MODES)
+def test_absent_adapters_stay_in_the_graph(base, monkeypatch, mode):
+ _use_mode(monkeypatch, mode)
model, _ = _batched(base)
ids = _ids()
clean = None
@@ -184,9 +225,13 @@ def test_absent_adapters_stay_in_the_graph(base):
logits = model(input_ids = ids, use_cache = False).logits
torch.testing.assert_close(logits, clean, atol = 0, rtol = 0)
logits.sum().backward()
+ assert _modes(model) == {mode}
for m in model.modules():
if isinstance(m, ml.MultiLoraExperts):
for adapter in m.adapters.values():
+ assert (
+ adapter.lora_B[2].grad is not None and adapter.lora_B[2].grad.abs().max() == 0
+ )
assert (
adapter.lora_A[2].grad is not None and adapter.lora_A[2].grad.abs().max() == 0
)
@@ -214,12 +259,11 @@ def input_major(base):
return model
[email protected]("mode", ["stash", "fold"])
[email protected]("mode", MODES)
def test_input_major_grouped_mm_experts_match_their_lone_peft_run(
base, input_major, monkeypatch, mode
):
- if mode == "fold":
- monkeypatch.setattr(ml, "_zoo_stash_read_store", lambda experts: None)
+ _use_mode(monkeypatch, mode)
ids = _ids()
# Zoo's forward reads the flag: the relaid experts compute what the original ones do.
torch.testing.assert_close(
@@ -228,14 +272,17 @@ def test_input_major_grouped_mm_experts_match_their_lone_peft_run(
model, _ = _batched(input_major)
ml.set_multi_lora_batch(model, torch.tensor([0, 1, 2], device = DEVICE), seq_len = T)
out = model(input_ids = ids, use_cache = False).logits
- assert {m.mode for m in model.modules() if isinstance(m, ml.MultiLoraExperts)} == {mode}
+ assert _modes(model) == {mode}
for u in range(3):
ref = _lone(input_major, model, u)(input_ids = ids[u : u + 1], use_cache = False).logits[0]
torch.testing.assert_close(out[u], ref, atol = TOL, rtol = 1e-4)
-def test_fold_mode_under_autocast_matches_the_lone_run(base, monkeypatch):
- monkeypatch.setattr(ml, "_zoo_stash_read_store", lambda experts: None)
[email protected]("mode", MODES)
+def test_autocast_matches_the_lone_run(base, monkeypatch, mode):
+ if DEVICE == "cuda" and not torch.cuda.is_bf16_supported(including_emulation = False):
+ pytest.skip("no bfloat16 autocast on this GPU")
+ _use_mode(monkeypatch, mode)
model, _ = _batched(base)
ids = _ids()
ml.set_multi_lora_batch(model, torch.tensor([0, 1, 2], device = DEVICE), seq_len = T)
@@ -245,6 +292,7 @@ def test_fold_mode_under_autocast_matches_the_lone_run(base, monkeypatch):
_lone(base, model, u)(input_ids = ids[u : u + 1], use_cache = False).logits[0]
for u in range(3)
]
+ assert _modes(model) == {mode}
for u in range(3):
# One bf16 ulp of the logits; the bf16 fold was off by about 0.1.
torch.testing.assert_close(out[u].float(), refs[u].float(), atol = 3e-2, rtol = 0)
@@ -267,3 +315,373 @@ def test_experts_read_directly_by_their_parent_are_refused():
with pytest.raises(ValueError, match = "reads experts.gate_up_proj itself"):
ml.inject_multi_lora_experts(FastBlock(), ["experts.gate_up_proj"], 2, 2, 2)
+
+
+class _Experts(torch.nn.Module):
+ """Fused experts in transformers' layout and eager forward: gate_up_proj (E, 2I, H), down_proj (E, H, I)."""
+
+ def __init__(
+ self,
+ E = 5,
+ H = 20,
+ I = 12,
+ bias = False,
+ ):
+ super().__init__()
+ g = torch.Generator().manual_seed(0)
+ self.gate_up_proj = torch.nn.Parameter(torch.randn(E, 2 * I, H, generator = g) * 0.3)
+ self.down_proj = torch.nn.Parameter(torch.randn(E, H, I, generator = g) * 0.3)
+ if bias:
+ self.gate_up_proj_bias = torch.nn.Parameter(torch.randn(E, 2 * I, generator = g))
+ self.act_fn = torch.nn.SiLU()
+
+ def forward(self, hidden_states, top_k_index, top_k_weights):
+ out = torch.zeros_like(hidden_states)
+ for t in range(hidden_states.shape[0]):
+ for j, e in enumerate(top_k_index[t].tolist()):
+ if e >= self.gate_up_proj.shape[0]:
+ continue
+ h = hidden_states[t] @ self.gate_up_proj[e].t()
+ if hasattr(self, "gate_up_proj_bias"):
+ h = h + self.gate_up_proj_bias[e]
+ gate, up = h.chunk(2, dim = -1)
+ y = (self.act_fn(gate) * up) @ self.down_proj[e].t()
+ out[t] = out[t] + y * top_k_weights[t, j]
+ return out
+
+
+def _routes(
+ T,
+ k,
+ E,
+ seed = 0,
+):
+ """Skewed routing: most routes on expert 0, expert E-2 never hit, one dropped route (id E)."""
+ g = torch.Generator().manual_seed(seed)
+ index = torch.randint(0, E, (T, k), generator = g)
+ index[torch.rand(T, k, generator = g) < 0.5] = 0
+ index[index == E - 2] = E - 1
+ index[0, -1] = E
+ weights = torch.rand(T, k, generator = g)
+ return index, weights
+
+
+def _wrapped(module, ranks, alphas, rslora, ids):
+ experts = ml.MultiLoraExperts(
+ module, ["gate_up_proj", "down_proj"], len(ranks), ranks, alphas, rslora
+ )
+ torch.manual_seed(0)
+ with torch.no_grad():
+ for adapter in experts.adapters.values():
+ adapter.reset_parameters()
+ for B in adapter.lora_B:
+ B.normal_(0, 0.5, generator = torch.Generator().manual_seed(B.numel()))
+ ml.set_multi_lora_batch(experts, ids)
+ return experts
+
+
+def _oracle(experts, hidden, index, weights, ids, rslora):
+ """float64, route by route: base plus the token's own adapter delta (scale alpha / r or alpha / sqrt(r))."""
+ base = experts.base_layer
+ E = base.gate_up_proj.shape[0]
+ factors = {
+ name: [(A.double(), B.double()) for A, B in zip(adapter.lora_A, adapter.lora_B)]
+ for name, adapter in experts.adapters.items()
+ }
+ out = []
+ for t in range(hidden.shape[0]):
+ u = int(ids[t])
+ acc = 0
+ for j in range(index.shape[1]):
+ e = int(index[t, j])
+ if e >= E:
+ continue
+
+ def proj(name, x):
+ W = getattr(base, name)[e].double()
+ if u >= 0:
+ A, B = factors[name][u]
+ r = A.shape[1]
+ scale = ALPHAS[u] / (r**0.5 if rslora else r)
+ W = W + scale * B[e] @ A[e]
+ return x @ W.t()
+
+ gate, up = proj("gate_up_proj", hidden[t]).chunk(2, dim = -1)
+ acc = acc + proj("down_proj", torch.nn.functional.silu(gate) * up) * weights[t, j]
+ out.append(acc)
+ return torch.stack(out), factors
+
+
[email protected]("rslora", [False, True])
[email protected]("present", ["all", "no_adapter_1"])
+def test_grouped_matches_a_float64_oracle(monkeypatch, rslora, present):
+ _need_grouped()
+ monkeypatch.delenv("UNSLOTH_MULTI_LORA_MOE_GROUPED", raising = False)
+ T, k, E = 24, 2, 5
+ ids = torch.tensor([0, 0, 2, -1, 0, 1, 2, 0] * 3)
+ if present == "no_adapter_1":
+ ids[ids == 1] = 2
+ experts = _wrapped(_Experts(E), RANKS, ALPHAS, rslora, ids)
+ assert not experts.transposed and not experts.reversed_io
+ index, weights = _routes(T, k, E)
+ hidden = torch.randn(T, 20, generator = torch.Generator().manual_seed(2))
+ hidden64, weights64 = hidden.double().requires_grad_(), weights.double().requires_grad_()
+ hidden, weights = hidden.requires_grad_(), weights.requires_grad_()
+ calls = ml.MULTI_LORA_MOE_STATS["grouped_calls"]
+ out = experts(hidden, index, weights)
+ assert experts.mode == "grouped" and ml.MULTI_LORA_MOE_STATS["grouped_calls"] == calls + 1
+ ref, factors = _oracle(experts, hidden64, index, weights64, ids, rslora)
+ torch.testing.assert_close(out.double(), ref, atol = 1e-5, rtol = 1e-4)
+ grad = torch.randn(out.shape, generator = torch.Generator().manual_seed(3))
+ (out * grad).sum().backward()
+ leaves = [hidden64, weights64] + [
+ t for pairs in factors.values() for pair in pairs for t in pair
+ ]
+ ref_grads = torch.autograd.grad((ref * grad.double()).sum(), leaves, allow_unused = True)
+ mine = [hidden.grad, weights.grad] + [
+ p.grad
+ for adapter in experts.adapters.values()
+ for pair in zip(adapter.lora_A, adapter.lora_B)
+ for p in pair
+ ]
+ for got, want in zip(mine, ref_grads):
+ want = torch.zeros_like(got, dtype = torch.float64) if want is None else want
+ assert got is not None
+ torch.testing.assert_close(got.double(), want, atol = 1e-5, rtol = 1e-4)
+ if present == "no_adapter_1":
+ for adapter in experts.adapters.values():
+ assert (
+ adapter.lora_A[1].grad.abs().max() == 0 and adapter.lora_B[1].grad.abs().max() == 0
+ )
+
+
[email protected]("U", [2, 3])
+def test_grouped_is_one_pass_on_zoo_gemms(monkeypatch, U):
+ """Every adapter in one pass: 2 base GEMMs through Zoo's base path and 4 LoRA grouped GEMMs, whatever U is."""
+ _need_grouped()
+ from unsloth_zoo.temporary_patches import moe_utils
+
+ monkeypatch.delenv("UNSLOTH_MULTI_LORA_MOE_GROUPED", raising = False)
+ T, k, E = 24, 2, 5
+ ids = torch.tensor([u % U for u in range(T)])
+ experts = _wrapped(_Experts(E), RANKS[:U], ALPHAS[:U], False, ids)
+ index, weights = _routes(T, k, E)
+ hidden = torch.randn(T, 20, generator = torch.Generator().manual_seed(2))
+ experts(hidden, index, weights)
+ assert experts.mode == "grouped"
+ calls = {"base": 0, "lora": 0}
+ base_mm, lora_mm = moe_utils._base_grouped_mm, moe_utils._grouped_mm_with_backward_fix
+
+ def count(kind, fn):
+ def wrapper(*args, **kwargs):
+ # LoRA GEMMs are the ones on the stacked (E * (U + 1), ...) factors; Zoo's base path may call the same op.
+ if kind == "base" or args[1].shape[0] == E * (U + 1):
+ calls[kind] += 1
+ return fn(*args, **kwargs)
+
+ return wrapper
+
+ monkeypatch.setattr(moe_utils, "_base_grouped_mm", count("base", base_mm))
+ monkeypatch.setattr(moe_utils, "_grouped_mm_with_backward_fix", count("lora", lora_mm))
+ experts(hidden, index, weights)
+ assert calls == {"base": 2, "lora": 4}
+
+
+def test_one_present_adapter_runs_its_own_pass(monkeypatch):
+ """A batch with one adapter's tokens has nothing to fuse: that call runs per adapter, with the same result."""
+ _need_grouped()
+ monkeypatch.delenv("UNSLOTH_MULTI_LORA_MOE_GROUPED", raising = False)
+ T, k, E = 24, 2, 5
+ index, weights = _routes(T, k, E)
+ hidden = torch.randn(T, 20, generator = torch.Generator().manual_seed(2))
+ experts = _wrapped(_Experts(E), RANKS, ALPHAS, False, torch.tensor([0, 1, 2, -1] * 6))
+ experts(hidden, index, weights)
+ assert experts.mode == "grouped"
+ ml.set_multi_lora_batch(experts, torch.tensor([1, -1] * 12))
+ before = dict(ml.MULTI_LORA_MOE_STATS)
+ out = experts(hidden, index, weights)
+ assert ml.MULTI_LORA_MOE_STATS["per_adapter_calls"] == before["per_adapter_calls"] + 1
+ assert ml.MULTI_LORA_MOE_STATS["grouped_calls"] == before["grouped_calls"]
+ torch.testing.assert_close(
+ out, experts._grouped(hidden, index, weights, torch.tensor([1, -1] * 12))
+ )
+
+
+def test_grouped_engages_and_the_switch_turns_it_off(base, monkeypatch):
+ _need_grouped()
+ ids = _ids()
+ counts = []
+ for value in ("1", "0"):
+ monkeypatch.setenv("UNSLOTH_MULTI_LORA_MOE_GROUPED", value)
+ model, wrapped = _batched(base)
+ ml.set_multi_lora_batch(model, torch.tensor([0, 1, 2], device = DEVICE), seq_len = T)
+ before = dict(ml.MULTI_LORA_MOE_STATS)
+ model(input_ids = ids, use_cache = False)
+ counts.append({key: ml.MULTI_LORA_MOE_STATS[key] - before[key] for key in before})
+ layers = len(wrapped) // 2
+ assert counts[0] == {"grouped_calls": layers, "per_adapter_calls": 0, "grouped_fallbacks": 0}
+ assert counts[1] == {"grouped_calls": 0, "per_adapter_calls": layers, "grouped_fallbacks": 0}
+
+
[email protected]("why", ["bias", "parity"])
+def test_unsupported_experts_fall_back_to_one_adapter_at_a_time(monkeypatch, why):
+ monkeypatch.delenv("UNSLOTH_MULTI_LORA_MOE_GROUPED", raising = False)
+ if why == "parity":
+ grouped = ml.MultiLoraExperts._grouped
+ monkeypatch.setattr(ml.MultiLoraExperts, "_grouped", lambda *a: grouped(*a) * 1.01)
+ T, k, E = 24, 2, 5
+ ids = torch.tensor([0, 1, 2, -1] * 6)
+ index, weights = _routes(T, k, E)
+ hidden = torch.randn(T, 20, generator = torch.Generator().manual_seed(2))
+ experts = _wrapped(_Experts(E, bias = why == "bias"), RANKS, ALPHAS, False, ids)
+ before = dict(ml.MULTI_LORA_MOE_STATS)
+ out = experts(hidden, index, weights)
+ assert experts.mode == "fold"
+ assert ml.MULTI_LORA_MOE_STATS["grouped_fallbacks"] == before["grouped_fallbacks"] + 1
+ assert ml.MULTI_LORA_MOE_STATS["grouped_calls"] == before["grouped_calls"]
+ monkeypatch.setenv("UNSLOTH_MULTI_LORA_MOE_GROUPED", "0")
+ experts.mode = experts.peft_mode = None
+ torch.testing.assert_close(out, experts(hidden, index, weights), atol = 0, rtol = 0)
+
+
+def test_grouped_reads_bnb_4bit_stacks_through_zoo(monkeypatch):
+ _need_grouped()
+ bnb = pytest.importorskip("bitsandbytes")
+ from unsloth_zoo.temporary_patches import moe_utils
+
+ monkeypatch.delenv("UNSLOTH_MULTI_LORA_MOE_GROUPED", raising = False)
+ # The synthetic forward cannot run a 4-bit stack, so there is no per-adapter path to probe.
+ monkeypatch.setattr(ml.MultiLoraExperts, "_run_stash", lambda *a: None)
+ monkeypatch.setattr(ml.MultiLoraExperts, "_grouped_parity", lambda *a: None)
+ T, k, E = 24, 2, 5
+ module = _Experts(E, H = 32, I = 16)
+ for name in ("gate_up_proj", "down_proj"):
+ W = getattr(module, name).data.to(torch.bfloat16)
+ packed, state = bnb.functional.quantize_4bit(W.reshape(-1, W.shape[-1]), quant_type = "nf4")
+ param = torch.Tensor._make_subclass(bnb.nn.Params4bit, packed, False)
+ param.quant_state, param._original_shape = state, W.shape
+ module._parameters[name] = param
+ ids = torch.tensor([0, 1, 2, -1] * 6)
+ experts = _wrapped(module, RANKS, ALPHAS, False, ids)
+ index, weights = _routes(T, k, E)
+ hidden = torch.randn(T, 32, generator = torch.Generator().manual_seed(2), requires_grad = True)
+ seen = []
+ get_base = moe_utils._get_base_weight
+ monkeypatch.setattr(
+ moe_utils, "_get_base_weight", lambda *a, **kw: seen.append(1) or get_base(*a, **kw)
+ )
+ out = experts(hidden, index, weights)
+ assert experts.mode == "grouped" and out.dtype == torch.float32
+ # Zoo's provider dequantizes both stacks in forward and, where Zoo recomputes, again in backward.
+ forward_reads = len(seen)
+ assert forward_reads >= 2
+ out.sum().backward()
+ if moe_utils._moe_recompute_enabled(experts.base_layer.gate_up_proj, dtype = torch.bfloat16):
+ assert len(seen) >= 2 * forward_reads
+ float_module = _Experts(E, H = 32, I = 16)
+ with torch.no_grad():
+ for name in ("gate_up_proj", "down_proj"):
+ # The oracle reads the stack as Zoo's provider does (CPU bitsandbytes slices differ from full reads).
+ param = getattr(experts.base_layer, name)
+ getattr(float_module, name).copy_(get_base(param, torch.bfloat16).float())
+ reference = _wrapped(float_module, RANKS, ALPHAS, False, ids)
+ ref, _ = _oracle(reference, hidden.detach().double(), index, weights.double(), ids, False)
+ # bfloat16 compute, as Unsloth Zoo runs 4-bit experts.
+ torch.testing.assert_close(out.double(), ref, atol = 2e-2 * ref.abs().max().item(), rtol = 0)
+
+
+def test_grouped_dropped_route_never_poisons_gradients(monkeypatch):
+ _need_grouped()
+ # A dropped route (expert id E) with a NaN weight must add exactly nothing, forward and backward.
+ monkeypatch.delenv("UNSLOTH_MULTI_LORA_MOE_GROUPED", raising = False)
+ T, k, E = 8, 2, 5
+ ids = torch.tensor([0, 1, 2, -1, 0, 1, 2, 0])
+ experts = _wrapped(_Experts(E), RANKS, ALPHAS, False, ids)
+ index, weights = _routes(T, k, E)
+ hidden = torch.randn(T, 20, generator = torch.Generator().manual_seed(2))
+ experts(hidden, index, weights)
+ assert experts.mode == "grouped"
+ weights[0, -1] = float("nan")
+ kept, clean = index.clone(), weights.clone()
+ kept[0, -1], clean[0, -1] = 0, 0.0
+
+ def run(index, weights):
+ h, w = hidden.clone().requires_grad_(), weights.clone().requires_grad_()
+ experts.zero_grad(set_to_none = True)
+ out = experts(h, index, w)
+ out.backward(torch.ones_like(out))
+ grads = [p.grad.clone() for p in experts.parameters() if p.grad is not None]
+ return out, h.grad, w.grad, grads
+
+ out, d_hidden, d_weights, grads = run(index, weights)
+ ref, ref_hidden, ref_weights, ref_grads = run(kept, clean)
+ torch.testing.assert_close(out, ref)
+ torch.testing.assert_close(d_hidden, ref_hidden)
+ assert d_weights[0, -1] == 0
+ torch.testing.assert_close(d_weights[:, :1], ref_weights[:, :1])
+ for g, r in zip(grads, ref_grads):
+ torch.testing.assert_close(g, r)
+
+
+def test_grouped_empty_top_k_gives_zeros(monkeypatch):
+ _need_grouped()
+ monkeypatch.delenv("UNSLOTH_MULTI_LORA_MOE_GROUPED", raising = False)
+ ids = torch.tensor([0, 1, 2, -1])
+ experts = _wrapped(_Experts(5), RANKS, ALPHAS, False, ids)
+ index, weights = _routes(4, 2, 5)
+ experts(torch.randn(4, 20), index, weights)
+ assert experts.mode == "grouped"
+ hidden = torch.randn(4, 20, requires_grad = True)
+ out = experts(hidden, index[:, :0], weights[:, :0])
+ assert out.shape == (4, 20) and not out.any()
+ out.sum().backward()
+ assert not hidden.grad.any()
+ for adapter in experts.adapters.values():
+ for p in (*adapter.lora_A, *adapter.lora_B):
+ assert p.grad is not None and not p.grad.any()
+
+
+class _StaticExperts(_Experts):
+ """The same experts with a data-independent forward (every expert, masked), so it traces with fullgraph."""
+
+ def forward(self, hidden_states, top_k_index, top_k_weights):
+ out = torch.zeros_like(hidden_states)
+ for e in range(self.gate_up_proj.shape[0]):
+ w = (top_k_weights * (top_k_index == e)).sum(-1, keepdim = True)
+ gate, up = (hidden_states @ self.gate_up_proj[e].t()).chunk(2, dim = -1)
+ out = out + ((self.act_fn(gate) * up) @ self.down_proj[e].t()) * w
+ return out
+
+
[email protected]("dtype", [torch.float32, torch.bfloat16])
+def test_grouped_under_torch_compile(monkeypatch, dtype):
+ # Grouped GEMM only traces in bfloat16; other dtypes take the per-adapter path inside a compiled graph.
+ _need_grouped(dtype)
+ monkeypatch.delenv("UNSLOTH_MULTI_LORA_MOE_GROUPED", raising = False)
+ T, k, E = 16, 2, 5
+ ids = torch.tensor([0, 1, 2, -1] * 4)
+ experts = _wrapped(_StaticExperts(E, H = 24, I = 16).to(dtype), RANKS, ALPHAS, False, ids).to(
+ DEVICE
+ )
+ for _, layer in ml._layers(experts):
+ layer.seen = None # a plain attribute, so .to() left it on the CPU
+ ml.set_multi_lora_batch(experts, ids.to(DEVICE))
+ index, weights = _routes(T, k, E)
+ index[0, -1] = 0
+ index, weights = index.to(DEVICE), weights.to(DEVICE)
+ hidden = (
+ torch.randn(T, 24, generator = torch.Generator().manual_seed(2))
+ .to(DEVICE, dtype)
+ .requires_grad_()
+ )
+ weights = weights.to(dtype)
+ eager = experts(hidden, index, weights)
+ assert experts.mode == "grouped"
+ torch._dynamo.reset()
+ # fullgraph on CUDA, where Zoo's grouped GEMM traces as a custom op; on CPU it is a compile-disabled loop.
+ compiled = torch.compile(experts, backend = "eager", fullgraph = DEVICE == "cuda")(
+ hidden, index, weights
+ )
+ torch.testing.assert_close(compiled.float(), eager.float(), atol = 2e-2, rtol = 2e-2)
+ compiled.sum().backward()
+ assert torch.isfinite(hidden.grad).all()
diff --git a/unsloth/kernels/multi_lora.py b/unsloth/kernels/multi_lora.py
index 7543c379cee..9e6a442df5e 100644
--- a/unsloth/kernels/multi_lora.py
+++ b/unsloth/kernels/multi_lora.py
@@ -10,6 +10,7 @@ from __future__ import annotations
import contextlib
import functools
import inspect
+import logging
import math
import os
import re
@@ -52,6 +53,7 @@ __all__ = [
"drop_absent_adapter_grads_",
"adapter_state_dict",
"set_multi_lora_triton",
+ "MULTI_LORA_MOE_STATS",
]
@@ -740,6 +742,20 @@ class MultiLoraExpertParam(nn.Module):
self.num_experts,
)
+ def grouped_stacks(self, reversed_io: bool, dtype: torch.dtype) -> tuple:
+ """Every adapter's factors as (E*(U+1), in, R) and (E*(U+1), R, out), group e*(U+1) + u, R the largest rank
+ rounded up to 8 (zero padding is exact), scale folded into the second; group e*(U+1) + U is zero."""
+ R = -(-max(self.ranks) // 8) * 8
+ firsts, seconds = [], []
+ for u in range(self.num_adapters):
+ first, second, scale, _ = self.grouped_mm_factors(u, reversed_io)
+ r = first.shape[-1]
+ firsts.append(F.pad(first.to(dtype), (0, R - r)))
+ seconds.append(F.pad(second.to(dtype) * scale, (0, 0, 0, R - r)))
+ firsts.append(torch.zeros_like(firsts[0]))
+ seconds.append(torch.zeros_like(seconds[0]))
+ return torch.stack(firsts, 1).flatten(0, 1), torch.stack(seconds, 1).flatten(0, 1)
+
def folded(self, W: torch.Tensor, adapter: int, transposed: bool) -> torch.Tensor:
"""W + delta in W's dtype, as PEFT's ParamWrapper folds it (one baddbmm, autocast off)."""
A, B = self.lora_A[adapter].to(W.dtype), self.lora_B[adapter].to(W.dtype)
@@ -819,9 +835,92 @@ def _adapter_groups(batch: MultiLoraBatch) -> list:
return groups
+def _token_adapters(batch: MultiLoraBatch) -> torch.Tensor:
+ """Each token's adapter id (-1 = none)."""
+ ids = batch.token_ids
+ if ids is None:
+ ids = batch.__dict__.get("_token_adapters")
+ if ids is None:
+ n, U = batch.num_tokens, batch.num_adapters
+ seg = batch.seg_start.long()
+ pos = torch.arange(n, device = seg.device)
+ ids = torch.full((n,), -1, dtype = torch.int64, device = seg.device)
+ ids = ids.scatter(0, batch.sorted_idx.long(), torch.searchsorted(seg[1:], pos, right = True))
+ ids = torch.where(ids >= U, torch.full_like(ids, -1), ids)
+ batch.__dict__["_token_adapters"] = ids
+ return ids
+
+
+MULTI_LORA_MOE_STATS = {"grouped_calls": 0, "per_adapter_calls": 0, "grouped_fallbacks": 0}
+_HALF = (torch.float16, torch.bfloat16)
+_FLOAT = (torch.float16, torch.bfloat16, torch.float32, torch.float64)
+_ZOO_GROUPED = (
+ "_base_grouped_mm",
+ "_get_base_weight",
+ "preprocess_weight",
+ "_moe_recompute_enabled",
+ "_selective_dequant_plan",
+ "_grouped_mm_with_backward_fix",
+ "_grouped_mm_supported_on",
+ "weighted_unpermute",
+ "combine_permuted_moe_outputs",
+)
+
+
+def _zoo_moe_utils():
+ """Unsloth Zoo's MoE module when it has the grouped GEMMs its single-job forward runs on, else None."""
+ try:
+ from unsloth_zoo.temporary_patches import moe_utils
+ except Exception: # noqa: BLE001
+ return None
+ return moe_utils if all(hasattr(moe_utils, n) for n in _ZOO_GROUPED) else None
+
+
+def _grouped_supported(device: torch.device, dtype: torch.dtype) -> bool:
+ """Zoo's grouped GEMM runs as a kernel here: its Triton grouped GEMM (CUDA, half) or torch's grouped GEMM.
+ Elsewhere it loops over groups, which at E * (U + 1) groups is slower than one adapter at a time."""
+ moe_utils = _zoo_moe_utils()
+ if moe_utils is None:
+ return False
+ if device.type != "cuda":
+ return True
+ try:
+ if dtype in _HALF:
+ from unsloth_zoo.temporary_patches import moe_grouped_fp16
+ if moe_grouped_fp16.triton_grouped_available(device):
+ return True
+ return bool(moe_utils._grouped_mm_supported_on(device))
+ except Exception: # noqa: BLE001
+ return False
+
+
+def _is_quantized(param) -> bool:
+ return getattr(param, "quant_state", None) is not None or param.dtype not in _FLOAT
+
+
+def _grouped_disabled() -> bool:
+ return os.environ.get("UNSLOTH_MULTI_LORA_MOE_GROUPED", "1") == "0"
+
+
+_WARNED: set = set()
+
+
+def _warn_once(message: str, level: int = logging.WARNING) -> None:
+ if message in _WARNED:
+ return
+ _WARNED.add(message)
+ try:
+ from unsloth_zoo.log import logger
+ except Exception: # noqa: BLE001
+ logger = logging.getLogger(__name__)
+ logger.log(level, message)
+
+
class MultiLoraExperts(nn.Module):
- """Frozen fused experts; each adapter's tokens run the experts' own forward with only its factors, as a lone
- PEFT job does: through Zoo's stash when its MoE forward reads one, else with the delta folded in."""
+ """Frozen fused experts. Mode "grouped": one pass for every adapter, rows sorted by (expert, adapter), the frozen
+ stacks in E-group GEMMs and each adapter's LoRA in E*(U+1)-group GEMMs over the same rows. Otherwise each
+ adapter's tokens run the experts' own forward with only its factors, as a lone PEFT job does: through Zoo's
+ stash when its MoE forward reads one ("stash"), else with the delta folded in ("fold")."""
def __init__(
self,
@@ -854,7 +953,10 @@ class MultiLoraExperts(nn.Module):
self.adapters[name] = MultiLoraExpertParam(
E, in_features, out_features, num_adapters, r, lora_alpha, use_rslora, device = device
)
- self.mode: Optional[str] = None # "stash" or "fold", decided on the first adapter call
+ # How a lone PEFT run of these experts applies LoRA ("stash" or "fold"), and how this module runs
+ # ("grouped", or the same as peft_mode); both decided on the first call with an adapter.
+ self.peft_mode: Optional[str] = None
+ self.mode: Optional[str] = None
def __getattr__(self, name: str):
try:
@@ -903,34 +1005,289 @@ class MultiLoraExperts(nn.Module):
def _run(self, adapter: Optional[int], args) -> torch.Tensor:
if adapter is None:
return self.base_layer(*args)
- if self.mode is None:
- # Probe on one token without autograd, so a checkpointed forward and its recompute run the same ops.
- with torch.no_grad():
- probe = self._run_stash(adapter, tuple(a[:1] for a in args))
- self.mode = "fold" if probe is None else "stash"
- if self.mode == "fold":
+ if self.peft_mode == "fold":
return self._run_fold(adapter, args)
out = self._run_stash(adapter, args)
if out is None:
raise RuntimeError("The experts' forward stopped reading the LoRA stash mid-run.")
return out
+ def _per_adapter(self, hidden_states: torch.Tensor, args, groups) -> torch.Tensor:
+ if len(groups) == 1:
+ return self._run(groups[0][0], (hidden_states, *args))
+ out = None
+ for u, rows in groups:
+ y = self._run(u, (hidden_states[rows], *(a[rows] for a in args)))
+ if out is None:
+ out = y.new_zeros((hidden_states.shape[0], *y.shape[1:]))
+ out = out.index_copy(0, rows, y)
+ return out
+
+ def _choose_mode(self, hidden_states: torch.Tensor, args, groups) -> None:
+ adapter, rows = next(((u, rows) for u, rows in groups if u is not None), (None, None))
+ if adapter is None:
+ return
+ # Probes run without autograd, so a checkpointed forward and its recompute run the same ops.
+ with torch.no_grad():
+ probe = self._run_stash(
+ adapter, (hidden_states[rows[:1]], *(a[rows[:1]] for a in args))
+ )
+ self.peft_mode = "fold" if probe is None else "stash"
+ self.mode = self.peft_mode
+ if _grouped_disabled():
+ return
+ try:
+ reason = self._grouped_unsupported(hidden_states, args) or self._grouped_parity(
+ hidden_states, args, groups
+ )
+ except Exception as error: # noqa: BLE001 - anything unexpected keeps the per-adapter path
+ reason = f"{type(error).__name__}: {error}"
+ if reason is None:
+ self.mode = "grouped"
+ _warn_once(
+ f"Unsloth: batched MoE LoRA on {type(self.base_layer).__name__} runs every adapter in one grouped pass.",
+ logging.INFO,
+ )
+ return
+ if not torch.compiler.is_compiling():
+ MULTI_LORA_MOE_STATS["grouped_fallbacks"] += 1
+ _warn_once(
+ f"Unsloth: batched MoE LoRA on {type(self.base_layer).__name__} runs one adapter at a time ({reason})."
+ )
+
+ def _in_out(self, name: str) -> tuple[int, int, int]:
+ """(E, in, out) of a fused stack, from its stored (E, d1, d2) and layout."""
+ base = self.base_layer
+ E, d1, d2 = _expert_stack_shape(getattr(base, name))
+ input_major = getattr(base, "is_transposed", False) or getattr(
+ base, "_unsloth_grouped_mm_format", False
+ )
+ return (E, d1, d2) if input_major else (E, d2, d1)
+
+ def _compute_dtype(self, hidden_states: torch.Tensor) -> torch.dtype:
+ """As Unsloth Zoo's MoE forward: a half activation runs as it is, half stacks set the dtype, quantized ones
+ take Zoo's ``moe_compute_dtype``; float32 stacks stay float32 even under autocast (grouped GEMM is not
+ autocast-cast)."""
+ if hidden_states.dtype in _HALF:
+ return hidden_states.dtype
+ param = self.base_layer.gate_up_proj
+ if param.dtype in _HALF:
+ return param.dtype
+ if _is_quantized(param):
+ from unsloth_zoo.temporary_patches.moe_utils import moe_compute_dtype
+ return moe_compute_dtype(hidden_states)
+ return hidden_states.dtype
+
+ def _grouped_unsupported(self, hidden_states: torch.Tensor, args) -> Optional[str]:
+ """Why the grouped path cannot run these experts, or None."""
+ base = self.base_layer
+ if not set(self.parameter_names) <= {"gate_up_proj", "down_proj"}:
+ return f"LoRA on {self.parameter_names}"
+ if "GptOss" in type(base).__name__ or getattr(base, "is_concatenated", True) is False:
+ return "interleaved gate_up_proj"
+ if base.__dict__.get("_unsloth_own_apply_gate") or getattr(
+ type(base), "_unsloth_own_apply_gate", False
+ ):
+ return "a custom gate"
+ if not callable(getattr(base, "act_fn", None)):
+ return "no act_fn"
+ names = [n for n, _ in base.named_parameters()] + [n for n, _ in base.named_buffers()]
+ if any("bias" in n for n in names):
+ return "expert biases"
+ params = [getattr(base, n, None) for n in ("gate_up_proj", "down_proj")]
+ if not all(isinstance(p, torch.Tensor) for p in params):
+ return "no fused gate_up_proj / down_proj"
+ if len(args) != 2 or hidden_states.dim() != 2:
+ return "an experts forward not called as (hidden_states, top_k_index, top_k_weights)"
+ top_k_index, top_k_weights = args
+ if (
+ top_k_index.dtype.is_floating_point
+ or not top_k_weights.dtype.is_floating_point
+ or top_k_index.shape != top_k_weights.shape
+ or top_k_index.dim() != 2
+ ):
+ return "an experts forward not called as (hidden_states, top_k_index, top_k_weights)"
+ moe_utils = _zoo_moe_utils()
+ if moe_utils is None:
+ return "an Unsloth Zoo without its grouped MoE GEMMs"
+ layout = getattr(moe_utils, "_mxfp4_expert_layout", None)
+ model_type = getattr(base, "_unsloth_model_type", None)
+ H = hidden_states.shape[-1]
+ if layout is not None and any(
+ layout(getattr(base, n), proj, H, model_type, base) is not None
+ for n, proj in (("gate_up_proj", "gate_up"), ("down_proj", "down"))
+ ):
+ return "MXFP4 experts"
+ dtype = self._compute_dtype(hidden_states)
+ (E, gate_in, gate_out), (E_down, down_in, down_out) = (
+ self._in_out("gate_up_proj"),
+ self._in_out("down_proj"),
+ )
+ if E != E_down or gate_in != H or down_out != H or gate_out != 2 * down_in:
+ return "unexpected expert shapes"
+ # Grouped GEMM needs 16-byte rows (the LoRA rank is padded to 8).
+ step = 16 // torch.empty((), dtype = dtype).element_size()
+ if any(d % step for d in (H, down_in)):
+ return "expert dims not a multiple of 16 bytes"
+ if not _grouped_supported(hidden_states.device, dtype):
+ return f"no grouped GEMM kernel in {dtype} on {hidden_states.device.type}"
+ return None
+
+ def _grouped_parity(self, hidden_states: torch.Tensor, args, groups) -> Optional[str]:
+ """A few rows per adapter through both paths; None when they agree."""
+ ids = _token_adapters(next(iter(self.adapters.values())).batch)
+ picks = [rows[:4] for _, rows in groups]
+ sel = torch.cat(picks)
+ with torch.no_grad():
+ got = self._grouped(hidden_states[sel], *(a[sel] for a in args), ids[sel]).float()
+ ref = torch.cat(
+ [
+ self._run(u, (hidden_states[r], *(a[r] for a in args))).float()
+ for (u, _), r in zip(groups, picks)
+ ]
+ )
+ err = (got - ref).abs().max().item()
+ # Under autocast the per-adapter path may itself run in half precision (Zoo's CPU fallback does).
+ half = self._compute_dtype(hidden_states) in _HALF or torch.is_autocast_enabled(
+ hidden_states.device.type
+ )
+ tol = 5e-2 if half else 1e-3
+ if err <= tol * max(ref.abs().max().item(), 1e-6):
+ return None
+ return f"the grouped probe differs by {err:.3g}"
+
+ def _project(
+ self, moe_utils, name: str, x, offs_expert, offs_pair, counts, plan, dtype
+ ) -> torch.Tensor:
+ """Frozen base over E expert groups through Zoo's base GEMM (its weight provider, recompute and selective
+ dequant, as the single-job forward), plus every adapter's LoRA over the E * (U + 1) pair groups."""
+ base = self.base_layer
+ src, proj = getattr(base, name), name.removesuffix("_proj")
+ hidden, model_type = (
+ self._in_out("gate_up_proj")[1],
+ getattr(base, "_unsloth_model_type", None),
+ )
+
+ def provider(experts = None):
+ weight = moe_utils._get_base_weight(src, dtype, counts, experts)
+ return moe_utils.preprocess_weight(
+ weight, proj, hidden, model_type, experts_module = base
+ )
+
+ recompute = moe_utils._moe_recompute_enabled(src, dtype = dtype)
+ y = moe_utils._base_grouped_mm(
+ x, offs_expert, provider, recompute, plan if recompute else None
+ )
+ if name in self.adapters:
+ first, second = self.adapters[name].grouped_stacks(self.reversed_io, dtype)
+ h = moe_utils._grouped_mm_with_backward_fix(x, first, offs_pair).contiguous()
+ y = y + moe_utils._grouped_mm_with_backward_fix(h, second, offs_pair)
+ return y
+
+ def _grouped(
+ self,
+ hidden_states: torch.Tensor,
+ top_k_index: torch.Tensor,
+ top_k_weights: torch.Tensor,
+ adapter_ids: torch.Tensor,
+ ) -> torch.Tensor:
+ T, k = top_k_index.shape
+ if T * k == 0:
+ # Nothing routed: zeros still tied to the hidden states and every adapter (empty slices, so exact 0 even
+ # for an inf / NaN adapter), as backward and default DDP need.
+ zero = hidden_states.reshape(-1)[:0].sum() + sum(
+ p.reshape(-1)[:0].sum()
+ for adapter in self.adapters.values()
+ for p in (*adapter.lora_A, *adapter.lora_B)
+ )
+ return hidden_states.new_zeros(hidden_states.shape) + zero.to(hidden_states.dtype)
+ E = _expert_stack_shape(self.base_layer.gate_up_proj)[0]
+ U = self.num_adapters
+ Ug = U + 1
+ dtype = self._compute_dtype(hidden_states)
+ # Route = expert * Ug + adapter, slot U for rows of no adapter (zero factors) and for dropped routes
+ # (expert id outside [0, E): they read a zero row and get weight 0, so they add exactly 0 and nothing can
+ # overflow or turn a gradient NaN); sorted, each expert's rows stay contiguous.
+ index = top_k_index.long()
+ valid = (index >= 0) & (index < E)
+ slot = torch.where(adapter_ids < 0, U, adapter_ids).to(index.device).unsqueeze(1)
+ key = (torch.where(valid, index, 0) * Ug + torch.where(valid, slot, U)).reshape(-1)
+ order = torch.argsort(key, stable = True)
+ # Fixed-size counts: bincount's output size depends on the data, so it syncs on CUDA and breaks a compiled graph.
+ counts = torch.zeros(E * Ug, dtype = torch.int32, device = key.device)
+ counts.scatter_add_(0, key, torch.ones_like(key, dtype = torch.int32))
+ offs_pair = torch.cumsum(counts, 0, dtype = torch.int32)
+ offs_expert = offs_pair.view(E, Ug)[:, -1].contiguous()
+ counts = counts.view(E, Ug).sum(1, dtype = torch.int32)
+ moe_utils = _zoo_moe_utils()
+ base = self.base_layer
+ plan = None
+ if moe_utils._moe_recompute_enabled(base.gate_up_proj, dtype = dtype):
+ plan = moe_utils._selective_dequant_plan(
+ (base.gate_up_proj, base.down_proj), counts, T * k
+ )
+ with torch.autocast(device_type = hidden_states.device.type, enabled = False):
+ rows = torch.where(valid.reshape(-1)[order], order // k, T)
+ x = torch.cat(
+ [
+ hidden_states.to(dtype),
+ hidden_states.new_zeros(1, hidden_states.shape[1], dtype = dtype),
+ ]
+ )
+ x = x[rows]
+ h = self._project(
+ moe_utils, "gate_up_proj", x, offs_expert, offs_pair, counts, plan, dtype
+ )
+ gate, up = h.chunk(2, dim = -1)
+ inter = base.act_fn(gate) * up
+ # From here on as Unsloth Zoo's MoE forward combines routes: its gate-gradient identity when enabled,
+ # then its fused weighted unpermute (deterministic fallback without Triton).
+ weights = torch.where(valid, top_k_weights, 0).reshape(-1)[order]
+ gategrad = getattr(moe_utils, "_moe_gategrad_enabled", lambda: False)()
+ gategrad = gategrad and hasattr(moe_utils, "_MoEGateGradIdentity")
+ gategrad = gategrad and top_k_weights.requires_grad and inter.dtype != torch.float16
+ if gategrad:
+ raw = weights.float() if weights.dtype == torch.float16 else weights
+ floored = torch.where(raw >= 0, raw.clamp(min = 1e-12), raw.clamp(max = -1e-12))
+ weights = raw + (floored - raw).detach()
+ inter = moe_utils._MoEGateGradIdentity.apply(inter, weights)
+ weights = weights.detach()
+ y = self._project(
+ moe_utils, "down_proj", inter, offs_expert, offs_pair, counts, plan, dtype
+ )
+ out = moe_utils.weighted_unpermute(
+ y, order, weights, T, k, out_dtype = hidden_states.dtype
+ )
+ if out is None:
+ out = moe_utils.combine_permuted_moe_outputs(
+ y * weights.unsqueeze(-1), order, T, k, out_dtype = hidden_states.dtype
+ )
+ return out
+
def forward(self, hidden_states: torch.Tensor, *args):
batch = next(iter(self.adapters.values())).batch
if batch is None or hidden_states.shape[0] != batch.num_tokens:
raise ValueError(
"Call set_multi_lora_batch(model, ...) with this batch's adapter ids first."
)
+ if self.mode is None:
+ self._choose_mode(hidden_states, args, _adapter_groups(batch))
+ grouped = self.mode == "grouped"
+ if grouped and torch.compiler.is_compiling():
+ # Grouped GEMM only traces in bfloat16 (its meta checks the dtype on every device).
+ grouped = self._compute_dtype(hidden_states) == torch.bfloat16
+ elif grouped:
+ # With one adapter's tokens there is nothing to fuse: its own pass is the same work in fewer groups.
+ grouped = sum(u is not None for u, _ in _adapter_groups(batch)) >= 2
+ if grouped:
+ if not torch.compiler.is_compiling():
+ MULTI_LORA_MOE_STATS["grouped_calls"] += 1
+ # Every adapter's factors are in the graph, so absent adapters get exact zero grads.
+ return self._grouped(hidden_states, *args, _token_adapters(batch))
+ if not torch.compiler.is_compiling():
+ MULTI_LORA_MOE_STATS["per_adapter_calls"] += 1
groups = _adapter_groups(batch)
- if len(groups) == 1:
- out = self._run(groups[0][0], (hidden_states, *args))
- else:
- out = None
- for u, rows in groups:
- y = self._run(u, (hidden_states[rows], *(a[rows] for a in args)))
- if out is None:
- out = y.new_zeros((hidden_states.shape[0], *y.shape[1:]))
- out = out.index_copy(0, rows, y)
+ out = self._per_adapter(hidden_states, args, groups)
present = {u for u, _ in groups}
absent = [u for u in range(self.num_adapters) if u not in present]
if absent and torch.is_grad_enabled():
@@ -1222,8 +1579,9 @@ def adapter_state_dict(
depth = len(experts.parameter_names)
for i, param in enumerate(experts.parameter_names):
key = f"{prefix}{name}" + ".base_layer" * (depth - 1 - i)
- # Folded experts load through PEFT's own ParamWrapper delta, which is rank-major whatever Zoo uses.
- layout = "rank_major" if experts.mode == "fold" else b_layout
+ # Experts whose forward ignores the stash load through PEFT's own ParamWrapper delta, which is
+ # rank-major whatever Zoo uses; how they trained here (grouped or not) does not matter.
+ layout = "rank_major" if experts.peft_mode == "fold" else b_layout
A, B = experts.adapters[param].peft_tensors(adapter, layout)
out[f"{key}.lora_A.weight"] = A
out[f"{key}.lora_B.weight"] = B