Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Multi-LoRA grouped experts: run the grouped support and parity probes only eagerly (#16415)

Daniel Hanbf38ae0ac66d
diff --git a/tests/test_multi_lora_cold_compile_cpu.py b/tests/test_multi_lora_cold_compile_cpu.py
new file mode 100644
index 00000000000..a4d68af0301
--- /dev/null
+++ b/tests/test_multi_lora_cold_compile_cpu.py
@@ -0,0 +1,128 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+
+from __future__ import annotations
+import ast
+import functools
+import logging
+import math
+import os
+from pathlib import Path
+import sys
+import types
+from typing import Iterable, Optional
+
+import pytest
+import torch
+from torch import nn
+from torch.nn import functional as F
+
+
+def _load_experts():
+ path = Path(__file__).resolve().parents[1] / "unsloth/kernels/multi_lora.py"
+ names = {
+ "MultiLoraBatch",
+ "MultiLoraExpertParam",
+ "MultiLoraExperts",
+ "_expert_stack_shape",
+ "_adapter_groups",
+ "_token_adapters",
+ "_grouped_disabled",
+ "_warn_once",
+ "_is_quantized",
+ }
+ tree = ast.parse(path.read_text(), filename = str(path))
+ tree.body = [
+ n for n in tree.body if isinstance(n, (ast.ClassDef, ast.FunctionDef)) and n.name in names
+ ]
+ mod = types.ModuleType("_multi_lora_compile_source")
+ from dataclasses import dataclass
+
+ mod.__dict__.update(
+ torch = torch,
+ nn = nn,
+ F = F,
+ math = math,
+ Optional = Optional,
+ Iterable = Iterable,
+ functools = functools,
+ dataclass = dataclass,
+ logging = logging,
+ os = os,
+ _HALF = (torch.float16, torch.bfloat16),
+ _FLOAT = (torch.float16, torch.bfloat16, torch.float32, torch.float64),
+ _WARNED = set(),
+ _peft_expert_in_axis = lambda transposed: 1 if transposed else 2,
+ _zoo_stash_read_store = lambda base: None,
+ MULTI_LORA_MOE_STATS = {"per_adapter_calls": 0, "grouped_fallbacks": 0, "grouped_calls": 0},
+ )
+ sys.modules[mod.__name__] = mod
+ exec(compile(tree, str(path), "exec"), mod.__dict__)
+ return mod
+
+
+class _StaticExperts(nn.Module):
+ def __init__(self, dtype):
+ super().__init__()
+ self.gate_up_proj = nn.Parameter(torch.full((1, 16, 8), 0.03, dtype = dtype))
+ self.down_proj = nn.Parameter(torch.full((1, 8, 8), 0.02, dtype = dtype))
+
+ def forward(self, hidden, index, weights):
+ gate, up = (hidden @ self.gate_up_proj[0].t()).chunk(2, -1)
+ return ((F.silu(gate) * up) @ self.down_proj[0].t()) * weights.sum(-1, keepdim = True)
+
+
+def _disabled_support_probe(self, hidden, args):
+ return None
+
+
+def _parity_grouped(self, hidden, index, weights, ids):
+ y = torch.zeros_like(hidden)
+ for u in range(self.num_adapters):
+ y = y + torch.where((ids == u).unsqueeze(-1), self._run(u, (hidden, index, weights)), 0)
+ return y
+
+
[email protected]("dtype", [torch.float32, torch.bfloat16])
[email protected]("probe", ["disabled_support", "data_dependent_parity"])
+def test_cold_fullgraph_does_not_run_grouped_probes(monkeypatch, dtype, probe):
+ ml = _load_experts()
+ monkeypatch.delenv("UNSLOTH_MULTI_LORA_MOE_GROUPED", raising = False)
+ monkeypatch.setattr(
+ ml.MultiLoraExperts,
+ "_grouped_unsupported",
+ _disabled_support_probe if probe == "disabled_support" else lambda *a: None,
+ )
+ monkeypatch.setattr(ml.MultiLoraExperts, "_grouped", _parity_grouped)
+ experts = ml.MultiLoraExperts(
+ _StaticExperts(dtype), ["gate_up_proj", "down_proj"], 2, r = 1, lora_alpha = 16
+ )
+ with torch.no_grad():
+ for adapter in experts.adapters.values():
+ for A, B in zip(adapter.lora_A, adapter.lora_B):
+ A.fill_(0.01)
+ B.fill_(0.02)
+ batch = ml.MultiLoraBatch.from_adapter_ids(torch.tensor([0, 1, 0, 1]), 2)
+ for adapter in experts.adapters.values():
+ adapter.batch = batch
+ ml._adapter_groups(batch)
+ hidden = torch.full((4, 8), 0.1, dtype = dtype, requires_grad = True)
+ index = torch.zeros(4, 1, dtype = torch.long)
+ weights = torch.ones(4, 1, dtype = dtype, requires_grad = True)
+ assert experts.mode is None
+ torch._dynamo.reset()
+ try:
+ with torch._dynamo.config.patch(capture_scalar_outputs = True):
+ out = torch.compile(experts, backend = "eager", fullgraph = True, dynamic = True)(
+ hidden, index, weights
+ )
+ leaves = [hidden, weights, *experts.adapters.parameters()]
+ gradients = torch.autograd.grad(out.sum(), leaves)
+ ref = experts._per_adapter(hidden, (index, weights), ml._adapter_groups(batch))
+ ref_gradients = torch.autograd.grad(ref.sum(), leaves)
+ torch.testing.assert_close(out, ref, rtol = 0, atol = 0)
+ for actual, expected in zip(gradients, ref_gradients):
+ assert actual.isfinite().all()
+ torch.testing.assert_close(actual, expected, rtol = 0, atol = 0)
+ finally:
+ torch._dynamo.reset()
diff --git a/unsloth/kernels/multi_lora.py b/unsloth/kernels/multi_lora.py
index 9e6a442df5e..8c5eccfd9b8 100644
--- a/unsloth/kernels/multi_lora.py
+++ b/unsloth/kernels/multi_lora.py
@@ -1024,7 +1024,11 @@ class MultiLoraExperts(nn.Module):
return out
def _choose_mode(self, hidden_states: torch.Tensor, args, groups) -> None:
- adapter, rows = next(((u, rows) for u, rows in groups if u is not None), (None, None))
+ adapter, rows = None, None
+ for u, candidate_rows in groups:
+ if u is not None:
+ adapter, rows = u, candidate_rows
+ break
if adapter is None:
return
# Probes run without autograd, so a checkpointed forward and its recompute run the same ops.
@@ -1034,7 +1038,8 @@ class MultiLoraExperts(nn.Module):
)
self.peft_mode = "fold" if probe is None else "stash"
self.mode = self.peft_mode
- if _grouped_disabled():
+ # Cold traces cannot run the data-dependent grouped parity probe.
+ if torch.compiler.is_compiling() or _grouped_disabled():
return
try:
reason = self._grouped_unsupported(hidden_states, args) or self._grouped_parity(