Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Multi-LoRA grouped experts: run the grouped support and parity probes only eagerly (#16415)
Daniel Hanbf38ae0ac66d
diff --git a/tests/test_multi_lora_cold_compile_cpu.py b/tests/test_multi_lora_cold_compile_cpu.pynew file mode 100644index 00000000000..a4d68af0301--- /dev/null+++ b/tests/test_multi_lora_cold_compile_cpu.py@@ -0,0 +1,128 @@+# SPDX-License-Identifier: AGPL-3.0-only++from __future__ import annotations+import ast+import functools+import logging+import math+import os+from pathlib import Path+import sys+import types+from typing import Iterable, Optional++import pytest+import torch+from torch import nn+from torch.nn import functional as F+++def _load_experts():+ path = Path(__file__).resolve().parents[1] / "unsloth/kernels/multi_lora.py"+ names = {+ "MultiLoraBatch",+ "MultiLoraExpertParam",+ "MultiLoraExperts",+ "_expert_stack_shape",+ "_adapter_groups",+ "_token_adapters",+ "_grouped_disabled",+ "_warn_once",+ "_is_quantized",+ }+ tree = ast.parse(path.read_text(), filename = str(path))+ tree.body = [+ n for n in tree.body if isinstance(n, (ast.ClassDef, ast.FunctionDef)) and n.name in names+ ]+ mod = types.ModuleType("_multi_lora_compile_source")+ from dataclasses import dataclass++ mod.__dict__.update(+ torch = torch,+ nn = nn,+ F = F,+ math = math,+ Optional = Optional,+ Iterable = Iterable,+ functools = functools,+ dataclass = dataclass,+ logging = logging,+ os = os,+ _HALF = (torch.float16, torch.bfloat16),+ _FLOAT = (torch.float16, torch.bfloat16, torch.float32, torch.float64),+ _WARNED = set(),+ _peft_expert_in_axis = lambda transposed: 1 if transposed else 2,+ _zoo_stash_read_store = lambda base: None,+ MULTI_LORA_MOE_STATS = {"per_adapter_calls": 0, "grouped_fallbacks": 0, "grouped_calls": 0},+ )+ sys.modules[mod.__name__] = mod+ exec(compile(tree, str(path), "exec"), mod.__dict__)+ return mod+++class _StaticExperts(nn.Module):+ def __init__(self, dtype):+ super().__init__()+ self.gate_up_proj = nn.Parameter(torch.full((1, 16, 8), 0.03, dtype = dtype))+ self.down_proj = nn.Parameter(torch.full((1, 8, 8), 0.02, dtype = dtype))++ def forward(self, hidden, index, weights):+ gate, up = (hidden @ self.gate_up_proj[0].t()).chunk(2, -1)+ return ((F.silu(gate) * up) @ self.down_proj[0].t()) * weights.sum(-1, keepdim = True)+++def _disabled_support_probe(self, hidden, args):+ return None+++def _parity_grouped(self, hidden, index, weights, ids):+ y = torch.zeros_like(hidden)+ for u in range(self.num_adapters):+ y = y + torch.where((ids == u).unsqueeze(-1), self._run(u, (hidden, index, weights)), 0)+ return y++[email protected]("dtype", [torch.float32, torch.bfloat16])[email protected]("probe", ["disabled_support", "data_dependent_parity"])+def test_cold_fullgraph_does_not_run_grouped_probes(monkeypatch, dtype, probe):+ ml = _load_experts()+ monkeypatch.delenv("UNSLOTH_MULTI_LORA_MOE_GROUPED", raising = False)+ monkeypatch.setattr(+ ml.MultiLoraExperts,+ "_grouped_unsupported",+ _disabled_support_probe if probe == "disabled_support" else lambda *a: None,+ )+ monkeypatch.setattr(ml.MultiLoraExperts, "_grouped", _parity_grouped)+ experts = ml.MultiLoraExperts(+ _StaticExperts(dtype), ["gate_up_proj", "down_proj"], 2, r = 1, lora_alpha = 16+ )+ with torch.no_grad():+ for adapter in experts.adapters.values():+ for A, B in zip(adapter.lora_A, adapter.lora_B):+ A.fill_(0.01)+ B.fill_(0.02)+ batch = ml.MultiLoraBatch.from_adapter_ids(torch.tensor([0, 1, 0, 1]), 2)+ for adapter in experts.adapters.values():+ adapter.batch = batch+ ml._adapter_groups(batch)+ hidden = torch.full((4, 8), 0.1, dtype = dtype, requires_grad = True)+ index = torch.zeros(4, 1, dtype = torch.long)+ weights = torch.ones(4, 1, dtype = dtype, requires_grad = True)+ assert experts.mode is None+ torch._dynamo.reset()+ try:+ with torch._dynamo.config.patch(capture_scalar_outputs = True):+ out = torch.compile(experts, backend = "eager", fullgraph = True, dynamic = True)(+ hidden, index, weights+ )+ leaves = [hidden, weights, *experts.adapters.parameters()]+ gradients = torch.autograd.grad(out.sum(), leaves)+ ref = experts._per_adapter(hidden, (index, weights), ml._adapter_groups(batch))+ ref_gradients = torch.autograd.grad(ref.sum(), leaves)+ torch.testing.assert_close(out, ref, rtol = 0, atol = 0)+ for actual, expected in zip(gradients, ref_gradients):+ assert actual.isfinite().all()+ torch.testing.assert_close(actual, expected, rtol = 0, atol = 0)+ finally:+ torch._dynamo.reset()diff --git a/unsloth/kernels/multi_lora.py b/unsloth/kernels/multi_lora.pyindex 9e6a442df5e..8c5eccfd9b8 100644--- a/unsloth/kernels/multi_lora.py+++ b/unsloth/kernels/multi_lora.py@@ -1024,7 +1024,11 @@ class MultiLoraExperts(nn.Module):return outdef _choose_mode(self, hidden_states: torch.Tensor, args, groups) -> None:- adapter, rows = next(((u, rows) for u, rows in groups if u is not None), (None, None))+ adapter, rows = None, None+ for u, candidate_rows in groups:+ if u is not None:+ adapter, rows = u, candidate_rows+ breakif adapter is None:return# Probes run without autograd, so a checkpointed forward and its recompute run the same ops.@@ -1034,7 +1038,8 @@ class MultiLoraExperts(nn.Module):)self.peft_mode = "fold" if probe is None else "stash"self.mode = self.peft_mode- if _grouped_disabled():+ # Cold traces cannot run the data-dependent grouped parity probe.+ if torch.compiler.is_compiling() or _grouped_disabled():returntry:reason = self._grouped_unsupported(hidden_states, args) or self._grouped_parity(