Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
MoE grouped GEMM: share autotuners across identical cards; fix the TMA test on one-GPU hosts and macOS (#14490)
Daniel Hanec80925abc32
diff --git a/tests/test_grouped_gemm_mixed_tma_device.py b/tests/test_grouped_gemm_mixed_tma_device.pyindex ab6a7a7262f..fbcdc531068 100644--- a/tests/test_grouped_gemm_mixed_tma_device.py+++ b/tests/test_grouped_gemm_mixed_tma_device.py@@ -14,7 +14,10 @@ from pathlib import Pathimport pytesttorch = pytest.importorskip("torch")-pytest.importorskip("triton")+triton = pytest.importorskip("triton")+if triton.__name__ != "triton":+ # unsloth_zoo's macOS stub answers every attribute, so the TMA API cannot be hidden.+ pytest.skip("triton is a stub", allow_module_level = True)import torch._dynamo # noqa: E402,F401 # import before custom_op is stubbed out belowfrom torch._subclasses.fake_tensor import FakeTensorMode # noqa: E402@@ -71,10 +74,18 @@ def _load("get_device_capability",lambda device = None: caps[state["current"] if device is None else device],)++ def get_device_properties(device = None):+ index = state["current"] if device is None else getattr(device, "index", device)+ major, minor = caps[index]+ return types.SimpleNamespace(+ name = f"sm{major}{minor}", major = major, minor = minor, multi_processor_count = 4+ )++ monkeypatch.setattr(torch.cuda, "get_device_properties", get_device_properties)+ # FakeTensorMode opens a real context on a fake tensor's device (cuda:1 is absent on one-GPU hosts).monkeypatch.setattr(- torch.cuda,- "get_device_properties",- lambda device = None: types.SimpleNamespace(multi_processor_count = 4),+ "torch._subclasses.fake_tensor.init_gpu_context", lambda device: None, raising = False)# No custom-op registration from the spoofed module.monkeypatch.setattr(torch.library, "custom_op", None, raising = False)@@ -191,6 +202,9 @@ class _Tuner:def __init__(self):self.cache = {}+ def run(self, *args, **kwargs):+ return self.cache+def test_each_card_autotunes_its_own_copy(monkeypatch):module, _ = _load(monkeypatch, [H100, A100])@@ -204,6 +218,32 @@ def test_each_card_autotunes_its_own_copy(monkeypatch):assert base.cache == {"shape": "h100-config"}+def test_identical_cards_share_one_autotune_cache(monkeypatch):+ # device_map="auto" over identical cards: tune once, as with one shared tuner. Each card keeps+ # its own tuner object: Triton writes per-run state (nargs) on it, and per-device autograd+ # threads can run two cards' kernels at once.+ module, _ = _load(monkeypatch, [H100, H100, A100])+ base = _Tuner()+ tuner0 = module._autotuner_for(base, torch.device("cuda", 0))+ tuner1 = module._autotuner_for(base, torch.device("cuda", 1))+ tuner2 = module._autotuner_for(base, torch.device("cuda", 2))+ assert tuner1 is not tuner0 and tuner1.cache is tuner0.cache+ assert tuner2.cache is not tuner0.cache+ tuner0.cache["shape"] = "h100-config"+ assert tuner1.cache == {"shape": "h100-config"} and tuner2.cache == {}+ assert module._autotuner_for(base, torch.device("cuda", 1)) is tuner1+++def test_per_card_autotuner_follows_reassigned_configs(monkeypatch):+ module, _ = _load(monkeypatch, [H100, A100])+ base = _Tuner()+ base.configs = ["a", "b", "c"]+ tuner = module._autotuner_for(base, torch.device("cuda", 1))+ base.configs = base.configs[:1]+ assert module._autotuner_for(base, torch.device("cuda", 1)).configs == ["a"]+ assert tuner.cache == {}++def test_single_card_keeps_the_shared_autotuner(monkeypatch):module, _ = _load(monkeypatch, [H100])monkeypatch.setattr(module.torch.cuda, "device_count", lambda: 1)@@ -285,3 +325,29 @@ def test_benchmark_postprocesses_the_per_card_tuner():for a in c.args[:1]}assert kernels and all(id(n) in wrapped for n in kernels)+++def test_identical_cards_tune_a_cold_shape_once(monkeypatch):+ # Two cards of one model missing the same shape at once: the second waits for the first's result.+ import threading+ import time++ module, _ = _load(monkeypatch, [H100, H100])+ tuned = []++ class Tuner(_Tuner):+ def run(self, key = "shape"):+ if key not in self.cache:+ tuned.append(key)+ time.sleep(0.2)+ self.cache[key] = "config"+ return self.cache[key]++ base = Tuner()+ tuners = [module._autotuner_for(base, torch.device("cuda", i)) for i in (0, 1)]+ threads = [threading.Thread(target = t.run) for t in tuners]+ for t in threads:+ t.start()+ for t in threads:+ t.join(5)+ assert tuned == ["shape"]diff --git a/unsloth/kernels/moe/grouped_gemm/interface.py b/unsloth/kernels/moe/grouped_gemm/interface.pyindex def12f79c72..837ffc4b28f 100644--- a/unsloth/kernels/moe/grouped_gemm/interface.py+++ b/unsloth/kernels/moe/grouped_gemm/interface.py@@ -6,6 +6,7 @@ import functoolsimport inspectimport loggingimport os+import threadingimport warningsfrom typing import Optionalfrom dataclasses import asdict@@ -59,18 +60,62 @@ def _check_tma_support():_SUPPORTS_TMA = _check_tma_support()_DEVICE_AUTOTUNERS = {}+_MODEL_AUTOTUNE_CACHES = {}+_MODEL_AUTOTUNE_LOCKS = {}+_DEVICE_MODELS = {}+++def _device_index(device):+ return device.index if device.index is not None else torch.cuda.current_device()+++def _device_model(device):+ index = _device_index(device)+ model = _DEVICE_MODELS.get(index)+ if model is None:+ try:+ p = torch.cuda.get_device_properties(index)+ model = (+ getattr(p, "name", None),+ getattr(p, "major", None),+ getattr(p, "minor", None),+ getattr(p, "multi_processor_count", None),+ )+ except Exception:+ model = index+ _DEVICE_MODELS[index] = model+ return modeldef _autotuner_for(kernel, device):- """Triton's autotune key omits the device, so on several GPUs each card tunes its own copy."""+ """Triton's autotune key omits the device, so on several GPUs each card runs its own copy. Copies on+ the same GPU model share one results cache, so identical cards tune once; a run's scratch state+ (nargs) stays per card, since backward runs one autograd thread per device."""if torch.cuda.device_count() <= 1 or not hasattr(kernel, "cache"):return kernel- key = (id(kernel), device.index)- if key not in _DEVICE_AUTOTUNERS:+ key = (id(kernel), _device_index(device))+ tuner = _DEVICE_AUTOTUNERS.get(key)+ if tuner is None:tuner = copy.copy(kernel)- tuner.cache = {}+ model_key = (id(kernel), _device_model(device))+ tuner.cache = _MODEL_AUTOTUNE_CACHES.setdefault(model_key, {})+ # One tuning per shape per model: a second card waits, then reads the first card's result+ # (concurrent benchmarking would also skew both timings). A cached launch holds it briefly.+ lock = _MODEL_AUTOTUNE_LOCKS.setdefault(model_key, threading.Lock())+ run = getattr(tuner, "run", None)+ if run is not None:++ def _run_single_flight(*args, _run = run, _lock = lock, **kwargs):+ with _lock:+ return _run(*args, **kwargs)++ tuner.run = _run_single_flight_DEVICE_AUTOTUNERS[key] = tuner- return _DEVICE_AUTOTUNERS[key]+ # Callers reassign kernel.configs (benchmarks, tests): follow it like the shared tuner would.+ configs = getattr(kernel, "configs", None)+ if configs is not None and getattr(tuner, "configs", None) is not configs:+ tuner.configs = configs+ return tuner_HAS_SET_ALLOCATOR = hasattr(triton, "set_allocator")