Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

MoE grouped GEMM: share autotuners across identical cards; fix the TMA test on one-GPU hosts and macOS (#14490)

Daniel Hanec80925abc32
diff --git a/tests/test_grouped_gemm_mixed_tma_device.py b/tests/test_grouped_gemm_mixed_tma_device.py
index ab6a7a7262f..fbcdc531068 100644
--- a/tests/test_grouped_gemm_mixed_tma_device.py
+++ b/tests/test_grouped_gemm_mixed_tma_device.py
@@ -14,7 +14,10 @@ from pathlib import Path
import pytest
torch = pytest.importorskip("torch")
-pytest.importorskip("triton")
+triton = pytest.importorskip("triton")
+if triton.__name__ != "triton":
+ # unsloth_zoo's macOS stub answers every attribute, so the TMA API cannot be hidden.
+ pytest.skip("triton is a stub", allow_module_level = True)
import torch._dynamo # noqa: E402,F401 # import before custom_op is stubbed out below
from torch._subclasses.fake_tensor import FakeTensorMode # noqa: E402
@@ -71,10 +74,18 @@ def _load(
"get_device_capability",
lambda device = None: caps[state["current"] if device is None else device],
)
+
+ def get_device_properties(device = None):
+ index = state["current"] if device is None else getattr(device, "index", device)
+ major, minor = caps[index]
+ return types.SimpleNamespace(
+ name = f"sm{major}{minor}", major = major, minor = minor, multi_processor_count = 4
+ )
+
+ monkeypatch.setattr(torch.cuda, "get_device_properties", get_device_properties)
+ # FakeTensorMode opens a real context on a fake tensor's device (cuda:1 is absent on one-GPU hosts).
monkeypatch.setattr(
- torch.cuda,
- "get_device_properties",
- lambda device = None: types.SimpleNamespace(multi_processor_count = 4),
+ "torch._subclasses.fake_tensor.init_gpu_context", lambda device: None, raising = False
)
# No custom-op registration from the spoofed module.
monkeypatch.setattr(torch.library, "custom_op", None, raising = False)
@@ -191,6 +202,9 @@ class _Tuner:
def __init__(self):
self.cache = {}
+ def run(self, *args, **kwargs):
+ return self.cache
+
def test_each_card_autotunes_its_own_copy(monkeypatch):
module, _ = _load(monkeypatch, [H100, A100])
@@ -204,6 +218,32 @@ def test_each_card_autotunes_its_own_copy(monkeypatch):
assert base.cache == {"shape": "h100-config"}
+def test_identical_cards_share_one_autotune_cache(monkeypatch):
+ # device_map="auto" over identical cards: tune once, as with one shared tuner. Each card keeps
+ # its own tuner object: Triton writes per-run state (nargs) on it, and per-device autograd
+ # threads can run two cards' kernels at once.
+ module, _ = _load(monkeypatch, [H100, H100, A100])
+ base = _Tuner()
+ tuner0 = module._autotuner_for(base, torch.device("cuda", 0))
+ tuner1 = module._autotuner_for(base, torch.device("cuda", 1))
+ tuner2 = module._autotuner_for(base, torch.device("cuda", 2))
+ assert tuner1 is not tuner0 and tuner1.cache is tuner0.cache
+ assert tuner2.cache is not tuner0.cache
+ tuner0.cache["shape"] = "h100-config"
+ assert tuner1.cache == {"shape": "h100-config"} and tuner2.cache == {}
+ assert module._autotuner_for(base, torch.device("cuda", 1)) is tuner1
+
+
+def test_per_card_autotuner_follows_reassigned_configs(monkeypatch):
+ module, _ = _load(monkeypatch, [H100, A100])
+ base = _Tuner()
+ base.configs = ["a", "b", "c"]
+ tuner = module._autotuner_for(base, torch.device("cuda", 1))
+ base.configs = base.configs[:1]
+ assert module._autotuner_for(base, torch.device("cuda", 1)).configs == ["a"]
+ assert tuner.cache == {}
+
+
def test_single_card_keeps_the_shared_autotuner(monkeypatch):
module, _ = _load(monkeypatch, [H100])
monkeypatch.setattr(module.torch.cuda, "device_count", lambda: 1)
@@ -285,3 +325,29 @@ def test_benchmark_postprocesses_the_per_card_tuner():
for a in c.args[:1]
}
assert kernels and all(id(n) in wrapped for n in kernels)
+
+
+def test_identical_cards_tune_a_cold_shape_once(monkeypatch):
+ # Two cards of one model missing the same shape at once: the second waits for the first's result.
+ import threading
+ import time
+
+ module, _ = _load(monkeypatch, [H100, H100])
+ tuned = []
+
+ class Tuner(_Tuner):
+ def run(self, key = "shape"):
+ if key not in self.cache:
+ tuned.append(key)
+ time.sleep(0.2)
+ self.cache[key] = "config"
+ return self.cache[key]
+
+ base = Tuner()
+ tuners = [module._autotuner_for(base, torch.device("cuda", i)) for i in (0, 1)]
+ threads = [threading.Thread(target = t.run) for t in tuners]
+ for t in threads:
+ t.start()
+ for t in threads:
+ t.join(5)
+ assert tuned == ["shape"]
diff --git a/unsloth/kernels/moe/grouped_gemm/interface.py b/unsloth/kernels/moe/grouped_gemm/interface.py
index def12f79c72..837ffc4b28f 100644
--- a/unsloth/kernels/moe/grouped_gemm/interface.py
+++ b/unsloth/kernels/moe/grouped_gemm/interface.py
@@ -6,6 +6,7 @@ import functools
import inspect
import logging
import os
+import threading
import warnings
from typing import Optional
from dataclasses import asdict
@@ -59,18 +60,62 @@ def _check_tma_support():
_SUPPORTS_TMA = _check_tma_support()
_DEVICE_AUTOTUNERS = {}
+_MODEL_AUTOTUNE_CACHES = {}
+_MODEL_AUTOTUNE_LOCKS = {}
+_DEVICE_MODELS = {}
+
+
+def _device_index(device):
+ return device.index if device.index is not None else torch.cuda.current_device()
+
+
+def _device_model(device):
+ index = _device_index(device)
+ model = _DEVICE_MODELS.get(index)
+ if model is None:
+ try:
+ p = torch.cuda.get_device_properties(index)
+ model = (
+ getattr(p, "name", None),
+ getattr(p, "major", None),
+ getattr(p, "minor", None),
+ getattr(p, "multi_processor_count", None),
+ )
+ except Exception:
+ model = index
+ _DEVICE_MODELS[index] = model
+ return model
def _autotuner_for(kernel, device):
- """Triton's autotune key omits the device, so on several GPUs each card tunes its own copy."""
+ """Triton's autotune key omits the device, so on several GPUs each card runs its own copy. Copies on
+ the same GPU model share one results cache, so identical cards tune once; a run's scratch state
+ (nargs) stays per card, since backward runs one autograd thread per device."""
if torch.cuda.device_count() <= 1 or not hasattr(kernel, "cache"):
return kernel
- key = (id(kernel), device.index)
- if key not in _DEVICE_AUTOTUNERS:
+ key = (id(kernel), _device_index(device))
+ tuner = _DEVICE_AUTOTUNERS.get(key)
+ if tuner is None:
tuner = copy.copy(kernel)
- tuner.cache = {}
+ model_key = (id(kernel), _device_model(device))
+ tuner.cache = _MODEL_AUTOTUNE_CACHES.setdefault(model_key, {})
+ # One tuning per shape per model: a second card waits, then reads the first card's result
+ # (concurrent benchmarking would also skew both timings). A cached launch holds it briefly.
+ lock = _MODEL_AUTOTUNE_LOCKS.setdefault(model_key, threading.Lock())
+ run = getattr(tuner, "run", None)
+ if run is not None:
+
+ def _run_single_flight(*args, _run = run, _lock = lock, **kwargs):
+ with _lock:
+ return _run(*args, **kwargs)
+
+ tuner.run = _run_single_flight
_DEVICE_AUTOTUNERS[key] = tuner
- return _DEVICE_AUTOTUNERS[key]
+ # Callers reassign kernel.configs (benchmarks, tests): follow it like the shared tuner would.
+ configs = getattr(kernel, "configs", None)
+ if configs is not None and getattr(tuner, "configs", None) is not configs:
+ tuner.configs = configs
+ return tuner
_HAS_SET_ALLOCATOR = hasattr(triton, "set_allocator")