Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
MoE grouped GEMM: no lock on cached autotuner launches; read the SM count once per card (#14605)
Daniel Hanb2ba86a51471
diff --git a/tests/test_grouped_gemm_mixed_tma_device.py b/tests/test_grouped_gemm_mixed_tma_device.pyindex 01a6c6c5368..c8ab81ceca0 100644--- a/tests/test_grouped_gemm_mixed_tma_device.py+++ b/tests/test_grouped_gemm_mixed_tma_device.py@@ -329,27 +329,51 @@ def test_benchmark_postprocesses_the_per_card_tuner():assert kernels and all(id(n) in wrapped for n in kernels)-def test_identical_cards_tune_a_cold_shape_once(monkeypatch):- # Two cards of one model missing the same shape at once: the second waits for the first's result.+def test_identical_cards_launch_concurrently(monkeypatch):+ # Per-device autograd threads launch on two identical cards at once: a cached launch on one card+ # must not wait on the other card's launch or tuning (no model-wide lock around run).import threading- import timemodule, _ = _load(monkeypatch, [H100, H100])- tuned = []+ barrier = threading.Barrier(2, timeout = 5)+ finished = []class Tuner(_Tuner):def run(self, key = "shape"):- if key not in self.cache:- tuned.append(key)- time.sleep(0.2)- self.cache[key] = "config"- return self.cache[key]+ barrier.wait() # both cards inside run at the same time, or BrokenBarrierError+ finished.append(self.cache.get(key))+ return self.cache.get(key)base = Tuner()tuners = [module._autotuner_for(base, torch.device("cuda", i)) for i in (0, 1)]- threads = [threading.Thread(target = t.run) for t in tuners]+ tuners[0].cache["shape"] = "h100-config" # warm: shared by both identical cards+ errors = []++ def launch(tuner):+ try:+ tuner.run()+ except threading.BrokenBarrierError as e:+ errors.append(e)++ threads = [threading.Thread(target = launch, args = (t,)) for t in tuners]for t in threads:t.start()for t in threads:- t.join(5)- assert tuned == ["shape"]+ t.join(10)+ assert not errors and finished == ["h100-config", "h100-config"]+++def test_sm_count_is_read_once_per_card(monkeypatch):+ module, _ = _load(monkeypatch, [H100, A100])+ reads = []+ real = module.torch.cuda.get_device_properties++ def counting(device = None):+ reads.append(device)+ return real(device)++ monkeypatch.setattr(module.torch.cuda, "get_device_properties", counting)+ for _ in range(3):+ assert module._num_sms(torch.device("cuda", 1)) == 4+ assert module._num_sms(torch.device("cuda", 0)) == 4+ assert sorted(reads) == [0, 1]diff --git a/unsloth/kernels/moe/grouped_gemm/interface.py b/unsloth/kernels/moe/grouped_gemm/interface.pyindex f310c9201a5..a444b578ecb 100644--- a/unsloth/kernels/moe/grouped_gemm/interface.py+++ b/unsloth/kernels/moe/grouped_gemm/interface.py@@ -6,7 +6,6 @@ import functoolsimport inspectimport loggingimport os-import threadingimport warningsfrom typing import Optionalfrom dataclasses import asdict@@ -63,7 +62,6 @@ _SUPPORTS_TMA = _check_tma_support()_DEVICE_AUTOTUNERS = {}_MODEL_AUTOTUNE_CACHES = {}-_MODEL_AUTOTUNE_LOCKS = {}_DEVICE_MODELS = {}@@ -89,10 +87,25 @@ def _device_model(device):return model+_DEVICE_SM_COUNTS = {}+++def _num_sms(device):+ """multi_processor_count of ``device``, read once per card (it cannot change in a process)."""+ index = _device_index(device)+ count = _DEVICE_SM_COUNTS.get(index)+ if count is None:+ count = _DEVICE_SM_COUNTS[index] = torch.cuda.get_device_properties(+ index+ ).multi_processor_count+ return count++def _autotuner_for(kernel, device):"""Triton's autotune key omits the device, so on several GPUs each card runs its own copy. Copies on- the same GPU model share one results cache, so identical cards tune once; a run's scratch state- (nargs) stays per card, since backward runs one autograd thread per device."""+ the same GPU model share one results cache, so identical cards reuse each other's tuning; a run's+ scratch state (nargs) stays per card, since backward runs one autograd thread per device. No lock:+ a cached launch never waits on another card's tuning."""if torch.cuda.device_count() <= 1 or not hasattr(kernel, "cache"):return kernelkey = (id(kernel), _device_index(device))@@ -101,22 +114,6 @@ def _autotuner_for(kernel, device):tuner = copy.copy(kernel)model_key = (id(kernel), _device_model(device))tuner.cache = _MODEL_AUTOTUNE_CACHES.setdefault(model_key, {})- # One tuning per shape per model: a second card waits, then reads the first card's result- # (concurrent benchmarking would also skew both timings). A cached launch holds it briefly.- lock = _MODEL_AUTOTUNE_LOCKS.setdefault(model_key, threading.Lock())- run = getattr(tuner, "run", None)- if run is not None:-- def _run_single_flight(- *args,- _run = run,- _lock = lock,- **kwargs,- ):- with _lock:- return _run(*args, **kwargs)-- tuner.run = _run_single_flight_DEVICE_AUTOTUNERS[key] = tuner# Callers reassign kernel.configs (benchmarks, tests): follow it like the shared tuner would.configs = getattr(kernel, "configs", None)@@ -283,7 +280,7 @@ def _grouped_gemm_forward_impl(y = torch.empty((total_tokens, N), device = X.device, dtype = X.dtype)- NUM_SMS = torch.cuda.get_device_properties(X.device).multi_processor_count+ NUM_SMS = _num_sms(X.device)def grid(META):return (NUM_SMS,)@@ -430,7 +427,7 @@ def _grouped_gemm_dX_impl(output_shape = (total_tokens, K)dX = torch.zeros(output_shape, device = dY.device, dtype = dY.dtype)- NUM_SMS = torch.cuda.get_device_properties(dY.device).multi_processor_count+ NUM_SMS = _num_sms(dY.device)def grid(META):return (NUM_SMS,)@@ -532,7 +529,7 @@ def _launch_grouped_gemm_dW("""assert not fuse_mul_pre, "fuse_mul_pre not supported"assert not fuse_mul_post, "fuse_mul_post not supported"- NUM_SMS = torch.cuda.get_device_properties(X.device).multi_processor_count if not debug else 1+ NUM_SMS = _num_sms(X.device) if not debug else 1X = X.view(-1, X.shape[-1]).contiguous()dY = dY.contiguous()m_sizes = m_sizes.contiguous()