Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

MoE grouped GEMM: no lock on cached autotuner launches; read the SM count once per card (#14605)

Daniel Hanb2ba86a51471
diff --git a/tests/test_grouped_gemm_mixed_tma_device.py b/tests/test_grouped_gemm_mixed_tma_device.py
index 01a6c6c5368..c8ab81ceca0 100644
--- a/tests/test_grouped_gemm_mixed_tma_device.py
+++ b/tests/test_grouped_gemm_mixed_tma_device.py
@@ -329,27 +329,51 @@ def test_benchmark_postprocesses_the_per_card_tuner():
assert kernels and all(id(n) in wrapped for n in kernels)
-def test_identical_cards_tune_a_cold_shape_once(monkeypatch):
- # Two cards of one model missing the same shape at once: the second waits for the first's result.
+def test_identical_cards_launch_concurrently(monkeypatch):
+ # Per-device autograd threads launch on two identical cards at once: a cached launch on one card
+ # must not wait on the other card's launch or tuning (no model-wide lock around run).
import threading
- import time
module, _ = _load(monkeypatch, [H100, H100])
- tuned = []
+ barrier = threading.Barrier(2, timeout = 5)
+ finished = []
class Tuner(_Tuner):
def run(self, key = "shape"):
- if key not in self.cache:
- tuned.append(key)
- time.sleep(0.2)
- self.cache[key] = "config"
- return self.cache[key]
+ barrier.wait() # both cards inside run at the same time, or BrokenBarrierError
+ finished.append(self.cache.get(key))
+ return self.cache.get(key)
base = Tuner()
tuners = [module._autotuner_for(base, torch.device("cuda", i)) for i in (0, 1)]
- threads = [threading.Thread(target = t.run) for t in tuners]
+ tuners[0].cache["shape"] = "h100-config" # warm: shared by both identical cards
+ errors = []
+
+ def launch(tuner):
+ try:
+ tuner.run()
+ except threading.BrokenBarrierError as e:
+ errors.append(e)
+
+ threads = [threading.Thread(target = launch, args = (t,)) for t in tuners]
for t in threads:
t.start()
for t in threads:
- t.join(5)
- assert tuned == ["shape"]
+ t.join(10)
+ assert not errors and finished == ["h100-config", "h100-config"]
+
+
+def test_sm_count_is_read_once_per_card(monkeypatch):
+ module, _ = _load(monkeypatch, [H100, A100])
+ reads = []
+ real = module.torch.cuda.get_device_properties
+
+ def counting(device = None):
+ reads.append(device)
+ return real(device)
+
+ monkeypatch.setattr(module.torch.cuda, "get_device_properties", counting)
+ for _ in range(3):
+ assert module._num_sms(torch.device("cuda", 1)) == 4
+ assert module._num_sms(torch.device("cuda", 0)) == 4
+ assert sorted(reads) == [0, 1]
diff --git a/unsloth/kernels/moe/grouped_gemm/interface.py b/unsloth/kernels/moe/grouped_gemm/interface.py
index f310c9201a5..a444b578ecb 100644
--- a/unsloth/kernels/moe/grouped_gemm/interface.py
+++ b/unsloth/kernels/moe/grouped_gemm/interface.py
@@ -6,7 +6,6 @@ import functools
import inspect
import logging
import os
-import threading
import warnings
from typing import Optional
from dataclasses import asdict
@@ -63,7 +62,6 @@ _SUPPORTS_TMA = _check_tma_support()
_DEVICE_AUTOTUNERS = {}
_MODEL_AUTOTUNE_CACHES = {}
-_MODEL_AUTOTUNE_LOCKS = {}
_DEVICE_MODELS = {}
@@ -89,10 +87,25 @@ def _device_model(device):
return model
+_DEVICE_SM_COUNTS = {}
+
+
+def _num_sms(device):
+ """multi_processor_count of ``device``, read once per card (it cannot change in a process)."""
+ index = _device_index(device)
+ count = _DEVICE_SM_COUNTS.get(index)
+ if count is None:
+ count = _DEVICE_SM_COUNTS[index] = torch.cuda.get_device_properties(
+ index
+ ).multi_processor_count
+ return count
+
+
def _autotuner_for(kernel, device):
"""Triton's autotune key omits the device, so on several GPUs each card runs its own copy. Copies on
- the same GPU model share one results cache, so identical cards tune once; a run's scratch state
- (nargs) stays per card, since backward runs one autograd thread per device."""
+ the same GPU model share one results cache, so identical cards reuse each other's tuning; a run's
+ scratch state (nargs) stays per card, since backward runs one autograd thread per device. No lock:
+ a cached launch never waits on another card's tuning."""
if torch.cuda.device_count() <= 1 or not hasattr(kernel, "cache"):
return kernel
key = (id(kernel), _device_index(device))
@@ -101,22 +114,6 @@ def _autotuner_for(kernel, device):
tuner = copy.copy(kernel)
model_key = (id(kernel), _device_model(device))
tuner.cache = _MODEL_AUTOTUNE_CACHES.setdefault(model_key, {})
- # One tuning per shape per model: a second card waits, then reads the first card's result
- # (concurrent benchmarking would also skew both timings). A cached launch holds it briefly.
- lock = _MODEL_AUTOTUNE_LOCKS.setdefault(model_key, threading.Lock())
- run = getattr(tuner, "run", None)
- if run is not None:
-
- def _run_single_flight(
- *args,
- _run = run,
- _lock = lock,
- **kwargs,
- ):
- with _lock:
- return _run(*args, **kwargs)
-
- tuner.run = _run_single_flight
_DEVICE_AUTOTUNERS[key] = tuner
# Callers reassign kernel.configs (benchmarks, tests): follow it like the shared tuner would.
configs = getattr(kernel, "configs", None)
@@ -283,7 +280,7 @@ def _grouped_gemm_forward_impl(
y = torch.empty((total_tokens, N), device = X.device, dtype = X.dtype)
- NUM_SMS = torch.cuda.get_device_properties(X.device).multi_processor_count
+ NUM_SMS = _num_sms(X.device)
def grid(META):
return (NUM_SMS,)
@@ -430,7 +427,7 @@ def _grouped_gemm_dX_impl(
output_shape = (total_tokens, K)
dX = torch.zeros(output_shape, device = dY.device, dtype = dY.dtype)
- NUM_SMS = torch.cuda.get_device_properties(dY.device).multi_processor_count
+ NUM_SMS = _num_sms(dY.device)
def grid(META):
return (NUM_SMS,)
@@ -532,7 +529,7 @@ def _launch_grouped_gemm_dW(
"""
assert not fuse_mul_pre, "fuse_mul_pre not supported"
assert not fuse_mul_post, "fuse_mul_post not supported"
- NUM_SMS = torch.cuda.get_device_properties(X.device).multi_processor_count if not debug else 1
+ NUM_SMS = _num_sms(X.device) if not debug else 1
X = X.view(-1, X.shape[-1]).contiguous()
dY = dY.contiguous()
m_sizes = m_sizes.contiguous()