Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

MoE grouped GEMM autotune: key, tune and fit tiles on the experts' own GPU (#14719)

Daniel Hanbd35fcc889c4
diff --git a/tests/test_moe_autotune_cache_hit.py b/tests/test_moe_autotune_cache_hit.py
index 760341f83af..c8c7e6b216d 100644
--- a/tests/test_moe_autotune_cache_hit.py
+++ b/tests/test_moe_autotune_cache_hit.py
@@ -46,6 +46,9 @@ def ac(monkeypatch, tmp_path):
module = importlib.import_module("unsloth.kernels.moe.autotune_cache")
# Ample SMEM so the load-time refit keeps tuned tiles as saved.
monkeypatch.setattr(module, "_device_smem_per_block", lambda: 1 << 20)
+ # One mocked H100: the cache key carries the GPU model, so pin it rather than read the host's card.
+ monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)
+ monkeypatch.setattr(module, "_device_model", lambda index: H100_MODEL)
module.clear_cache()
yield module
module.clear_cache()
@@ -54,6 +57,7 @@ def ac(monkeypatch, tmp_path):
sys.modules.pop(name, None)
+H100_MODEL = ("NVIDIA H100 80GB HBM3", 9, 0, 132)
DIMS = dict(num_experts = 128, hidden_dim = 2048, intermediate_dim = 768, top_k = 8)
@@ -79,7 +83,9 @@ def _tuned_configs():
def _key(ac, capability = (9, 0)):
- return ac._get_cache_key(dtype = torch.bfloat16, device_capability = capability, **DIMS)
+ return ac._get_cache_key(
+ dtype = torch.bfloat16, device_capability = capability, device_model = H100_MODEL, **DIMS
+ )
def _no_autotune(*args, **kwargs):
diff --git a/tests/test_moe_autotune_cache_validate.py b/tests/test_moe_autotune_cache_validate.py
index 621718ce413..c6383ceb589 100644
--- a/tests/test_moe_autotune_cache_validate.py
+++ b/tests/test_moe_autotune_cache_validate.py
@@ -48,6 +48,9 @@ def ac(monkeypatch, tmp_path):
monkeypatch.setattr(torch.version, "hip", None)
module = importlib.import_module("unsloth.kernels.moe.autotune_cache")
monkeypatch.setattr(module, "_device_smem_per_block", lambda: SM86_SMEM)
+ # No readable GPU model, so the keys these tests write are the ones get_or_autotune_moe_kernels reads.
+ monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)
+ monkeypatch.setattr(module, "_device_model", lambda index: None)
module.clear_cache()
yield module
module.clear_cache()
diff --git a/tests/test_moe_autotune_operand_device.py b/tests/test_moe_autotune_operand_device.py
index 5c2d2cca404..5ccb2da7795 100644
--- a/tests/test_moe_autotune_operand_device.py
+++ b/tests/test_moe_autotune_operand_device.py
@@ -4,11 +4,15 @@
"""MoE kernel configs follow the operands' GPU, not the current one (H100 + L4 in one process).
The launcher enters X.device, so configs picked from the current card's capability, shared memory
-or disk cache entry could exceed the operand card's limit. CPU only: device properties are mocked."""
+or disk cache entry could exceed the operand card's limit. The cache key also carries the GPU model
+(name, capability, SM count), so two models sharing a capability (L40S + RTX 4090) tune separately,
+and a pre-model-key disk entry is never reused (nothing records which model wrote it). CPU only: device properties
+are mocked."""
import importlib
import json
import sys
+import time
import types
from pathlib import Path
@@ -21,8 +25,8 @@ pytest.importorskip("triton")
MOE_DIR = ROOT / "unsloth" / "kernels" / "moe"
-# index -> (capability, shared_memory_per_block_optin)
-H100, L4 = ((9, 0), 232448), ((8, 9), 101376)
+# index -> (capability, shared_memory_per_block_optin[, name, SM count])
+H100, L4 = ((9, 0), 232448, "NVIDIA H100", 132), ((8, 9), 101376, "NVIDIA L4", 58)
F32, BF16, F16 = torch.float32, torch.bfloat16, torch.float16
DTYPES = [(BF16, None), (F16, None), (F32, BF16), (F32, F16)]
SHAPES = [(2048, 768), (2048, 1408), (1024, 352), (768, 2048)]
@@ -69,10 +73,27 @@ def _gpus(
monkeypatch.setattr(torch.cuda, "device_count", lambda: len(cards))
monkeypatch.setattr(torch.cuda, "current_device", lambda: current)
monkeypatch.setattr(torch.cuda, "get_device_capability", lambda d = None: cards[index(d)][0])
- monkeypatch.setattr(
- torch.cuda,
- "get_device_properties",
- lambda d = None: types.SimpleNamespace(shared_memory_per_block_optin = cards[index(d)][1]),
+ reads = []
+
+ def properties(d = None):
+ card = cards[index(d)]
+ reads.append(index(d))
+ return types.SimpleNamespace(
+ shared_memory_per_block_optin = card[1],
+ name = card[2] if len(card) > 2 else f"GPU sm{card[0][0]}{card[0][1]}",
+ major = card[0][0],
+ minor = card[0][1],
+ multi_processor_count = card[3] if len(card) > 3 else 80,
+ )
+
+ monkeypatch.setattr(torch.cuda, "get_device_properties", properties)
+ return reads
+
+
+def _key(ac, card, index):
+ # The key get_or_autotune_moe_kernels writes for 8 x 2048 x 768 top-2 bf16 on card ``index``.
+ return ac._get_cache_key(
+ 8, 2048, 768, 2, BF16, card[0], 8192, None, device_model = ac._device_model(index)
)
@@ -151,7 +172,7 @@ def test_cache_lookup_and_tuning_follow_the_operand_card(ac, monkeypatch):
# The H100 entry is not reused for the L4 operand; tuning runs on (and is saved for) the L4.
assert tuned_on == [operand]
assert max(_smem(ac, configs, BF16, None, L4)) <= L4[1]
- l4_key = ac._get_cache_key(8, 2048, 768, 2, BF16, L4[0], 8192, None)
+ l4_key = _key(ac, L4, 1)
saved = json.loads(Path(ac._get_cache_file_path(l4_key)).read_text())
assert tuple(saved["device_capability"]) == L4[0]
# Both entries survive: the capability check reads the operand card, not the current one.
@@ -162,7 +183,7 @@ def test_cache_lookup_and_tuning_follow_the_operand_card(ac, monkeypatch):
def test_disk_hit_refits_to_the_operand_card(ac, monkeypatch):
_gpus(monkeypatch, [H100, L4], current = 0)
operand = torch.device("cuda", 1)
- l4_key = ac._get_cache_key(8, 2048, 768, 2, BF16, L4[0], 8192, None)
+ l4_key = _key(ac, L4, 1)
ac.save_cached_config(l4_key, *_big_configs(), device = operand)
monkeypatch.setattr(ac, "_run_moe_autotuning", lambda *a, **k: pytest.fail("cache miss"))
configs = ac.get_or_autotune_moe_kernels(
@@ -191,3 +212,116 @@ def test_single_gpu_configs_unchanged(ac, monkeypatch, card):
assert _dicts(fn(device = torch.device("cuda", 0), **kw)) == base
assert _dicts(fn(device = torch.device("cuda"), **kw)) == base
assert _dicts(fn(device = 0, **kw)) == base
+
+
+DIMS = dict(num_experts = 8, hidden_dim = 2048, intermediate_dim = 768, top_k = 2, dtype = BF16)
+
+
+def _record_tuning(ac, monkeypatch):
+ seen = []
+
+ def tune(
+ *args,
+ device = None,
+ **kwargs,
+ ):
+ seen.append(None if device is None else ac._cuda_index(device))
+ return ac._get_heuristic_configs(n_dim = 1536, k_dim = 2048, dtype = BF16, device = device)
+
+ monkeypatch.setattr(ac, "_run_moe_autotuning", tune)
+ monkeypatch.setattr(ac, "save_cached_config", lambda *a, **k: None)
+ return seen
+
+
+def test_each_card_tunes_and_keys_for_itself(ac, monkeypatch):
+ _gpus(monkeypatch, [H100, L4], current = 0)
+ seen = _record_tuning(ac, monkeypatch)
+ ac.get_or_autotune_moe_kernels(**DIMS, device = torch.device("cuda", 1))
+ ac.get_or_autotune_moe_kernels(**DIMS, device = torch.device("cuda", 0))
+ assert seen == [1, 0]
+
+
+def test_no_device_or_the_current_card_shares_the_current_entry(ac, monkeypatch):
+ _gpus(monkeypatch, [H100, L4], current = 0)
+ seen = _record_tuning(ac, monkeypatch)
+ first = ac.get_or_autotune_moe_kernels(**DIMS)
+ second = ac.get_or_autotune_moe_kernels(**DIMS, device = torch.device("cuda", 0))
+ assert seen == [None] and first is second
+
+
+def test_cpu_or_unindexed_devices_keep_the_current_path(ac, monkeypatch):
+ _gpus(monkeypatch, [H100, L4], current = 0)
+ seen = _record_tuning(ac, monkeypatch)
+ first = ac.get_or_autotune_moe_kernels(**DIMS, device = torch.device("cpu"))
+ assert ac.get_or_autotune_moe_kernels(**DIMS, device = "cuda") is first
+ assert seen == [None]
+
+
+L40S = ((8, 9), 101376, "NVIDIA L40S", 142)
+RTX_4090 = ((8, 9), 101376, "NVIDIA GeForce RTX 4090", 128)
+
+
+def test_two_models_with_one_capability_tune_separately(ac, monkeypatch):
+ _gpus(monkeypatch, [L40S, RTX_4090], current = 0)
+ seen = _record_tuning(ac, monkeypatch)
+ ac.get_or_autotune_moe_kernels(**DIMS, device = torch.device("cuda", 0))
+ ac.get_or_autotune_moe_kernels(**DIMS, device = torch.device("cuda", 1))
+ assert seen == [0, 1]
+
+
+def test_identical_cards_share_one_entry_without_repeated_property_reads(ac, monkeypatch):
+ reads = _gpus(monkeypatch, [L40S, L40S], current = 0)
+ seen = _record_tuning(ac, monkeypatch)
+ first = ac.get_or_autotune_moe_kernels(**DIMS, device = torch.device("cuda", 0))
+ second = ac.get_or_autotune_moe_kernels(**DIMS, device = torch.device("cuda", 1))
+ assert seen == [0] and first is second
+ before = list(reads)
+ for index in (0, 1, 0, 1):
+ ac.get_or_autotune_moe_kernels(**DIMS, device = torch.device("cuda", index))
+ assert reads == before and seen == [0]
+
+
+def _write_legacy_entry(ac, card):
+ fwd, dx, dw = ac._get_default_configs(n_dim = 1536, k_dim = 2048, dtype = BF16)
+ legacy = ac._get_cache_key(8, 2048, 768, 2, BF16, card[0], 8192, None)
+ Path(ac._get_cache_file_path(legacy)).write_text(
+ json.dumps(
+ {
+ "timestamp": time.time(),
+ "device_capability": list(card[0]),
+ "schema_version": ac._CACHE_SCHEMA_VERSION,
+ "triton_version": getattr(ac.triton, "__version__", None),
+ "config_fwd": fwd.__dict__,
+ "config_bwd_dx": dx.__dict__,
+ "config_bwd_dw": dw.__dict__,
+ }
+ ),
+ encoding = "utf-8",
+ )
+
+
+def test_a_pre_model_key_entry_is_retuned_once_and_saved_under_the_model_key(ac, monkeypatch):
+ # Every visible card matching cannot prove which model wrote the entry (shared ~/.cache, swapped card).
+ _gpus(monkeypatch, [L40S, L40S], current = 0)
+ real_save = ac.save_cached_config
+ seen = _record_tuning(ac, monkeypatch)
+ monkeypatch.setattr(ac, "save_cached_config", real_save)
+ _write_legacy_entry(ac, L40S)
+ ac.get_or_autotune_moe_kernels(**DIMS)
+ ac.get_or_autotune_moe_kernels(**DIMS, device = torch.device("cuda", 1))
+ assert seen == [None]
+ assert Path(ac._get_cache_file_path(_key(ac, L40S, 0))).exists()
+ # A fresh process loads the model-keyed entry instead of retuning or reading the legacy file.
+ ac._kernel_config_cache.clear()
+ ac._autotune_completed.clear()
+ ac.get_or_autotune_moe_kernels(**DIMS, device = torch.device("cuda", 1))
+ assert seen == [None]
+
+
+def test_a_pre_model_key_entry_is_not_trusted_on_a_mixed_model_host(ac, monkeypatch):
+ # Same capability, different models: the old key cannot say which card wrote it.
+ _gpus(monkeypatch, [L40S, RTX_4090], current = 0)
+ seen = _record_tuning(ac, monkeypatch)
+ _write_legacy_entry(ac, L40S)
+ ac.get_or_autotune_moe_kernels(**DIMS, device = torch.device("cuda", 1))
+ assert seen == [1]
diff --git a/unsloth/kernels/moe/autotune_cache.py b/unsloth/kernels/moe/autotune_cache.py
index f992b7f5de3..f534a401d0c 100644
--- a/unsloth/kernels/moe/autotune_cache.py
+++ b/unsloth/kernels/moe/autotune_cache.py
@@ -42,8 +42,9 @@ def _get_cache_key(
device_capability: Tuple[int, int],
seq_len: int = 8192, # Default sequence length for tuning
weight_dtype: Optional[torch.dtype] = None,
+ device_model: Optional[Tuple[Any, ...]] = None,
) -> str:
- """Generate a unique cache key based on model configuration."""
+ """Cache key for this MoE shape on this GPU model; ``device_model=None`` gives the pre-model key."""
key_data = {
"num_experts": num_experts,
"hidden_dim": hidden_dim,
@@ -57,6 +58,8 @@ def _get_cache_key(
}
if weight_dtype is not None and weight_dtype != dtype:
key_data["weight_dtype"] = str(weight_dtype)
+ if device_model is not None:
+ key_data["device_model"] = list(device_model)
key_str = json.dumps(key_data, sort_keys = True)
return hashlib.md5(key_str.encode()).hexdigest()
@@ -172,14 +175,17 @@ def get_or_autotune_moe_kernels(
seq_len: Sequence length to use for tuning benchmarks
device: The operands' CUDA device (default: current). On a mixed-GPU host the configs
must fit the card the kernels launch on, so capability, shared memory, the cache key
- and tuning all follow it.
+ (which also carries the GPU model) and tuning all follow it. A non-CUDA device keeps
+ the current device.
Returns:
Tuple of (config_fwd, config_bwd_dx, config_bwd_dw)
"""
+ device = _operand_cuda_device(device)
device_capability = torch.cuda.get_device_capability(device)
dev_kw = _device_kwargs(device)
- cache_key = _get_cache_key(
+ device_model = _device_model(device)
+ key_args = (
num_experts,
hidden_dim,
intermediate_dim,
@@ -189,6 +195,7 @@ def get_or_autotune_moe_kernels(
seq_len,
weight_dtype,
)
+ cache_key = _get_cache_key(*key_args, device_model = device_model)
# Env override to disable autotuning
if os.environ.get("UNSLOTH_MOE_DISABLE_AUTOTUNE", "0") == "1":
@@ -207,6 +214,7 @@ def get_or_autotune_moe_kernels(
return _kernel_config_cache[cache_key]
if not force_autotune:
+ # Pre-model-key entries are never read: nothing records which GPU model tuned them.
cached_data = load_cached_config(cache_key, **dev_kw)
if cached_data is not None:
try:
@@ -280,6 +288,40 @@ def get_or_autotune_moe_kernels(
)
+def _operand_cuda_device(device):
+ """``device`` when it names a CUDA card (torch.device, str or int), else None (current device)."""
+ if device is None or isinstance(device, int):
+ return device
+ try:
+ return device if torch.device(device).type == "cuda" else None
+ except (TypeError, RuntimeError):
+ return None
+
+
+_DEVICE_MODELS: Dict[int, Optional[Tuple[Any, ...]]] = {}
+
+
+def _device_model(device = None) -> Optional[Tuple[Any, ...]]:
+ """(name, major, minor, SM count) of the card, as grouped_gemm.interface keys its caches; None if unreadable."""
+ try:
+ index = torch.cuda.current_device() if device is None else _cuda_index(device)
+ except Exception:
+ return None
+ if index not in _DEVICE_MODELS:
+ try:
+ p = torch.cuda.get_device_properties(index)
+ model = (
+ getattr(p, "name", None),
+ getattr(p, "major", None),
+ getattr(p, "minor", None),
+ getattr(p, "multi_processor_count", None),
+ )
+ except Exception:
+ model = None
+ _DEVICE_MODELS[index] = model
+ return _DEVICE_MODELS[index]
+
+
def _run_moe_autotuning(
num_experts: int,
hidden_dim: int,
@@ -742,6 +784,7 @@ def clear_cache() -> None:
"""Clear all cached kernel configurations."""
global _kernel_config_cache, _autotune_completed
_kernel_config_cache.clear()
+ _DEVICE_MODELS.clear()
_autotune_completed.clear()
logger.info("Cleared MoE kernel cache")