Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
MoE grouped GEMM autotune: key, tune and fit tiles on the experts' own GPU (#14719)
Daniel Hanbd35fcc889c4
diff --git a/tests/test_moe_autotune_cache_hit.py b/tests/test_moe_autotune_cache_hit.pyindex 760341f83af..c8c7e6b216d 100644--- a/tests/test_moe_autotune_cache_hit.py+++ b/tests/test_moe_autotune_cache_hit.py@@ -46,6 +46,9 @@ def ac(monkeypatch, tmp_path):module = importlib.import_module("unsloth.kernels.moe.autotune_cache")# Ample SMEM so the load-time refit keeps tuned tiles as saved.monkeypatch.setattr(module, "_device_smem_per_block", lambda: 1 << 20)+ # One mocked H100: the cache key carries the GPU model, so pin it rather than read the host's card.+ monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)+ monkeypatch.setattr(module, "_device_model", lambda index: H100_MODEL)module.clear_cache()yield modulemodule.clear_cache()@@ -54,6 +57,7 @@ def ac(monkeypatch, tmp_path):sys.modules.pop(name, None)+H100_MODEL = ("NVIDIA H100 80GB HBM3", 9, 0, 132)DIMS = dict(num_experts = 128, hidden_dim = 2048, intermediate_dim = 768, top_k = 8)@@ -79,7 +83,9 @@ def _tuned_configs():def _key(ac, capability = (9, 0)):- return ac._get_cache_key(dtype = torch.bfloat16, device_capability = capability, **DIMS)+ return ac._get_cache_key(+ dtype = torch.bfloat16, device_capability = capability, device_model = H100_MODEL, **DIMS+ )def _no_autotune(*args, **kwargs):diff --git a/tests/test_moe_autotune_cache_validate.py b/tests/test_moe_autotune_cache_validate.pyindex 621718ce413..c6383ceb589 100644--- a/tests/test_moe_autotune_cache_validate.py+++ b/tests/test_moe_autotune_cache_validate.py@@ -48,6 +48,9 @@ def ac(monkeypatch, tmp_path):monkeypatch.setattr(torch.version, "hip", None)module = importlib.import_module("unsloth.kernels.moe.autotune_cache")monkeypatch.setattr(module, "_device_smem_per_block", lambda: SM86_SMEM)+ # No readable GPU model, so the keys these tests write are the ones get_or_autotune_moe_kernels reads.+ monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)+ monkeypatch.setattr(module, "_device_model", lambda index: None)module.clear_cache()yield modulemodule.clear_cache()diff --git a/tests/test_moe_autotune_operand_device.py b/tests/test_moe_autotune_operand_device.pyindex 5c2d2cca404..5ccb2da7795 100644--- a/tests/test_moe_autotune_operand_device.py+++ b/tests/test_moe_autotune_operand_device.py@@ -4,11 +4,15 @@"""MoE kernel configs follow the operands' GPU, not the current one (H100 + L4 in one process).The launcher enters X.device, so configs picked from the current card's capability, shared memory-or disk cache entry could exceed the operand card's limit. CPU only: device properties are mocked."""+or disk cache entry could exceed the operand card's limit. The cache key also carries the GPU model+(name, capability, SM count), so two models sharing a capability (L40S + RTX 4090) tune separately,+and a pre-model-key disk entry is never reused (nothing records which model wrote it). CPU only: device properties+are mocked."""import importlibimport jsonimport sys+import timeimport typesfrom pathlib import Path@@ -21,8 +25,8 @@ pytest.importorskip("triton")MOE_DIR = ROOT / "unsloth" / "kernels" / "moe"-# index -> (capability, shared_memory_per_block_optin)-H100, L4 = ((9, 0), 232448), ((8, 9), 101376)+# index -> (capability, shared_memory_per_block_optin[, name, SM count])+H100, L4 = ((9, 0), 232448, "NVIDIA H100", 132), ((8, 9), 101376, "NVIDIA L4", 58)F32, BF16, F16 = torch.float32, torch.bfloat16, torch.float16DTYPES = [(BF16, None), (F16, None), (F32, BF16), (F32, F16)]SHAPES = [(2048, 768), (2048, 1408), (1024, 352), (768, 2048)]@@ -69,10 +73,27 @@ def _gpus(monkeypatch.setattr(torch.cuda, "device_count", lambda: len(cards))monkeypatch.setattr(torch.cuda, "current_device", lambda: current)monkeypatch.setattr(torch.cuda, "get_device_capability", lambda d = None: cards[index(d)][0])- monkeypatch.setattr(- torch.cuda,- "get_device_properties",- lambda d = None: types.SimpleNamespace(shared_memory_per_block_optin = cards[index(d)][1]),+ reads = []++ def properties(d = None):+ card = cards[index(d)]+ reads.append(index(d))+ return types.SimpleNamespace(+ shared_memory_per_block_optin = card[1],+ name = card[2] if len(card) > 2 else f"GPU sm{card[0][0]}{card[0][1]}",+ major = card[0][0],+ minor = card[0][1],+ multi_processor_count = card[3] if len(card) > 3 else 80,+ )++ monkeypatch.setattr(torch.cuda, "get_device_properties", properties)+ return reads+++def _key(ac, card, index):+ # The key get_or_autotune_moe_kernels writes for 8 x 2048 x 768 top-2 bf16 on card ``index``.+ return ac._get_cache_key(+ 8, 2048, 768, 2, BF16, card[0], 8192, None, device_model = ac._device_model(index))@@ -151,7 +172,7 @@ def test_cache_lookup_and_tuning_follow_the_operand_card(ac, monkeypatch):# The H100 entry is not reused for the L4 operand; tuning runs on (and is saved for) the L4.assert tuned_on == [operand]assert max(_smem(ac, configs, BF16, None, L4)) <= L4[1]- l4_key = ac._get_cache_key(8, 2048, 768, 2, BF16, L4[0], 8192, None)+ l4_key = _key(ac, L4, 1)saved = json.loads(Path(ac._get_cache_file_path(l4_key)).read_text())assert tuple(saved["device_capability"]) == L4[0]# Both entries survive: the capability check reads the operand card, not the current one.@@ -162,7 +183,7 @@ def test_cache_lookup_and_tuning_follow_the_operand_card(ac, monkeypatch):def test_disk_hit_refits_to_the_operand_card(ac, monkeypatch):_gpus(monkeypatch, [H100, L4], current = 0)operand = torch.device("cuda", 1)- l4_key = ac._get_cache_key(8, 2048, 768, 2, BF16, L4[0], 8192, None)+ l4_key = _key(ac, L4, 1)ac.save_cached_config(l4_key, *_big_configs(), device = operand)monkeypatch.setattr(ac, "_run_moe_autotuning", lambda *a, **k: pytest.fail("cache miss"))configs = ac.get_or_autotune_moe_kernels(@@ -191,3 +212,116 @@ def test_single_gpu_configs_unchanged(ac, monkeypatch, card):assert _dicts(fn(device = torch.device("cuda", 0), **kw)) == baseassert _dicts(fn(device = torch.device("cuda"), **kw)) == baseassert _dicts(fn(device = 0, **kw)) == base+++DIMS = dict(num_experts = 8, hidden_dim = 2048, intermediate_dim = 768, top_k = 2, dtype = BF16)+++def _record_tuning(ac, monkeypatch):+ seen = []++ def tune(+ *args,+ device = None,+ **kwargs,+ ):+ seen.append(None if device is None else ac._cuda_index(device))+ return ac._get_heuristic_configs(n_dim = 1536, k_dim = 2048, dtype = BF16, device = device)++ monkeypatch.setattr(ac, "_run_moe_autotuning", tune)+ monkeypatch.setattr(ac, "save_cached_config", lambda *a, **k: None)+ return seen+++def test_each_card_tunes_and_keys_for_itself(ac, monkeypatch):+ _gpus(monkeypatch, [H100, L4], current = 0)+ seen = _record_tuning(ac, monkeypatch)+ ac.get_or_autotune_moe_kernels(**DIMS, device = torch.device("cuda", 1))+ ac.get_or_autotune_moe_kernels(**DIMS, device = torch.device("cuda", 0))+ assert seen == [1, 0]+++def test_no_device_or_the_current_card_shares_the_current_entry(ac, monkeypatch):+ _gpus(monkeypatch, [H100, L4], current = 0)+ seen = _record_tuning(ac, monkeypatch)+ first = ac.get_or_autotune_moe_kernels(**DIMS)+ second = ac.get_or_autotune_moe_kernels(**DIMS, device = torch.device("cuda", 0))+ assert seen == [None] and first is second+++def test_cpu_or_unindexed_devices_keep_the_current_path(ac, monkeypatch):+ _gpus(monkeypatch, [H100, L4], current = 0)+ seen = _record_tuning(ac, monkeypatch)+ first = ac.get_or_autotune_moe_kernels(**DIMS, device = torch.device("cpu"))+ assert ac.get_or_autotune_moe_kernels(**DIMS, device = "cuda") is first+ assert seen == [None]+++L40S = ((8, 9), 101376, "NVIDIA L40S", 142)+RTX_4090 = ((8, 9), 101376, "NVIDIA GeForce RTX 4090", 128)+++def test_two_models_with_one_capability_tune_separately(ac, monkeypatch):+ _gpus(monkeypatch, [L40S, RTX_4090], current = 0)+ seen = _record_tuning(ac, monkeypatch)+ ac.get_or_autotune_moe_kernels(**DIMS, device = torch.device("cuda", 0))+ ac.get_or_autotune_moe_kernels(**DIMS, device = torch.device("cuda", 1))+ assert seen == [0, 1]+++def test_identical_cards_share_one_entry_without_repeated_property_reads(ac, monkeypatch):+ reads = _gpus(monkeypatch, [L40S, L40S], current = 0)+ seen = _record_tuning(ac, monkeypatch)+ first = ac.get_or_autotune_moe_kernels(**DIMS, device = torch.device("cuda", 0))+ second = ac.get_or_autotune_moe_kernels(**DIMS, device = torch.device("cuda", 1))+ assert seen == [0] and first is second+ before = list(reads)+ for index in (0, 1, 0, 1):+ ac.get_or_autotune_moe_kernels(**DIMS, device = torch.device("cuda", index))+ assert reads == before and seen == [0]+++def _write_legacy_entry(ac, card):+ fwd, dx, dw = ac._get_default_configs(n_dim = 1536, k_dim = 2048, dtype = BF16)+ legacy = ac._get_cache_key(8, 2048, 768, 2, BF16, card[0], 8192, None)+ Path(ac._get_cache_file_path(legacy)).write_text(+ json.dumps(+ {+ "timestamp": time.time(),+ "device_capability": list(card[0]),+ "schema_version": ac._CACHE_SCHEMA_VERSION,+ "triton_version": getattr(ac.triton, "__version__", None),+ "config_fwd": fwd.__dict__,+ "config_bwd_dx": dx.__dict__,+ "config_bwd_dw": dw.__dict__,+ }+ ),+ encoding = "utf-8",+ )+++def test_a_pre_model_key_entry_is_retuned_once_and_saved_under_the_model_key(ac, monkeypatch):+ # Every visible card matching cannot prove which model wrote the entry (shared ~/.cache, swapped card).+ _gpus(monkeypatch, [L40S, L40S], current = 0)+ real_save = ac.save_cached_config+ seen = _record_tuning(ac, monkeypatch)+ monkeypatch.setattr(ac, "save_cached_config", real_save)+ _write_legacy_entry(ac, L40S)+ ac.get_or_autotune_moe_kernels(**DIMS)+ ac.get_or_autotune_moe_kernels(**DIMS, device = torch.device("cuda", 1))+ assert seen == [None]+ assert Path(ac._get_cache_file_path(_key(ac, L40S, 0))).exists()+ # A fresh process loads the model-keyed entry instead of retuning or reading the legacy file.+ ac._kernel_config_cache.clear()+ ac._autotune_completed.clear()+ ac.get_or_autotune_moe_kernels(**DIMS, device = torch.device("cuda", 1))+ assert seen == [None]+++def test_a_pre_model_key_entry_is_not_trusted_on_a_mixed_model_host(ac, monkeypatch):+ # Same capability, different models: the old key cannot say which card wrote it.+ _gpus(monkeypatch, [L40S, RTX_4090], current = 0)+ seen = _record_tuning(ac, monkeypatch)+ _write_legacy_entry(ac, L40S)+ ac.get_or_autotune_moe_kernels(**DIMS, device = torch.device("cuda", 1))+ assert seen == [1]diff --git a/unsloth/kernels/moe/autotune_cache.py b/unsloth/kernels/moe/autotune_cache.pyindex f992b7f5de3..f534a401d0c 100644--- a/unsloth/kernels/moe/autotune_cache.py+++ b/unsloth/kernels/moe/autotune_cache.py@@ -42,8 +42,9 @@ def _get_cache_key(device_capability: Tuple[int, int],seq_len: int = 8192, # Default sequence length for tuningweight_dtype: Optional[torch.dtype] = None,+ device_model: Optional[Tuple[Any, ...]] = None,) -> str:- """Generate a unique cache key based on model configuration."""+ """Cache key for this MoE shape on this GPU model; ``device_model=None`` gives the pre-model key."""key_data = {"num_experts": num_experts,"hidden_dim": hidden_dim,@@ -57,6 +58,8 @@ def _get_cache_key(}if weight_dtype is not None and weight_dtype != dtype:key_data["weight_dtype"] = str(weight_dtype)+ if device_model is not None:+ key_data["device_model"] = list(device_model)key_str = json.dumps(key_data, sort_keys = True)return hashlib.md5(key_str.encode()).hexdigest()@@ -172,14 +175,17 @@ def get_or_autotune_moe_kernels(seq_len: Sequence length to use for tuning benchmarksdevice: The operands' CUDA device (default: current). On a mixed-GPU host the configsmust fit the card the kernels launch on, so capability, shared memory, the cache key- and tuning all follow it.+ (which also carries the GPU model) and tuning all follow it. A non-CUDA device keeps+ the current device.Returns:Tuple of (config_fwd, config_bwd_dx, config_bwd_dw)"""+ device = _operand_cuda_device(device)device_capability = torch.cuda.get_device_capability(device)dev_kw = _device_kwargs(device)- cache_key = _get_cache_key(+ device_model = _device_model(device)+ key_args = (num_experts,hidden_dim,intermediate_dim,@@ -189,6 +195,7 @@ def get_or_autotune_moe_kernels(seq_len,weight_dtype,)+ cache_key = _get_cache_key(*key_args, device_model = device_model)# Env override to disable autotuningif os.environ.get("UNSLOTH_MOE_DISABLE_AUTOTUNE", "0") == "1":@@ -207,6 +214,7 @@ def get_or_autotune_moe_kernels(return _kernel_config_cache[cache_key]if not force_autotune:+ # Pre-model-key entries are never read: nothing records which GPU model tuned them.cached_data = load_cached_config(cache_key, **dev_kw)if cached_data is not None:try:@@ -280,6 +288,40 @@ def get_or_autotune_moe_kernels()+def _operand_cuda_device(device):+ """``device`` when it names a CUDA card (torch.device, str or int), else None (current device)."""+ if device is None or isinstance(device, int):+ return device+ try:+ return device if torch.device(device).type == "cuda" else None+ except (TypeError, RuntimeError):+ return None+++_DEVICE_MODELS: Dict[int, Optional[Tuple[Any, ...]]] = {}+++def _device_model(device = None) -> Optional[Tuple[Any, ...]]:+ """(name, major, minor, SM count) of the card, as grouped_gemm.interface keys its caches; None if unreadable."""+ try:+ index = torch.cuda.current_device() if device is None else _cuda_index(device)+ except Exception:+ return None+ if index not in _DEVICE_MODELS:+ try:+ p = torch.cuda.get_device_properties(index)+ model = (+ getattr(p, "name", None),+ getattr(p, "major", None),+ getattr(p, "minor", None),+ getattr(p, "multi_processor_count", None),+ )+ except Exception:+ model = None+ _DEVICE_MODELS[index] = model+ return _DEVICE_MODELS[index]++def _run_moe_autotuning(num_experts: int,hidden_dim: int,@@ -742,6 +784,7 @@ def clear_cache() -> None:"""Clear all cached kernel configurations."""global _kernel_config_cache, _autotune_completed_kernel_config_cache.clear()+ _DEVICE_MODELS.clear()_autotune_completed.clear()logger.info("Cleared MoE kernel cache")