Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
MoE autotune cache: version disk entries and refit loaded configs (#14617)
Daniel Han465263ef9210
diff --git a/tests/test_moe_autotune_cache_hit.py b/tests/test_moe_autotune_cache_hit.pyindex b9b4e660d3f..760341f83af 100644--- a/tests/test_moe_autotune_cache_hit.py+++ b/tests/test_moe_autotune_cache_hit.py@@ -44,6 +44,8 @@ def ac(monkeypatch, tmp_path):monkeypatch.delenv("UNSLOTH_MOE_DISABLE_AUTOTUNE", raising = False)monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *a, **k: (9, 0))module = importlib.import_module("unsloth.kernels.moe.autotune_cache")+ # Ample SMEM so the load-time refit keeps tuned tiles as saved.+ monkeypatch.setattr(module, "_device_smem_per_block", lambda: 1 << 20)module.clear_cache()yield modulemodule.clear_cache()diff --git a/tests/test_moe_autotune_cache_validate.py b/tests/test_moe_autotune_cache_validate.pynew file mode 100644index 00000000000..b3df981d187--- /dev/null+++ b/tests/test_moe_autotune_cache_validate.py@@ -0,0 +1,180 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Disk cache hits in the MoE autotune cache are returned without re-tuning. Entries written+before the cache carried a schema / Triton identity (when the loader still discarded them) or+before a kernel change (the per-expert dW kernel needs about 2x the shared memory of the legacy+dW kernel for the same tile) must not be launched unvalidated: unversioned entries are misses,+and every loaded entry is refitted to the tensor dims and the device shared memory."""++import hashlib+import importlib+import json+import os+import sys+import types+from pathlib import Path++import pytest++ROOT = Path(__file__).resolve().parents[1]++torch = pytest.importorskip("torch")+pytest.importorskip("triton")++MOE_DIR = ROOT / "unsloth" / "kernels" / "moe"+SM86_SMEM = 101376+++def ac(monkeypatch, tmp_path):+ # Import autotune_cache without running unsloth/__init__.py (needs a GPU).+ for name, path in (+ ("unsloth", ROOT / "unsloth"),+ ("unsloth.kernels", ROOT / "unsloth" / "kernels"),+ ("unsloth.kernels.moe", MOE_DIR),+ ):+ pkg = types.ModuleType(name)+ pkg.__path__ = [str(path)]+ monkeypatch.setitem(sys.modules, name, pkg)+ for name in list(sys.modules):+ if name.startswith("unsloth.kernels.moe."):+ monkeypatch.delitem(sys.modules, name)+ monkeypatch.setenv("HOME", str(tmp_path))+ monkeypatch.setenv("USERPROFILE", str(tmp_path))+ monkeypatch.delenv("UNSLOTH_MOE_DISABLE_AUTOTUNE", raising = False)+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *a, **k: (8, 6))+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ monkeypatch.setattr(torch.version, "hip", None)+ module = importlib.import_module("unsloth.kernels.moe.autotune_cache")+ monkeypatch.setattr(module, "_device_smem_per_block", lambda: SM86_SMEM)+ module.clear_cache()+ yield module+ module.clear_cache()+ for name in list(sys.modules):+ if name.startswith("unsloth.kernels.moe."):+ sys.modules.pop(name, None)+++DIMS = dict(num_experts = 128, hidden_dim = 2048, intermediate_dim = 768, top_k = 8)+++def _configs(fwd, dx, dw):+ from unsloth.kernels.moe.grouped_gemm.kernels.tuning import (+ KernelConfigBackward_dW,+ KernelConfigBackward_dX,+ KernelConfigForward,+ )++ def kw(t):+ return dict(zip(("BLOCK_SIZE_M", "BLOCK_SIZE_N", "BLOCK_SIZE_K", "num_warps", "num_stages"), t))++ return (+ KernelConfigForward(**kw(fwd)),+ KernelConfigBackward_dX(**kw(dx)),+ KernelConfigBackward_dW(**kw(dw)),+ )+++SMALL = (64, 64, 64, 4, 2) # fits sm86 for every kind and divides every dim used here+++def _tile(c):+ return (c.BLOCK_SIZE_M, c.BLOCK_SIZE_N, c.BLOCK_SIZE_K, c.num_warps, c.num_stages)+++class _Autotune:+ def __init__(self, ac):+ self.calls = 0+ self.ac = ac++ def __call__(self, *args, **kwargs):+ self.calls += 1+ return _configs(SMALL, SMALL, SMALL)+++def _legacy_key(dims, dtype = torch.bfloat16, capability = (8, 6), seq_len = 8192):+ key_data = dict(+ num_experts = dims["num_experts"],+ hidden_dim = dims["hidden_dim"],+ intermediate_dim = dims["intermediate_dim"],+ top_k = dims["top_k"],+ dtype = str(dtype),+ device_capability = capability,+ seq_len = seq_len,+ )+ return hashlib.md5(json.dumps(key_data, sort_keys = True).encode()).hexdigest()+++def _legacy_payload(configs):+ # Exactly what the pre-schema save_cached_config wrote.+ return {+ "timestamp": 0.0,+ "device_capability": [8, 6],+ "config_fwd": configs[0].__dict__,+ "config_bwd_dx": configs[1].__dict__,+ "config_bwd_dw": configs[2].__dict__,+ "metadata": {},+ }+++def test_legacy_unversioned_cache_file_is_ignored(ac, monkeypatch):+ legacy = _configs(SMALL, SMALL, (128, 256, 256, 8, 4))+ payload = _legacy_payload(legacy)+ cache_dir = Path(ac._get_cache_file_path("x")).parent+ legacy_file = cache_dir / f"{_legacy_key(DIMS)}.json"+ current_key = ac._get_cache_key(+ dtype = torch.bfloat16, device_capability = (8, 6), **DIMS+ )+ for path in {legacy_file, Path(ac._get_cache_file_path(current_key))}:+ path.write_text(json.dumps(payload), encoding = "utf-8")++ tuner = _Autotune(ac)+ monkeypatch.setattr(ac, "_run_moe_autotuning", tuner)+ configs = ac.get_or_autotune_moe_kernels(dtype = torch.bfloat16, **DIMS)+ assert tuner.calls == 1, "an unversioned disk entry was reused without re-tuning"+ assert [_tile(c) for c in configs] == [SMALL, SMALL, SMALL]+ assert legacy_file.exists(), "a legacy file under another key must be left alone"+++def test_oversized_dw_tile_from_disk_is_fitted(ac, monkeypatch):+ # Legacy-dW-valid tile: 132,096 B on the per-expert dW kernel vs 101,376 B on sm86.+ key = ac._get_cache_key(dtype = torch.bfloat16, device_capability = (8, 6), **DIMS)+ ac.save_cached_config(key, *_configs(SMALL, SMALL, (128, 256, 256, 8, 4)))+ tuner = _Autotune(ac)+ monkeypatch.setattr(ac, "_run_moe_autotuning", tuner)+ configs = ac.get_or_autotune_moe_kernels(dtype = torch.bfloat16, **DIMS)+ assert tuner.calls == 0+ assert _tile(configs[2]) == (64, 256, 256, 8, 3)+ for kind, config in zip(("fwd", "dX", "dW"), configs):+ assert ac._estimate_smem(kind, config, 2, 2, (8, 6)) <= SM86_SMEM+++def test_block_k_larger_than_dim_from_disk_is_fitted(ac, monkeypatch):+ # N = 2 * 64 = 128, K = 64: a saved BLOCK_SIZE_K = 128 fails tl.static_assert(K % BLOCK_SIZE_K == 0).+ dims = dict(num_experts = 8, hidden_dim = 64, intermediate_dim = 64, top_k = 2)+ key = ac._get_cache_key(dtype = torch.bfloat16, device_capability = (8, 6), **dims)+ tile = (64, 128, 128, 4, 2)+ ac.save_cached_config(key, *_configs(tile, tile, (32, 128, 128, 4, 2)))+ tuner = _Autotune(ac)+ monkeypatch.setattr(ac, "_run_moe_autotuning", tuner)+ configs = ac.get_or_autotune_moe_kernels(dtype = torch.bfloat16, **dims)+ assert tuner.calls == 0+ for config in configs:+ assert 128 % config.BLOCK_SIZE_N == 0+ assert 64 % config.BLOCK_SIZE_K == 0+++def test_valid_current_entry_hits_without_autotune(ac, monkeypatch):+ saved = _configs((64, 128, 64, 4, 3), (64, 64, 128, 4, 3), (64, 64, 64, 8, 2))+ key = ac._get_cache_key(dtype = torch.bfloat16, device_capability = (8, 6), **DIMS)+ ac.save_cached_config(key, *saved)+ tuner = _Autotune(ac)+ monkeypatch.setattr(ac, "_run_moe_autotuning", tuner)+ first = ac.get_or_autotune_moe_kernels(dtype = torch.bfloat16, **DIMS)+ assert tuner.calls == 0+ assert [c.__dict__ for c in first] == [c.__dict__ for c in saved]+ assert os.path.exists(ac._get_cache_file_path(key))+ monkeypatch.setattr(ac, "load_cached_config", lambda *a, **k: pytest.fail("disk re-read"))+ assert ac.get_or_autotune_moe_kernels(dtype = torch.bfloat16, **DIMS) is first+ assert tuner.calls == 0diff --git a/unsloth/kernels/moe/autotune_cache.py b/unsloth/kernels/moe/autotune_cache.pyindex e12a4030a7a..6fb9f289cb2 100644--- a/unsloth/kernels/moe/autotune_cache.py+++ b/unsloth/kernels/moe/autotune_cache.py@@ -29,6 +29,9 @@ logger = logging.getLogger(__name__)_kernel_config_cache: Dict[str, Any] = {}_autotune_completed: Dict[str, bool] = {}+# Bump when a kernel change alters which tiles fit; other revisions / Triton versions are misses.+_CACHE_SCHEMA_VERSION = 2+def _get_cache_key(num_experts: int,@@ -49,6 +52,8 @@ def _get_cache_key("dtype": str(dtype),"device_capability": device_capability,"seq_len": seq_len,+ "schema_version": _CACHE_SCHEMA_VERSION,+ "triton_version": getattr(triton, "__version__", None),}if weight_dtype is not None and weight_dtype != dtype:key_data["weight_dtype"] = str(weight_dtype)@@ -83,6 +88,12 @@ def load_cached_config(cache_key: str) -> Optional[Dict[str, Any]]:os.remove(cache_file)return None+ if cached_data.get("schema_version") != _CACHE_SCHEMA_VERSION or cached_data.get(+ "triton_version"+ ) != getattr(triton, "__version__", None):+ logger.info("MoE kernel cache schema or Triton version changed, ignoring cache")+ return None+logger.info(f"Loaded cached MoE kernel config: {cache_key}")return cached_dataexcept Exception as e:@@ -107,6 +118,8 @@ def save_cached_config(cache_data = {"timestamp": time.time(),"device_capability": torch.cuda.get_device_capability(),+ "schema_version": _CACHE_SCHEMA_VERSION,+ "triton_version": getattr(triton, "__version__", None),"config_fwd": config_fwd.__dict__ if hasattr(config_fwd, "__dict__") else str(config_fwd),"config_bwd_dx": config_bwd_dx.__dict__if hasattr(config_bwd_dx, "__dict__")@@ -192,7 +205,11 @@ def get_or_autotune_moe_kernels(config_bwd_dx = KernelConfigBackward_dX(**cached_data["config_bwd_dx"])config_bwd_dw = KernelConfigBackward_dW(**cached_data["config_bwd_dw"])- configs = (config_fwd, config_bwd_dx, config_bwd_dw)+ # A disk entry may predate a kernel change: refit to dims and SMEM.+ configs = _fit_configs_to_dims(+ (config_fwd, config_bwd_dx, config_bwd_dw), 2 * intermediate_dim, hidden_dim+ )+ configs = _fit_configs_to_smem(configs, dtype, weight_dtype)_kernel_config_cache[cache_key] = configsreturn configsexcept Exception as e: