Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
MoE grouped GEMM: keep the legacy dW kernel when an explicit tile overflows per-expert shared memory (#14723)
Daniel Hane66f408eb5fa
diff --git a/tests/test_moe_dw_explicit_tile_smem.py b/tests/test_moe_dw_explicit_tile_smem.pynew file mode 100644index 00000000000..ba90fd40d15--- /dev/null+++ b/tests/test_moe_dw_explicit_tile_smem.py@@ -0,0 +1,130 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""An explicit dW tile the legacy kernel launched must still launch with the per-expert kernel as default.++128x256x256 at 4 stages needs 131,584 B (sm_80/86/89/120) / 262,656 B (sm_90) per-expert vs 69,632 /+131,072 B legacy (Triton 3.8 metadata.shared), over the 101,376 / 232,448 B limits."""++import importlib+import sys+import types+from pathlib import Path++import pytest++ROOT = Path(__file__).resolve().parents[1]++torch = pytest.importorskip("torch")+pytest.importorskip("triton")++GPUS = {+ "A100": ((8, 0), 166912),+ "A10": ((8, 6), 101376),+ "L4": ((8, 9), 101376),+ "H100": ((9, 0), 232448),+ "B200": ((10, 0), 232448),+ "RTX5090": ((12, 0), 101376),+}+EXPLICIT = (128, 256, 256, 4) # fits the legacy kernel on every card above+HEURISTIC = (64, 64, 256, 4)+++def gg(monkeypatch):+ stubs = {+ "unsloth": ROOT / "unsloth",+ "unsloth.kernels": ROOT / "unsloth" / "kernels",+ "unsloth.kernels.moe": ROOT / "unsloth" / "kernels" / "moe",+ }+ for name, path in stubs.items():+ pkg = types.ModuleType(name)+ pkg.__path__ = [str(path)]+ monkeypatch.setitem(sys.modules, name, pkg)+ sys.modules["unsloth"].DEVICE_TYPE = "cuda"+ device_type = types.ModuleType("unsloth.device_type")+ device_type.cuda_visible_capabilities = lambda: ()+ monkeypatch.setitem(sys.modules, device_type.__name__, device_type)+ utils = types.ModuleType("unsloth.kernels.utils")+ utils.torch_gpu_device = lambda device: torch.cuda.device(device)+ monkeypatch.setitem(sys.modules, utils.__name__, utils)+ for name in list(sys.modules):+ if name.startswith("unsloth.kernels.moe."):+ monkeypatch.delitem(sys.modules, name)+ try:+ module = importlib.import_module("unsloth.kernels.moe.grouped_gemm.interface")+ except Exception as e: # pragma: no cover - depends on the installed Triton+ pytest.skip(f"grouped_gemm interface is unimportable here: {type(e).__name__}: {e}")+ getattr(module, "_PER_EXPERT_DW_FITS", {}).clear()+ yield module+ for name in list(sys.modules):+ if name.startswith("unsloth.kernels.moe."):+ sys.modules.pop(name, None)+++def _card(monkeypatch, gg, capability, smem):+ props = types.SimpleNamespace(+ shared_memory_per_block_optin = smem, major = capability[0], minor = capability[1]+ )+ monkeypatch.setattr(torch.cuda, "get_device_properties", lambda *a, **k: props)+ monkeypatch.setattr(torch.version, "hip", None, raising = False)+ getattr(gg, "_PER_EXPERT_DW_FITS", {}).clear()++[email protected]("gpu", list(GPUS))+def test_explicit_tile_routes_by_operand_card(monkeypatch, gg, gpu):+ capability, smem = GPUS[gpu]+ _card(monkeypatch, gg, capability, smem)+ device = torch.device("cuda", 0)+ M, N, K, stages = EXPLICIT+ assert gg._per_expert_dW_fits(device, 2, M, N, K, stages) == (gpu == "A100")+ assert not gg._per_expert_dW_fits(device, 4, M, N, K, stages)+ M, N, K, stages = HEURISTIC+ assert gg._per_expert_dW_fits(device, 2, M, N, K, stages)+ assert gg._per_expert_dW_fits(device, 4, M, N, K, stages)+++def test_unknown_device_keeps_the_per_expert_kernel(monkeypatch, gg):+ def boom(*a, **k):+ raise RuntimeError("no CUDA")++ monkeypatch.setattr(torch.cuda, "get_device_properties", boom)+ assert gg._per_expert_dW_fits(torch.device("cuda", 0), 2, *EXPLICIT)+++def test_explicit_tile_launches_on_this_gpu(monkeypatch, gg):+ if not torch.cuda.is_available():+ pytest.skip("needs a CUDA device")+ if getattr(torch.version, "hip", None):+ pytest.skip("ROCm reports shared memory differently")+ smem = torch.cuda.get_device_properties(0).shared_memory_per_block_optin+ if smem > 232448:+ pytest.skip(f"{smem} B shared memory fits the per-expert explicit tile")+ E, N, K, T, topk = 128, 1536, 2048, 64, 8+ g = torch.Generator(device = "cuda").manual_seed(4)+ X = torch.randn(T, K, device = "cuda", dtype = torch.bfloat16, generator = g)+ dY = torch.randn(T * topk, N, device = "cuda", dtype = torch.bfloat16, generator = g)+ m_sizes = torch.full((E,), T * topk // E, device = "cuda", dtype = torch.int32)+ m_sizes[0] += m_sizes[1]+ m_sizes[1] = 0+ gather = torch.randperm(T * topk, device = "cuda", generator = g).to(torch.int32)+ M, BN, BK, stages = EXPLICIT+ kw = dict(+ BLOCK_SIZE_M = M,+ BLOCK_SIZE_N = BN,+ BLOCK_SIZE_K = BK,+ num_warps = 8,+ num_stages = stages,+ permute_x = True,+ )+ monkeypatch.setenv("UNSLOTH_MOE_DW_KERNEL", "legacy")+ legacy = gg._grouped_gemm_dW_impl(X, dY, m_sizes, gather, topk, **kw)+ monkeypatch.delenv("UNSLOTH_MOE_DW_KERNEL")+ default = gg._grouped_gemm_dW_impl(X, dY, m_sizes, gather, topk, **kw)+ torch.cuda.synchronize()+ assert torch.equal(default, legacy)+ assert bool((default[1] == 0).all())+ acc = default.float()+ gg._grouped_gemm_dW_accumulate_impl(X, dY, m_sizes, gather, acc, topk, **kw)+ torch.cuda.synchronize()+ torch.testing.assert_close(acc, 2 * default.float(), rtol = 1e-2, atol = 1e-2)diff --git a/unsloth/kernels/moe/grouped_gemm/interface.py b/unsloth/kernels/moe/grouped_gemm/interface.pyindex a444b578ecb..fbedaf0d507 100644--- a/unsloth/kernels/moe/grouped_gemm/interface.py+++ b/unsloth/kernels/moe/grouped_gemm/interface.py@@ -6,6 +6,7 @@ import functoolsimport inspectimport loggingimport os+import typesimport warningsfrom typing import Optionalfrom dataclasses import asdict@@ -497,6 +498,43 @@ def _use_per_expert_dW_kernel():return os.environ.get("UNSLOTH_MOE_DW_KERNEL", "").strip().lower() != "legacy"+_PER_EXPERT_DW_FITS = {}+++def _per_expert_dW_fits(device, act_bytes, BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, num_stages):+ """False when this tile overflows ``device`` shared memory in the per-expert dW kernel.++ It double buffers both operands, so a caller-supplied tile the legacy kernel launched can raise OutOfResources."""+ index = _device_index(device)+ key = (index, act_bytes, BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, num_stages)+ fits = _PER_EXPERT_DW_FITS.get(key)+ if fits is None:+ fits = True+ try:+ if not getattr(torch.version, "hip", None):+ from ..autotune_cache import _estimate_smem++ props = torch.cuda.get_device_properties(index)+ smem = int(props.shared_memory_per_block_optin)+ tile = types.SimpleNamespace(+ BLOCK_SIZE_M = BLOCK_SIZE_M,+ BLOCK_SIZE_N = BLOCK_SIZE_N,+ BLOCK_SIZE_K = BLOCK_SIZE_K,+ num_stages = num_stages,+ )+ need = _estimate_smem("dW", tile, act_bytes, act_bytes, (props.major, props.minor))+ fits = need <= smem+ if not fits:+ logger.info(+ f"Unsloth: MoE dW tile (M, N, K, stages) {key[2:]} needs ~{need} B shared "+ f"memory per-expert, over {smem} B on cuda:{index}; using the legacy dW kernel."+ )+ except Exception:+ fits = True+ _PER_EXPERT_DW_FITS[key] = fits+ return fits++def _launch_grouped_gemm_dW(X: torch.Tensor,dY: torch.Tensor,@@ -574,6 +612,14 @@ def _launch_grouped_gemm_dW(assert M_grad == total_tokens, f"dY M ({M_grad}) != total_tokens ({total_tokens})"per_expert = _use_per_expert_dW_kernel()+ if (+ per_expert+ and not autotune+ and not _per_expert_dW_fits(+ X.device, X.element_size(), BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, num_stages+ )+ ):+ per_expert = Falseif accumulate:# Read-modify-write into an existing gradient: one program owns each (expert, n, k) tile.