Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

MoE grouped GEMM: keep the legacy dW kernel when an explicit tile overflows per-expert shared memory (#14723)

Daniel Hane66f408eb5fa
diff --git a/tests/test_moe_dw_explicit_tile_smem.py b/tests/test_moe_dw_explicit_tile_smem.py
new file mode 100644
index 00000000000..ba90fd40d15
--- /dev/null
+++ b/tests/test_moe_dw_explicit_tile_smem.py
@@ -0,0 +1,130 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""An explicit dW tile the legacy kernel launched must still launch with the per-expert kernel as default.
+
+128x256x256 at 4 stages needs 131,584 B (sm_80/86/89/120) / 262,656 B (sm_90) per-expert vs 69,632 /
+131,072 B legacy (Triton 3.8 metadata.shared), over the 101,376 / 232,448 B limits."""
+
+import importlib
+import sys
+import types
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[1]
+
+torch = pytest.importorskip("torch")
+pytest.importorskip("triton")
+
+GPUS = {
+ "A100": ((8, 0), 166912),
+ "A10": ((8, 6), 101376),
+ "L4": ((8, 9), 101376),
+ "H100": ((9, 0), 232448),
+ "B200": ((10, 0), 232448),
+ "RTX5090": ((12, 0), 101376),
+}
+EXPLICIT = (128, 256, 256, 4) # fits the legacy kernel on every card above
+HEURISTIC = (64, 64, 256, 4)
+
+
+def gg(monkeypatch):
+ stubs = {
+ "unsloth": ROOT / "unsloth",
+ "unsloth.kernels": ROOT / "unsloth" / "kernels",
+ "unsloth.kernels.moe": ROOT / "unsloth" / "kernels" / "moe",
+ }
+ for name, path in stubs.items():
+ pkg = types.ModuleType(name)
+ pkg.__path__ = [str(path)]
+ monkeypatch.setitem(sys.modules, name, pkg)
+ sys.modules["unsloth"].DEVICE_TYPE = "cuda"
+ device_type = types.ModuleType("unsloth.device_type")
+ device_type.cuda_visible_capabilities = lambda: ()
+ monkeypatch.setitem(sys.modules, device_type.__name__, device_type)
+ utils = types.ModuleType("unsloth.kernels.utils")
+ utils.torch_gpu_device = lambda device: torch.cuda.device(device)
+ monkeypatch.setitem(sys.modules, utils.__name__, utils)
+ for name in list(sys.modules):
+ if name.startswith("unsloth.kernels.moe."):
+ monkeypatch.delitem(sys.modules, name)
+ try:
+ module = importlib.import_module("unsloth.kernels.moe.grouped_gemm.interface")
+ except Exception as e: # pragma: no cover - depends on the installed Triton
+ pytest.skip(f"grouped_gemm interface is unimportable here: {type(e).__name__}: {e}")
+ getattr(module, "_PER_EXPERT_DW_FITS", {}).clear()
+ yield module
+ for name in list(sys.modules):
+ if name.startswith("unsloth.kernels.moe."):
+ sys.modules.pop(name, None)
+
+
+def _card(monkeypatch, gg, capability, smem):
+ props = types.SimpleNamespace(
+ shared_memory_per_block_optin = smem, major = capability[0], minor = capability[1]
+ )
+ monkeypatch.setattr(torch.cuda, "get_device_properties", lambda *a, **k: props)
+ monkeypatch.setattr(torch.version, "hip", None, raising = False)
+ getattr(gg, "_PER_EXPERT_DW_FITS", {}).clear()
+
+
[email protected]("gpu", list(GPUS))
+def test_explicit_tile_routes_by_operand_card(monkeypatch, gg, gpu):
+ capability, smem = GPUS[gpu]
+ _card(monkeypatch, gg, capability, smem)
+ device = torch.device("cuda", 0)
+ M, N, K, stages = EXPLICIT
+ assert gg._per_expert_dW_fits(device, 2, M, N, K, stages) == (gpu == "A100")
+ assert not gg._per_expert_dW_fits(device, 4, M, N, K, stages)
+ M, N, K, stages = HEURISTIC
+ assert gg._per_expert_dW_fits(device, 2, M, N, K, stages)
+ assert gg._per_expert_dW_fits(device, 4, M, N, K, stages)
+
+
+def test_unknown_device_keeps_the_per_expert_kernel(monkeypatch, gg):
+ def boom(*a, **k):
+ raise RuntimeError("no CUDA")
+
+ monkeypatch.setattr(torch.cuda, "get_device_properties", boom)
+ assert gg._per_expert_dW_fits(torch.device("cuda", 0), 2, *EXPLICIT)
+
+
+def test_explicit_tile_launches_on_this_gpu(monkeypatch, gg):
+ if not torch.cuda.is_available():
+ pytest.skip("needs a CUDA device")
+ if getattr(torch.version, "hip", None):
+ pytest.skip("ROCm reports shared memory differently")
+ smem = torch.cuda.get_device_properties(0).shared_memory_per_block_optin
+ if smem > 232448:
+ pytest.skip(f"{smem} B shared memory fits the per-expert explicit tile")
+ E, N, K, T, topk = 128, 1536, 2048, 64, 8
+ g = torch.Generator(device = "cuda").manual_seed(4)
+ X = torch.randn(T, K, device = "cuda", dtype = torch.bfloat16, generator = g)
+ dY = torch.randn(T * topk, N, device = "cuda", dtype = torch.bfloat16, generator = g)
+ m_sizes = torch.full((E,), T * topk // E, device = "cuda", dtype = torch.int32)
+ m_sizes[0] += m_sizes[1]
+ m_sizes[1] = 0
+ gather = torch.randperm(T * topk, device = "cuda", generator = g).to(torch.int32)
+ M, BN, BK, stages = EXPLICIT
+ kw = dict(
+ BLOCK_SIZE_M = M,
+ BLOCK_SIZE_N = BN,
+ BLOCK_SIZE_K = BK,
+ num_warps = 8,
+ num_stages = stages,
+ permute_x = True,
+ )
+ monkeypatch.setenv("UNSLOTH_MOE_DW_KERNEL", "legacy")
+ legacy = gg._grouped_gemm_dW_impl(X, dY, m_sizes, gather, topk, **kw)
+ monkeypatch.delenv("UNSLOTH_MOE_DW_KERNEL")
+ default = gg._grouped_gemm_dW_impl(X, dY, m_sizes, gather, topk, **kw)
+ torch.cuda.synchronize()
+ assert torch.equal(default, legacy)
+ assert bool((default[1] == 0).all())
+ acc = default.float()
+ gg._grouped_gemm_dW_accumulate_impl(X, dY, m_sizes, gather, acc, topk, **kw)
+ torch.cuda.synchronize()
+ torch.testing.assert_close(acc, 2 * default.float(), rtol = 1e-2, atol = 1e-2)
diff --git a/unsloth/kernels/moe/grouped_gemm/interface.py b/unsloth/kernels/moe/grouped_gemm/interface.py
index a444b578ecb..fbedaf0d507 100644
--- a/unsloth/kernels/moe/grouped_gemm/interface.py
+++ b/unsloth/kernels/moe/grouped_gemm/interface.py
@@ -6,6 +6,7 @@ import functools
import inspect
import logging
import os
+import types
import warnings
from typing import Optional
from dataclasses import asdict
@@ -497,6 +498,43 @@ def _use_per_expert_dW_kernel():
return os.environ.get("UNSLOTH_MOE_DW_KERNEL", "").strip().lower() != "legacy"
+_PER_EXPERT_DW_FITS = {}
+
+
+def _per_expert_dW_fits(device, act_bytes, BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, num_stages):
+ """False when this tile overflows ``device`` shared memory in the per-expert dW kernel.
+
+ It double buffers both operands, so a caller-supplied tile the legacy kernel launched can raise OutOfResources."""
+ index = _device_index(device)
+ key = (index, act_bytes, BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, num_stages)
+ fits = _PER_EXPERT_DW_FITS.get(key)
+ if fits is None:
+ fits = True
+ try:
+ if not getattr(torch.version, "hip", None):
+ from ..autotune_cache import _estimate_smem
+
+ props = torch.cuda.get_device_properties(index)
+ smem = int(props.shared_memory_per_block_optin)
+ tile = types.SimpleNamespace(
+ BLOCK_SIZE_M = BLOCK_SIZE_M,
+ BLOCK_SIZE_N = BLOCK_SIZE_N,
+ BLOCK_SIZE_K = BLOCK_SIZE_K,
+ num_stages = num_stages,
+ )
+ need = _estimate_smem("dW", tile, act_bytes, act_bytes, (props.major, props.minor))
+ fits = need <= smem
+ if not fits:
+ logger.info(
+ f"Unsloth: MoE dW tile (M, N, K, stages) {key[2:]} needs ~{need} B shared "
+ f"memory per-expert, over {smem} B on cuda:{index}; using the legacy dW kernel."
+ )
+ except Exception:
+ fits = True
+ _PER_EXPERT_DW_FITS[key] = fits
+ return fits
+
+
def _launch_grouped_gemm_dW(
X: torch.Tensor,
dY: torch.Tensor,
@@ -574,6 +612,14 @@ def _launch_grouped_gemm_dW(
assert M_grad == total_tokens, f"dY M ({M_grad}) != total_tokens ({total_tokens})"
per_expert = _use_per_expert_dW_kernel()
+ if (
+ per_expert
+ and not autotune
+ and not _per_expert_dW_fits(
+ X.device, X.element_size(), BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, num_stages
+ )
+ ):
+ per_expert = False
if accumulate:
# Read-modify-write into an existing gradient: one program owns each (expert, n, k) tile.