Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

MoE dW: fall back to the legacy kernel when a per-expert tile overflows and no smaller tile is known (#15238)

Daniel Han2b3a1b33f70d
diff --git a/tests/test_moe_dw_rocm_lds_fallback.py b/tests/test_moe_dw_rocm_lds_fallback.py
new file mode 100644
index 00000000000..1b5873c5918
--- /dev/null
+++ b/tests/test_moe_dw_rocm_lds_fallback.py
@@ -0,0 +1,127 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Per-expert dW tile over the card's shared memory with no smaller tile known (ROCm: no shared memory model
+here, so _card_smem is None). The heuristic 64x64x256 4-stage tile needs ~80 KiB of LDS on gfx1151 (64 KiB):
+it must fall back to the legacy kernel (which fit it before the per-expert kernel) instead of raising
+OutOfResources, give the legacy result bit for bit, and not retry the per-expert launch on the next call.
+Simulated on NVIDIA: _card_smem -> None and a per-expert launch that raises OutOfResources."""
+
+import importlib
+import sys
+import types
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[1]
+
+torch = pytest.importorskip("torch")
+pytest.importorskip("triton")
+pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs a GPU")
+
+HEURISTIC = (64, 64, 256, 4)
+
+
+def gg(monkeypatch):
+ stubs = {
+ "unsloth": ROOT / "unsloth",
+ "unsloth.kernels": ROOT / "unsloth" / "kernels",
+ "unsloth.kernels.moe": ROOT / "unsloth" / "kernels" / "moe",
+ }
+ for name, path in stubs.items():
+ pkg = types.ModuleType(name)
+ pkg.__path__ = [str(path)]
+ monkeypatch.setitem(sys.modules, name, pkg)
+ sys.modules["unsloth"].DEVICE_TYPE = "cuda"
+ device_type = types.ModuleType("unsloth.device_type")
+ device_type.cuda_visible_capabilities = lambda: ()
+ monkeypatch.setitem(sys.modules, device_type.__name__, device_type)
+ utils = types.ModuleType("unsloth.kernels.utils")
+ utils.torch_gpu_device = lambda device: torch.cuda.device(device)
+ monkeypatch.setitem(sys.modules, utils.__name__, utils)
+ for name in list(sys.modules):
+ if name.startswith("unsloth.kernels.moe."):
+ monkeypatch.delitem(sys.modules, name)
+ try:
+ module = importlib.import_module("unsloth.kernels.moe.grouped_gemm.interface")
+ except Exception as e: # pragma: no cover - depends on the installed Triton
+ pytest.skip(f"grouped_gemm interface is unimportable here: {type(e).__name__}: {e}")
+ getattr(module, "_PER_EXPERT_DW_FITS", {}).clear()
+ yield module
+ for name in list(sys.modules):
+ if name.startswith("unsloth.kernels.moe."):
+ sys.modules.pop(name, None)
+
+
+def _ref(X, dY, m_sizes, E, N, K):
+ X, dY = X.double(), dY.double()
+ out = torch.zeros(E, N, K, dtype = torch.float64, device = X.device)
+ start = 0
+ for e, m in enumerate(m_sizes.tolist()):
+ out[e] = dY[start : start + m].T @ X[start : start + m]
+ start += m
+ return out
+
+
+class _OverLds:
+ """Stands in for the per-expert kernel on a card whose LDS it overflows."""
+
+ def __init__(self, error):
+ self.calls = 0
+ self.error = error
+
+ def __getitem__(self, grid):
+ def launch(*args, **kwargs):
+ self.calls += 1
+ raise self.error(81920, 65536, "shared memory")
+
+ return launch
+
+
+def _inputs(dtype):
+ E, N, K, T, topk = 16, 256, 512, 64, 4
+ g = torch.Generator(device = "cuda").manual_seed(7)
+ X = torch.randn(T, K, device = "cuda", dtype = dtype, generator = g)
+ dY = torch.randn(T * topk, N, device = "cuda", dtype = dtype, generator = g)
+ m_sizes = torch.full((E,), T * topk // E, device = "cuda", dtype = torch.int32)
+ m_sizes[0] += m_sizes[1]
+ m_sizes[1] = 0 # an empty expert: the legacy kernel skips it, so dW must start at zero
+ gather = torch.randperm(T * topk, device = "cuda", generator = g).to(torch.int32)
+ return X, dY, m_sizes, gather, topk, E, N, K
+
+
[email protected]("accumulate", [False, True], ids = ["fresh", "accumulate"])
+def test_lds_overflow_falls_back_to_legacy(monkeypatch, gg, accumulate):
+ dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16
+ X, dY, m_sizes, gather, topk, E, N, K = _inputs(dtype)
+ M, BN, BK, stages = HEURISTIC
+ kw = dict(
+ BLOCK_SIZE_M = M, BLOCK_SIZE_N = BN, BLOCK_SIZE_K = BK, num_warps = 8, num_stages = stages,
+ permute_x = True,
+ ) # fmt: skip
+ monkeypatch.setenv("UNSLOTH_MOE_DW_KERNEL", "legacy")
+ legacy = gg._grouped_gemm_dW_impl(X, dY, m_sizes, gather, topk, **kw)
+ monkeypatch.delenv("UNSLOTH_MOE_DW_KERNEL")
+ torch.cuda.synchronize()
+
+ over = _OverLds(gg.OutOfResources)
+ monkeypatch.setattr(gg, "_grouped_gemm_dW_per_expert_kernel", over)
+ monkeypatch.setattr(gg, "_card_smem", lambda index: None) # as on ROCm
+ gg._PER_EXPERT_DW_FITS.clear()
+ gg._DW_CARD_FITS.clear()
+ for _ in range(2):
+ if accumulate:
+ got = torch.full((E, N, K), 0.5, device = "cuda", dtype = torch.float32)
+ gg._grouped_gemm_dW_accumulate_impl(X, dY, m_sizes, gather, got, topk, **kw)
+ torch.cuda.synchronize()
+ torch.testing.assert_close(got, legacy.float() + 0.5, rtol = 1e-2, atol = 1e-2)
+ else:
+ got = gg._grouped_gemm_dW_impl(X, dY, m_sizes, gather, topk, **kw)
+ torch.cuda.synchronize()
+ assert torch.equal(got, legacy)
+ assert bool((got[1] == 0).all())
+ assert over.calls == 1, over.calls # the second call goes straight to the legacy kernel
+ rel = (legacy.double() - _ref(X[gather.long() // topk], dY, m_sizes, E, N, K)).norm()
+ assert rel / _ref(X[gather.long() // topk], dY, m_sizes, E, N, K).norm() < 5e-3
diff --git a/unsloth/kernels/moe/grouped_gemm/interface.py b/unsloth/kernels/moe/grouped_gemm/interface.py
index f6539b9a509..2e16d20f0ab 100644
--- a/unsloth/kernels/moe/grouped_gemm/interface.py
+++ b/unsloth/kernels/moe/grouped_gemm/interface.py
@@ -775,6 +775,7 @@ def _launch_grouped_gemm_dW(
BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, num_stages = _fit_tiles_to_operand_device(
"dW", X, X.dtype, BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, num_stages
)
+ legacy_stages = num_stages
per_expert_stages = (
_per_expert_dW_stages(
X.device,
@@ -912,12 +913,31 @@ def _launch_grouped_gemm_dW(
tiles = fit_args[3:]
fitted = _fit_dW_tiles_to_card(*fit_args)
if fitted == tiles:
- raise
- logger.info(
- f"Unsloth: MoE dW tile (M, N, K, stages) {tiles} is over the shared memory of "
- f"cuda:{_device_index(X.device)}; using {fitted}."
- )
- _set_dW_tiles(kernel_args, fitted)
+ if not per_expert:
+ raise
+ # No smaller tile known for this card (ROCm has no shared memory model here: the
+ # heuristic 64x64x256 4-stage tile needs ~80 KiB of LDS on gfx1151's 64 KiB): run the
+ # legacy kernel, which launched this tile before the per-expert kernel, and remember.
+ logger.info(
+ f"Unsloth: MoE per-expert dW tile (M, N, K, stages) {tiles} is over the shared "
+ f"memory of {X.device}; using the legacy dW kernel."
+ )
+ _PER_EXPERT_DW_FITS[
+ (_device_index(X.device), X.element_size(), *tiles[:3], legacy_stages,
+ not (permute_x or permute_y))
+ ] = None # fmt: skip
+ per_expert = False # read by grid()
+ kernel = _grouped_gemm_dW_kernel
+ kernel_args.pop("EXPERTS_POW2", None)
+ kernel_args["num_stages"] = legacy_stages
+ if not accumulate:
+ dW.zero_() # the legacy kernel skips empty experts
+ else:
+ logger.info(
+ f"Unsloth: MoE dW tile (M, N, K, stages) {tiles} is over the shared memory of "
+ f"cuda:{_device_index(X.device)}; using {fitted}."
+ )
+ _set_dW_tiles(kernel_args, fitted)
compiled_kernel = kernel[grid](**kernel_args, enable_fp_fusion = True)
if autotune: