Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
MoE dW: fall back to the legacy kernel when a per-expert tile overflows and no smaller tile is known (#15238)
Daniel Han2b3a1b33f70d
diff --git a/tests/test_moe_dw_rocm_lds_fallback.py b/tests/test_moe_dw_rocm_lds_fallback.pynew file mode 100644index 00000000000..1b5873c5918--- /dev/null+++ b/tests/test_moe_dw_rocm_lds_fallback.py@@ -0,0 +1,127 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Per-expert dW tile over the card's shared memory with no smaller tile known (ROCm: no shared memory model+here, so _card_smem is None). The heuristic 64x64x256 4-stage tile needs ~80 KiB of LDS on gfx1151 (64 KiB):+it must fall back to the legacy kernel (which fit it before the per-expert kernel) instead of raising+OutOfResources, give the legacy result bit for bit, and not retry the per-expert launch on the next call.+Simulated on NVIDIA: _card_smem -> None and a per-expert launch that raises OutOfResources."""++import importlib+import sys+import types+from pathlib import Path++import pytest++ROOT = Path(__file__).resolve().parents[1]++torch = pytest.importorskip("torch")+pytest.importorskip("triton")+pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs a GPU")++HEURISTIC = (64, 64, 256, 4)+++def gg(monkeypatch):+ stubs = {+ "unsloth": ROOT / "unsloth",+ "unsloth.kernels": ROOT / "unsloth" / "kernels",+ "unsloth.kernels.moe": ROOT / "unsloth" / "kernels" / "moe",+ }+ for name, path in stubs.items():+ pkg = types.ModuleType(name)+ pkg.__path__ = [str(path)]+ monkeypatch.setitem(sys.modules, name, pkg)+ sys.modules["unsloth"].DEVICE_TYPE = "cuda"+ device_type = types.ModuleType("unsloth.device_type")+ device_type.cuda_visible_capabilities = lambda: ()+ monkeypatch.setitem(sys.modules, device_type.__name__, device_type)+ utils = types.ModuleType("unsloth.kernels.utils")+ utils.torch_gpu_device = lambda device: torch.cuda.device(device)+ monkeypatch.setitem(sys.modules, utils.__name__, utils)+ for name in list(sys.modules):+ if name.startswith("unsloth.kernels.moe."):+ monkeypatch.delitem(sys.modules, name)+ try:+ module = importlib.import_module("unsloth.kernels.moe.grouped_gemm.interface")+ except Exception as e: # pragma: no cover - depends on the installed Triton+ pytest.skip(f"grouped_gemm interface is unimportable here: {type(e).__name__}: {e}")+ getattr(module, "_PER_EXPERT_DW_FITS", {}).clear()+ yield module+ for name in list(sys.modules):+ if name.startswith("unsloth.kernels.moe."):+ sys.modules.pop(name, None)+++def _ref(X, dY, m_sizes, E, N, K):+ X, dY = X.double(), dY.double()+ out = torch.zeros(E, N, K, dtype = torch.float64, device = X.device)+ start = 0+ for e, m in enumerate(m_sizes.tolist()):+ out[e] = dY[start : start + m].T @ X[start : start + m]+ start += m+ return out+++class _OverLds:+ """Stands in for the per-expert kernel on a card whose LDS it overflows."""++ def __init__(self, error):+ self.calls = 0+ self.error = error++ def __getitem__(self, grid):+ def launch(*args, **kwargs):+ self.calls += 1+ raise self.error(81920, 65536, "shared memory")++ return launch+++def _inputs(dtype):+ E, N, K, T, topk = 16, 256, 512, 64, 4+ g = torch.Generator(device = "cuda").manual_seed(7)+ X = torch.randn(T, K, device = "cuda", dtype = dtype, generator = g)+ dY = torch.randn(T * topk, N, device = "cuda", dtype = dtype, generator = g)+ m_sizes = torch.full((E,), T * topk // E, device = "cuda", dtype = torch.int32)+ m_sizes[0] += m_sizes[1]+ m_sizes[1] = 0 # an empty expert: the legacy kernel skips it, so dW must start at zero+ gather = torch.randperm(T * topk, device = "cuda", generator = g).to(torch.int32)+ return X, dY, m_sizes, gather, topk, E, N, K++[email protected]("accumulate", [False, True], ids = ["fresh", "accumulate"])+def test_lds_overflow_falls_back_to_legacy(monkeypatch, gg, accumulate):+ dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16+ X, dY, m_sizes, gather, topk, E, N, K = _inputs(dtype)+ M, BN, BK, stages = HEURISTIC+ kw = dict(+ BLOCK_SIZE_M = M, BLOCK_SIZE_N = BN, BLOCK_SIZE_K = BK, num_warps = 8, num_stages = stages,+ permute_x = True,+ ) # fmt: skip+ monkeypatch.setenv("UNSLOTH_MOE_DW_KERNEL", "legacy")+ legacy = gg._grouped_gemm_dW_impl(X, dY, m_sizes, gather, topk, **kw)+ monkeypatch.delenv("UNSLOTH_MOE_DW_KERNEL")+ torch.cuda.synchronize()++ over = _OverLds(gg.OutOfResources)+ monkeypatch.setattr(gg, "_grouped_gemm_dW_per_expert_kernel", over)+ monkeypatch.setattr(gg, "_card_smem", lambda index: None) # as on ROCm+ gg._PER_EXPERT_DW_FITS.clear()+ gg._DW_CARD_FITS.clear()+ for _ in range(2):+ if accumulate:+ got = torch.full((E, N, K), 0.5, device = "cuda", dtype = torch.float32)+ gg._grouped_gemm_dW_accumulate_impl(X, dY, m_sizes, gather, got, topk, **kw)+ torch.cuda.synchronize()+ torch.testing.assert_close(got, legacy.float() + 0.5, rtol = 1e-2, atol = 1e-2)+ else:+ got = gg._grouped_gemm_dW_impl(X, dY, m_sizes, gather, topk, **kw)+ torch.cuda.synchronize()+ assert torch.equal(got, legacy)+ assert bool((got[1] == 0).all())+ assert over.calls == 1, over.calls # the second call goes straight to the legacy kernel+ rel = (legacy.double() - _ref(X[gather.long() // topk], dY, m_sizes, E, N, K)).norm()+ assert rel / _ref(X[gather.long() // topk], dY, m_sizes, E, N, K).norm() < 5e-3diff --git a/unsloth/kernels/moe/grouped_gemm/interface.py b/unsloth/kernels/moe/grouped_gemm/interface.pyindex f6539b9a509..2e16d20f0ab 100644--- a/unsloth/kernels/moe/grouped_gemm/interface.py+++ b/unsloth/kernels/moe/grouped_gemm/interface.py@@ -775,6 +775,7 @@ def _launch_grouped_gemm_dW(BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, num_stages = _fit_tiles_to_operand_device("dW", X, X.dtype, BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, num_stages)+ legacy_stages = num_stagesper_expert_stages = (_per_expert_dW_stages(X.device,@@ -912,12 +913,31 @@ def _launch_grouped_gemm_dW(tiles = fit_args[3:]fitted = _fit_dW_tiles_to_card(*fit_args)if fitted == tiles:- raise- logger.info(- f"Unsloth: MoE dW tile (M, N, K, stages) {tiles} is over the shared memory of "- f"cuda:{_device_index(X.device)}; using {fitted}."- )- _set_dW_tiles(kernel_args, fitted)+ if not per_expert:+ raise+ # No smaller tile known for this card (ROCm has no shared memory model here: the+ # heuristic 64x64x256 4-stage tile needs ~80 KiB of LDS on gfx1151's 64 KiB): run the+ # legacy kernel, which launched this tile before the per-expert kernel, and remember.+ logger.info(+ f"Unsloth: MoE per-expert dW tile (M, N, K, stages) {tiles} is over the shared "+ f"memory of {X.device}; using the legacy dW kernel."+ )+ _PER_EXPERT_DW_FITS[+ (_device_index(X.device), X.element_size(), *tiles[:3], legacy_stages,+ not (permute_x or permute_y))+ ] = None # fmt: skip+ per_expert = False # read by grid()+ kernel = _grouped_gemm_dW_kernel+ kernel_args.pop("EXPERTS_POW2", None)+ kernel_args["num_stages"] = legacy_stages+ if not accumulate:+ dW.zero_() # the legacy kernel skips empty experts+ else:+ logger.info(+ f"Unsloth: MoE dW tile (M, N, K, stages) {tiles} is over the shared memory of "+ f"cuda:{_device_index(X.device)}; using {fitted}."+ )+ _set_dW_tiles(kernel_args, fitted)compiled_kernel = kernel[grid](**kernel_args, enable_fp_fusion = True)if autotune: