Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
MoE per-expert dW: count pipelined shared memory, fit tiles that overflow the card (#15070)
Daniel Han5f3d662cdca5
diff --git a/tests/test_moe_dw_explicit_tile_smem.py b/tests/test_moe_dw_explicit_tile_smem.pyindex ba90fd40d15..4c8b2cf6132 100644--- a/tests/test_moe_dw_explicit_tile_smem.py+++ b/tests/test_moe_dw_explicit_tile_smem.py@@ -1,7 +1,8 @@# SPDX-License-Identifier: AGPL-3.0-only# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.-"""An explicit dW tile the legacy kernel launched must still launch with the per-expert kernel as default.+"""An explicit dW tile the legacy kernel launched must still launch with the per-expert kernel as default, and+a tile too big for this card (T4: 64 KiB) launches with a fitted copy instead of raising OutOfResources.128x256x256 at 4 stages needs 131,584 B (sm_80/86/89/120) / 262,656 B (sm_90) per-expert vs 69,632 /131,072 B legacy (Triton 3.8 metadata.shared), over the 101,376 / 232,448 B limits."""@@ -128,3 +129,82 @@ def test_explicit_tile_launches_on_this_gpu(monkeypatch, gg):gg._grouped_gemm_dW_accumulate_impl(X, dY, m_sizes, gather, acc, topk, **kw)torch.cuda.synchronize()torch.testing.assert_close(acc, 2 * default.float(), rtol = 1e-2, atol = 1e-2)+++# Triton 3.6 and 3.8 metadata.shared (identical) of the per-expert dW kernel, bf16, unpermuted operands: its one+# M loop is software pipelined, so X is buffered num_stages - 1 times (sm_80-class) / both tiles per stage (sm_90).+PIPELINED_SHARED = {+ (8, 9): {+ (64, 64, 256, 2): 40960,+ (64, 64, 256, 3): 73728,+ (64, 64, 256, 4): 106496,+ (32, 64, 256, 4): 53248,+ (128, 128, 64, 3): 65536,+ (128, 128, 64, 4): 81920,+ },+ (9, 0): {+ (64, 64, 256, 2): 81920,+ (64, 64, 256, 4): 163840,+ (32, 64, 256, 3): 61440,+ (128, 128, 64, 4): 196608,+ },+}++[email protected]("capability", list(PIPELINED_SHARED))+def test_pipelined_estimate_covers_compiled_shared_memory(gg, capability):+ from unsloth.kernels.moe.autotune_cache import _estimate_smem++ for (M, N, K, stages), shared in PIPELINED_SHARED[capability].items():+ tile = types.SimpleNamespace(+ BLOCK_SIZE_M = M, BLOCK_SIZE_N = N, BLOCK_SIZE_K = K, num_stages = stages+ )+ need = _estimate_smem("dW", tile, 2, 2, capability, True)+ assert shared <= need <= shared + 2048, (capability, M, N, K, stages, need, shared)++[email protected]("gpu", ["A10", "L4", "RTX5090"])+def test_unpermuted_heuristic_tile_drops_a_stage(monkeypatch, gg, gpu):+ # 64x64x256 x 4 stages unpermuted needs 106,496 B on a 101,376 B card: run 3 stages, same tile.+ capability, smem = GPUS[gpu]+ _card(monkeypatch, gg, capability, smem)+ device = torch.device("cuda", 0)+ assert gg._per_expert_dW_stages(device, 2, *HEURISTIC, pipelined = True) == 3+ assert gg._per_expert_dW_stages(device, 2, *HEURISTIC) == 4 # gathered loads do not pipeline++[email protected]("gpu", ["A100", "H100", "B200"])+def test_unpermuted_heuristic_tile_unchanged_with_enough_smem(monkeypatch, gg, gpu):+ capability, smem = GPUS[gpu]+ _card(monkeypatch, gg, capability, smem)+ device = torch.device("cuda", 0)+ assert gg._per_expert_dW_stages(device, 2, *HEURISTIC, pipelined = True) == 4+++def _ref(X, dY, m_sizes, E, N, K):+ X, dY = X.double(), dY.double()+ out = torch.zeros(E, N, K, dtype = torch.float64, device = X.device)+ start = 0+ for e, m in enumerate(m_sizes.tolist()):+ out[e] = dY[start : start + m].T @ X[start : start + m]+ start += m+ return out+++def test_unpermuted_heuristic_tile_launches_on_this_gpu(gg):+ # L4 / A10 on main: OutOfResources(106496, 101376) for the Qwen3-30B gate_up dW (UNSLOTH_MOE_DISABLE_AUTOTUNE=1).+ if not torch.cuda.is_available():+ pytest.skip("needs a CUDA device")+ E, N, K, T, topk = 128, 1536, 2048, 256, 8+ dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16+ g = torch.Generator(device = "cuda").manual_seed(5)+ X = torch.randn(T * topk, K, device = "cuda", dtype = dtype, generator = g)+ dY = torch.randn(T * topk, N, device = "cuda", dtype = dtype, generator = g)+ m_sizes = torch.full((E,), T * topk // E, device = "cuda", dtype = torch.int32)+ gather = torch.arange(T * topk, device = "cuda", dtype = torch.int32)+ M, BN, BK, stages = HEURISTIC+ kw = dict(BLOCK_SIZE_M = M, BLOCK_SIZE_N = BN, BLOCK_SIZE_K = BK, num_warps = 8, num_stages = stages)+ out = gg._grouped_gemm_dW_impl(X, dY, m_sizes, gather, topk, **kw)+ torch.cuda.synchronize()+ ref = _ref(X, dY, m_sizes, E, N, K)+ assert ((out.double() - ref).norm() / ref.norm()) < 5e-3diff --git a/tests/test_moe_fallback_tile_dims.py b/tests/test_moe_fallback_tile_dims.pyindex 141334ba37b..af3e44239da 100644--- a/tests/test_moe_fallback_tile_dims.py+++ b/tests/test_moe_fallback_tile_dims.py@@ -78,9 +78,16 @@ def _divides(old, n, k):return all(n % t[1] == 0 and k % t[2] == 0 for t in old)+def no_smem_fit(ac, monkeypatch):+ # The dims fit alone: on a card with less shared memory (L4, A10: 101,376 B) the smem fit also steps the+ # heuristic dX down to 3 stages, which the smem tests below cover.+ monkeypatch.setattr(ac, "_device_smem_per_block", lambda *a, **k: None)++@pytest.mark.parametrize("hidden,inter", SHAPES + [(2048, 1024)])@pytest.mark.parametrize("which", ["heuristic", "default"])-def test_fallback_tiles_divide_each_gemm(ac, hidden, inter, which):+def test_fallback_tiles_divide_each_gemm(ac, no_smem_fit, hidden, inter, which):fn = ac._get_heuristic_configs if which == "heuristic" else ac._get_default_configsold = OLD_HEURISTIC if which == "heuristic" else OLD_DEFAULTfor kwargs, (n, k) in _gemms(hidden, inter):@@ -104,7 +111,7 @@ def test_working_shapes_keep_exact_configs(ac, hidden, inter, which):assert [c.__dict__ for c in new] == [c.__dict__ for c in fn()]-def test_no_dims_keeps_old_configs(ac):+def test_no_dims_keeps_old_configs(ac, no_smem_fit):assert _tiles(ac._get_heuristic_configs()) == OLD_HEURISTICassert _tiles(ac._get_default_configs()) == OLD_DEFAULTdiff --git a/unsloth/kernels/moe/autotune_cache.py b/unsloth/kernels/moe/autotune_cache.pyindex be20a9321f0..b33337d7bda 100644--- a/unsloth/kernels/moe/autotune_cache.py+++ b/unsloth/kernels/moe/autotune_cache.py@@ -507,9 +507,17 @@ def _itemsize(dtype: Optional[torch.dtype], default: int = 2) -> int:def _estimate_smem(- kind: str, config: Any, act_bytes: int, weight_bytes: int, capability: Tuple[int, int]+ kind: str,+ config: Any,+ act_bytes: int,+ weight_bytes: int,+ capability: Tuple[int, int],+ pipelined_dW: bool = False,) -> int:- """Shared memory (B) per launch; matches compiled metadata.shared (Triton 3.6-3.8, sm_75..120) or exceeds it by <= 1 KiB."""+ """Shared memory (B) per launch; matches compiled metadata.shared (Triton 3.6-3.8, sm_75..120) or exceeds it by <= 1 KiB.++ pipelined_dW: the per-expert dW kernel with unpermuted operands. Its one M loop is software pipelined (the+ legacy kernel's nested loop and a gathered load never are), so 16-bit shared memory grows with num_stages."""M, N, K = config.BLOCK_SIZE_M, config.BLOCK_SIZE_N, config.BLOCK_SIZE_Kstages = max(int(config.num_stages), 1)if kind == "fwd":@@ -534,7 +542,12 @@ def _estimate_smem(buffers = stages if hopper_like else stages - 1else:buffers = min(stages, 2) if hopper_like else 1- return buffers * (a + b) + (2048 if hopper_like else 1024)+ need = buffers * (a + b) + (2048 if hopper_like else 1024)+ if pipelined_dW and stages > 1:+ # sm_90+ buffers both tiles per stage; sm_80-class cp.async buffers X (b) stages - 1 times beside one dY tile.+ piped = stages * (a + b) + 2048 if hopper_like else (stages - 1) * b + a + 1024+ need = max(need, piped)+ return needif hopper_like:# A cast operand keeps num_stages - 1 buffers plus one copy in the dot dtype.if a_cast:@@ -554,10 +567,14 @@ def _fit_config_to_smem(weight_bytes: int,capability: Tuple[int, int],smem: int,+ pipelined_dW: bool = False,) -> Any:# Halving a power-of-two tile keeps it dividing N / K.def fits():- return _estimate_smem(kind, config, act_bytes, weight_bytes, capability) <= smem+ return (+ _estimate_smem(kind, config, act_bytes, weight_bytes, capability, pipelined_dW)+ <= smem+ )before = (config.BLOCK_SIZE_M, config.BLOCK_SIZE_N, config.BLOCK_SIZE_K, config.num_stages)loop_dim = _LOOP_DIM[kind]diff --git a/unsloth/kernels/moe/grouped_gemm/interface.py b/unsloth/kernels/moe/grouped_gemm/interface.pyindex 0ca2d5291a1..cd638e9bbab 100644--- a/unsloth/kernels/moe/grouped_gemm/interface.py+++ b/unsloth/kernels/moe/grouped_gemm/interface.py@@ -17,6 +17,11 @@ from unsloth.kernels.utils import torch_gpu_deviceimport torchimport triton+try:+ from triton.runtime.errors import OutOfResources+except ImportError: # older Triton+ from triton.runtime import OutOfResources+from .kernels.backward import (_autotuned_grouped_gemm_dW_kernel,_autotuned_grouped_gemm_dW_per_expert_kernel,@@ -570,40 +575,109 @@ def _use_per_expert_dW_kernel():_PER_EXPERT_DW_FITS = {}+_DW_CARD_FITS = {}++def _card_smem(index):+ """(shared_memory_per_block_optin, (major, minor)) of cuda:index, else None (ROCm, no CUDA)."""+ if getattr(torch.version, "hip", None):+ return None+ props = torch.cuda.get_device_properties(index)+ return int(props.shared_memory_per_block_optin), (props.major, props.minor)-def _per_expert_dW_fits(device, act_bytes, BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, num_stages):- """False when this tile overflows ``device`` shared memory in the per-expert dW kernel.- It double buffers both operands, so a caller-supplied tile the legacy kernel launched can raise OutOfResources."""+def _per_expert_dW_stages(+ device, act_bytes, BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, num_stages, pipelined = False+):+ """Largest num_stages <= the requested one at which the per-expert dW kernel fits ``device`` shared memory, or+ None (use the legacy kernel). Unpermuted operands (``pipelined``) multi-buffer X per stage, so 4 stages of the+ heuristic 64x64x256 tile need 106,496 B: only the stage count drops, the tile (and so the bits) stay."""index = _device_index(device)- key = (index, act_bytes, BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, num_stages)- fits = _PER_EXPERT_DW_FITS.get(key)- if fits is None:- fits = True- try:- if not getattr(torch.version, "hip", None):- from ..autotune_cache import _estimate_smem+ key = (index, act_bytes, BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, num_stages, pipelined)+ if key in _PER_EXPERT_DW_FITS:+ return _PER_EXPERT_DW_FITS[key]+ stages = num_stages+ try:+ card = _card_smem(index)+ if card is not None:+ from ..autotune_cache import _estimate_smem- props = torch.cuda.get_device_properties(index)- smem = int(props.shared_memory_per_block_optin)+ smem, capability = card+ stages = None+ for candidate in range(max(int(num_stages), 1), 0, -1):tile = types.SimpleNamespace(+ BLOCK_SIZE_M = BLOCK_SIZE_M,+ BLOCK_SIZE_N = BLOCK_SIZE_N,+ BLOCK_SIZE_K = BLOCK_SIZE_K,+ num_stages = candidate,+ )+ need = _estimate_smem("dW", tile, act_bytes, act_bytes, capability, pipelined)+ if need <= smem:+ stages = candidate+ break+ if not pipelined:+ break # stage count does not change a gathered launch's footprint+ if stages is None:+ logger.info(+ f"Unsloth: MoE dW tile (M, N, K, stages) {key[2:6]} needs ~{need} B shared "+ f"memory per-expert, over {smem} B on cuda:{index}; using the legacy dW kernel."+ )+ elif stages != num_stages:+ logger.info(+ f"Unsloth: MoE dW tile (M, N, K) {key[2:5]} runs {stages} stages instead of "+ f"{num_stages} to fit {smem} B shared memory on cuda:{index}."+ )+ except Exception:+ stages = num_stages+ _PER_EXPERT_DW_FITS[key] = stages+ return stages+++def _per_expert_dW_fits(+ device, act_bytes, BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, num_stages, pipelined = False+):+ """False when this tile overflows ``device`` shared memory in the per-expert dW kernel at every stage count.++ A caller-supplied tile the legacy kernel launched can raise OutOfResources there (gathered loads double buffer)."""+ return (+ _per_expert_dW_stages(+ device, act_bytes, BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, num_stages, pipelined+ )+ is not None+ )+++def _fit_dW_tiles_to_card(device, act_bytes, pipelined, BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, num_stages):+ """Shrunk copy of a dW tile that raised OutOfResources on ``device`` (an explicit tile sized for a bigger card,+ e.g. 128x256x256 on a 64 KiB T4, where the legacy kernel overflows too). Only called after a launch failed, so a+ tile the estimate over-counts but that launches is never touched."""+ tiles = (BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, num_stages)+ key = (_device_index(device), act_bytes, pipelined, tiles)+ fitted = _DW_CARD_FITS.get(key)+ if fitted is None:+ fitted = tiles+ try:+ card = _card_smem(key[0])+ if card is not None:+ config = types.SimpleNamespace(BLOCK_SIZE_M = BLOCK_SIZE_M,BLOCK_SIZE_N = BLOCK_SIZE_N,BLOCK_SIZE_K = BLOCK_SIZE_K,num_stages = num_stages,)- need = _estimate_smem("dW", tile, act_bytes, act_bytes, (props.major, props.minor))- fits = need <= smem- if not fits:- logger.info(- f"Unsloth: MoE dW tile (M, N, K, stages) {key[2:]} needs ~{need} B shared "- f"memory per-expert, over {smem} B on cuda:{index}; using the legacy dW kernel."- )+ _autotune_cache()._fit_config_to_smem(+ "dW", config, act_bytes, act_bytes, card[1], card[0], pipelined+ )+ fitted = (+ config.BLOCK_SIZE_M,+ config.BLOCK_SIZE_N,+ config.BLOCK_SIZE_K,+ config.num_stages,+ )except Exception:- fits = True- _PER_EXPERT_DW_FITS[key] = fits- return fits+ fitted = tiles+ _DW_CARD_FITS[key] = fitted+ return fitteddef _launch_grouped_gemm_dW(@@ -683,14 +757,27 @@ def _launch_grouped_gemm_dW(assert M_grad == total_tokens, f"dY M ({M_grad}) != total_tokens ({total_tokens})"per_expert = _use_per_expert_dW_kernel()- if (- per_expert- and not autotune- and not _per_expert_dW_fits(- X.device, X.element_size(), BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, num_stages+ if not autotune:+ BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, num_stages = _fit_tiles_to_operand_device(+ "dW", X, X.dtype, BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, num_stages)- ):- per_expert = False+ per_expert_stages = (+ _per_expert_dW_stages(+ X.device,+ X.element_size(),+ BLOCK_SIZE_M,+ BLOCK_SIZE_N,+ BLOCK_SIZE_K,+ num_stages,+ pipelined = not (permute_x or permute_y),+ )+ if per_expert+ else None+ )+ if per_expert_stages is None:+ per_expert = False+ else:+ num_stages = per_expert_stagesif accumulate:# Read-modify-write into an existing gradient: one program owns each (expert, n, k) tile.@@ -709,11 +796,6 @@ def _launch_grouped_gemm_dW(# sizes assume an output no wider than X.use_tma_store = False- if not autotune:- BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, num_stages = _fit_tiles_to_operand_device(- "dW", X, X.dtype, BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, num_stages- )-def grid(META):if per_expert:return (@@ -798,9 +880,31 @@ def _launch_grouped_gemm_dW(is_fake = _is_tracing(X, dY)if not is_fake:with torch_gpu_device(X.device):- compiled_kernel: triton.compiler.CompiledKernel = kernel[grid](- **kernel_args, enable_fp_fusion = True- )+ fit_args = None+ if not autotune:+ tiles = (BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, kernel_args["num_stages"])+ pipelined = per_expert and not (permute_x or permute_y)+ fit_args = (X.device, X.element_size(), pipelined, *tiles)+ if (_device_index(X.device), X.element_size(), pipelined, tiles) in _DW_CARD_FITS:+ _set_dW_tiles(kernel_args, _fit_dW_tiles_to_card(*fit_args)) # failed before+ try:+ compiled_kernel: triton.compiler.CompiledKernel = kernel[grid](+ **kernel_args, enable_fp_fusion = True+ )+ except OutOfResources:+ # Raised before the kernel runs (dW untouched): retry once with a tile fitted to this card.+ if fit_args is None:+ raise+ tiles = fit_args[3:]+ fitted = _fit_dW_tiles_to_card(*fit_args)+ if fitted == tiles:+ raise+ logger.info(+ f"Unsloth: MoE dW tile (M, N, K, stages) {tiles} is over the shared memory of "+ f"cuda:{_device_index(X.device)}; using {fitted}."+ )+ _set_dW_tiles(kernel_args, fitted)+ compiled_kernel = kernel[grid](**kernel_args, enable_fp_fusion = True)if autotune:log_kernel_info(compiled_kernel, kernel.best_config)@@ -810,6 +914,15 @@ def _launch_grouped_gemm_dW(return dW+def _set_dW_tiles(kernel_args, tiles):+ (+ kernel_args["BLOCK_SIZE_M"],+ kernel_args["BLOCK_SIZE_N"],+ kernel_args["BLOCK_SIZE_K"],+ kernel_args["num_stages"],+ ) = tiles++def _grouped_gemm_dW_impl(X: torch.Tensor,dY: torch.Tensor,