Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

MoE per-expert dW: count pipelined shared memory, fit tiles that overflow the card (#15070)

Daniel Han5f3d662cdca5
diff --git a/tests/test_moe_dw_explicit_tile_smem.py b/tests/test_moe_dw_explicit_tile_smem.py
index ba90fd40d15..4c8b2cf6132 100644
--- a/tests/test_moe_dw_explicit_tile_smem.py
+++ b/tests/test_moe_dw_explicit_tile_smem.py
@@ -1,7 +1,8 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
-"""An explicit dW tile the legacy kernel launched must still launch with the per-expert kernel as default.
+"""An explicit dW tile the legacy kernel launched must still launch with the per-expert kernel as default, and
+a tile too big for this card (T4: 64 KiB) launches with a fitted copy instead of raising OutOfResources.
128x256x256 at 4 stages needs 131,584 B (sm_80/86/89/120) / 262,656 B (sm_90) per-expert vs 69,632 /
131,072 B legacy (Triton 3.8 metadata.shared), over the 101,376 / 232,448 B limits."""
@@ -128,3 +129,82 @@ def test_explicit_tile_launches_on_this_gpu(monkeypatch, gg):
gg._grouped_gemm_dW_accumulate_impl(X, dY, m_sizes, gather, acc, topk, **kw)
torch.cuda.synchronize()
torch.testing.assert_close(acc, 2 * default.float(), rtol = 1e-2, atol = 1e-2)
+
+
+# Triton 3.6 and 3.8 metadata.shared (identical) of the per-expert dW kernel, bf16, unpermuted operands: its one
+# M loop is software pipelined, so X is buffered num_stages - 1 times (sm_80-class) / both tiles per stage (sm_90).
+PIPELINED_SHARED = {
+ (8, 9): {
+ (64, 64, 256, 2): 40960,
+ (64, 64, 256, 3): 73728,
+ (64, 64, 256, 4): 106496,
+ (32, 64, 256, 4): 53248,
+ (128, 128, 64, 3): 65536,
+ (128, 128, 64, 4): 81920,
+ },
+ (9, 0): {
+ (64, 64, 256, 2): 81920,
+ (64, 64, 256, 4): 163840,
+ (32, 64, 256, 3): 61440,
+ (128, 128, 64, 4): 196608,
+ },
+}
+
+
[email protected]("capability", list(PIPELINED_SHARED))
+def test_pipelined_estimate_covers_compiled_shared_memory(gg, capability):
+ from unsloth.kernels.moe.autotune_cache import _estimate_smem
+
+ for (M, N, K, stages), shared in PIPELINED_SHARED[capability].items():
+ tile = types.SimpleNamespace(
+ BLOCK_SIZE_M = M, BLOCK_SIZE_N = N, BLOCK_SIZE_K = K, num_stages = stages
+ )
+ need = _estimate_smem("dW", tile, 2, 2, capability, True)
+ assert shared <= need <= shared + 2048, (capability, M, N, K, stages, need, shared)
+
+
[email protected]("gpu", ["A10", "L4", "RTX5090"])
+def test_unpermuted_heuristic_tile_drops_a_stage(monkeypatch, gg, gpu):
+ # 64x64x256 x 4 stages unpermuted needs 106,496 B on a 101,376 B card: run 3 stages, same tile.
+ capability, smem = GPUS[gpu]
+ _card(monkeypatch, gg, capability, smem)
+ device = torch.device("cuda", 0)
+ assert gg._per_expert_dW_stages(device, 2, *HEURISTIC, pipelined = True) == 3
+ assert gg._per_expert_dW_stages(device, 2, *HEURISTIC) == 4 # gathered loads do not pipeline
+
+
[email protected]("gpu", ["A100", "H100", "B200"])
+def test_unpermuted_heuristic_tile_unchanged_with_enough_smem(monkeypatch, gg, gpu):
+ capability, smem = GPUS[gpu]
+ _card(monkeypatch, gg, capability, smem)
+ device = torch.device("cuda", 0)
+ assert gg._per_expert_dW_stages(device, 2, *HEURISTIC, pipelined = True) == 4
+
+
+def _ref(X, dY, m_sizes, E, N, K):
+ X, dY = X.double(), dY.double()
+ out = torch.zeros(E, N, K, dtype = torch.float64, device = X.device)
+ start = 0
+ for e, m in enumerate(m_sizes.tolist()):
+ out[e] = dY[start : start + m].T @ X[start : start + m]
+ start += m
+ return out
+
+
+def test_unpermuted_heuristic_tile_launches_on_this_gpu(gg):
+ # L4 / A10 on main: OutOfResources(106496, 101376) for the Qwen3-30B gate_up dW (UNSLOTH_MOE_DISABLE_AUTOTUNE=1).
+ if not torch.cuda.is_available():
+ pytest.skip("needs a CUDA device")
+ E, N, K, T, topk = 128, 1536, 2048, 256, 8
+ dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16
+ g = torch.Generator(device = "cuda").manual_seed(5)
+ X = torch.randn(T * topk, K, device = "cuda", dtype = dtype, generator = g)
+ dY = torch.randn(T * topk, N, device = "cuda", dtype = dtype, generator = g)
+ m_sizes = torch.full((E,), T * topk // E, device = "cuda", dtype = torch.int32)
+ gather = torch.arange(T * topk, device = "cuda", dtype = torch.int32)
+ M, BN, BK, stages = HEURISTIC
+ kw = dict(BLOCK_SIZE_M = M, BLOCK_SIZE_N = BN, BLOCK_SIZE_K = BK, num_warps = 8, num_stages = stages)
+ out = gg._grouped_gemm_dW_impl(X, dY, m_sizes, gather, topk, **kw)
+ torch.cuda.synchronize()
+ ref = _ref(X, dY, m_sizes, E, N, K)
+ assert ((out.double() - ref).norm() / ref.norm()) < 5e-3
diff --git a/tests/test_moe_fallback_tile_dims.py b/tests/test_moe_fallback_tile_dims.py
index 141334ba37b..af3e44239da 100644
--- a/tests/test_moe_fallback_tile_dims.py
+++ b/tests/test_moe_fallback_tile_dims.py
@@ -78,9 +78,16 @@ def _divides(old, n, k):
return all(n % t[1] == 0 and k % t[2] == 0 for t in old)
+def no_smem_fit(ac, monkeypatch):
+ # The dims fit alone: on a card with less shared memory (L4, A10: 101,376 B) the smem fit also steps the
+ # heuristic dX down to 3 stages, which the smem tests below cover.
+ monkeypatch.setattr(ac, "_device_smem_per_block", lambda *a, **k: None)
+
+
@pytest.mark.parametrize("hidden,inter", SHAPES + [(2048, 1024)])
@pytest.mark.parametrize("which", ["heuristic", "default"])
-def test_fallback_tiles_divide_each_gemm(ac, hidden, inter, which):
+def test_fallback_tiles_divide_each_gemm(ac, no_smem_fit, hidden, inter, which):
fn = ac._get_heuristic_configs if which == "heuristic" else ac._get_default_configs
old = OLD_HEURISTIC if which == "heuristic" else OLD_DEFAULT
for kwargs, (n, k) in _gemms(hidden, inter):
@@ -104,7 +111,7 @@ def test_working_shapes_keep_exact_configs(ac, hidden, inter, which):
assert [c.__dict__ for c in new] == [c.__dict__ for c in fn()]
-def test_no_dims_keeps_old_configs(ac):
+def test_no_dims_keeps_old_configs(ac, no_smem_fit):
assert _tiles(ac._get_heuristic_configs()) == OLD_HEURISTIC
assert _tiles(ac._get_default_configs()) == OLD_DEFAULT
diff --git a/unsloth/kernels/moe/autotune_cache.py b/unsloth/kernels/moe/autotune_cache.py
index be20a9321f0..b33337d7bda 100644
--- a/unsloth/kernels/moe/autotune_cache.py
+++ b/unsloth/kernels/moe/autotune_cache.py
@@ -507,9 +507,17 @@ def _itemsize(dtype: Optional[torch.dtype], default: int = 2) -> int:
def _estimate_smem(
- kind: str, config: Any, act_bytes: int, weight_bytes: int, capability: Tuple[int, int]
+ kind: str,
+ config: Any,
+ act_bytes: int,
+ weight_bytes: int,
+ capability: Tuple[int, int],
+ pipelined_dW: bool = False,
) -> int:
- """Shared memory (B) per launch; matches compiled metadata.shared (Triton 3.6-3.8, sm_75..120) or exceeds it by <= 1 KiB."""
+ """Shared memory (B) per launch; matches compiled metadata.shared (Triton 3.6-3.8, sm_75..120) or exceeds it by <= 1 KiB.
+
+ pipelined_dW: the per-expert dW kernel with unpermuted operands. Its one M loop is software pipelined (the
+ legacy kernel's nested loop and a gathered load never are), so 16-bit shared memory grows with num_stages."""
M, N, K = config.BLOCK_SIZE_M, config.BLOCK_SIZE_N, config.BLOCK_SIZE_K
stages = max(int(config.num_stages), 1)
if kind == "fwd":
@@ -534,7 +542,12 @@ def _estimate_smem(
buffers = stages if hopper_like else stages - 1
else:
buffers = min(stages, 2) if hopper_like else 1
- return buffers * (a + b) + (2048 if hopper_like else 1024)
+ need = buffers * (a + b) + (2048 if hopper_like else 1024)
+ if pipelined_dW and stages > 1:
+ # sm_90+ buffers both tiles per stage; sm_80-class cp.async buffers X (b) stages - 1 times beside one dY tile.
+ piped = stages * (a + b) + 2048 if hopper_like else (stages - 1) * b + a + 1024
+ need = max(need, piped)
+ return need
if hopper_like:
# A cast operand keeps num_stages - 1 buffers plus one copy in the dot dtype.
if a_cast:
@@ -554,10 +567,14 @@ def _fit_config_to_smem(
weight_bytes: int,
capability: Tuple[int, int],
smem: int,
+ pipelined_dW: bool = False,
) -> Any:
# Halving a power-of-two tile keeps it dividing N / K.
def fits():
- return _estimate_smem(kind, config, act_bytes, weight_bytes, capability) <= smem
+ return (
+ _estimate_smem(kind, config, act_bytes, weight_bytes, capability, pipelined_dW)
+ <= smem
+ )
before = (config.BLOCK_SIZE_M, config.BLOCK_SIZE_N, config.BLOCK_SIZE_K, config.num_stages)
loop_dim = _LOOP_DIM[kind]
diff --git a/unsloth/kernels/moe/grouped_gemm/interface.py b/unsloth/kernels/moe/grouped_gemm/interface.py
index 0ca2d5291a1..cd638e9bbab 100644
--- a/unsloth/kernels/moe/grouped_gemm/interface.py
+++ b/unsloth/kernels/moe/grouped_gemm/interface.py
@@ -17,6 +17,11 @@ from unsloth.kernels.utils import torch_gpu_device
import torch
import triton
+try:
+ from triton.runtime.errors import OutOfResources
+except ImportError: # older Triton
+ from triton.runtime import OutOfResources
+
from .kernels.backward import (
_autotuned_grouped_gemm_dW_kernel,
_autotuned_grouped_gemm_dW_per_expert_kernel,
@@ -570,40 +575,109 @@ def _use_per_expert_dW_kernel():
_PER_EXPERT_DW_FITS = {}
+_DW_CARD_FITS = {}
+
+def _card_smem(index):
+ """(shared_memory_per_block_optin, (major, minor)) of cuda:index, else None (ROCm, no CUDA)."""
+ if getattr(torch.version, "hip", None):
+ return None
+ props = torch.cuda.get_device_properties(index)
+ return int(props.shared_memory_per_block_optin), (props.major, props.minor)
-def _per_expert_dW_fits(device, act_bytes, BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, num_stages):
- """False when this tile overflows ``device`` shared memory in the per-expert dW kernel.
- It double buffers both operands, so a caller-supplied tile the legacy kernel launched can raise OutOfResources."""
+def _per_expert_dW_stages(
+ device, act_bytes, BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, num_stages, pipelined = False
+):
+ """Largest num_stages <= the requested one at which the per-expert dW kernel fits ``device`` shared memory, or
+ None (use the legacy kernel). Unpermuted operands (``pipelined``) multi-buffer X per stage, so 4 stages of the
+ heuristic 64x64x256 tile need 106,496 B: only the stage count drops, the tile (and so the bits) stay."""
index = _device_index(device)
- key = (index, act_bytes, BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, num_stages)
- fits = _PER_EXPERT_DW_FITS.get(key)
- if fits is None:
- fits = True
- try:
- if not getattr(torch.version, "hip", None):
- from ..autotune_cache import _estimate_smem
+ key = (index, act_bytes, BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, num_stages, pipelined)
+ if key in _PER_EXPERT_DW_FITS:
+ return _PER_EXPERT_DW_FITS[key]
+ stages = num_stages
+ try:
+ card = _card_smem(index)
+ if card is not None:
+ from ..autotune_cache import _estimate_smem
- props = torch.cuda.get_device_properties(index)
- smem = int(props.shared_memory_per_block_optin)
+ smem, capability = card
+ stages = None
+ for candidate in range(max(int(num_stages), 1), 0, -1):
tile = types.SimpleNamespace(
+ BLOCK_SIZE_M = BLOCK_SIZE_M,
+ BLOCK_SIZE_N = BLOCK_SIZE_N,
+ BLOCK_SIZE_K = BLOCK_SIZE_K,
+ num_stages = candidate,
+ )
+ need = _estimate_smem("dW", tile, act_bytes, act_bytes, capability, pipelined)
+ if need <= smem:
+ stages = candidate
+ break
+ if not pipelined:
+ break # stage count does not change a gathered launch's footprint
+ if stages is None:
+ logger.info(
+ f"Unsloth: MoE dW tile (M, N, K, stages) {key[2:6]} needs ~{need} B shared "
+ f"memory per-expert, over {smem} B on cuda:{index}; using the legacy dW kernel."
+ )
+ elif stages != num_stages:
+ logger.info(
+ f"Unsloth: MoE dW tile (M, N, K) {key[2:5]} runs {stages} stages instead of "
+ f"{num_stages} to fit {smem} B shared memory on cuda:{index}."
+ )
+ except Exception:
+ stages = num_stages
+ _PER_EXPERT_DW_FITS[key] = stages
+ return stages
+
+
+def _per_expert_dW_fits(
+ device, act_bytes, BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, num_stages, pipelined = False
+):
+ """False when this tile overflows ``device`` shared memory in the per-expert dW kernel at every stage count.
+
+ A caller-supplied tile the legacy kernel launched can raise OutOfResources there (gathered loads double buffer)."""
+ return (
+ _per_expert_dW_stages(
+ device, act_bytes, BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, num_stages, pipelined
+ )
+ is not None
+ )
+
+
+def _fit_dW_tiles_to_card(device, act_bytes, pipelined, BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, num_stages):
+ """Shrunk copy of a dW tile that raised OutOfResources on ``device`` (an explicit tile sized for a bigger card,
+ e.g. 128x256x256 on a 64 KiB T4, where the legacy kernel overflows too). Only called after a launch failed, so a
+ tile the estimate over-counts but that launches is never touched."""
+ tiles = (BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, num_stages)
+ key = (_device_index(device), act_bytes, pipelined, tiles)
+ fitted = _DW_CARD_FITS.get(key)
+ if fitted is None:
+ fitted = tiles
+ try:
+ card = _card_smem(key[0])
+ if card is not None:
+ config = types.SimpleNamespace(
BLOCK_SIZE_M = BLOCK_SIZE_M,
BLOCK_SIZE_N = BLOCK_SIZE_N,
BLOCK_SIZE_K = BLOCK_SIZE_K,
num_stages = num_stages,
)
- need = _estimate_smem("dW", tile, act_bytes, act_bytes, (props.major, props.minor))
- fits = need <= smem
- if not fits:
- logger.info(
- f"Unsloth: MoE dW tile (M, N, K, stages) {key[2:]} needs ~{need} B shared "
- f"memory per-expert, over {smem} B on cuda:{index}; using the legacy dW kernel."
- )
+ _autotune_cache()._fit_config_to_smem(
+ "dW", config, act_bytes, act_bytes, card[1], card[0], pipelined
+ )
+ fitted = (
+ config.BLOCK_SIZE_M,
+ config.BLOCK_SIZE_N,
+ config.BLOCK_SIZE_K,
+ config.num_stages,
+ )
except Exception:
- fits = True
- _PER_EXPERT_DW_FITS[key] = fits
- return fits
+ fitted = tiles
+ _DW_CARD_FITS[key] = fitted
+ return fitted
def _launch_grouped_gemm_dW(
@@ -683,14 +757,27 @@ def _launch_grouped_gemm_dW(
assert M_grad == total_tokens, f"dY M ({M_grad}) != total_tokens ({total_tokens})"
per_expert = _use_per_expert_dW_kernel()
- if (
- per_expert
- and not autotune
- and not _per_expert_dW_fits(
- X.device, X.element_size(), BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, num_stages
+ if not autotune:
+ BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, num_stages = _fit_tiles_to_operand_device(
+ "dW", X, X.dtype, BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, num_stages
)
- ):
- per_expert = False
+ per_expert_stages = (
+ _per_expert_dW_stages(
+ X.device,
+ X.element_size(),
+ BLOCK_SIZE_M,
+ BLOCK_SIZE_N,
+ BLOCK_SIZE_K,
+ num_stages,
+ pipelined = not (permute_x or permute_y),
+ )
+ if per_expert
+ else None
+ )
+ if per_expert_stages is None:
+ per_expert = False
+ else:
+ num_stages = per_expert_stages
if accumulate:
# Read-modify-write into an existing gradient: one program owns each (expert, n, k) tile.
@@ -709,11 +796,6 @@ def _launch_grouped_gemm_dW(
# sizes assume an output no wider than X.
use_tma_store = False
- if not autotune:
- BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, num_stages = _fit_tiles_to_operand_device(
- "dW", X, X.dtype, BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, num_stages
- )
-
def grid(META):
if per_expert:
return (
@@ -798,9 +880,31 @@ def _launch_grouped_gemm_dW(
is_fake = _is_tracing(X, dY)
if not is_fake:
with torch_gpu_device(X.device):
- compiled_kernel: triton.compiler.CompiledKernel = kernel[grid](
- **kernel_args, enable_fp_fusion = True
- )
+ fit_args = None
+ if not autotune:
+ tiles = (BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, kernel_args["num_stages"])
+ pipelined = per_expert and not (permute_x or permute_y)
+ fit_args = (X.device, X.element_size(), pipelined, *tiles)
+ if (_device_index(X.device), X.element_size(), pipelined, tiles) in _DW_CARD_FITS:
+ _set_dW_tiles(kernel_args, _fit_dW_tiles_to_card(*fit_args)) # failed before
+ try:
+ compiled_kernel: triton.compiler.CompiledKernel = kernel[grid](
+ **kernel_args, enable_fp_fusion = True
+ )
+ except OutOfResources:
+ # Raised before the kernel runs (dW untouched): retry once with a tile fitted to this card.
+ if fit_args is None:
+ raise
+ tiles = fit_args[3:]
+ fitted = _fit_dW_tiles_to_card(*fit_args)
+ if fitted == tiles:
+ raise
+ logger.info(
+ f"Unsloth: MoE dW tile (M, N, K, stages) {tiles} is over the shared memory of "
+ f"cuda:{_device_index(X.device)}; using {fitted}."
+ )
+ _set_dW_tiles(kernel_args, fitted)
+ compiled_kernel = kernel[grid](**kernel_args, enable_fp_fusion = True)
if autotune:
log_kernel_info(compiled_kernel, kernel.best_config)
@@ -810,6 +914,15 @@ def _launch_grouped_gemm_dW(
return dW
+def _set_dW_tiles(kernel_args, tiles):
+ (
+ kernel_args["BLOCK_SIZE_M"],
+ kernel_args["BLOCK_SIZE_N"],
+ kernel_args["BLOCK_SIZE_K"],
+ kernel_args["num_stages"],
+ ) = tiles
+
+
def _grouped_gemm_dW_impl(
X: torch.Tensor,
dY: torch.Tensor,