Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

MoE grouped GEMM: tl.range for the dX expert loop, drop the dead dW FLATTEN (#13831)

Daniel Hand64b08027fbe
diff --git a/tests/test_grouped_gemm_flatten.py b/tests/test_grouped_gemm_flatten.py
new file mode 100644
index 00000000000..d3c0d526d70
--- /dev/null
+++ b/tests/test_grouped_gemm_flatten.py
@@ -0,0 +1,83 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""The dX expert loop used the builtin range(..., flatten = FLATTEN): Triton drops keywords there."""
+
+import ast
+import sys
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT))
+
+torch = pytest.importorskip("torch")
+pytest.importorskip("triton")
+import triton # noqa: E402
+import triton.language as tl # noqa: E402
+
+KERNELS_DIR = ROOT / "unsloth" / "kernels" / "moe" / "grouped_gemm" / "kernels"
+
+
[email protected](scope = "module")
+def bw():
+ try:
+ from unsloth.kernels.moe.grouped_gemm.kernels import backward
+ except Exception as exc:
+ if torch.cuda.is_available():
+ raise # a real import regression, not an unsupported host
+ pytest.skip(f"grouped_gemm needs a GPU build: {exc}")
+ return backward
+
+
+def test_no_builtin_range_receives_keywords():
+ offenders = [
+ f"{path.name}:{node.lineno}"
+ for path in sorted(KERNELS_DIR.glob("*.py"))
+ for node in ast.walk(ast.parse(path.read_text()))
+ if isinstance(node, ast.Call)
+ and isinstance(node.func, ast.Name)
+ and node.func.id == "range"
+ and node.keywords
+ ]
+ assert not offenders, offenders
+
+
+def test_dw_kernel_has_no_dead_flatten_constexpr(bw):
+ assert "FLATTEN" not in bw._grouped_gemm_dW_kernel.arg_names
+
+
+# Triton < 3.4 resolves the constexpr-off tl.make_tensor_descriptor branches while hashing.
[email protected](not hasattr(tl, "make_tensor_descriptor"), reason = "needs Triton >= 3.4")
[email protected]("flatten", [True, False])
+def test_dx_expert_loop_ir_carries_flatten(bw, flatten):
+ from triton.backends import backends
+ from triton.backends.compiler import GPUTarget
+ from triton.compiler import ASTSource
+
+ if torch.version.hip is not None or "nvidia" not in backends:
+ pytest.skip("needs Triton's CUDA backend")
+
+ fn = bw._grouped_gemm_dX_kernel
+ ptrs = {"dY_ptr": "*bf16", "w_ptr": "*bf16", "dX_ptr": "*bf16"}
+ ptrs |= {"m_sizes_ptr": "*i32", "gather_indices_ptr": "*i32"}
+ consts = dict(NUM_EXPERTS = 8, TOPK = 1, N = 128, K = 256, BLOCK_SIZE_M = 64, BLOCK_SIZE_N = 64)
+ consts |= dict(BLOCK_SIZE_K = 64, FLATTEN = flatten)
+ sig, constexprs = {}, {}
+ for p in fn.params:
+ if p.name in ptrs:
+ sig[p.name] = ptrs[p.name]
+ elif p.is_constexpr:
+ constexprs[p.name] = consts.get(p.name, p.default)
+ sig[p.name] = "constexpr"
+ else:
+ sig[p.name] = "i32"
+ try:
+ src = ASTSource(fn, sig, constexprs = constexprs)
+ ttir = triton.compile(src, target = GPUTarget("cuda", 80, 32)).asm["ttir"]
+ except Exception as exc: # pragma: no cover
+ if "driver" in str(exc).lower():
+ pytest.skip(f"no Triton CUDA driver: {exc}")
+ raise
+ assert ttir.count("tt.flatten") == int(flatten)
diff --git a/unsloth/kernels/moe/grouped_gemm/interface.py b/unsloth/kernels/moe/grouped_gemm/interface.py
index 3b935e5e5c3..7f31b700c39 100644
--- a/unsloth/kernels/moe/grouped_gemm/interface.py
+++ b/unsloth/kernels/moe/grouped_gemm/interface.py
@@ -109,7 +109,6 @@ def _grouped_gemm_forward_impl(
permute_x: bool = False,
permute_y: bool = False,
fuse_mul_post: bool = False,
- # Autotuning -- overrides manual kernel params when True
autotune: bool = False,
# Kernel tuning params: must be tuned, else poor performance.
BLOCK_SIZE_M: int = 32,
@@ -533,7 +532,6 @@ def _grouped_gemm_dW_impl(
"NUM_SMS": NUM_SMS,
"PERMUTE_X": permute_x,
"PERMUTE_Y": permute_y,
- "FLATTEN": flatten,
}
if not autotune:
diff --git a/unsloth/kernels/moe/grouped_gemm/kernels/backward.py b/unsloth/kernels/moe/grouped_gemm/kernels/backward.py
index bf00bd3eaa2..f19bcc4907f 100644
--- a/unsloth/kernels/moe/grouped_gemm/kernels/backward.py
+++ b/unsloth/kernels/moe/grouped_gemm/kernels/backward.py
@@ -99,7 +99,7 @@ def _grouped_gemm_dX_kernel(
n_block_range = tl.arange(0, BLOCK_SIZE_N)
k_block_range = tl.arange(0, BLOCK_SIZE_K)
- for expert_idx in range(NUM_EXPERTS, flatten = FLATTEN):
+ for expert_idx in tl.range(NUM_EXPERTS, flatten = FLATTEN):
m_start = m_end
m_size = tl.load(m_sizes_ptr + expert_idx).to(tl.int32)
m_end = m_start + m_size
@@ -199,7 +199,6 @@ def _grouped_gemm_dX_kernel(
dX = accumulator.to(output_dtype)
- # A BLOCK_M x BLOCK_K tile is written out, so stride by K.
if USE_TMA_STORE:
offset_m = tile_m_idx * BLOCK_SIZE_M
offset_k = tile_k_idx * BLOCK_SIZE_K
@@ -213,7 +212,6 @@ def _grouped_gemm_dX_kernel(
tidx += NUM_SMS
- # Update the total tiles count for the next expert group
processed_tiles += num_tiles_per_expert
@@ -266,7 +264,6 @@ def _grouped_gemm_dW_kernel(
USE_TMA_LOAD_dY: tl.constexpr = False,
USE_TMA_LOAD_X: tl.constexpr = False,
USE_TMA_STORE: tl.constexpr = False,
- FLATTEN: tl.constexpr = True,
acc_dtype: tl.constexpr = tl.float32,
) -> None:
TOTAL_TOKENS = NUM_TOKENS * TOPK
@@ -290,7 +287,6 @@ def _grouped_gemm_dW_kernel(
strides = [K, 1],
block_shape = [BLOCK_SIZE_M, BLOCK_SIZE_K],
)
- # Output tiles per expert, since each expert weight matrix is [N, K].
num_n_tiles = tl.cdiv(N, BLOCK_SIZE_N)
num_k_tiles = tl.cdiv(K, BLOCK_SIZE_K)
output_tiles_per_expert = num_n_tiles * num_k_tiles
@@ -323,7 +319,6 @@ def _grouped_gemm_dW_kernel(
m_end = 0
for expert_idx in range(NUM_EXPERTS):
- # A fresh accumulator per expert.
accumulator = tl.zeros((BLOCK_SIZE_N, BLOCK_SIZE_K), dtype = acc_dtype)
m_start = m_end
@@ -414,7 +409,6 @@ def _grouped_gemm_dW_kernel(
y = accumulator.to(output_dtype)
if USE_TMA_STORE:
- # Expand dims to match the [E, N, K] shape.
y = tl.expand_dims(y, 0)
dW_desc.store([expert_idx, n_offset, k_offset], y)
else: