Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
MoE grouped GEMM: tl.range for the dX expert loop, drop the dead dW FLATTEN (#13831)
Daniel Hand64b08027fbe
diff --git a/tests/test_grouped_gemm_flatten.py b/tests/test_grouped_gemm_flatten.pynew file mode 100644index 00000000000..d3c0d526d70--- /dev/null+++ b/tests/test_grouped_gemm_flatten.py@@ -0,0 +1,83 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""The dX expert loop used the builtin range(..., flatten = FLATTEN): Triton drops keywords there."""++import ast+import sys+from pathlib import Path++import pytest++ROOT = Path(__file__).resolve().parents[1]+sys.path.insert(0, str(ROOT))++torch = pytest.importorskip("torch")+pytest.importorskip("triton")+import triton # noqa: E402+import triton.language as tl # noqa: E402++KERNELS_DIR = ROOT / "unsloth" / "kernels" / "moe" / "grouped_gemm" / "kernels"++[email protected](scope = "module")+def bw():+ try:+ from unsloth.kernels.moe.grouped_gemm.kernels import backward+ except Exception as exc:+ if torch.cuda.is_available():+ raise # a real import regression, not an unsupported host+ pytest.skip(f"grouped_gemm needs a GPU build: {exc}")+ return backward+++def test_no_builtin_range_receives_keywords():+ offenders = [+ f"{path.name}:{node.lineno}"+ for path in sorted(KERNELS_DIR.glob("*.py"))+ for node in ast.walk(ast.parse(path.read_text()))+ if isinstance(node, ast.Call)+ and isinstance(node.func, ast.Name)+ and node.func.id == "range"+ and node.keywords+ ]+ assert not offenders, offenders+++def test_dw_kernel_has_no_dead_flatten_constexpr(bw):+ assert "FLATTEN" not in bw._grouped_gemm_dW_kernel.arg_names+++# Triton < 3.4 resolves the constexpr-off tl.make_tensor_descriptor branches while hashing.[email protected](not hasattr(tl, "make_tensor_descriptor"), reason = "needs Triton >= 3.4")[email protected]("flatten", [True, False])+def test_dx_expert_loop_ir_carries_flatten(bw, flatten):+ from triton.backends import backends+ from triton.backends.compiler import GPUTarget+ from triton.compiler import ASTSource++ if torch.version.hip is not None or "nvidia" not in backends:+ pytest.skip("needs Triton's CUDA backend")++ fn = bw._grouped_gemm_dX_kernel+ ptrs = {"dY_ptr": "*bf16", "w_ptr": "*bf16", "dX_ptr": "*bf16"}+ ptrs |= {"m_sizes_ptr": "*i32", "gather_indices_ptr": "*i32"}+ consts = dict(NUM_EXPERTS = 8, TOPK = 1, N = 128, K = 256, BLOCK_SIZE_M = 64, BLOCK_SIZE_N = 64)+ consts |= dict(BLOCK_SIZE_K = 64, FLATTEN = flatten)+ sig, constexprs = {}, {}+ for p in fn.params:+ if p.name in ptrs:+ sig[p.name] = ptrs[p.name]+ elif p.is_constexpr:+ constexprs[p.name] = consts.get(p.name, p.default)+ sig[p.name] = "constexpr"+ else:+ sig[p.name] = "i32"+ try:+ src = ASTSource(fn, sig, constexprs = constexprs)+ ttir = triton.compile(src, target = GPUTarget("cuda", 80, 32)).asm["ttir"]+ except Exception as exc: # pragma: no cover+ if "driver" in str(exc).lower():+ pytest.skip(f"no Triton CUDA driver: {exc}")+ raise+ assert ttir.count("tt.flatten") == int(flatten)diff --git a/unsloth/kernels/moe/grouped_gemm/interface.py b/unsloth/kernels/moe/grouped_gemm/interface.pyindex 3b935e5e5c3..7f31b700c39 100644--- a/unsloth/kernels/moe/grouped_gemm/interface.py+++ b/unsloth/kernels/moe/grouped_gemm/interface.py@@ -109,7 +109,6 @@ def _grouped_gemm_forward_impl(permute_x: bool = False,permute_y: bool = False,fuse_mul_post: bool = False,- # Autotuning -- overrides manual kernel params when Trueautotune: bool = False,# Kernel tuning params: must be tuned, else poor performance.BLOCK_SIZE_M: int = 32,@@ -533,7 +532,6 @@ def _grouped_gemm_dW_impl("NUM_SMS": NUM_SMS,"PERMUTE_X": permute_x,"PERMUTE_Y": permute_y,- "FLATTEN": flatten,}if not autotune:diff --git a/unsloth/kernels/moe/grouped_gemm/kernels/backward.py b/unsloth/kernels/moe/grouped_gemm/kernels/backward.pyindex bf00bd3eaa2..f19bcc4907f 100644--- a/unsloth/kernels/moe/grouped_gemm/kernels/backward.py+++ b/unsloth/kernels/moe/grouped_gemm/kernels/backward.py@@ -99,7 +99,7 @@ def _grouped_gemm_dX_kernel(n_block_range = tl.arange(0, BLOCK_SIZE_N)k_block_range = tl.arange(0, BLOCK_SIZE_K)- for expert_idx in range(NUM_EXPERTS, flatten = FLATTEN):+ for expert_idx in tl.range(NUM_EXPERTS, flatten = FLATTEN):m_start = m_endm_size = tl.load(m_sizes_ptr + expert_idx).to(tl.int32)m_end = m_start + m_size@@ -199,7 +199,6 @@ def _grouped_gemm_dX_kernel(dX = accumulator.to(output_dtype)- # A BLOCK_M x BLOCK_K tile is written out, so stride by K.if USE_TMA_STORE:offset_m = tile_m_idx * BLOCK_SIZE_Moffset_k = tile_k_idx * BLOCK_SIZE_K@@ -213,7 +212,6 @@ def _grouped_gemm_dX_kernel(tidx += NUM_SMS- # Update the total tiles count for the next expert groupprocessed_tiles += num_tiles_per_expert@@ -266,7 +264,6 @@ def _grouped_gemm_dW_kernel(USE_TMA_LOAD_dY: tl.constexpr = False,USE_TMA_LOAD_X: tl.constexpr = False,USE_TMA_STORE: tl.constexpr = False,- FLATTEN: tl.constexpr = True,acc_dtype: tl.constexpr = tl.float32,) -> None:TOTAL_TOKENS = NUM_TOKENS * TOPK@@ -290,7 +287,6 @@ def _grouped_gemm_dW_kernel(strides = [K, 1],block_shape = [BLOCK_SIZE_M, BLOCK_SIZE_K],)- # Output tiles per expert, since each expert weight matrix is [N, K].num_n_tiles = tl.cdiv(N, BLOCK_SIZE_N)num_k_tiles = tl.cdiv(K, BLOCK_SIZE_K)output_tiles_per_expert = num_n_tiles * num_k_tiles@@ -323,7 +319,6 @@ def _grouped_gemm_dW_kernel(m_end = 0for expert_idx in range(NUM_EXPERTS):- # A fresh accumulator per expert.accumulator = tl.zeros((BLOCK_SIZE_N, BLOCK_SIZE_K), dtype = acc_dtype)m_start = m_end@@ -414,7 +409,6 @@ def _grouped_gemm_dW_kernel(y = accumulator.to(output_dtype)if USE_TMA_STORE:- # Expand dims to match the [E, N, K] shape.y = tl.expand_dims(y, 0)dW_desc.store([expert_idx, n_offset, k_offset], y)else: