Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Keep the Triton MoE grouped GEMM in compiled graphs and index weights past 2^31 elements (#12114)
Daniel Hanb5bae411bd57
diff --git a/tests/test_grouped_gemm_compile_and_int64.py b/tests/test_grouped_gemm_compile_and_int64.pynew file mode 100644index 00000000000..c0da2e4506a--- /dev/null+++ b/tests/test_grouped_gemm_compile_and_int64.py@@ -0,0 +1,124 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""The Triton MoE grouped GEMM must run under torch.compile and index weights past 2^31 elements.++1. The kernels were marked `allow_in_graph`, so AOT traced their Python bodies on fake+ tensors: the graph kept the `torch.empty` output and dropped the Triton launch, and the+ compiled forward / backward returned uninitialised memory. They are now custom ops.+2. Weight offsets (`expert_idx * N * K`) were int32, so an (E, N, K) weight holding more+ than 2^31 elements read and wrote the wrong experts (Step-3.7 at 288 experts).+"""++import sys+from pathlib import Path++import pytest++ROOT = Path(__file__).resolve().parents[1]+sys.path.insert(0, str(ROOT))++import torch # noqa: E402++pytest.importorskip("triton", reason = "the grouped GEMM is a Triton kernel")++try:+ from unsloth.kernels.moe.grouped_gemm import interface as gg+ from unsloth.kernels.moe.grouped_gemm.kernels.tuning import (+ KernelConfigBackward_dW,+ KernelConfigBackward_dX,+ KernelConfigForward,+ )+except Exception as exc: # pragma: no cover - depends on the installed stack+ pytest.skip(reason = f"grouped_gemm is unimportable here: {exc}", allow_module_level = True)++requires_cuda = pytest.mark.skipif(+ not torch.cuda.is_available(), reason = "grouped GEMM needs a real CUDA device"+)++E, TOPK, T, H, I = 4, 2, 64, 256, 128+FWD = KernelConfigForward(+ BLOCK_SIZE_M = 64, BLOCK_SIZE_N = 64, BLOCK_SIZE_K = 64, num_warps = 4, num_stages = 2+)+DX = KernelConfigBackward_dX(+ BLOCK_SIZE_M = 64, BLOCK_SIZE_N = 64, BLOCK_SIZE_K = 64, num_warps = 4, num_stages = 2+)+DW = KernelConfigBackward_dW(+ BLOCK_SIZE_M = 64, BLOCK_SIZE_N = 64, BLOCK_SIZE_K = 64, num_warps = 4, num_stages = 2+)+++def _moe_mlp(x, w1, w2, m_sizes, gather_indices):+ h = gg.grouped_gemm(+ X = x, W = w1, m_sizes = m_sizes, topk = TOPK, gather_indices = gather_indices,+ permute_x = True, kernel_config_fwd = FWD, kernel_config_bwd_dX = DX,+ kernel_config_bwd_dW = DW, is_first_gemm = True,+ ) # fmt: skip+ gate, up = h.chunk(2, dim = -1)+ return gg.grouped_gemm(+ X = torch.nn.functional.silu(gate) * up, W = w2, m_sizes = m_sizes, topk = TOPK,+ gather_indices = gather_indices, permute_y = True, kernel_config_fwd = FWD,+ kernel_config_bwd_dX = DX, kernel_config_bwd_dW = DW, is_first_gemm = False,+ ) # fmt: skip+++def _routing(device):+ idx = torch.topk(torch.randn(T, E, device = device), TOPK).indices.view(-1)+ m_sizes = torch.bincount(idx, minlength = E).to(torch.int32)+ return m_sizes, torch.argsort(idx, stable = True).to(torch.int32)+++def test_kernels_are_opaque_ops_for_the_compiler():+ if not hasattr(torch.library, "custom_op"):+ pytest.skip(+ reason = "torch < 2.4 has no torch.library.custom_op; the kernels fall back to dynamo.disable"+ )+ for name in ("grouped_gemm_forward", "grouped_gemm_dX", "grouped_gemm_dW"):+ assert hasattr(torch.ops.unsloth, name)+++@requires_cuda+def test_compiled_matches_eager_forward_and_backward():+ torch.manual_seed(0)+ dev = "cuda"+ w1 = (torch.randn(E, 2 * I, H, device = dev) * 0.05).bfloat16().requires_grad_(True)+ w2 = (torch.randn(E, H, I, device = dev) * 0.05).bfloat16().requires_grad_(True)+ compiled = torch.compile(_moe_mlp, fullgraph = True)+ # Several calls: the stale buffer only shows up once the allocator reuses memory.+ for step in range(3):+ m_sizes, gather_indices = _routing(dev)+ x = torch.randn(T, H, device = dev).bfloat16() * (step + 1)+ grad = torch.randn(T * TOPK, H, device = dev)+ outs = []+ for fn in (_moe_mlp, compiled):+ xi = x.clone().requires_grad_(True)+ w1.grad = w2.grad = None+ y = fn(xi, w1, w2, m_sizes, gather_indices)+ (y.float() * grad).sum().backward()+ outs.append([t.detach().float().clone() for t in (y, xi.grad, w1.grad, w2.grad)])+ for name, e, c in zip(("y", "dx", "dw1", "dw2"), *outs):+ rel = ((c - e).norm() / e.norm()).item()+ assert rel < 2e-2, f"call {step}: compiled {name} differs from eager, rel {rel:.3g}"+++@requires_cuda+def test_weight_past_2_pow_31_elements_reads_the_right_expert():+ n_experts, N, K, tokens = 129, 4096, 4096, 256 # 129 * 4096 * 4096 > 2^31+ need = n_experts * N * K * 2 + (1 << 30)+ if torch.cuda.mem_get_info()[0] < need:+ pytest.skip(reason = f"needs {need / 2**30:.1f} GiB free for a > 2^31 element bf16 weight")+ torch.manual_seed(0)+ W = torch.empty(n_experts, N, K, device = "cuda", dtype = torch.bfloat16)+ W[:-1].zero_()+ W[-1].normal_(0, 0.02)+ # Every token goes to the last expert, whose weights start past element 2^31.+ m_sizes = torch.zeros(n_experts, device = "cuda", dtype = torch.int32)+ m_sizes[-1] = tokens+ X = torch.randn(tokens, K, device = "cuda", dtype = torch.bfloat16)+ y = gg.grouped_gemm_forward(+ X, W, 1, m_sizes, BLOCK_SIZE_M = 64, BLOCK_SIZE_N = 128, BLOCK_SIZE_K = 64+ )+ ref = X.float() @ W[-1].float().T+ assert (y.float() - ref).abs().max().item() < 0.05 * ref.abs().max().item()+ del W+ torch.cuda.empty_cache()diff --git a/unsloth/kernels/moe/grouped_gemm/interface.py b/unsloth/kernels/moe/grouped_gemm/interface.pyindex 07fd12ab935..62d76ddbe3a 100644--- a/unsloth/kernels/moe/grouped_gemm/interface.py+++ b/unsloth/kernels/moe/grouped_gemm/interface.py@@ -1,8 +1,11 @@# SPDX-License-Identifier: GNU Affero General Public License v3.0# Copyright 2023-present the Unsloth team. All rights reserved.+import functools+import inspectimport loggingimport warnings+from typing import Optionalfrom dataclasses import asdictfrom unsloth import DEVICE_TYPE@@ -56,12 +59,6 @@ def supports_tma():return _SUPPORTS_TMA-try:- from torch.compiler import allow_in_graph-except ImportError:- from torch._dynamo import allow_in_graph--def _is_tracing(*tensors):"""True if tensors are fake tensors used during torch.compile tracing (Triton cannot run). Not torch.compiler.is_compiling(): that is True during both tracing AND execution, and only tracing must skip the kernels."""for t in tensors:@@ -72,6 +69,7 @@ def _is_tracing(*tensors):_per_device_alloc_fns = {}+_FUSED_MUL_WARN = Falsedef get_per_device_per_stream_alloc_fn(device):@@ -101,8 +99,7 @@ def log_kernel_info(logger.debug(f"{kernel_name} autotuned best_config: {best_config}")-@allow_in_graph-def grouped_gemm_forward(+def _grouped_gemm_forward_impl(X: torch.Tensor,W: torch.Tensor,topk: int,@@ -276,8 +273,7 @@ def grouped_gemm_forward(return y-@allow_in_graph-def grouped_gemm_dX(+def _grouped_gemm_dX_impl(dY: torch.Tensor,W: torch.Tensor,gather_indices: torch.Tensor,@@ -417,8 +413,7 @@ def grouped_gemm_dX(return dX-@allow_in_graph-def grouped_gemm_dW(+def _grouped_gemm_dW_impl(X: torch.Tensor,dY: torch.Tensor,m_sizes: torch.Tensor,@@ -565,6 +560,100 @@ def grouped_gemm_dW(return dW+# Opaque ops: allow_in_graph let AOT trace the body on fake tensors, keeping torch.empty and dropping the launch.+# Not triton_op: torch.compile rejects prune_configs_by on triton.autotune.+def _fwd_fake(X, W, topk, m_sizes, gather_indices, topk_weights, permute_x, permute_y, *args):+ N = W.shape[1] if W.ndim == 3 else W.shape[0] // m_sizes.shape[0]+ total_tokens = gather_indices.shape[0] if (permute_x or permute_y) else X.numel() // X.shape[-1]+ return X.new_empty((total_tokens, N))+++def _dX_fake(dY, W, gather_indices, m_sizes, topk, *args):+ total_tokens = (+ gather_indices.shape[0] if gather_indices is not None else dY.numel() // dY.shape[-1]+ )+ return dY.new_empty((total_tokens, W.shape[-1]))+++def _dW_fake(X, dY, m_sizes, *args):+ return X.new_empty((m_sizes.shape[0], dY.shape[-1], X.shape[-1]))+++_OPTIONAL_TENSORS = ("gather_indices", "topk_weights")+++def _make_op(name, impl, fake):+ """torch.library.custom_op (torch >= 2.4) over impl; torch.compiler.disable where it is missing."""+ custom_op = getattr(getattr(torch, "library", None), "custom_op", None)+ if custom_op is None:+ return torch._dynamo.disable(impl)+ existing = getattr(getattr(torch.ops, "unsloth", None), name, None)+ if existing is not None:+ # grouped_gemm is also importable as a top-level package (glm4_moe, tests): reuse the registration.+ return existing+ params = [+ p.replace(+ default = inspect.Parameter.empty,+ annotation = Optional[torch.Tensor] if p.name in _OPTIONAL_TENSORS else p.annotation,+ )+ for p in inspect.signature(impl).parameters.values()+ ]+ names = [p.name for p in params]++ def op(*args):+ return impl(**dict(zip(names, args)))++ op.__signature__ = inspect.Signature(params, return_annotation = torch.Tensor)+ op.__name__ = name+ op = custom_op(f"unsloth::{name}", op, mutates_args = ())+ op.register_fake(fake)+ return op+++_grouped_gemm_forward_op = _make_op("grouped_gemm_forward", _grouped_gemm_forward_impl, _fwd_fake)+_grouped_gemm_dX_op = _make_op("grouped_gemm_dX", _grouped_gemm_dX_impl, _dX_fake)+_grouped_gemm_dW_op = _make_op("grouped_gemm_dW", _grouped_gemm_dW_impl, _dW_fake)+++def _signature(impl):+ params = inspect.signature(impl).parameters.values()+ return tuple(p.name for p in params), {+ p.name: p.default for p in params if p.default is not p.empty+ }+++_SIGNATURES = {+ impl: _signature(impl)+ for impl in (_grouped_gemm_forward_impl, _grouped_gemm_dX_impl, _grouped_gemm_dW_impl)+}+++def _dispatch(impl, op, args, kwargs):+ names, defaults = _SIGNATURES[impl]+ bound = dict(defaults)+ bound.update(zip(names, args))+ bound.update(kwargs)+ # Eager calls skip the custom-op dispatcher; traced calls (dynamo, or fake tensors under make_fx) need the op.+ if torch.compiler.is_compiling() or _is_tracing(bound[names[0]], bound[names[1]]):+ return op(*[bound[n] for n in names])+ return impl(**bound)++[email protected](_grouped_gemm_forward_impl)+def grouped_gemm_forward(*args, **kwargs) -> torch.Tensor:+ return _dispatch(_grouped_gemm_forward_impl, _grouped_gemm_forward_op, args, kwargs)++[email protected](_grouped_gemm_dX_impl)+def grouped_gemm_dX(*args, **kwargs) -> torch.Tensor:+ return _dispatch(_grouped_gemm_dX_impl, _grouped_gemm_dX_op, args, kwargs)++[email protected](_grouped_gemm_dW_impl)+def grouped_gemm_dW(*args, **kwargs) -> torch.Tensor:+ return _dispatch(_grouped_gemm_dW_impl, _grouped_gemm_dW_op, args, kwargs)++class GroupedGemm(torch.autograd.Function):@staticmethoddef forward(diff --git a/unsloth/kernels/moe/grouped_gemm/kernels/backward.py b/unsloth/kernels/moe/grouped_gemm/kernels/backward.pyindex 9e60c24342e..bf00bd3eaa2 100644--- a/unsloth/kernels/moe/grouped_gemm/kernels/backward.py+++ b/unsloth/kernels/moe/grouped_gemm/kernels/backward.py@@ -105,7 +105,8 @@ def _grouped_gemm_dX_kernel(m_end = m_start + m_sizeif m_size > 0:- n_start = expert_idx * N+ # int64: expert_idx * N * K passes 2^31 once the weight holds 2^31 elements.+ n_start = expert_idx.to(tl.int64) * Nnum_m_tiles = tl.cdiv(m_size, BLOCK_SIZE_M)num_k_tiles = tl.cdiv(K, BLOCK_SIZE_K)num_tiles_per_expert = num_m_tiles * num_k_tiles@@ -137,7 +138,8 @@ def _grouped_gemm_dX_kernel(gather_indices_ptr + indices_to_gather,mask = indices_to_gather < TOTAL_TOKENS,)- expert_token_offsets = expert_token_idx[:, None]+ expert_token_offsets = expert_token_idx.to(tl.int64)[:, None]+ indices_to_gather_64 = indices_to_gather.to(tl.int64)row_mask = gather_offsets < m_sizerow_mask = row_mask[:, None]@@ -146,18 +148,18 @@ def _grouped_gemm_dX_kernel(if PERMUTE_X:# Permuted on load in the forward pass (typically the first grouped GEMM in the MoE MLP), so load contiguous and permute on store.- load_a_idx = indices_to_gather[:, None] * N+ load_a_idx = indices_to_gather_64[:, None] * Nstore_idx = expert_token_offsets * Kelse:# Permuted on store in the forward pass (typically the second grouped GEMM), so permute on load and store contiguous.load_a_idx = expert_token_offsets * N- store_idx = indices_to_gather[:, None] * K+ store_idx = indices_to_gather_64[:, None] * Kelse:# Offsets relative to the CURRENT expert; m_start then advances to this expert's start token.offs_am = tile_m_idx * BLOCK_SIZE_M + m_block_range# [M, N] @ [N, K] -> [M, K], so A strides by N and B by K, plus m_start for A's expert start token and n_start for B's slice of the [E, N, K] weight matrix.- row_offsets_a = m_start + offs_am[:, None]+ row_offsets_a = (m_start + offs_am[:, None]).to(tl.int64)load_a_idx = row_offsets_a * Nstore_idx = row_offsets_a * Krow_mask = offs_am[:, None] < m_size@@ -330,7 +332,7 @@ def _grouped_gemm_dW_kernel(m_end = m_start + m_size# Offset by n_start: the result goes into this expert's slice of the global [E, N, K] weight matrix.- n_start = expert_idx * N+ n_start = expert_idx.to(tl.int64) * Nstore_row_offs = n_start + n_offset + block_range_nif m_size > 0:@@ -354,7 +356,7 @@ def _grouped_gemm_dW_kernel(if m_block_size > 0:m_global_offset = m_start + tile_m_idx- m_offsets = m_global_offset + block_range_m+ m_offsets = (m_global_offset + block_range_m).to(tl.int64)if PERMUTE_X or PERMUTE_Y:gather_offsets = tile_m_idx + block_range_m@@ -367,7 +369,7 @@ def _grouped_gemm_dW_kernel(gather_indices_ptr + indices_to_gather,mask = indices_to_gather < TOTAL_TOKENS,)- expert_token_offsets = expert_token_idx[:, None]+ expert_token_offsets = expert_token_idx.to(tl.int64)[:, None]row_load_mask = gather_offsets < m_size@@ -378,7 +380,7 @@ def _grouped_gemm_dW_kernel() # Permute on load: token to expert order, /TOPK for the original count.dY_row_load_idx = m_offsets[:, None] * Nelse:- x_row_load_idx = indices_to_gather[:, None] * K+ x_row_load_idx = indices_to_gather.to(tl.int64)[:, None] * KdY_row_load_idx = expert_token_offsets * Nelse:diff --git a/unsloth/kernels/moe/grouped_gemm/kernels/forward.py b/unsloth/kernels/moe/grouped_gemm/kernels/forward.pyindex 0a4010e2881..6960cb33d90 100644--- a/unsloth/kernels/moe/grouped_gemm/kernels/forward.py+++ b/unsloth/kernels/moe/grouped_gemm/kernels/forward.py@@ -82,7 +82,8 @@ def _grouped_gemm_forward_kernel(m_end = m_start + m_sizeif m_size > 0:- n_start = expert_idx * N+ # int64: expert_idx * N * K passes 2^31 once the weight holds 2^31 elements.+ n_start = expert_idx.to(tl.int64) * Nnum_m_tiles = tl.cdiv(m_size, BLOCK_SIZE_M)num_n_tiles = tl.cdiv(N, BLOCK_SIZE_N)@@ -116,7 +117,8 @@ def _grouped_gemm_forward_kernel(gather_indices_ptr + indices_to_gather,mask = indices_to_gather < TOTAL_TOKENS,)- expert_token_offsets = expert_token_idx[:, None]+ expert_token_offsets = expert_token_idx.to(tl.int64)[:, None]+ indices_to_gather_64 = indices_to_gather.to(tl.int64)# Masks for permuted load and storerow_mask = gather_offsets < m_size@@ -128,21 +130,21 @@ def _grouped_gemm_forward_kernel(load_idx = ((expert_token_offsets // TOPK) * K) # Permute on load from token to expert order, dividing by TOPK to index the original token count.- store_idx = indices_to_gather[:, None] * N+ store_idx = indices_to_gather_64[:, None] * Nelse:off_am = tile_m_idx * BLOCK_SIZE_Mif not PERMUTE_Y:# These will already be computed if permuting yoffs_am = off_am + m_block_rangerow_mask = offs_am[:, None] < m_size- row_idx = m_start + offs_am[:, None]+ row_idx = (m_start + offs_am[:, None]).to(tl.int64)store_idx = row_idx * Nif not USE_TMA_LOAD_X:load_idx = row_idx * Kif PERMUTE_Y:if not USE_TMA_LOAD_X:- load_idx = indices_to_gather[:, None] * K+ load_idx = indices_to_gather_64[:, None] * Kstore_idx = expert_token_offsets * N# topk weights are always loaded in expert order: pre-multiplication scales hidden states before