Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Keep the Triton MoE grouped GEMM in compiled graphs and index weights past 2^31 elements (#12114)

Daniel Hanb5bae411bd57
diff --git a/tests/test_grouped_gemm_compile_and_int64.py b/tests/test_grouped_gemm_compile_and_int64.py
new file mode 100644
index 00000000000..c0da2e4506a
--- /dev/null
+++ b/tests/test_grouped_gemm_compile_and_int64.py
@@ -0,0 +1,124 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""The Triton MoE grouped GEMM must run under torch.compile and index weights past 2^31 elements.
+
+1. The kernels were marked `allow_in_graph`, so AOT traced their Python bodies on fake
+ tensors: the graph kept the `torch.empty` output and dropped the Triton launch, and the
+ compiled forward / backward returned uninitialised memory. They are now custom ops.
+2. Weight offsets (`expert_idx * N * K`) were int32, so an (E, N, K) weight holding more
+ than 2^31 elements read and wrote the wrong experts (Step-3.7 at 288 experts).
+"""
+
+import sys
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT))
+
+import torch # noqa: E402
+
+pytest.importorskip("triton", reason = "the grouped GEMM is a Triton kernel")
+
+try:
+ from unsloth.kernels.moe.grouped_gemm import interface as gg
+ from unsloth.kernels.moe.grouped_gemm.kernels.tuning import (
+ KernelConfigBackward_dW,
+ KernelConfigBackward_dX,
+ KernelConfigForward,
+ )
+except Exception as exc: # pragma: no cover - depends on the installed stack
+ pytest.skip(reason = f"grouped_gemm is unimportable here: {exc}", allow_module_level = True)
+
+requires_cuda = pytest.mark.skipif(
+ not torch.cuda.is_available(), reason = "grouped GEMM needs a real CUDA device"
+)
+
+E, TOPK, T, H, I = 4, 2, 64, 256, 128
+FWD = KernelConfigForward(
+ BLOCK_SIZE_M = 64, BLOCK_SIZE_N = 64, BLOCK_SIZE_K = 64, num_warps = 4, num_stages = 2
+)
+DX = KernelConfigBackward_dX(
+ BLOCK_SIZE_M = 64, BLOCK_SIZE_N = 64, BLOCK_SIZE_K = 64, num_warps = 4, num_stages = 2
+)
+DW = KernelConfigBackward_dW(
+ BLOCK_SIZE_M = 64, BLOCK_SIZE_N = 64, BLOCK_SIZE_K = 64, num_warps = 4, num_stages = 2
+)
+
+
+def _moe_mlp(x, w1, w2, m_sizes, gather_indices):
+ h = gg.grouped_gemm(
+ X = x, W = w1, m_sizes = m_sizes, topk = TOPK, gather_indices = gather_indices,
+ permute_x = True, kernel_config_fwd = FWD, kernel_config_bwd_dX = DX,
+ kernel_config_bwd_dW = DW, is_first_gemm = True,
+ ) # fmt: skip
+ gate, up = h.chunk(2, dim = -1)
+ return gg.grouped_gemm(
+ X = torch.nn.functional.silu(gate) * up, W = w2, m_sizes = m_sizes, topk = TOPK,
+ gather_indices = gather_indices, permute_y = True, kernel_config_fwd = FWD,
+ kernel_config_bwd_dX = DX, kernel_config_bwd_dW = DW, is_first_gemm = False,
+ ) # fmt: skip
+
+
+def _routing(device):
+ idx = torch.topk(torch.randn(T, E, device = device), TOPK).indices.view(-1)
+ m_sizes = torch.bincount(idx, minlength = E).to(torch.int32)
+ return m_sizes, torch.argsort(idx, stable = True).to(torch.int32)
+
+
+def test_kernels_are_opaque_ops_for_the_compiler():
+ if not hasattr(torch.library, "custom_op"):
+ pytest.skip(
+ reason = "torch < 2.4 has no torch.library.custom_op; the kernels fall back to dynamo.disable"
+ )
+ for name in ("grouped_gemm_forward", "grouped_gemm_dX", "grouped_gemm_dW"):
+ assert hasattr(torch.ops.unsloth, name)
+
+
+@requires_cuda
+def test_compiled_matches_eager_forward_and_backward():
+ torch.manual_seed(0)
+ dev = "cuda"
+ w1 = (torch.randn(E, 2 * I, H, device = dev) * 0.05).bfloat16().requires_grad_(True)
+ w2 = (torch.randn(E, H, I, device = dev) * 0.05).bfloat16().requires_grad_(True)
+ compiled = torch.compile(_moe_mlp, fullgraph = True)
+ # Several calls: the stale buffer only shows up once the allocator reuses memory.
+ for step in range(3):
+ m_sizes, gather_indices = _routing(dev)
+ x = torch.randn(T, H, device = dev).bfloat16() * (step + 1)
+ grad = torch.randn(T * TOPK, H, device = dev)
+ outs = []
+ for fn in (_moe_mlp, compiled):
+ xi = x.clone().requires_grad_(True)
+ w1.grad = w2.grad = None
+ y = fn(xi, w1, w2, m_sizes, gather_indices)
+ (y.float() * grad).sum().backward()
+ outs.append([t.detach().float().clone() for t in (y, xi.grad, w1.grad, w2.grad)])
+ for name, e, c in zip(("y", "dx", "dw1", "dw2"), *outs):
+ rel = ((c - e).norm() / e.norm()).item()
+ assert rel < 2e-2, f"call {step}: compiled {name} differs from eager, rel {rel:.3g}"
+
+
+@requires_cuda
+def test_weight_past_2_pow_31_elements_reads_the_right_expert():
+ n_experts, N, K, tokens = 129, 4096, 4096, 256 # 129 * 4096 * 4096 > 2^31
+ need = n_experts * N * K * 2 + (1 << 30)
+ if torch.cuda.mem_get_info()[0] < need:
+ pytest.skip(reason = f"needs {need / 2**30:.1f} GiB free for a > 2^31 element bf16 weight")
+ torch.manual_seed(0)
+ W = torch.empty(n_experts, N, K, device = "cuda", dtype = torch.bfloat16)
+ W[:-1].zero_()
+ W[-1].normal_(0, 0.02)
+ # Every token goes to the last expert, whose weights start past element 2^31.
+ m_sizes = torch.zeros(n_experts, device = "cuda", dtype = torch.int32)
+ m_sizes[-1] = tokens
+ X = torch.randn(tokens, K, device = "cuda", dtype = torch.bfloat16)
+ y = gg.grouped_gemm_forward(
+ X, W, 1, m_sizes, BLOCK_SIZE_M = 64, BLOCK_SIZE_N = 128, BLOCK_SIZE_K = 64
+ )
+ ref = X.float() @ W[-1].float().T
+ assert (y.float() - ref).abs().max().item() < 0.05 * ref.abs().max().item()
+ del W
+ torch.cuda.empty_cache()
diff --git a/unsloth/kernels/moe/grouped_gemm/interface.py b/unsloth/kernels/moe/grouped_gemm/interface.py
index 07fd12ab935..62d76ddbe3a 100644
--- a/unsloth/kernels/moe/grouped_gemm/interface.py
+++ b/unsloth/kernels/moe/grouped_gemm/interface.py
@@ -1,8 +1,11 @@
# SPDX-License-Identifier: GNU Affero General Public License v3.0
# Copyright 2023-present the Unsloth team. All rights reserved.
+import functools
+import inspect
import logging
import warnings
+from typing import Optional
from dataclasses import asdict
from unsloth import DEVICE_TYPE
@@ -56,12 +59,6 @@ def supports_tma():
return _SUPPORTS_TMA
-try:
- from torch.compiler import allow_in_graph
-except ImportError:
- from torch._dynamo import allow_in_graph
-
-
def _is_tracing(*tensors):
"""True if tensors are fake tensors used during torch.compile tracing (Triton cannot run). Not torch.compiler.is_compiling(): that is True during both tracing AND execution, and only tracing must skip the kernels."""
for t in tensors:
@@ -72,6 +69,7 @@ def _is_tracing(*tensors):
_per_device_alloc_fns = {}
+_FUSED_MUL_WARN = False
def get_per_device_per_stream_alloc_fn(device):
@@ -101,8 +99,7 @@ def log_kernel_info(
logger.debug(f"{kernel_name} autotuned best_config: {best_config}")
-@allow_in_graph
-def grouped_gemm_forward(
+def _grouped_gemm_forward_impl(
X: torch.Tensor,
W: torch.Tensor,
topk: int,
@@ -276,8 +273,7 @@ def grouped_gemm_forward(
return y
-@allow_in_graph
-def grouped_gemm_dX(
+def _grouped_gemm_dX_impl(
dY: torch.Tensor,
W: torch.Tensor,
gather_indices: torch.Tensor,
@@ -417,8 +413,7 @@ def grouped_gemm_dX(
return dX
-@allow_in_graph
-def grouped_gemm_dW(
+def _grouped_gemm_dW_impl(
X: torch.Tensor,
dY: torch.Tensor,
m_sizes: torch.Tensor,
@@ -565,6 +560,100 @@ def grouped_gemm_dW(
return dW
+# Opaque ops: allow_in_graph let AOT trace the body on fake tensors, keeping torch.empty and dropping the launch.
+# Not triton_op: torch.compile rejects prune_configs_by on triton.autotune.
+def _fwd_fake(X, W, topk, m_sizes, gather_indices, topk_weights, permute_x, permute_y, *args):
+ N = W.shape[1] if W.ndim == 3 else W.shape[0] // m_sizes.shape[0]
+ total_tokens = gather_indices.shape[0] if (permute_x or permute_y) else X.numel() // X.shape[-1]
+ return X.new_empty((total_tokens, N))
+
+
+def _dX_fake(dY, W, gather_indices, m_sizes, topk, *args):
+ total_tokens = (
+ gather_indices.shape[0] if gather_indices is not None else dY.numel() // dY.shape[-1]
+ )
+ return dY.new_empty((total_tokens, W.shape[-1]))
+
+
+def _dW_fake(X, dY, m_sizes, *args):
+ return X.new_empty((m_sizes.shape[0], dY.shape[-1], X.shape[-1]))
+
+
+_OPTIONAL_TENSORS = ("gather_indices", "topk_weights")
+
+
+def _make_op(name, impl, fake):
+ """torch.library.custom_op (torch >= 2.4) over impl; torch.compiler.disable where it is missing."""
+ custom_op = getattr(getattr(torch, "library", None), "custom_op", None)
+ if custom_op is None:
+ return torch._dynamo.disable(impl)
+ existing = getattr(getattr(torch.ops, "unsloth", None), name, None)
+ if existing is not None:
+ # grouped_gemm is also importable as a top-level package (glm4_moe, tests): reuse the registration.
+ return existing
+ params = [
+ p.replace(
+ default = inspect.Parameter.empty,
+ annotation = Optional[torch.Tensor] if p.name in _OPTIONAL_TENSORS else p.annotation,
+ )
+ for p in inspect.signature(impl).parameters.values()
+ ]
+ names = [p.name for p in params]
+
+ def op(*args):
+ return impl(**dict(zip(names, args)))
+
+ op.__signature__ = inspect.Signature(params, return_annotation = torch.Tensor)
+ op.__name__ = name
+ op = custom_op(f"unsloth::{name}", op, mutates_args = ())
+ op.register_fake(fake)
+ return op
+
+
+_grouped_gemm_forward_op = _make_op("grouped_gemm_forward", _grouped_gemm_forward_impl, _fwd_fake)
+_grouped_gemm_dX_op = _make_op("grouped_gemm_dX", _grouped_gemm_dX_impl, _dX_fake)
+_grouped_gemm_dW_op = _make_op("grouped_gemm_dW", _grouped_gemm_dW_impl, _dW_fake)
+
+
+def _signature(impl):
+ params = inspect.signature(impl).parameters.values()
+ return tuple(p.name for p in params), {
+ p.name: p.default for p in params if p.default is not p.empty
+ }
+
+
+_SIGNATURES = {
+ impl: _signature(impl)
+ for impl in (_grouped_gemm_forward_impl, _grouped_gemm_dX_impl, _grouped_gemm_dW_impl)
+}
+
+
+def _dispatch(impl, op, args, kwargs):
+ names, defaults = _SIGNATURES[impl]
+ bound = dict(defaults)
+ bound.update(zip(names, args))
+ bound.update(kwargs)
+ # Eager calls skip the custom-op dispatcher; traced calls (dynamo, or fake tensors under make_fx) need the op.
+ if torch.compiler.is_compiling() or _is_tracing(bound[names[0]], bound[names[1]]):
+ return op(*[bound[n] for n in names])
+ return impl(**bound)
+
+
[email protected](_grouped_gemm_forward_impl)
+def grouped_gemm_forward(*args, **kwargs) -> torch.Tensor:
+ return _dispatch(_grouped_gemm_forward_impl, _grouped_gemm_forward_op, args, kwargs)
+
+
[email protected](_grouped_gemm_dX_impl)
+def grouped_gemm_dX(*args, **kwargs) -> torch.Tensor:
+ return _dispatch(_grouped_gemm_dX_impl, _grouped_gemm_dX_op, args, kwargs)
+
+
[email protected](_grouped_gemm_dW_impl)
+def grouped_gemm_dW(*args, **kwargs) -> torch.Tensor:
+ return _dispatch(_grouped_gemm_dW_impl, _grouped_gemm_dW_op, args, kwargs)
+
+
class GroupedGemm(torch.autograd.Function):
@staticmethod
def forward(
diff --git a/unsloth/kernels/moe/grouped_gemm/kernels/backward.py b/unsloth/kernels/moe/grouped_gemm/kernels/backward.py
index 9e60c24342e..bf00bd3eaa2 100644
--- a/unsloth/kernels/moe/grouped_gemm/kernels/backward.py
+++ b/unsloth/kernels/moe/grouped_gemm/kernels/backward.py
@@ -105,7 +105,8 @@ def _grouped_gemm_dX_kernel(
m_end = m_start + m_size
if m_size > 0:
- n_start = expert_idx * N
+ # int64: expert_idx * N * K passes 2^31 once the weight holds 2^31 elements.
+ n_start = expert_idx.to(tl.int64) * N
num_m_tiles = tl.cdiv(m_size, BLOCK_SIZE_M)
num_k_tiles = tl.cdiv(K, BLOCK_SIZE_K)
num_tiles_per_expert = num_m_tiles * num_k_tiles
@@ -137,7 +138,8 @@ def _grouped_gemm_dX_kernel(
gather_indices_ptr + indices_to_gather,
mask = indices_to_gather < TOTAL_TOKENS,
)
- expert_token_offsets = expert_token_idx[:, None]
+ expert_token_offsets = expert_token_idx.to(tl.int64)[:, None]
+ indices_to_gather_64 = indices_to_gather.to(tl.int64)
row_mask = gather_offsets < m_size
row_mask = row_mask[:, None]
@@ -146,18 +148,18 @@ def _grouped_gemm_dX_kernel(
if PERMUTE_X:
# Permuted on load in the forward pass (typically the first grouped GEMM in the MoE MLP), so load contiguous and permute on store.
- load_a_idx = indices_to_gather[:, None] * N
+ load_a_idx = indices_to_gather_64[:, None] * N
store_idx = expert_token_offsets * K
else:
# Permuted on store in the forward pass (typically the second grouped GEMM), so permute on load and store contiguous.
load_a_idx = expert_token_offsets * N
- store_idx = indices_to_gather[:, None] * K
+ store_idx = indices_to_gather_64[:, None] * K
else:
# Offsets relative to the CURRENT expert; m_start then advances to this expert's start token.
offs_am = tile_m_idx * BLOCK_SIZE_M + m_block_range
# [M, N] @ [N, K] -> [M, K], so A strides by N and B by K, plus m_start for A's expert start token and n_start for B's slice of the [E, N, K] weight matrix.
- row_offsets_a = m_start + offs_am[:, None]
+ row_offsets_a = (m_start + offs_am[:, None]).to(tl.int64)
load_a_idx = row_offsets_a * N
store_idx = row_offsets_a * K
row_mask = offs_am[:, None] < m_size
@@ -330,7 +332,7 @@ def _grouped_gemm_dW_kernel(
m_end = m_start + m_size
# Offset by n_start: the result goes into this expert's slice of the global [E, N, K] weight matrix.
- n_start = expert_idx * N
+ n_start = expert_idx.to(tl.int64) * N
store_row_offs = n_start + n_offset + block_range_n
if m_size > 0:
@@ -354,7 +356,7 @@ def _grouped_gemm_dW_kernel(
if m_block_size > 0:
m_global_offset = m_start + tile_m_idx
- m_offsets = m_global_offset + block_range_m
+ m_offsets = (m_global_offset + block_range_m).to(tl.int64)
if PERMUTE_X or PERMUTE_Y:
gather_offsets = tile_m_idx + block_range_m
@@ -367,7 +369,7 @@ def _grouped_gemm_dW_kernel(
gather_indices_ptr + indices_to_gather,
mask = indices_to_gather < TOTAL_TOKENS,
)
- expert_token_offsets = expert_token_idx[:, None]
+ expert_token_offsets = expert_token_idx.to(tl.int64)[:, None]
row_load_mask = gather_offsets < m_size
@@ -378,7 +380,7 @@ def _grouped_gemm_dW_kernel(
) # Permute on load: token to expert order, /TOPK for the original count.
dY_row_load_idx = m_offsets[:, None] * N
else:
- x_row_load_idx = indices_to_gather[:, None] * K
+ x_row_load_idx = indices_to_gather.to(tl.int64)[:, None] * K
dY_row_load_idx = expert_token_offsets * N
else:
diff --git a/unsloth/kernels/moe/grouped_gemm/kernels/forward.py b/unsloth/kernels/moe/grouped_gemm/kernels/forward.py
index 0a4010e2881..6960cb33d90 100644
--- a/unsloth/kernels/moe/grouped_gemm/kernels/forward.py
+++ b/unsloth/kernels/moe/grouped_gemm/kernels/forward.py
@@ -82,7 +82,8 @@ def _grouped_gemm_forward_kernel(
m_end = m_start + m_size
if m_size > 0:
- n_start = expert_idx * N
+ # int64: expert_idx * N * K passes 2^31 once the weight holds 2^31 elements.
+ n_start = expert_idx.to(tl.int64) * N
num_m_tiles = tl.cdiv(m_size, BLOCK_SIZE_M)
num_n_tiles = tl.cdiv(N, BLOCK_SIZE_N)
@@ -116,7 +117,8 @@ def _grouped_gemm_forward_kernel(
gather_indices_ptr + indices_to_gather,
mask = indices_to_gather < TOTAL_TOKENS,
)
- expert_token_offsets = expert_token_idx[:, None]
+ expert_token_offsets = expert_token_idx.to(tl.int64)[:, None]
+ indices_to_gather_64 = indices_to_gather.to(tl.int64)
# Masks for permuted load and store
row_mask = gather_offsets < m_size
@@ -128,21 +130,21 @@ def _grouped_gemm_forward_kernel(
load_idx = (
(expert_token_offsets // TOPK) * K
) # Permute on load from token to expert order, dividing by TOPK to index the original token count.
- store_idx = indices_to_gather[:, None] * N
+ store_idx = indices_to_gather_64[:, None] * N
else:
off_am = tile_m_idx * BLOCK_SIZE_M
if not PERMUTE_Y:
# These will already be computed if permuting y
offs_am = off_am + m_block_range
row_mask = offs_am[:, None] < m_size
- row_idx = m_start + offs_am[:, None]
+ row_idx = (m_start + offs_am[:, None]).to(tl.int64)
store_idx = row_idx * N
if not USE_TMA_LOAD_X:
load_idx = row_idx * K
if PERMUTE_Y:
if not USE_TMA_LOAD_X:
- load_idx = indices_to_gather[:, None] * K
+ load_idx = indices_to_gather_64[:, None] * K
store_idx = expert_token_offsets * N
# topk weights are always loaded in expert order: pre-multiplication scales hidden states before