Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

MoE grouped GEMM: per-expert dW kernel (#14550)

Daniel Han4a5a3e5828c2
diff --git a/tests/test_grouped_gemm_mixed_tma_device.py b/tests/test_grouped_gemm_mixed_tma_device.py
index fbcdc531068..01a6c6c5368 100644
--- a/tests/test_grouped_gemm_mixed_tma_device.py
+++ b/tests/test_grouped_gemm_mixed_tma_device.py
@@ -111,8 +111,10 @@ def _load(
n: _Kernel(launches, guard_stack)
for n in (
"_autotuned_grouped_gemm_dW_kernel",
+ "_autotuned_grouped_gemm_dW_per_expert_kernel",
"_autotuned_grouped_gemm_dX_kernel",
"_grouped_gemm_dW_kernel",
+ "_grouped_gemm_dW_per_expert_kernel",
"_grouped_gemm_dX_kernel",
)
}
diff --git a/tests/test_moe_dw_per_expert.py b/tests/test_moe_dw_per_expert.py
new file mode 100644
index 00000000000..2de3b385601
--- /dev/null
+++ b/tests/test_moe_dw_per_expert.py
@@ -0,0 +1,159 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Per-expert grouped GEMM dW kernel vs the legacy kernel (UNSLOTH_MOE_DW_KERNEL=legacy) and fp64."""
+
+import sys
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT))
+
+import torch # noqa: E402
+
+
+def _gg():
+ pytest.importorskip("triton", reason = "the grouped GEMM is a Triton kernel")
+ if not torch.cuda.is_available():
+ pytest.skip("grouped GEMM needs a CUDA device")
+ try:
+ from unsloth.kernels.moe.grouped_gemm import interface as gg
+ except Exception as exc: # pragma: no cover - depends on the installed stack
+ pytest.skip(f"grouped_gemm is unimportable here: {exc}")
+ return gg
+
+
+def _routing(T, E, topk, empty, seed):
+ g = torch.Generator(device = "cuda").manual_seed(seed)
+ logits = torch.randn(T, E, device = "cuda", generator = g)
+ for e in empty:
+ logits[:, e] = -float("inf")
+ if empty:
+ logits[:, (max(empty) + 1) % E] += 4.0 # one hot expert next to the empty ones
+ idx = torch.topk(logits, topk).indices.view(-1)
+ m_sizes = torch.bincount(idx, minlength = E).to(torch.int32)
+ gather = torch.argsort(idx, stable = True).to(torch.int32)
+ return m_sizes, gather
+
+
+def _ref(X, dY, m_sizes, gather, topk, mode, E, N, K):
+ X, dY, gather = X.double(), dY.double(), gather.long()
+ if mode == "permute_x":
+ xs, dys = X[gather // topk], dY
+ elif mode == "permute_y":
+ xs, dys = X, dY[gather]
+ else:
+ xs, dys = X, dY
+ out = torch.zeros(E, N, K, dtype = torch.float64, device = X.device)
+ start = 0
+ for e, m in enumerate(m_sizes.tolist()):
+ out[e] = dys[start : start + m].T @ xs[start : start + m]
+ start += m
+ return out
+
+
+# (E, N, K, T, topk): Qwen3-30B-A3B down, Qwen1.5-MoE gate_up (ragged N tail), Mixtral-like few experts.
+SHAPES = [(128, 2048, 768, 256, 8), (60, 2816, 2048, 128, 4), (8, 704, 320, 300, 2)]
+
+
[email protected]("shape", SHAPES)
[email protected]("mode", ["permute_x", "permute_y", "none"])
[email protected]("dtype", [torch.bfloat16, torch.float16])
[email protected]("accumulate", [False, True])
+def test_per_expert_matches_legacy_and_fp64(monkeypatch, shape, mode, dtype, accumulate):
+ gg = _gg()
+ E, N, K, T, topk = shape
+ empty = (0, 3, E - 1)
+ m_sizes, gather = _routing(T, E, topk, empty, seed = E)
+ g = torch.Generator(device = "cuda").manual_seed(1)
+ X = torch.randn(T if mode == "permute_x" else T * topk, K, device = "cuda", generator = g).to(dtype)
+ dY = torch.randn(T * topk, N, device = "cuda", generator = g).to(dtype)
+ kw = dict(
+ BLOCK_SIZE_M = 64, BLOCK_SIZE_N = 64, BLOCK_SIZE_K = 64, num_warps = 4, num_stages = 3,
+ permute_x = mode == "permute_x", permute_y = mode == "permute_y",
+ ) # fmt: skip
+ prev = torch.randn(E, N, K, device = "cuda", generator = g).to(dtype)
+ outs = {}
+ for kernel in ("auto", "legacy"):
+ monkeypatch.setenv("UNSLOTH_MOE_DW_KERNEL", kernel)
+ if accumulate:
+ out = prev.clone()
+ gg._grouped_gemm_dW_accumulate_impl(X, dY, m_sizes, gather, out, topk, **kw)
+ else:
+ out = gg._grouped_gemm_dW_impl(X, dY, m_sizes, gather, topk, **kw)
+ outs[kernel] = out
+ # Same reduction order per (expert, tile): bit-identical to the legacy kernel.
+ assert torch.equal(outs["auto"], outs["legacy"])
+ ref = _ref(X, dY, m_sizes, gather, topk, mode, E, N, K) + (prev.double() if accumulate else 0)
+ rel = (outs["auto"].double() - ref).norm() / ref.norm()
+ assert rel < (3e-3 if dtype == torch.bfloat16 else 5e-4), rel
+ for e in empty: # an empty expert is exactly zero, or exactly the previous gradient
+ assert torch.equal(outs["auto"][e], prev[e] if accumulate else torch.zeros_like(prev[e]))
+
+
+def test_per_expert_fp32_dw_dtype_and_poisoned_output(monkeypatch):
+ gg = _gg()
+ E, N, K, T, topk = 16, 256, 192, 64, 2
+ m_sizes, gather = _routing(T, E, topk, (2, 5), seed = 3)
+ X = torch.randn(T, K, device = "cuda").bfloat16()
+ dY = torch.randn(T * topk, N, device = "cuda").bfloat16()
+ # The per-expert path allocates dW with torch.empty: poison the allocator's next block first.
+ torch.full((E * N * K,), float("nan"), device = "cuda", dtype = torch.float32)
+ monkeypatch.setenv("UNSLOTH_MOE_DW_KERNEL", "auto")
+ out = gg._grouped_gemm_dW_impl(
+ X, dY, m_sizes, gather, topk, dW_dtype = torch.float32, permute_x = True,
+ BLOCK_SIZE_M = 32, BLOCK_SIZE_N = 64, BLOCK_SIZE_K = 64,
+ ) # fmt: skip
+ assert out.dtype == torch.float32 and torch.isfinite(out).all()
+ ref = _ref(X, dY, m_sizes, gather, topk, "permute_x", E, N, K)
+ assert ((out.double() - ref).norm() / ref.norm()) < 1e-5
+ assert (out[2] == 0).all() and (out[5] == 0).all()
+
+
[email protected]("accumulate", [False, True])
+def test_per_expert_fp32_activations_bf16_weight(monkeypatch, accumulate):
+ # bf16 full finetune: fp32 RMSNorm output into a bf16 expert weight, dW stored in bf16.
+ gg = _gg()
+ E, N, K, T, topk = 32, 512, 384, 256, 4
+ m_sizes, gather = _routing(T, E, topk, (1, 7), seed = 11)
+ g = torch.Generator(device = "cuda").manual_seed(2)
+ X = torch.randn(T, K, device = "cuda", generator = g)
+ dY = torch.randn(T * topk, N, device = "cuda", generator = g)
+ prev = torch.randn(E, N, K, device = "cuda", generator = g).bfloat16()
+ kw = dict(
+ BLOCK_SIZE_M = 128,
+ BLOCK_SIZE_N = 128,
+ BLOCK_SIZE_K = 64,
+ num_warps = 8,
+ num_stages = 3,
+ permute_x = True,
+ )
+ outs = {}
+ for kernel in ("auto", "legacy"):
+ monkeypatch.setenv("UNSLOTH_MOE_DW_KERNEL", kernel)
+ if accumulate:
+ outs[kernel] = prev.clone()
+ gg._grouped_gemm_dW_accumulate_impl(X, dY, m_sizes, gather, outs[kernel], topk, **kw)
+ else:
+ outs[kernel] = gg._grouped_gemm_dW_impl(
+ X, dY, m_sizes, gather, topk, dW_dtype = torch.bfloat16, **kw
+ )
+ assert outs["auto"].dtype == torch.bfloat16
+ ref = _ref(X, dY, m_sizes, gather, topk, "permute_x", E, N, K) + (
+ prev.double() if accumulate else 0
+ )
+ for kernel, out in outs.items():
+ assert ((out.double() - ref).norm() / ref.norm()) < 5e-3, kernel
+ assert torch.equal(outs["auto"], outs["legacy"])
+
+
+def test_kill_switch_selects_legacy(monkeypatch):
+ gg = _gg()
+ monkeypatch.setenv("UNSLOTH_MOE_DW_KERNEL", "legacy")
+ assert not gg._use_per_expert_dW_kernel()
+ monkeypatch.setenv("UNSLOTH_MOE_DW_KERNEL", "per_expert")
+ assert gg._use_per_expert_dW_kernel()
+ monkeypatch.delenv("UNSLOTH_MOE_DW_KERNEL")
+ assert gg._use_per_expert_dW_kernel()
diff --git a/unsloth/kernels/moe/autotune_cache.py b/unsloth/kernels/moe/autotune_cache.py
index 0135c16b8c8..bdd04244474 100644
--- a/unsloth/kernels/moe/autotune_cache.py
+++ b/unsloth/kernels/moe/autotune_cache.py
@@ -267,17 +267,12 @@ def _run_moe_autotuning(
_autotuner_for,
grouped_gemm_forward,
grouped_gemm_dX,
- grouped_gemm_dW,
)
from .grouped_gemm.kernels.forward import _autotuned_grouped_gemm_forward_kernel
- from .grouped_gemm.kernels.backward import (
- _autotuned_grouped_gemm_dX_kernel,
- _autotuned_grouped_gemm_dW_kernel,
- )
+ from .grouped_gemm.kernels.backward import _autotuned_grouped_gemm_dX_kernel
from .grouped_gemm.kernels.tuning import (
KernelConfigForward,
KernelConfigBackward_dX,
- KernelConfigBackward_dW,
)
logger.info("Autotuning forward kernel (first GEMM)...")
@@ -334,6 +329,23 @@ def _run_moe_autotuning(
use_tma_store = triton_config_bwd_dx.kwargs.get("USE_TMA_STORE", False),
)
+ config_bwd_dw = _autotune_dw_config(
+ hidden_states, dummy_grad, m_sizes, gather_indices, top_k, num_experts
+ )
+
+ return config_fwd, config_bwd_dx, config_bwd_dw
+
+
+def _autotune_dw_config(hidden_states, dummy_grad, m_sizes, gather_indices, top_k, num_experts):
+ """Autotune the dW kernel the launcher dispatches to and read the config from that kernel's tuner."""
+ from .grouped_gemm.interface import _autotuner_for, _use_per_expert_dW_kernel, grouped_gemm_dW
+ from .grouped_gemm.kernels.backward import (
+ _autotuned_grouped_gemm_dW_kernel,
+ _autotuned_grouped_gemm_dW_per_expert_kernel,
+ )
+ from .grouped_gemm.kernels.tuning import KernelConfigBackward_dW
+
+ per_expert = _use_per_expert_dW_kernel()
logger.info("Autotuning backward dW kernel...")
_ = grouped_gemm_dW(
X = hidden_states,
@@ -345,11 +357,16 @@ def _run_moe_autotuning(
permute_y = False,
autotune = True,
)
- triton_config_bwd_dw = _autotuner_for(
- _autotuned_grouped_gemm_dW_kernel, hidden_states.device
- ).best_config
-
- config_bwd_dw = KernelConfigBackward_dW(
+ if per_expert:
+ triton_config_bwd_dw = _autotuner_for(
+ _autotuned_grouped_gemm_dW_per_expert_kernel, hidden_states.device
+ ).best_config
+ else:
+ triton_config_bwd_dw = _autotuner_for(
+ _autotuned_grouped_gemm_dW_kernel, hidden_states.device
+ ).best_config
+
+ return KernelConfigBackward_dW(
BLOCK_SIZE_M = triton_config_bwd_dw.kwargs["BLOCK_SIZE_M"],
BLOCK_SIZE_N = triton_config_bwd_dw.kwargs["BLOCK_SIZE_N"],
BLOCK_SIZE_K = triton_config_bwd_dw.kwargs["BLOCK_SIZE_K"],
@@ -360,8 +377,6 @@ def _run_moe_autotuning(
use_tma_store = triton_config_bwd_dw.kwargs.get("USE_TMA_STORE", False),
)
- return config_fwd, config_bwd_dx, config_bwd_dw
-
_MIN_FALLBACK_BLOCK_SIZE = 16
diff --git a/unsloth/kernels/moe/grouped_gemm/interface.py b/unsloth/kernels/moe/grouped_gemm/interface.py
index d867ba72c77..f310c9201a5 100644
--- a/unsloth/kernels/moe/grouped_gemm/interface.py
+++ b/unsloth/kernels/moe/grouped_gemm/interface.py
@@ -19,8 +19,10 @@ import triton
from .kernels.backward import (
_autotuned_grouped_gemm_dW_kernel,
+ _autotuned_grouped_gemm_dW_per_expert_kernel,
_autotuned_grouped_gemm_dX_kernel,
_grouped_gemm_dW_kernel,
+ _grouped_gemm_dW_per_expert_kernel,
_grouped_gemm_dX_kernel,
)
from .kernels.forward import (
@@ -491,6 +493,13 @@ def _grouped_gemm_dX_impl(
return dX
+def _use_per_expert_dW_kernel():
+ """Per-expert dW grid (one program per (expert, n, k) tile); UNSLOTH_MOE_DW_KERNEL=legacy restores the
+ kernel that walks every expert inside each (n, k) program. Measured faster on every shape, dtype and GPU
+ tried (A100, L4, RTX PRO 6000, B200), so there is no shape routing."""
+ return os.environ.get("UNSLOTH_MOE_DW_KERNEL", "").strip().lower() != "legacy"
+
+
def _launch_grouped_gemm_dW(
X: torch.Tensor,
dY: torch.Tensor,
@@ -567,6 +576,7 @@ def _launch_grouped_gemm_dW(
M_grad, N = dY.shape
assert M_grad == total_tokens, f"dY M ({M_grad}) != total_tokens ({total_tokens})"
+ per_expert = _use_per_expert_dW_kernel()
if accumulate:
# Read-modify-write into an existing gradient: one program owns each (expert, n, k) tile.
@@ -576,7 +586,8 @@ def _launch_grouped_gemm_dW(
assert dW_out.is_contiguous() and dW_out.device == X.device
dW = dW_out
else:
- dW = torch.zeros(
+ # The per-expert kernel writes every tile (zeros for an empty expert); the legacy one skips them.
+ dW = (torch.empty if per_expert else torch.zeros)(
(num_experts, N, K), device = X.device, dtype = X.dtype if dW_dtype is None else dW_dtype
)
if use_tma_store and (dW.element_size() > X.element_size() or dW.data_ptr() % 16 != 0):
@@ -588,6 +599,12 @@ def _launch_grouped_gemm_dW(
pass
def grid(META):
+ if per_expert:
+ return (
+ num_experts
+ * triton.cdiv(N, META["BLOCK_SIZE_N"])
+ * triton.cdiv(K, META["BLOCK_SIZE_K"]),
+ )
return (NUM_SMS,)
if debug:
@@ -627,6 +644,13 @@ def _launch_grouped_gemm_dW(
}
if accumulate:
kernel_args["ACCUMULATE"] = True
+ if per_expert:
+ kernel_args["EXPERTS_POW2"] = triton.next_power_of_2(num_experts)
+ if X.element_size() >= 4:
+ # fp32 operands (fp32 RMSNorm outputs in a bf16 full finetune): the tf32 dot reads both tiles
+ # M-major, and a multi-stage pipeline doubles shared memory and halves occupancy; one stage is
+ # faster for every tile measured. The legacy kernel never pipelines this loop either.
+ num_stages = 1
if not autotune:
kernel_args.update(
@@ -642,11 +666,18 @@ def _launch_grouped_gemm_dW(
}
)
- kernel = (
- _autotuner_for(_autotuned_grouped_gemm_dW_kernel, X.device)
- if autotune
- else _grouped_gemm_dW_kernel
- )
+ if per_expert:
+ kernel = (
+ _autotuner_for(_autotuned_grouped_gemm_dW_per_expert_kernel, X.device)
+ if autotune
+ else _grouped_gemm_dW_per_expert_kernel
+ )
+ else:
+ kernel = (
+ _autotuner_for(_autotuned_grouped_gemm_dW_kernel, X.device)
+ if autotune
+ else _grouped_gemm_dW_kernel
+ )
is_fake = _is_tracing(X, dY)
if not is_fake:
diff --git a/unsloth/kernels/moe/grouped_gemm/kernels/backward.py b/unsloth/kernels/moe/grouped_gemm/kernels/backward.py
index 45e2efd6ef8..9e2cd97ba6b 100644
--- a/unsloth/kernels/moe/grouped_gemm/kernels/backward.py
+++ b/unsloth/kernels/moe/grouped_gemm/kernels/backward.py
@@ -426,3 +426,185 @@ _autotuned_grouped_gemm_dW_kernel = triton.autotune(
prune_configs_by = {"early_config_prune": prune_kernel_configs_backward_dW},
key = ["NUM_EXPERTS", "N", "K", "PERMUTE_X", "PERMUTE_Y"],
)(_grouped_gemm_dW_kernel)
+
+
+def _grouped_gemm_dW_per_expert_kernel(
+ x_ptr,
+ dY_ptr,
+ dW_ptr,
+ m_sizes_ptr,
+ gather_indices_ptr,
+ NUM_TOKENS,
+ TOPK: tl.constexpr,
+ NUM_EXPERTS: tl.constexpr,
+ N: tl.constexpr,
+ K: tl.constexpr,
+ NUM_SMS,
+ BLOCK_SIZE_N: tl.constexpr,
+ BLOCK_SIZE_K: tl.constexpr,
+ BLOCK_SIZE_M: tl.constexpr,
+ PERMUTE_X: tl.constexpr = False,
+ PERMUTE_Y: tl.constexpr = False,
+ USE_TMA_LOAD_dY: tl.constexpr = False,
+ USE_TMA_LOAD_X: tl.constexpr = False,
+ USE_TMA_STORE: tl.constexpr = False,
+ acc_dtype: tl.constexpr = tl.float32,
+ ACCUMULATE: tl.constexpr = False,
+ EXPERTS_POW2: tl.constexpr = 1,
+) -> None:
+ """dW[e] = dY_e^T X_e with one program per (expert, n tile, k tile).
+
+ The legacy kernel walks every expert inside each (n, k) program, so only N * K / (BN * BK)
+ programs exist and each restarts its pipeline per expert. Here the grid is
+ NUM_EXPERTS * n_tiles * k_tiles and a program reduces over its own expert's rows only. Every
+ program owns its output tile: no atomics, deterministic. An empty expert writes zeros (the
+ output is not pre-zeroed) or, when accumulating, leaves the existing gradient untouched.
+ """
+ TOTAL_TOKENS = NUM_TOKENS * TOPK
+ output_dtype = dW_ptr.dtype.element_ty
+
+ num_n_tiles = tl.cdiv(N, BLOCK_SIZE_N)
+ num_k_tiles = tl.cdiv(K, BLOCK_SIZE_K)
+ tiles_per_expert = num_n_tiles * num_k_tiles
+
+ pid = tl.program_id(0)
+ expert_idx = pid // tiles_per_expert
+ tile_idx = pid % tiles_per_expert
+ tile_n_idx = tile_idx % num_n_tiles
+ tile_k_idx = tile_idx // num_n_tiles
+ n_offset = tile_n_idx * BLOCK_SIZE_N
+ k_offset = tile_k_idx * BLOCK_SIZE_K
+
+ # This expert's row range from the per-expert counts (prefix sum in registers).
+ e_range = tl.arange(0, EXPERTS_POW2)
+ sizes = tl.load(m_sizes_ptr + e_range, mask = e_range < NUM_EXPERTS, other = 0).to(tl.int32)
+ m_start = tl.sum(tl.where(e_range < expert_idx, sizes, 0), axis = 0)
+ m_size = tl.sum(tl.where(e_range == expert_idx, sizes, 0), axis = 0)
+ m_end = m_start + m_size
+
+ if ACCUMULATE:
+ if m_size == 0:
+ return
+
+ block_range_m = tl.arange(0, BLOCK_SIZE_M)
+ block_range_n = tl.arange(0, BLOCK_SIZE_N)
+ block_range_k = tl.arange(0, BLOCK_SIZE_K)
+ n_mask = block_range_n + n_offset < N
+ k_mask = block_range_k + k_offset < K
+
+ if USE_TMA_LOAD_dY:
+ # Rows past this expert's end read as zero, so the other operand needs no masking either.
+ dY_desc = tl.make_tensor_descriptor(
+ dY_ptr,
+ shape = [m_end, N],
+ strides = [N, 1],
+ block_shape = [BLOCK_SIZE_M, BLOCK_SIZE_N],
+ )
+ if USE_TMA_LOAD_X:
+ x_desc = tl.make_tensor_descriptor(
+ x_ptr,
+ shape = [m_end, K],
+ strides = [K, 1],
+ block_shape = [BLOCK_SIZE_M, BLOCK_SIZE_K],
+ )
+
+ accumulator = tl.zeros((BLOCK_SIZE_N, BLOCK_SIZE_K), dtype = acc_dtype)
+ for tile_m in range(0, m_size, BLOCK_SIZE_M):
+ rows = m_start + tile_m + block_range_m
+ row_mask = rows < m_end
+ rows64 = rows.to(tl.int64)
+ if PERMUTE_X or PERMUTE_Y:
+ token_idx = tl.load(gather_indices_ptr + rows64, mask = row_mask, other = 0).to(tl.int64)
+ if PERMUTE_X:
+ x_rows = token_idx // TOPK
+ dY_rows = rows64
+ else:
+ x_rows = rows64
+ dY_rows = token_idx
+ else:
+ x_rows = rows64
+ dY_rows = rows64
+
+ if USE_TMA_LOAD_X:
+ x = x_desc.load([m_start + tile_m, k_offset])
+ else:
+ x = tl.load(
+ x_ptr + x_rows[:, None] * K + (k_offset + block_range_k)[None, :],
+ mask = row_mask[:, None] & k_mask[None, :],
+ other = 0.0,
+ )
+ if USE_TMA_LOAD_dY:
+ dY = dY_desc.load([m_start + tile_m, n_offset])
+ else:
+ dY = tl.load(
+ dY_ptr + dY_rows[:, None] * N + (n_offset + block_range_n)[None, :],
+ mask = row_mask[:, None] & n_mask[None, :],
+ other = 0.0,
+ )
+ accumulator += tl.dot(dY.T.to(x.dtype), x)
+
+ store_row_offs = expert_idx.to(tl.int64) * N + n_offset + block_range_n
+ nk_mask = n_mask[:, None] & k_mask[None, :]
+ if ACCUMULATE:
+ # Same rule as the legacy kernel: the old gradient is added after the reduction, one rounding.
+ if USE_TMA_STORE:
+ prev = tl.make_tensor_descriptor(
+ dW_ptr,
+ shape = [NUM_EXPERTS, N, K],
+ strides = [N * K, K, 1],
+ block_shape = [1, BLOCK_SIZE_N, BLOCK_SIZE_K],
+ ).load([expert_idx, n_offset, k_offset])
+ prev = tl.reshape(prev, (BLOCK_SIZE_N, BLOCK_SIZE_K))
+ else:
+ prev = tl.load(
+ dW_ptr + store_row_offs[:, None] * K + (k_offset + block_range_k)[None, :],
+ mask = nk_mask,
+ other = 0.0,
+ )
+ accumulator += prev.to(acc_dtype)
+ y = accumulator.to(output_dtype)
+ if USE_TMA_STORE:
+ tl.static_assert(N % BLOCK_SIZE_N == 0, "N must be divisible by BLOCK_SIZE_N")
+ tl.static_assert(K % BLOCK_SIZE_K == 0, "K must be divisible by BLOCK_SIZE_K")
+ dW_desc = tl.make_tensor_descriptor(
+ dW_ptr,
+ shape = [NUM_EXPERTS, N, K],
+ strides = [N * K, K, 1],
+ block_shape = [1, BLOCK_SIZE_N, BLOCK_SIZE_K],
+ )
+ dW_desc.store([expert_idx, n_offset, k_offset], tl.expand_dims(y, 0))
+ else:
+ tl.store(
+ dW_ptr + store_row_offs[:, None] * K + (k_offset + block_range_k)[None, :],
+ y,
+ mask = nk_mask,
+ )
+
+
+def _prune_dW_per_expert_configs(configs, args, **kwargs):
+ pruned = prune_kernel_configs_backward_dW(configs, args, **kwargs)
+ if kwargs["x_ptr"].element_size() < 4:
+ return pruned
+ # fp32 operands: single stage (see _launch_grouped_gemm_dW); drop the duplicates that leaves.
+ seen, single = set(), []
+ for config in pruned:
+ key = (tuple(sorted(config.kwargs.items())), config.num_warps)
+ if key not in seen:
+ seen.add(key)
+ single.append(
+ triton.Config(
+ dict(config.kwargs),
+ num_warps = config.num_warps,
+ num_stages = 1,
+ num_ctas = config.num_ctas,
+ )
+ )
+ return single
+
+
+_autotuned_grouped_gemm_dW_per_expert_kernel = triton.autotune(
+ configs = get_dW_kernel_configs(),
+ prune_configs_by = {"early_config_prune": _prune_dW_per_expert_configs},
+ key = ["NUM_EXPERTS", "N", "K", "PERMUTE_X", "PERMUTE_Y"],
+)(_grouped_gemm_dW_per_expert_kernel)