Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
MoE grouped GEMM: per-expert dW kernel (#14550)
Daniel Han4a5a3e5828c2
diff --git a/tests/test_grouped_gemm_mixed_tma_device.py b/tests/test_grouped_gemm_mixed_tma_device.pyindex fbcdc531068..01a6c6c5368 100644--- a/tests/test_grouped_gemm_mixed_tma_device.py+++ b/tests/test_grouped_gemm_mixed_tma_device.py@@ -111,8 +111,10 @@ def _load(n: _Kernel(launches, guard_stack)for n in ("_autotuned_grouped_gemm_dW_kernel",+ "_autotuned_grouped_gemm_dW_per_expert_kernel","_autotuned_grouped_gemm_dX_kernel","_grouped_gemm_dW_kernel",+ "_grouped_gemm_dW_per_expert_kernel","_grouped_gemm_dX_kernel",)}diff --git a/tests/test_moe_dw_per_expert.py b/tests/test_moe_dw_per_expert.pynew file mode 100644index 00000000000..2de3b385601--- /dev/null+++ b/tests/test_moe_dw_per_expert.py@@ -0,0 +1,159 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Per-expert grouped GEMM dW kernel vs the legacy kernel (UNSLOTH_MOE_DW_KERNEL=legacy) and fp64."""++import sys+from pathlib import Path++import pytest++ROOT = Path(__file__).resolve().parents[1]+sys.path.insert(0, str(ROOT))++import torch # noqa: E402+++def _gg():+ pytest.importorskip("triton", reason = "the grouped GEMM is a Triton kernel")+ if not torch.cuda.is_available():+ pytest.skip("grouped GEMM needs a CUDA device")+ try:+ from unsloth.kernels.moe.grouped_gemm import interface as gg+ except Exception as exc: # pragma: no cover - depends on the installed stack+ pytest.skip(f"grouped_gemm is unimportable here: {exc}")+ return gg+++def _routing(T, E, topk, empty, seed):+ g = torch.Generator(device = "cuda").manual_seed(seed)+ logits = torch.randn(T, E, device = "cuda", generator = g)+ for e in empty:+ logits[:, e] = -float("inf")+ if empty:+ logits[:, (max(empty) + 1) % E] += 4.0 # one hot expert next to the empty ones+ idx = torch.topk(logits, topk).indices.view(-1)+ m_sizes = torch.bincount(idx, minlength = E).to(torch.int32)+ gather = torch.argsort(idx, stable = True).to(torch.int32)+ return m_sizes, gather+++def _ref(X, dY, m_sizes, gather, topk, mode, E, N, K):+ X, dY, gather = X.double(), dY.double(), gather.long()+ if mode == "permute_x":+ xs, dys = X[gather // topk], dY+ elif mode == "permute_y":+ xs, dys = X, dY[gather]+ else:+ xs, dys = X, dY+ out = torch.zeros(E, N, K, dtype = torch.float64, device = X.device)+ start = 0+ for e, m in enumerate(m_sizes.tolist()):+ out[e] = dys[start : start + m].T @ xs[start : start + m]+ start += m+ return out+++# (E, N, K, T, topk): Qwen3-30B-A3B down, Qwen1.5-MoE gate_up (ragged N tail), Mixtral-like few experts.+SHAPES = [(128, 2048, 768, 256, 8), (60, 2816, 2048, 128, 4), (8, 704, 320, 300, 2)]++[email protected]("shape", SHAPES)[email protected]("mode", ["permute_x", "permute_y", "none"])[email protected]("dtype", [torch.bfloat16, torch.float16])[email protected]("accumulate", [False, True])+def test_per_expert_matches_legacy_and_fp64(monkeypatch, shape, mode, dtype, accumulate):+ gg = _gg()+ E, N, K, T, topk = shape+ empty = (0, 3, E - 1)+ m_sizes, gather = _routing(T, E, topk, empty, seed = E)+ g = torch.Generator(device = "cuda").manual_seed(1)+ X = torch.randn(T if mode == "permute_x" else T * topk, K, device = "cuda", generator = g).to(dtype)+ dY = torch.randn(T * topk, N, device = "cuda", generator = g).to(dtype)+ kw = dict(+ BLOCK_SIZE_M = 64, BLOCK_SIZE_N = 64, BLOCK_SIZE_K = 64, num_warps = 4, num_stages = 3,+ permute_x = mode == "permute_x", permute_y = mode == "permute_y",+ ) # fmt: skip+ prev = torch.randn(E, N, K, device = "cuda", generator = g).to(dtype)+ outs = {}+ for kernel in ("auto", "legacy"):+ monkeypatch.setenv("UNSLOTH_MOE_DW_KERNEL", kernel)+ if accumulate:+ out = prev.clone()+ gg._grouped_gemm_dW_accumulate_impl(X, dY, m_sizes, gather, out, topk, **kw)+ else:+ out = gg._grouped_gemm_dW_impl(X, dY, m_sizes, gather, topk, **kw)+ outs[kernel] = out+ # Same reduction order per (expert, tile): bit-identical to the legacy kernel.+ assert torch.equal(outs["auto"], outs["legacy"])+ ref = _ref(X, dY, m_sizes, gather, topk, mode, E, N, K) + (prev.double() if accumulate else 0)+ rel = (outs["auto"].double() - ref).norm() / ref.norm()+ assert rel < (3e-3 if dtype == torch.bfloat16 else 5e-4), rel+ for e in empty: # an empty expert is exactly zero, or exactly the previous gradient+ assert torch.equal(outs["auto"][e], prev[e] if accumulate else torch.zeros_like(prev[e]))+++def test_per_expert_fp32_dw_dtype_and_poisoned_output(monkeypatch):+ gg = _gg()+ E, N, K, T, topk = 16, 256, 192, 64, 2+ m_sizes, gather = _routing(T, E, topk, (2, 5), seed = 3)+ X = torch.randn(T, K, device = "cuda").bfloat16()+ dY = torch.randn(T * topk, N, device = "cuda").bfloat16()+ # The per-expert path allocates dW with torch.empty: poison the allocator's next block first.+ torch.full((E * N * K,), float("nan"), device = "cuda", dtype = torch.float32)+ monkeypatch.setenv("UNSLOTH_MOE_DW_KERNEL", "auto")+ out = gg._grouped_gemm_dW_impl(+ X, dY, m_sizes, gather, topk, dW_dtype = torch.float32, permute_x = True,+ BLOCK_SIZE_M = 32, BLOCK_SIZE_N = 64, BLOCK_SIZE_K = 64,+ ) # fmt: skip+ assert out.dtype == torch.float32 and torch.isfinite(out).all()+ ref = _ref(X, dY, m_sizes, gather, topk, "permute_x", E, N, K)+ assert ((out.double() - ref).norm() / ref.norm()) < 1e-5+ assert (out[2] == 0).all() and (out[5] == 0).all()++[email protected]("accumulate", [False, True])+def test_per_expert_fp32_activations_bf16_weight(monkeypatch, accumulate):+ # bf16 full finetune: fp32 RMSNorm output into a bf16 expert weight, dW stored in bf16.+ gg = _gg()+ E, N, K, T, topk = 32, 512, 384, 256, 4+ m_sizes, gather = _routing(T, E, topk, (1, 7), seed = 11)+ g = torch.Generator(device = "cuda").manual_seed(2)+ X = torch.randn(T, K, device = "cuda", generator = g)+ dY = torch.randn(T * topk, N, device = "cuda", generator = g)+ prev = torch.randn(E, N, K, device = "cuda", generator = g).bfloat16()+ kw = dict(+ BLOCK_SIZE_M = 128,+ BLOCK_SIZE_N = 128,+ BLOCK_SIZE_K = 64,+ num_warps = 8,+ num_stages = 3,+ permute_x = True,+ )+ outs = {}+ for kernel in ("auto", "legacy"):+ monkeypatch.setenv("UNSLOTH_MOE_DW_KERNEL", kernel)+ if accumulate:+ outs[kernel] = prev.clone()+ gg._grouped_gemm_dW_accumulate_impl(X, dY, m_sizes, gather, outs[kernel], topk, **kw)+ else:+ outs[kernel] = gg._grouped_gemm_dW_impl(+ X, dY, m_sizes, gather, topk, dW_dtype = torch.bfloat16, **kw+ )+ assert outs["auto"].dtype == torch.bfloat16+ ref = _ref(X, dY, m_sizes, gather, topk, "permute_x", E, N, K) + (+ prev.double() if accumulate else 0+ )+ for kernel, out in outs.items():+ assert ((out.double() - ref).norm() / ref.norm()) < 5e-3, kernel+ assert torch.equal(outs["auto"], outs["legacy"])+++def test_kill_switch_selects_legacy(monkeypatch):+ gg = _gg()+ monkeypatch.setenv("UNSLOTH_MOE_DW_KERNEL", "legacy")+ assert not gg._use_per_expert_dW_kernel()+ monkeypatch.setenv("UNSLOTH_MOE_DW_KERNEL", "per_expert")+ assert gg._use_per_expert_dW_kernel()+ monkeypatch.delenv("UNSLOTH_MOE_DW_KERNEL")+ assert gg._use_per_expert_dW_kernel()diff --git a/unsloth/kernels/moe/autotune_cache.py b/unsloth/kernels/moe/autotune_cache.pyindex 0135c16b8c8..bdd04244474 100644--- a/unsloth/kernels/moe/autotune_cache.py+++ b/unsloth/kernels/moe/autotune_cache.py@@ -267,17 +267,12 @@ def _run_moe_autotuning(_autotuner_for,grouped_gemm_forward,grouped_gemm_dX,- grouped_gemm_dW,)from .grouped_gemm.kernels.forward import _autotuned_grouped_gemm_forward_kernel- from .grouped_gemm.kernels.backward import (- _autotuned_grouped_gemm_dX_kernel,- _autotuned_grouped_gemm_dW_kernel,- )+ from .grouped_gemm.kernels.backward import _autotuned_grouped_gemm_dX_kernelfrom .grouped_gemm.kernels.tuning import (KernelConfigForward,KernelConfigBackward_dX,- KernelConfigBackward_dW,)logger.info("Autotuning forward kernel (first GEMM)...")@@ -334,6 +329,23 @@ def _run_moe_autotuning(use_tma_store = triton_config_bwd_dx.kwargs.get("USE_TMA_STORE", False),)+ config_bwd_dw = _autotune_dw_config(+ hidden_states, dummy_grad, m_sizes, gather_indices, top_k, num_experts+ )++ return config_fwd, config_bwd_dx, config_bwd_dw+++def _autotune_dw_config(hidden_states, dummy_grad, m_sizes, gather_indices, top_k, num_experts):+ """Autotune the dW kernel the launcher dispatches to and read the config from that kernel's tuner."""+ from .grouped_gemm.interface import _autotuner_for, _use_per_expert_dW_kernel, grouped_gemm_dW+ from .grouped_gemm.kernels.backward import (+ _autotuned_grouped_gemm_dW_kernel,+ _autotuned_grouped_gemm_dW_per_expert_kernel,+ )+ from .grouped_gemm.kernels.tuning import KernelConfigBackward_dW++ per_expert = _use_per_expert_dW_kernel()logger.info("Autotuning backward dW kernel...")_ = grouped_gemm_dW(X = hidden_states,@@ -345,11 +357,16 @@ def _run_moe_autotuning(permute_y = False,autotune = True,)- triton_config_bwd_dw = _autotuner_for(- _autotuned_grouped_gemm_dW_kernel, hidden_states.device- ).best_config-- config_bwd_dw = KernelConfigBackward_dW(+ if per_expert:+ triton_config_bwd_dw = _autotuner_for(+ _autotuned_grouped_gemm_dW_per_expert_kernel, hidden_states.device+ ).best_config+ else:+ triton_config_bwd_dw = _autotuner_for(+ _autotuned_grouped_gemm_dW_kernel, hidden_states.device+ ).best_config++ return KernelConfigBackward_dW(BLOCK_SIZE_M = triton_config_bwd_dw.kwargs["BLOCK_SIZE_M"],BLOCK_SIZE_N = triton_config_bwd_dw.kwargs["BLOCK_SIZE_N"],BLOCK_SIZE_K = triton_config_bwd_dw.kwargs["BLOCK_SIZE_K"],@@ -360,8 +377,6 @@ def _run_moe_autotuning(use_tma_store = triton_config_bwd_dw.kwargs.get("USE_TMA_STORE", False),)- return config_fwd, config_bwd_dx, config_bwd_dw-_MIN_FALLBACK_BLOCK_SIZE = 16diff --git a/unsloth/kernels/moe/grouped_gemm/interface.py b/unsloth/kernels/moe/grouped_gemm/interface.pyindex d867ba72c77..f310c9201a5 100644--- a/unsloth/kernels/moe/grouped_gemm/interface.py+++ b/unsloth/kernels/moe/grouped_gemm/interface.py@@ -19,8 +19,10 @@ import tritonfrom .kernels.backward import (_autotuned_grouped_gemm_dW_kernel,+ _autotuned_grouped_gemm_dW_per_expert_kernel,_autotuned_grouped_gemm_dX_kernel,_grouped_gemm_dW_kernel,+ _grouped_gemm_dW_per_expert_kernel,_grouped_gemm_dX_kernel,)from .kernels.forward import (@@ -491,6 +493,13 @@ def _grouped_gemm_dX_impl(return dX+def _use_per_expert_dW_kernel():+ """Per-expert dW grid (one program per (expert, n, k) tile); UNSLOTH_MOE_DW_KERNEL=legacy restores the+ kernel that walks every expert inside each (n, k) program. Measured faster on every shape, dtype and GPU+ tried (A100, L4, RTX PRO 6000, B200), so there is no shape routing."""+ return os.environ.get("UNSLOTH_MOE_DW_KERNEL", "").strip().lower() != "legacy"++def _launch_grouped_gemm_dW(X: torch.Tensor,dY: torch.Tensor,@@ -567,6 +576,7 @@ def _launch_grouped_gemm_dW(M_grad, N = dY.shapeassert M_grad == total_tokens, f"dY M ({M_grad}) != total_tokens ({total_tokens})"+ per_expert = _use_per_expert_dW_kernel()if accumulate:# Read-modify-write into an existing gradient: one program owns each (expert, n, k) tile.@@ -576,7 +586,8 @@ def _launch_grouped_gemm_dW(assert dW_out.is_contiguous() and dW_out.device == X.devicedW = dW_outelse:- dW = torch.zeros(+ # The per-expert kernel writes every tile (zeros for an empty expert); the legacy one skips them.+ dW = (torch.empty if per_expert else torch.zeros)((num_experts, N, K), device = X.device, dtype = X.dtype if dW_dtype is None else dW_dtype)if use_tma_store and (dW.element_size() > X.element_size() or dW.data_ptr() % 16 != 0):@@ -588,6 +599,12 @@ def _launch_grouped_gemm_dW(passdef grid(META):+ if per_expert:+ return (+ num_experts+ * triton.cdiv(N, META["BLOCK_SIZE_N"])+ * triton.cdiv(K, META["BLOCK_SIZE_K"]),+ )return (NUM_SMS,)if debug:@@ -627,6 +644,13 @@ def _launch_grouped_gemm_dW(}if accumulate:kernel_args["ACCUMULATE"] = True+ if per_expert:+ kernel_args["EXPERTS_POW2"] = triton.next_power_of_2(num_experts)+ if X.element_size() >= 4:+ # fp32 operands (fp32 RMSNorm outputs in a bf16 full finetune): the tf32 dot reads both tiles+ # M-major, and a multi-stage pipeline doubles shared memory and halves occupancy; one stage is+ # faster for every tile measured. The legacy kernel never pipelines this loop either.+ num_stages = 1if not autotune:kernel_args.update(@@ -642,11 +666,18 @@ def _launch_grouped_gemm_dW(})- kernel = (- _autotuner_for(_autotuned_grouped_gemm_dW_kernel, X.device)- if autotune- else _grouped_gemm_dW_kernel- )+ if per_expert:+ kernel = (+ _autotuner_for(_autotuned_grouped_gemm_dW_per_expert_kernel, X.device)+ if autotune+ else _grouped_gemm_dW_per_expert_kernel+ )+ else:+ kernel = (+ _autotuner_for(_autotuned_grouped_gemm_dW_kernel, X.device)+ if autotune+ else _grouped_gemm_dW_kernel+ )is_fake = _is_tracing(X, dY)if not is_fake:diff --git a/unsloth/kernels/moe/grouped_gemm/kernels/backward.py b/unsloth/kernels/moe/grouped_gemm/kernels/backward.pyindex 45e2efd6ef8..9e2cd97ba6b 100644--- a/unsloth/kernels/moe/grouped_gemm/kernels/backward.py+++ b/unsloth/kernels/moe/grouped_gemm/kernels/backward.py@@ -426,3 +426,185 @@ _autotuned_grouped_gemm_dW_kernel = triton.autotune(prune_configs_by = {"early_config_prune": prune_kernel_configs_backward_dW},key = ["NUM_EXPERTS", "N", "K", "PERMUTE_X", "PERMUTE_Y"],)(_grouped_gemm_dW_kernel)+++def _grouped_gemm_dW_per_expert_kernel(+ x_ptr,+ dY_ptr,+ dW_ptr,+ m_sizes_ptr,+ gather_indices_ptr,+ NUM_TOKENS,+ TOPK: tl.constexpr,+ NUM_EXPERTS: tl.constexpr,+ N: tl.constexpr,+ K: tl.constexpr,+ NUM_SMS,+ BLOCK_SIZE_N: tl.constexpr,+ BLOCK_SIZE_K: tl.constexpr,+ BLOCK_SIZE_M: tl.constexpr,+ PERMUTE_X: tl.constexpr = False,+ PERMUTE_Y: tl.constexpr = False,+ USE_TMA_LOAD_dY: tl.constexpr = False,+ USE_TMA_LOAD_X: tl.constexpr = False,+ USE_TMA_STORE: tl.constexpr = False,+ acc_dtype: tl.constexpr = tl.float32,+ ACCUMULATE: tl.constexpr = False,+ EXPERTS_POW2: tl.constexpr = 1,+) -> None:+ """dW[e] = dY_e^T X_e with one program per (expert, n tile, k tile).++ The legacy kernel walks every expert inside each (n, k) program, so only N * K / (BN * BK)+ programs exist and each restarts its pipeline per expert. Here the grid is+ NUM_EXPERTS * n_tiles * k_tiles and a program reduces over its own expert's rows only. Every+ program owns its output tile: no atomics, deterministic. An empty expert writes zeros (the+ output is not pre-zeroed) or, when accumulating, leaves the existing gradient untouched.+ """+ TOTAL_TOKENS = NUM_TOKENS * TOPK+ output_dtype = dW_ptr.dtype.element_ty++ num_n_tiles = tl.cdiv(N, BLOCK_SIZE_N)+ num_k_tiles = tl.cdiv(K, BLOCK_SIZE_K)+ tiles_per_expert = num_n_tiles * num_k_tiles++ pid = tl.program_id(0)+ expert_idx = pid // tiles_per_expert+ tile_idx = pid % tiles_per_expert+ tile_n_idx = tile_idx % num_n_tiles+ tile_k_idx = tile_idx // num_n_tiles+ n_offset = tile_n_idx * BLOCK_SIZE_N+ k_offset = tile_k_idx * BLOCK_SIZE_K++ # This expert's row range from the per-expert counts (prefix sum in registers).+ e_range = tl.arange(0, EXPERTS_POW2)+ sizes = tl.load(m_sizes_ptr + e_range, mask = e_range < NUM_EXPERTS, other = 0).to(tl.int32)+ m_start = tl.sum(tl.where(e_range < expert_idx, sizes, 0), axis = 0)+ m_size = tl.sum(tl.where(e_range == expert_idx, sizes, 0), axis = 0)+ m_end = m_start + m_size++ if ACCUMULATE:+ if m_size == 0:+ return++ block_range_m = tl.arange(0, BLOCK_SIZE_M)+ block_range_n = tl.arange(0, BLOCK_SIZE_N)+ block_range_k = tl.arange(0, BLOCK_SIZE_K)+ n_mask = block_range_n + n_offset < N+ k_mask = block_range_k + k_offset < K++ if USE_TMA_LOAD_dY:+ # Rows past this expert's end read as zero, so the other operand needs no masking either.+ dY_desc = tl.make_tensor_descriptor(+ dY_ptr,+ shape = [m_end, N],+ strides = [N, 1],+ block_shape = [BLOCK_SIZE_M, BLOCK_SIZE_N],+ )+ if USE_TMA_LOAD_X:+ x_desc = tl.make_tensor_descriptor(+ x_ptr,+ shape = [m_end, K],+ strides = [K, 1],+ block_shape = [BLOCK_SIZE_M, BLOCK_SIZE_K],+ )++ accumulator = tl.zeros((BLOCK_SIZE_N, BLOCK_SIZE_K), dtype = acc_dtype)+ for tile_m in range(0, m_size, BLOCK_SIZE_M):+ rows = m_start + tile_m + block_range_m+ row_mask = rows < m_end+ rows64 = rows.to(tl.int64)+ if PERMUTE_X or PERMUTE_Y:+ token_idx = tl.load(gather_indices_ptr + rows64, mask = row_mask, other = 0).to(tl.int64)+ if PERMUTE_X:+ x_rows = token_idx // TOPK+ dY_rows = rows64+ else:+ x_rows = rows64+ dY_rows = token_idx+ else:+ x_rows = rows64+ dY_rows = rows64++ if USE_TMA_LOAD_X:+ x = x_desc.load([m_start + tile_m, k_offset])+ else:+ x = tl.load(+ x_ptr + x_rows[:, None] * K + (k_offset + block_range_k)[None, :],+ mask = row_mask[:, None] & k_mask[None, :],+ other = 0.0,+ )+ if USE_TMA_LOAD_dY:+ dY = dY_desc.load([m_start + tile_m, n_offset])+ else:+ dY = tl.load(+ dY_ptr + dY_rows[:, None] * N + (n_offset + block_range_n)[None, :],+ mask = row_mask[:, None] & n_mask[None, :],+ other = 0.0,+ )+ accumulator += tl.dot(dY.T.to(x.dtype), x)++ store_row_offs = expert_idx.to(tl.int64) * N + n_offset + block_range_n+ nk_mask = n_mask[:, None] & k_mask[None, :]+ if ACCUMULATE:+ # Same rule as the legacy kernel: the old gradient is added after the reduction, one rounding.+ if USE_TMA_STORE:+ prev = tl.make_tensor_descriptor(+ dW_ptr,+ shape = [NUM_EXPERTS, N, K],+ strides = [N * K, K, 1],+ block_shape = [1, BLOCK_SIZE_N, BLOCK_SIZE_K],+ ).load([expert_idx, n_offset, k_offset])+ prev = tl.reshape(prev, (BLOCK_SIZE_N, BLOCK_SIZE_K))+ else:+ prev = tl.load(+ dW_ptr + store_row_offs[:, None] * K + (k_offset + block_range_k)[None, :],+ mask = nk_mask,+ other = 0.0,+ )+ accumulator += prev.to(acc_dtype)+ y = accumulator.to(output_dtype)+ if USE_TMA_STORE:+ tl.static_assert(N % BLOCK_SIZE_N == 0, "N must be divisible by BLOCK_SIZE_N")+ tl.static_assert(K % BLOCK_SIZE_K == 0, "K must be divisible by BLOCK_SIZE_K")+ dW_desc = tl.make_tensor_descriptor(+ dW_ptr,+ shape = [NUM_EXPERTS, N, K],+ strides = [N * K, K, 1],+ block_shape = [1, BLOCK_SIZE_N, BLOCK_SIZE_K],+ )+ dW_desc.store([expert_idx, n_offset, k_offset], tl.expand_dims(y, 0))+ else:+ tl.store(+ dW_ptr + store_row_offs[:, None] * K + (k_offset + block_range_k)[None, :],+ y,+ mask = nk_mask,+ )+++def _prune_dW_per_expert_configs(configs, args, **kwargs):+ pruned = prune_kernel_configs_backward_dW(configs, args, **kwargs)+ if kwargs["x_ptr"].element_size() < 4:+ return pruned+ # fp32 operands: single stage (see _launch_grouped_gemm_dW); drop the duplicates that leaves.+ seen, single = set(), []+ for config in pruned:+ key = (tuple(sorted(config.kwargs.items())), config.num_warps)+ if key not in seen:+ seen.add(key)+ single.append(+ triton.Config(+ dict(config.kwargs),+ num_warps = config.num_warps,+ num_stages = 1,+ num_ctas = config.num_ctas,+ )+ )+ return single+++_autotuned_grouped_gemm_dW_per_expert_kernel = triton.autotune(+ configs = get_dW_kernel_configs(),+ prune_configs_by = {"early_config_prune": _prune_dW_per_expert_configs},+ key = ["NUM_EXPERTS", "N", "K", "PERMUTE_X", "PERMUTE_Y"],+)(_grouped_gemm_dW_per_expert_kernel)