Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
MoE grouped GEMM: accumulate expert dW into an existing param.grad (#14480)
Daniel Hand2f83d51d16b
diff --git a/tests/test_moe_dw_accumulate.py b/tests/test_moe_dw_accumulate.pynew file mode 100644index 00000000000..c83d36ea95c--- /dev/null+++ b/tests/test_moe_dw_accumulate.py@@ -0,0 +1,577 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""In-place grouped GEMM dW accumulation vs UNSLOTH_MOE_DW_ACCUMULATE=0 and an fp64 reference."""++import os+import re+import socket+import sys+from pathlib import Path++import pytest++sys.path.insert(0, str(Path(__file__).resolve().parents[1]))++import torch # noqa: E402++pytest.importorskip("triton", reason = "the grouped GEMM is a Triton kernel")+if not torch.cuda.is_available():+ pytest.skip("grouped GEMM needs a CUDA device", allow_module_level = True)++try:+ from unsloth.kernels.moe.grouped_gemm import interface as gg+ from unsloth.kernels.moe.grouped_gemm.kernels.tuning import (+ KernelConfigBackward_dW,+ KernelConfigBackward_dX,+ KernelConfigForward,+ )+except Exception as exc: # pragma: no cover - depends on the installed stack+ pytest.skip(f"grouped_gemm is unimportable here: {exc}", allow_module_level = True)++DEV = "cuda"+TMA_STORE = [False, True] if gg.supports_tma() else [False]+++def _cfgs(+ bn = 64,+ bk = 64,+ tma_store = False,+):+ fwd = KernelConfigForward(BLOCK_SIZE_M = 64, BLOCK_SIZE_N = bn, BLOCK_SIZE_K = bk)+ dx = KernelConfigBackward_dX(BLOCK_SIZE_M = 64, BLOCK_SIZE_N = bn, BLOCK_SIZE_K = bk)+ dw = KernelConfigBackward_dW(+ BLOCK_SIZE_M = 64, BLOCK_SIZE_N = bn, BLOCK_SIZE_K = bk, use_tma_store = tma_store+ )+ return fwd, dx, dw+++def _routing(+ T,+ E,+ topk,+ empty = (),+):+ logits = torch.randn(T, E, device = DEV)+ for e in empty:+ logits[:, e] = -float("inf")+ idx = torch.topk(logits, topk).indices.view(-1)+ m_sizes = torch.bincount(idx, minlength = E).to(torch.int32)+ gather = torch.argsort(idx, stable = True).to(torch.int32)+ return m_sizes, gather+++def _gemm(X, W, m_sizes, gather, topk, mode, cfgs):+ fwd, dx, dw = cfgs+ return gg.grouped_gemm(+ X = X, W = W, m_sizes = m_sizes, topk = topk, gather_indices = gather,+ permute_x = mode == "permute_x", permute_y = mode == "permute_y",+ kernel_config_fwd = fwd, kernel_config_bwd_dX = dx, kernel_config_bwd_dW = dw,+ is_first_gemm = mode != "permute_y",+ ) # fmt: skip+++def _ref_dW(X, G, m_sizes, gather, topk, mode, E, N, K):+ """fp64 dW = sum over the expert's rows of dY^T X, in the kernel's expert-grouped row order."""+ X, G, gather = X.double(), G.double(), gather.long()+ if mode == "permute_x":+ xs, dys = X[gather // topk], G+ elif mode == "permute_y":+ xs, dys = X, G[gather]+ else:+ xs, dys = X, G+ out = torch.zeros(E, N, K, dtype = torch.float64, device = X.device)+ start = 0+ for e, m in enumerate(m_sizes.tolist()):+ out[e] = dys[start : start + m].T @ xs[start : start + m]+ start += m+ return out+++class _Case:+ def __init__(+ self,+ E,+ N,+ K,+ T,+ topk,+ mode,+ dtype,+ empty = (),+ seed = 0,+ ):+ g = torch.Generator(device = DEV).manual_seed(seed)+ torch.manual_seed(seed)+ self.E, self.N, self.K, self.topk, self.mode = E, N, K, topk, mode+ self.m_sizes, self.gather = _routing(T, E, topk, empty)+ rows = T if mode == "permute_x" else T * topk+ self.X = torch.randn(rows, K, device = DEV, generator = g).to(dtype)+ self.G = torch.randn(T * topk, N, device = DEV, generator = g).to(dtype)+ self.ref = _ref_dW(self.X, self.G, self.m_sizes, self.gather, topk, mode, E, N, K)+ self.absref = _ref_dW(+ self.X.abs(), self.G.abs(), self.m_sizes, self.gather, topk, mode, E, N, K+ )++ def backward(self, W, cfgs, **bw):+ Y = _gemm(self.X, W, self.m_sizes, self.gather, self.topk, self.mode, cfgs)+ Y.backward(self.G, **bw)+++def _param(+ E,+ N,+ K,+ dtype,+ seed = 1,+):+ torch.manual_seed(seed)+ return torch.nn.Parameter(torch.randn(E, N, K, device = DEV).to(dtype) * 0.05)+++def _run_steps(cases, W, cfgs, inplace, g0, monkeypatch):+ monkeypatch.setenv("UNSLOTH_MOE_DW_ACCUMULATE", "1" if inplace else "0")+ W.grad = None if g0 is None else g0.clone()+ before = dict(gg.DW_ACCUMULATE_STATS)+ steps = []+ for c in cases:+ prev = None if W.grad is None else W.grad.clone()+ c.backward(W, cfgs)+ steps.append((prev, W.grad.clone(), c.ref, c.absref))+ used = gg.DW_ACCUMULATE_STATS["inplace"] - before["inplace"]+ _run_steps.steps = steps+ return W.grad.clone(), used++[email protected]("mode", ["permute_x", "permute_y", "none"])[email protected]("dtype", [torch.bfloat16, torch.float16, torch.float32])[email protected]("ga", [1, 4])[email protected]("grad_start", ["none", "zeros", "random"])[email protected]("tma_store", TMA_STORE)+def test_matches_returned_gradient_path_and_is_closer_to_fp64(+ mode, dtype, ga, grad_start, tma_store, monkeypatch+):+ E, N, K, T, topk = 8, 128, 192, 200, 2+ cfgs = _cfgs(tma_store = tma_store)+ cases = [_Case(E, N, K, T, topk, mode, dtype, seed = 10 + i) for i in range(ga)]+ W = _param(E, N, K, dtype)+ g0 = {+ "none": None,+ "zeros": torch.zeros_like(W),+ "random": (torch.randn_like(W.float()) * 3).to(dtype),+ }[grad_start]+ stock, used_stock = _run_steps(cases, W, cfgs, False, g0, monkeypatch)+ ours, used_ours = _run_steps(cases, W, cfgs, True, g0, monkeypatch)+ ours_steps = _run_steps.steps+ assert used_stock == 0+ assert used_ours == (ga - 1 if grad_start == "none" else ga)++ if ga == 1 and grad_start in ("none", "zeros"):+ assert torch.equal(ours, stock)+ return+ ref = sum(c.ref for c in cases) + (0 if g0 is None else g0.double())+ err_o, err_s = (ours.double() - ref).abs(), (stock.double() - ref).abs()+ if dtype == torch.float32:+ assert torch.equal(ours, stock)+ return+ assert err_o.mean() <= err_s.mean(), (err_o.mean().item(), err_s.mean().item())+ closer = (err_o < err_s).sum().item()+ farther = (err_o > err_s).sum().item()+ assert closer >= farther, (closer, farther)+ for prev, after, step_ref, absref in ours_steps:+ if prev is None:+ continue+ r = prev.double() + step_ref+ bound = _half_ulp(r, after, dtype) + _FP32_SLACK * (absref + prev.double().abs())+ assert ((after.double() - r).abs() <= bound).all()+++_FP32_SLACK = 2.0**-20+++def _half_ulp(ref, out, dtype):+ mant = {torch.bfloat16: 7, torch.float16: 10}[dtype]+ mag = torch.maximum(ref.abs(), out.double().abs()).clamp_min(torch.finfo(dtype).tiny)+ _, exp = torch.frexp(mag)+ return torch.ldexp(torch.ones_like(mag), exp - 1 - mant) / 2++[email protected]("mode", ["permute_x", "permute_y"])+def test_bf16_weight_with_fp32_activations(mode, monkeypatch):+ E, N, K, T, topk = 8, 128, 192, 200, 2+ cfgs = _cfgs()+ cases = [_Case(E, N, K, T, topk, mode, torch.float32, seed = 20 + i) for i in range(3)]+ kw = dict(BLOCK_SIZE_M = 64, BLOCK_SIZE_N = 64, BLOCK_SIZE_K = 64,+ permute_x = mode == "permute_x", permute_y = mode == "permute_y") # fmt: skip+ accs = [gg.grouped_gemm_dW(c.X, c.G, c.m_sizes, c.gather, topk, **kw) for c in cases]+ assert accs[0].dtype == torch.float32+ W = _param(E, N, K, torch.bfloat16)+ out = {}+ for inplace in (False, True):+ _run_steps(cases, W, cfgs, inplace, None, monkeypatch)+ out[inplace] = _run_steps.steps+ for inplace in (False, True):+ assert torch.equal(out[inplace][0][1], accs[0].to(torch.bfloat16)), inplace+ assert gg.DW_ACCUMULATE_STATS["inplace"] > 0+ errs = {True: [], False: []}+ for k in (1, 2):+ for inplace in (False, True):+ prev, after, _, absref = out[inplace][k]+ r = prev.double() + accs[k].double()+ err = (after.double() - r).abs()+ errs[inplace].append(err.mean().item())+ if inplace:+ bound = _half_ulp(r, after, torch.bfloat16) + _FP32_SLACK * (+ absref + prev.double().abs()+ )+ assert (err <= bound).all()+ assert sum(errs[True]) <= sum(errs[False]), errs++[email protected]("tma_store", TMA_STORE)+def test_empty_experts_keep_their_gradient(tma_store, monkeypatch):+ E, N, K, T, topk = 8, 128, 128, 64, 2+ cfgs = _cfgs(tma_store = tma_store)+ case = _Case(E, N, K, T, topk, "permute_x", torch.bfloat16, empty = (1, 5))+ assert case.m_sizes[1].item() == 0 and case.m_sizes[5].item() == 0+ W = _param(E, N, K, torch.bfloat16)+ g0 = torch.randn_like(W)+ g0[1, 0, 0] = -0.0+ ours, used = _run_steps([case], W, cfgs, True, g0, monkeypatch)+ assert used == 1+ for e in (1, 5):+ assert torch.equal(ours[e], g0[e]) and torch.signbit(ours[1, 0, 0])+ fresh, _ = _run_steps([case], W, cfgs, True, None, monkeypatch)+ assert fresh[1].abs().sum() == 0 and fresh[5].abs().sum() == 0++[email protected]("N, K, T", [(96, 80, 37), (40, 72, 1), (130, 48, 301)])+def test_odd_shapes_direct_op(N, K, T):+ E, topk = 5, 2+ for mode in ("permute_x", "permute_y", "none"):+ c = _Case(E, N, K, T, topk, mode, torch.bfloat16, seed = N + K + T)+ g0 = torch.randn(E, N, K, device = DEV).bfloat16()+ out = g0.clone()+ kw = dict(BLOCK_SIZE_M = 32, BLOCK_SIZE_N = 32, BLOCK_SIZE_K = 32,+ permute_x = mode == "permute_x", permute_y = mode == "permute_y") # fmt: skip+ gg.grouped_gemm_dW_accumulate(c.X, c.G, c.m_sizes, c.gather, out, topk, **kw)+ fresh = gg.grouped_gemm_dW(c.X, c.G, c.m_sizes, c.gather, topk, **kw)+ ref = c.ref + g0.double()+ err_o = (out.double() - ref).abs()+ err_s = ((g0 + fresh).double() - ref).abs()+ assert err_o.mean() <= err_s.mean(), mode+ bound = _half_ulp(ref, out, torch.bfloat16) + _FP32_SLACK * (c.absref + g0.double().abs())+ assert (err_o <= bound).all(), mode+++def _fallback_check(W, cfgs, case, fn):+ g0 = torch.randn_like(W)+ W.grad = g0.clone()+ before = gg.DW_ACCUMULATE_STATS["inplace"]+ out = fn()+ assert gg.DW_ACCUMULATE_STATS["inplace"] == before, "must not write param.grad in place"+ return g0, out+++def test_autograd_grad_and_backward_inputs_do_not_touch_param_grad(monkeypatch):+ monkeypatch.setenv("UNSLOTH_MOE_DW_ACCUMULATE", "1")+ E, N, K, T, topk = 4, 64, 64, 32, 2+ cfgs = _cfgs()+ c = _Case(E, N, K, T, topk, "permute_x", torch.bfloat16)+ W = _param(E, N, K, torch.bfloat16)++ def grad_call():+ Y = _gemm(c.X, W, c.m_sizes, c.gather, topk, c.mode, cfgs)+ return torch.autograd.grad(Y, [W], c.G)[0]++ g0, dW = _fallback_check(W, cfgs, c, grad_call)+ assert torch.equal(W.grad, g0)+ torch.testing.assert_close(dW.double(), c.ref, rtol = 2e-2, atol = 2e-2 * c.ref.abs().max().item())++ X = c.X.clone().requires_grad_(True)++ def inputs_call():+ _gemm(X, W, c.m_sizes, c.gather, topk, c.mode, cfgs).backward(c.G, inputs = [X])++ g0, _ = _fallback_check(W, cfgs, c, inputs_call)+ assert torch.equal(W.grad, g0) and X.grad is not None++ W.grad = g0.clone()+ before = gg.DW_ACCUMULATE_STATS["inplace"]+ _gemm(X, W, c.m_sizes, c.gather, topk, c.mode, cfgs).backward(c.G, inputs = [W])+ assert gg.DW_ACCUMULATE_STATS["inplace"] == before + 1+ torch.testing.assert_close(+ (W.grad.double() - g0.double()), c.ref, rtol = 3e-2, atol = 3e-2 * c.ref.abs().max().item()+ )+++def test_hooks_create_graph_kill_switch_and_non_leaf_fall_back(monkeypatch):+ monkeypatch.setenv("UNSLOTH_MOE_DW_ACCUMULATE", "1")+ E, N, K, T, topk = 4, 64, 64, 32, 2+ cfgs = _cfgs()+ c = _Case(E, N, K, T, topk, "permute_x", torch.bfloat16)++ W = _param(E, N, K, torch.bfloat16)+ seen = []+ h = W.register_hook(lambda g: seen.append(g is not None))+ _fallback_check(W, cfgs, c, lambda: c.backward(W, cfgs))+ assert seen == [True]+ h.remove()++ W = _param(E, N, K, torch.bfloat16)+ W.register_post_accumulate_grad_hook(lambda p: seen.append("post"))+ _fallback_check(W, cfgs, c, lambda: c.backward(W, cfgs))+ assert seen[-1] == "post"++ W = _param(E, N, K, torch.bfloat16)+ _fallback_check(W, cfgs, c, lambda: c.backward(W, cfgs, create_graph = True))++ W = _param(E, N, K, torch.bfloat16)+ with torch.autograd.detect_anomaly(check_nan = True):+ _fallback_check(W, cfgs, c, lambda: c.backward(W, cfgs))++ monkeypatch.setenv("UNSLOTH_MOE_DW_ACCUMULATE", "0")+ _fallback_check(W, cfgs, c, lambda: c.backward(W, cfgs))+ monkeypatch.setenv("UNSLOTH_MOE_DW_ACCUMULATE", "1")++ W = _param(E, N, K, torch.bfloat16)+ g0, _ = _fallback_check(W, cfgs, c, lambda: c.backward(W * 1.0, cfgs))+ torch.testing.assert_close(+ (W.grad.double() - g0.double()), c.ref, rtol = 3e-2, atol = 3e-2 * c.ref.abs().max().item()+ )++ W = _param(E, N, K, torch.bfloat16).requires_grad_(False)+ X = c.X.clone().requires_grad_(True)+ before = dict(gg.DW_ACCUMULATE_STATS)+ _gemm(X, W, c.m_sizes, c.gather, topk, c.mode, cfgs).backward(c.G)+ assert gg.DW_ACCUMULATE_STATS == before and W.grad is None+++def test_retain_graph_tied_weight_and_inf_propagation(monkeypatch):+ E, N, K, T, topk = 4, 64, 128, 48, 2+ cfgs = _cfgs()+ c = _Case(E, N, K, T, topk, "permute_x", torch.bfloat16)+ c2 = _Case(E, N, K, T, topk, "permute_x", torch.bfloat16, seed = 7)+ g0 = torch.randn(E, N, K, device = DEV).bfloat16()+ results = {}+ for inplace in (False, True):+ monkeypatch.setenv("UNSLOTH_MOE_DW_ACCUMULATE", "1" if inplace else "0")+ W = _param(E, N, K, torch.bfloat16)+ W.grad = g0.clone()+ Y = _gemm(c.X, W, c.m_sizes, c.gather, topk, c.mode, cfgs)+ Y.backward(c.G, retain_graph = True)+ Y.backward(c.G)+ Y1 = _gemm(c.X, W, c.m_sizes, c.gather, topk, c.mode, cfgs)+ Y2 = _gemm(c2.X, W, c2.m_sizes, c2.gather, topk, c2.mode, cfgs)+ ((Y1 * c.G).sum() + (Y2 * c2.G).sum()).backward()+ results[inplace] = W.grad.clone()+ ref = g0.double() + 3 * c.ref + c2.ref+ err_o = (results[True].double() - ref).abs()+ err_s = (results[False].double() - ref).abs()+ assert err_o.mean() <= err_s.mean()++ monkeypatch.setenv("UNSLOTH_MOE_DW_ACCUMULATE", "1")+ W = _param(E, N, K, torch.bfloat16)+ W.grad = g0.clone()+ G = c.G.clone()+ G[3, 5] = float("inf")+ _gemm(c.X, W, c.m_sizes, c.gather, topk, c.mode, cfgs).backward(G)+ assert not torch.isfinite(W.grad).all()+++def test_compiled_fullgraph_zero_breaks_and_matches_eager(monkeypatch):+ monkeypatch.setenv("UNSLOTH_MOE_DW_ACCUMULATE", "1")+ from torch._dynamo.utils import counters++ E, N, K, T, topk = 4, 128, 128, 64, 2+ cfgs = _cfgs()+ c = _Case(E, N, K, T, topk, "permute_x", torch.bfloat16)++ def fn(X, W):+ return _gemm(X, W, c.m_sizes, c.gather, topk, c.mode, cfgs)++ torch._dynamo.reset()+ counters.clear()+ compiled = torch.compile(fn, fullgraph = True)+ grads = []+ for f in (fn, compiled):+ W = _param(E, N, K, torch.bfloat16)+ g0 = torch.randn_like(W)+ W.grad = g0.clone()+ X = c.X.clone().requires_grad_(True)+ for _ in range(2):+ f(X, W).backward(c.G)+ grads.append(W.grad.double() - g0.double())+ assert sum(counters["graph_break"].values()) == 0, dict(counters["graph_break"])+ for g in grads:+ torch.testing.assert_close(g, 2 * c.ref, rtol = 3e-2, atol = 3e-2 * c.ref.abs().max().item())++ def acc(X, dY, buf):+ gg.grouped_gemm_dW_accumulate(+ X, dY, c.m_sizes, c.gather, buf, topk, BLOCK_SIZE_M = 64, BLOCK_SIZE_N = 64,+ BLOCK_SIZE_K = 64, permute_x = True,+ ) # fmt: skip+ return buf++ counters.clear()+ buf_e = torch.randn(E, N, K, device = DEV).bfloat16()+ buf_c = buf_e.clone()+ acc(c.X, c.G, buf_e)+ torch.compile(acc, fullgraph = True)(c.X, c.G, buf_c)+ assert sum(counters["graph_break"].values()) == 0+ assert torch.equal(buf_e, buf_c)+ schema = str(torch.ops.unsloth.grouped_gemm_dW_accumulate.default._schema)+ assert re.search(r"Tensor\(a\d*!\) dW", schema), schema+++def _free_port():+ with socket.socket() as s:+ s.bind(("127.0.0.1", 0))+ return s.getsockname()[1]+++def _ddp_worker(rank, world, port, inplace, bucket_view, out_path):+ import torch.distributed as dist++ os.environ.update(MASTER_ADDR = "127.0.0.1", MASTER_PORT = str(port))+ os.environ["UNSLOTH_MOE_DW_ACCUMULATE"] = "1" if inplace else "0"+ dist.init_process_group("gloo", rank = rank, world_size = world)+ E, N, K, T, topk = 4, 128, 128, 64, 2+ cfgs = _cfgs()++ class Block(torch.nn.Module):+ def __init__(self):+ super().__init__()+ torch.manual_seed(0)+ self.w = torch.nn.Parameter(torch.randn(E, N, K, device = DEV).bfloat16() * 0.05)+ self.head = torch.nn.Linear(N, 1, device = DEV, dtype = torch.bfloat16)++ def forward(self, x, m_sizes, gather):+ return self.head(_gemm(x, self.w, m_sizes, gather, topk, "permute_x", cfgs))++ model = Block()+ ddp = torch.nn.parallel.DistributedDataParallel(model, gradient_as_bucket_view = bucket_view)+ cases = [+ _Case(E, N, K, T, topk, "permute_x", torch.bfloat16, seed = 100 * rank + i) for i in range(4)+ ]+ for i, case in enumerate(cases):+ loss = ddp(case.X, case.m_sizes, case.gather).float().pow(2).mean()+ if i < 3:+ with ddp.no_sync():+ loss.backward()+ else:+ loss.backward()+ torch.save(+ {"grad": model.w.grad.float().cpu(), "inplace": gg.DW_ACCUMULATE_STATS["inplace"]},+ f"{out_path}.{rank}",+ )+ dist.destroy_process_group()++[email protected]("bucket_view", [False, True])+def test_ddp_gradient_accumulation_with_bucket_hooks(bucket_view, tmp_path):+ import torch.multiprocessing as mp++ got = {}+ for inplace in (False, True):+ out = str(tmp_path / f"g_{inplace}")+ mp.spawn(_ddp_worker, args = (2, _free_port(), inplace, bucket_view, out), nprocs = 2)+ got[inplace] = [torch.load(f"{out}.{r}") for r in range(2)]+ for r in range(2):+ assert got[False][r]["inplace"] == 0+ assert got[True][r]["inplace"] == 3+ assert torch.equal(got[True][0]["grad"], got[True][1]["grad"])+ a, b = got[True][0]["grad"], got[False][0]["grad"]+ rel = ((a - b).norm() / b.norm()).item()+ assert rel < 1e-2, rel+++def _fsdp2_worker(rank, inplace, out_path):+ import torch.distributed as dist++ try:+ from torch.distributed.fsdp import fully_shard+ except ImportError:+ from torch.distributed._composable.fsdp import fully_shard++ os.environ.update(MASTER_ADDR = "127.0.0.1", MASTER_PORT = str(_free_port()))+ os.environ["UNSLOTH_MOE_DW_ACCUMULATE"] = "1" if inplace else "0"+ torch.cuda.set_device(0)+ dist.init_process_group("nccl", rank = 0, world_size = 1)+ E, N, K, T, topk = 4, 128, 128, 64, 2+ cfgs = _cfgs()++ class Block(torch.nn.Module):+ def __init__(self):+ super().__init__()+ torch.manual_seed(0)+ self.w = torch.nn.Parameter(torch.randn(E, N, K, device = DEV).bfloat16() * 0.05)++ def forward(self, x, m_sizes, gather):+ return _gemm(x, self.w, m_sizes, gather, topk, "permute_x", cfgs)++ model = Block()+ fully_shard(model)+ cases = [_Case(E, N, K, T, topk, "permute_x", torch.bfloat16, seed = i) for i in range(4)]+ for i, case in enumerate(cases):+ model.set_requires_gradient_sync(i == 3)+ model(case.X, case.m_sizes, case.gather).float().pow(2).mean().backward()+ grad = model.w.grad.full_tensor() if hasattr(model.w.grad, "full_tensor") else model.w.grad+ torch.save({"grad": grad.float().cpu(), "inplace": gg.DW_ACCUMULATE_STATS["inplace"]}, out_path)+ dist.destroy_process_group()+++def test_fsdp2_no_sync_gradient_accumulation(tmp_path):+ import torch.multiprocessing as mp++ if not torch.distributed.is_nccl_available():+ pytest.skip("FSDP2 smoke needs NCCL")+ got = {}+ for inplace in (False, True):+ out = str(tmp_path / f"f_{inplace}.pt")+ mp.spawn(_fsdp2_worker, args = (inplace, out), nprocs = 1)+ got[inplace] = torch.load(out)+ assert got[False]["inplace"] == 0+ a, b = got[True]["grad"], got[False]["grad"]+ rel = ((a - b).norm() / b.norm()).item()+ assert rel < 1e-2, rel+ print(f"FSDP2 in-place launches: {got[True]['inplace']}, rel vs stock {rel:.2e}")+++def test_existing_gradient_survives_in_microbatch_cancellation(monkeypatch):+ monkeypatch.setenv("UNSLOTH_MOE_DW_ACCUMULATE", "1")+ E, N, K, T = 1, 64, 64, 128+ dtype, big = torch.bfloat16, 1e4+ X = torch.zeros(T, K, device = DEV, dtype = dtype)+ G = torch.zeros(T, N, device = DEV, dtype = dtype)+ X[0], X[64] = big, big+ G[0], G[64] = big, -big+ m_sizes = torch.tensor([T], device = DEV, dtype = torch.int32)+ gather = torch.arange(T, device = DEV, dtype = torch.int32)+ W = _param(E, N, K, dtype)+ W.grad = torch.ones_like(W)+ before = gg.DW_ACCUMULATE_STATS["inplace"]+ _gemm(X, W, m_sizes, gather, 1, "none", _cfgs()).backward(G)+ assert gg.DW_ACCUMULATE_STATS["inplace"] == before + 1+ assert torch.equal(W.grad, torch.ones_like(W))+++def test_side_stream_use_keeps_the_returned_gradient_path(monkeypatch):+ monkeypatch.setenv("UNSLOTH_MOE_DW_ACCUMULATE", "1")+ E, N, K, T, topk = 4, 64, 64, 32, 2+ cfgs = _cfgs()+ c = _Case(E, N, K, T, topk, "permute_x", torch.bfloat16)+ W = _param(E, N, K, torch.bfloat16)+ side = torch.cuda.Stream()+ side.wait_stream(torch.cuda.current_stream())++ def side_call():+ with torch.cuda.stream(side):+ c.backward(W, cfgs)+ torch.cuda.current_stream().wait_stream(side)++ g0, _ = _fallback_check(W, cfgs, c, side_call)+ torch.testing.assert_close(+ (W.grad.double() - g0.double()), c.ref, rtol = 3e-2, atol = 3e-2 * c.ref.abs().max().item()+ )+ _fallback_check(W, cfgs, c, lambda: c.backward(W, cfgs))diff --git a/unsloth/kernels/moe/grouped_gemm/interface.py b/unsloth/kernels/moe/grouped_gemm/interface.pyindex 5e2870f7ca3..def12f79c72 100644--- a/unsloth/kernels/moe/grouped_gemm/interface.py+++ b/unsloth/kernels/moe/grouped_gemm/interface.py@@ -5,6 +5,7 @@ import copyimport functoolsimport inspectimport logging+import osimport warningsfrom typing import Optionalfrom dataclasses import asdict@@ -39,7 +40,6 @@ ch.setFormatter(formatter)logger.addHandler(ch)-# Precompute TMA support (NVIDIA capability >= 9 plus a triton carrying make_tensor_descriptor) to avoid graph breaks.# Process-wide, so every visible card must be sm90+ (experts may sit on any); an unreadable card means False.def _check_tma_support():if DEVICE_TYPE in ("xpu", "hip"):@@ -73,7 +73,6 @@ def _autotuner_for(kernel, device):return _DEVICE_AUTOTUNERS[key]-# triton.set_allocator is Triton 3.0+._HAS_SET_ALLOCATOR = hasattr(triton, "set_allocator")@@ -132,7 +131,6 @@ def _grouped_gemm_forward_impl(permute_y: bool = False,fuse_mul_post: bool = False,autotune: bool = False,- # Kernel tuning params: must be tuned, else poor performance.BLOCK_SIZE_M: int = 32,BLOCK_SIZE_N: int = 32,BLOCK_SIZE_K: int = 32,@@ -141,7 +139,6 @@ def _grouped_gemm_forward_impl(use_tma_load_w: bool = False,use_tma_load_x: bool = False,use_tma_store: bool = False,- # Software pipelining; no effect until the loop is re-written.flatten: bool = True,debug: bool = False,) -> torch.Tensor:@@ -167,7 +164,6 @@ def _grouped_gemm_forward_impl(assert not (permute_y and use_tma_store), "Cannot use both TMA store and permute_y"if use_tma_load_x:- # TMA load for activations; TMA gather is Blackwell+ only.assert not permute_x, "Cannot use both use_tma_load_x and permute_x"use_tma = use_tma_load_w or use_tma_load_x or use_tma_store@@ -289,7 +285,6 @@ def _grouped_gemm_forward_impl(is_fake = _is_tracing(X, W)if not is_fake:- # Triton launches on the current device: pin it to the operands' card.with torch_gpu_device(X.device):compiled_kernel: triton.compiler.CompiledKernel = kernel[grid](**kernel_args, enable_fp_fusion = True@@ -338,7 +333,6 @@ def _grouped_gemm_dX_impl(assert m_sizes.ndim == 1assert not (permute_x and permute_y), "Cannot permute both X and Y"- # Flipped from the forward pass: a y permuted in the forward must be permuted on load in the backward.assert not (permute_y and use_tma_load_dy), "Cannot use both TMA load and permute_y"assert not (permute_x and use_tma_store), "Cannot use both TMA store and permute_x"@@ -374,7 +368,6 @@ def _grouped_gemm_dX_impl(assert M_total % topk == 0, f"M_total ({M_total}) must be divisible by topk ({topk})"num_tokens = M_total // topk- # The kernel only reads gather_indices under permute_x / permute_y, so it stays optional otherwise.if permute_x or permute_y:assert (gather_indices is not None@@ -382,7 +375,6 @@ def _grouped_gemm_dX_impl(total_tokens = gather_indices.shape[0] if gather_indices is not None else M_totalassert total_tokens == M_total, f"Total tokens ({total_tokens}) must match M_total ({M_total})"- # The output stays [NUM_TOKENS * TOPK, K] even under permute_x: gradients accumulate across every expert a token chose, reduced in a post-processing step.output_shape = (total_tokens, K)dX = torch.zeros(output_shape, device = dY.device, dtype = dY.dtype)@@ -449,12 +441,15 @@ def _grouped_gemm_dX_impl(return dX-def _grouped_gemm_dW_impl(+def _launch_grouped_gemm_dW(X: torch.Tensor,dY: torch.Tensor,m_sizes: torch.Tensor,gather_indices: torch.Tensor,topk: int,+ dW_out: Optional[torch.Tensor] = None,+ accumulate: bool = False,+ dW_dtype: Optional[torch.dtype] = None,BLOCK_SIZE_M: int = 32,BLOCK_SIZE_N: int = 32,BLOCK_SIZE_K: int = 32,@@ -523,7 +518,21 @@ def _grouped_gemm_dW_impl(assert M_grad == total_tokens, f"dY M ({M_grad}) != total_tokens ({total_tokens})"- dW = torch.zeros((num_experts, N, K), device = X.device, dtype = X.dtype)+ if accumulate:+ # Read-modify-write into an existing gradient: one program owns each (expert, n, k) tile.+ assert not autotune, "autotune benchmarks re-run the kernel and would add dW several times"+ assert dW_out is not None+ assert dW_out.numel() == num_experts * N * K, f"{dW_out.shape} != {(num_experts, N, K)}"+ assert dW_out.is_contiguous() and dW_out.device == X.device+ dW = dW_out+ else:+ dW = torch.zeros(+ (num_experts, N, K), device = X.device, dtype = X.dtype if dW_dtype is None else dW_dtype+ )+ if use_tma_store and (dW.element_size() > X.element_size() or dW.data_ptr() % 16 != 0):+ # TMA needs a 16 byte aligned base (a DDP bucket view may not be), and the tuned tile+ # sizes assume an output no wider than X.+ use_tma_store = Falseif not autotune:pass@@ -566,6 +575,8 @@ def _grouped_gemm_dW_impl("PERMUTE_X": permute_x,"PERMUTE_Y": permute_y,}+ if accumulate:+ kernel_args["ACCUMULATE"] = Trueif not autotune:kernel_args.update(@@ -602,6 +613,58 @@ def _grouped_gemm_dW_impl(return dW+def _grouped_gemm_dW_impl(+ X: torch.Tensor,+ dY: torch.Tensor,+ m_sizes: torch.Tensor,+ gather_indices: torch.Tensor,+ topk: int,+ BLOCK_SIZE_M: int = 32,+ BLOCK_SIZE_N: int = 32,+ BLOCK_SIZE_K: int = 32,+ permute_x: bool = False,+ permute_y: bool = False,+ use_tma_load_dy: bool = False,+ use_tma_load_x: bool = False,+ use_tma_store: bool = False,+ fuse_mul_pre: bool = False,+ fuse_mul_post: bool = False,+ num_warps: int = 4,+ num_stages: int = 2,+ flatten: bool = True,+ autotune: bool = False,+ debug: bool = False,+ dW_dtype: Optional[torch.dtype] = None,+) -> torch.Tensor:+ """dW into a fresh zeroed (E, N, K) buffer of dW_dtype (default X.dtype)."""+ kwargs = dict(locals())+ return _launch_grouped_gemm_dW(dW_out = None, accumulate = False, **kwargs)+++def _grouped_gemm_dW_accumulate_impl(+ X: torch.Tensor,+ dY: torch.Tensor,+ m_sizes: torch.Tensor,+ gather_indices: torch.Tensor,+ dW: torch.Tensor,+ topk: int,+ BLOCK_SIZE_M: int = 32,+ BLOCK_SIZE_N: int = 32,+ BLOCK_SIZE_K: int = 32,+ permute_x: bool = False,+ permute_y: bool = False,+ use_tma_load_dy: bool = False,+ use_tma_load_x: bool = False,+ use_tma_store: bool = False,+ num_warps: int = 4,+ num_stages: int = 2,+) -> None:+ """dW += X^T dY per expert, in place, added in fp32 and rounded once to dW.dtype."""+ kwargs = dict(locals())+ dW_out = kwargs.pop("dW")+ _launch_grouped_gemm_dW(dW_out = dW_out, accumulate = True, **kwargs)++# Opaque ops: allow_in_graph let AOT trace the body on fake tensors, keeping torch.empty and dropping the launch.# Not triton_op: torch.compile rejects prune_configs_by on triton.autotune.def _fwd_fake(X, W, topk, m_sizes, gather_indices, topk_weights, permute_x, permute_y, *args):@@ -618,20 +681,34 @@ def _dX_fake(dY, W, gather_indices, m_sizes, topk, *args):def _dW_fake(X, dY, m_sizes, *args):- return X.new_empty((m_sizes.shape[0], dY.shape[-1], X.shape[-1]))+ dW_dtype = dict(zip(_DW_PARAM_NAMES[3:], args)).get("dW_dtype")+ return X.new_empty(+ (m_sizes.shape[0], dY.shape[-1], X.shape[-1]),+ dtype = X.dtype if dW_dtype is None else dW_dtype,+ )+++def _dW_accumulate_fake(X, dY, m_sizes, gather_indices, dW, *args):+ return None+_DW_PARAM_NAMES = tuple(inspect.signature(_grouped_gemm_dW_impl).parameters)_OPTIONAL_TENSORS = ("gather_indices", "topk_weights")-def _make_op(name, impl, fake):+def _make_op(+ name,+ impl,+ fake,+ mutates_args = (),+ returns = torch.Tensor,+):"""torch.library.custom_op (torch >= 2.4) over impl; torch.compiler.disable where it is missing."""custom_op = getattr(getattr(torch, "library", None), "custom_op", None)if custom_op is None:return torch._dynamo.disable(impl)existing = getattr(getattr(torch.ops, "unsloth", None), name, None)if existing is not None:- # grouped_gemm is also importable as a top-level package (glm4_moe, tests): reuse the registration.return existingparams = [p.replace(@@ -645,9 +722,9 @@ def _make_op(name, impl, fake):def op(*args):return impl(**dict(zip(names, args)))- op.__signature__ = inspect.Signature(params, return_annotation = torch.Tensor)+ op.__signature__ = inspect.Signature(params, return_annotation = returns)op.__name__ = name- op = custom_op(f"unsloth::{name}", op, mutates_args = ())+ op = custom_op(f"unsloth::{name}", op, mutates_args = tuple(mutates_args))op.register_fake(fake)return op@@ -655,6 +732,13 @@ def _make_op(name, impl, fake):_grouped_gemm_forward_op = _make_op("grouped_gemm_forward", _grouped_gemm_forward_impl, _fwd_fake)_grouped_gemm_dX_op = _make_op("grouped_gemm_dX", _grouped_gemm_dX_impl, _dX_fake)_grouped_gemm_dW_op = _make_op("grouped_gemm_dW", _grouped_gemm_dW_impl, _dW_fake)+_grouped_gemm_dW_accumulate_op = _make_op(+ "grouped_gemm_dW_accumulate",+ _grouped_gemm_dW_accumulate_impl,+ _dW_accumulate_fake,+ mutates_args = ("dW",),+ returns = None,+)def _signature(impl):@@ -666,7 +750,12 @@ def _signature(impl):_SIGNATURES = {impl: _signature(impl)- for impl in (_grouped_gemm_forward_impl, _grouped_gemm_dX_impl, _grouped_gemm_dW_impl)+ for impl in (+ _grouped_gemm_forward_impl,+ _grouped_gemm_dX_impl,+ _grouped_gemm_dW_impl,+ _grouped_gemm_dW_accumulate_impl,+ )}@@ -675,7 +764,6 @@ def _dispatch(impl, op, args, kwargs):bound = dict(defaults)bound.update(zip(names, args))bound.update(kwargs)- # Eager calls skip the custom-op dispatcher; traced calls (dynamo, or fake tensors under make_fx) need the op.if torch.compiler.is_compiling() or _is_tracing(bound[names[0]], bound[names[1]]):return op(*[bound[n] for n in names])return impl(**bound)@@ -696,6 +784,71 @@ def grouped_gemm_dW(*args, **kwargs) -> torch.Tensor:return _dispatch(_grouped_gemm_dW_impl, _grouped_gemm_dW_op, args, kwargs)[email protected](_grouped_gemm_dW_accumulate_impl)+def grouped_gemm_dW_accumulate(*args, **kwargs) -> None:+ return _dispatch(_grouped_gemm_dW_accumulate_impl, _grouped_gemm_dW_accumulate_op, args, kwargs)+++DW_ACCUMULATE_STATS = {"inplace": 0, "fallback": 0}+++_DW_DTYPES = (torch.float16, torch.bfloat16, torch.float32)+++def _dW_accumulate_enabled():+ return os.environ.get("UNSLOTH_MOE_DW_ACCUMULATE", "1") != "0"+++def _dW_accumulate_target(ctx, X, autotune):+ """The leaf parameter whose existing .grad the dW kernel may add into, else None.++ Any case where AccumulateGrad would do more than `param.grad += dW` falls back to returning dW.+ Hooks registered directly on autograd Nodes (Node.register_hook / register_prehook) cannot be+ enumerated from Python and see None for W here; UNSLOTH_MOE_DW_ACCUMULATE=0 restores the+ returned-gradient path."""+ if torch.compiler.is_compiling() or _is_tracing(X) or torch.is_grad_enabled():+ return None+ # Anomaly mode checks the returned gradients for NaN; an in-place add returns None for W.+ if torch.is_anomaly_enabled():+ return None+ if autotune or not _dW_accumulate_enabled():+ return None+ try:+ acc = ctx.next_functions[1][0]+ except (AttributeError, IndexError, TypeError):+ return None+ if acc is None or type(acc).__name__ != "AccumulateGrad":+ return None+ param = acc.variable+ if type(param) not in (torch.Tensor, torch.nn.Parameter):+ return None # tensor subclasses (DTensor, quantized) own their gradient layout+ grad = param.grad+ if grad is None or grad.layout != torch.strided or type(grad) is not torch.Tensor:+ return None+ if (+ grad.dtype not in _DW_DTYPES+ or grad.dtype != param.dtype+ or grad.device != X.device+ or grad.shape != param.shape+ or not grad.is_contiguous()+ ):+ return None+ if getattr(param, "_backward_hooks", None) or getattr(+ param, "_post_accumulate_grad_hooks", None+ ):+ return None+ if getattr(param, "_unsloth_dw_side_stream", False):+ return None+ if torch.cuda.current_stream(X.device) != torch.cuda.default_stream(X.device):+ return None+ try:+ if not torch._C._will_engine_execute_node(acc):+ return None+ except RuntimeError:+ return None # autograd.grad(): W's gradient is captured, never accumulated+ return param++class GroupedGemm(torch.autograd.Function):@staticmethoddef forward(@@ -727,8 +880,12 @@ class GroupedGemm(torch.autograd.Function):ctx.dX_only = dX_onlyctx.dW_only = dW_only- # topk_weights is not saved for backward: training with fused_mul is unsupported.ctx.save_for_backward(X, W, m_sizes, gather_indices)+ if W.requires_grad and X.is_cuda and not torch.compiler.is_compiling():+ if torch.cuda.current_stream(X.device) != torch.cuda.default_stream(X.device):+ # A use on a side stream may run its backward concurrently with another use of W:+ # in-place accumulation would race, so this weight keeps the returned-gradient path.+ W._unsloth_dw_side_stream = Truefwd_config = {}if kernel_config_fwd is not None:@@ -768,7 +925,6 @@ class GroupedGemm(torch.autograd.Function):assert (not ctx.fuse_mul_post), "fused_mul should only be used for inference, not for training"- # Frozen W (LoRA, dequantized experts) or X: skip its kernel and buffer, as autograd does for _grouped_mm.dX_only = ctx.dX_only or not ctx.needs_input_grad[1]dW_only = ctx.dW_only or not ctx.needs_input_grad[0]if dX_only and dW_only:@@ -797,17 +953,38 @@ class GroupedGemm(torch.autograd.Function):bwd_dW_config["num_warps"] = kernel_config_bwd_dW.num_warpsbwd_dW_config["num_stages"] = kernel_config_bwd_dW.num_stages- dW = grouped_gemm_dW(- X = X,- dY = dY,- m_sizes = m_sizes,- gather_indices = gather_indices,- topk = topk,- permute_x = permute_x,- permute_y = permute_y,- autotune = autotune,- **bwd_dW_config,- )+ target = _dW_accumulate_target(ctx, X, autotune)+ if target is not None:+ grouped_gemm_dW_accumulate(+ X = X,+ dY = dY,+ m_sizes = m_sizes,+ gather_indices = gather_indices,+ dW = target.grad,+ topk = topk,+ permute_x = permute_x,+ permute_y = permute_y,+ **bwd_dW_config,+ )+ # Already in param.grad: returning it as well would add it twice.+ dW = None+ DW_ACCUMULATE_STATS["inplace"] += 1+ else:+ if W.dtype != X.dtype and W.dtype in _DW_DTYPES and _dW_accumulate_enabled():+ bwd_dW_config["dW_dtype"] = W.dtype+ dW = grouped_gemm_dW(+ X = X,+ dY = dY,+ m_sizes = m_sizes,+ gather_indices = gather_indices,+ topk = topk,+ permute_x = permute_x,+ permute_y = permute_y,+ autotune = autotune,+ **bwd_dW_config,+ )+ if not torch.compiler.is_compiling():+ DW_ACCUMULATE_STATS["fallback"] += 1else:dW = Nonediff --git a/unsloth/kernels/moe/grouped_gemm/kernels/backward.py b/unsloth/kernels/moe/grouped_gemm/kernels/backward.pyindex f19bcc4907f..45e2efd6ef8 100644--- a/unsloth/kernels/moe/grouped_gemm/kernels/backward.py+++ b/unsloth/kernels/moe/grouped_gemm/kernels/backward.py@@ -71,11 +71,9 @@ def _grouped_gemm_dX_kernel(output_dtype = dX_ptr.dtype.element_tytidx = tl.program_id(0)- # Removes the need for predication along N in the GEMM main loop.tl.static_assert(N % BLOCK_SIZE_N == 0, "N must be divisible by BLOCK_SIZE_N")tl.static_assert(K % BLOCK_SIZE_K == 0, "K must be divisible by BLOCK_SIZE_K")- # A single global TMA descriptor with one block shape; TMA load never permutes x, so the shape is [TOTAL_TOKENS, K]. Unverified across expert boundaries.if USE_TMA_LOAD_dY:dY_desc = tl.make_tensor_descriptor(dY_ptr,@@ -112,7 +110,6 @@ def _grouped_gemm_dX_kernel(num_tiles_per_expert = num_m_tiles * num_k_tilesif USE_TMA_STORE:- # Define the descriptor inside the loop to predicate the store along M.tl.static_assert(K % BLOCK_SIZE_K == 0, "K must be divisible by BLOCK_SIZE_K")dX_desc = tl.make_tensor_descriptor(dX_ptr,@@ -121,7 +118,6 @@ def _grouped_gemm_dX_kernel(block_shape = [BLOCK_SIZE_M, BLOCK_SIZE_K],)- # Bounds are relative to tiles processed so far, so only this expert's tiles are handled and the total across experts is never exceeded.while tidx >= processed_tiles and tidx < (processed_tiles + num_tiles_per_expert):group_index = tidx - processed_tiles@@ -144,21 +140,15 @@ def _grouped_gemm_dX_kernel(row_mask = gather_offsets < m_sizerow_mask = row_mask[:, None]- # Only (PERMUTE_X and not PERMUTE_Y) and (not PERMUTE_X and PERMUTE_Y) occur, so load/store offsets are simply flipped between the two cases, with the strides adjusted to match.-if PERMUTE_X:- # Permuted on load in the forward pass (typically the first grouped GEMM in the MoE MLP), so load contiguous and permute on store.load_a_idx = indices_to_gather_64[:, None] * Nstore_idx = expert_token_offsets * Kelse:- # Permuted on store in the forward pass (typically the second grouped GEMM), so permute on load and store contiguous.load_a_idx = expert_token_offsets * Nstore_idx = indices_to_gather_64[:, None] * Kelse:- # Offsets relative to the CURRENT expert; m_start then advances to this expert's start token.offs_am = tile_m_idx * BLOCK_SIZE_M + m_block_range- # [M, N] @ [N, K] -> [M, K], so A strides by N and B by K, plus m_start for A's expert start token and n_start for B's slice of the [E, N, K] weight matrix.row_offsets_a = (m_start + offs_am[:, None]).to(tl.int64)load_a_idx = row_offsets_a * Nstore_idx = row_offsets_a * K@@ -193,7 +183,6 @@ def _grouped_gemm_dX_kernel(if not USE_TMA_LOAD_dY:dY_ptrs += BLOCK_SIZE_N- # B is no longer advanced along the contiguous dimension: weights are [N, K], so stride by K to reach the next [N_BLOCK_SIZE, K_BLOCK_SIZE] tile.if not USE_TMA_LOAD_W:w_ptrs += BLOCK_SIZE_N * K@@ -218,7 +207,6 @@ def _grouped_gemm_dX_kernel(_autotuned_grouped_gemm_dX_kernel = triton.autotune(configs = get_dX_kernel_configs(),prune_configs_by = {"early_config_prune": prune_dX_configs},- # NUM_TOKENS is left out of the key to avoid recompiling for every sequence length.key = ["NUM_EXPERTS", "N", "K", "PERMUTE_X", "PERMUTE_Y"],)(_grouped_gemm_dX_kernel)@@ -265,6 +253,10 @@ def _grouped_gemm_dW_kernel(USE_TMA_LOAD_X: tl.constexpr = False,USE_TMA_STORE: tl.constexpr = False,acc_dtype: tl.constexpr = tl.float32,+ ACCUMULATE: tl.constexpr = False,+ # Seed the fp32 accumulator with the existing tile (load overlaps the main loop) instead of adding+ # it after the loop. Only for a bf16 / fp16 dW, whose final rounding dwarfs the fp32 order change;+ # an fp32 dW adds at the end, the same single fp32 add autograd would do.) -> None:TOTAL_TOKENS = NUM_TOKENS * TOPKTMA_LOAD_BOTH: tl.constexpr = USE_TMA_LOAD_X and USE_TMA_LOAD_dY@@ -295,7 +287,6 @@ def _grouped_gemm_dW_kernel(block_range_n = tl.arange(0, BLOCK_SIZE_N)block_range_k = tl.arange(0, BLOCK_SIZE_K)- # N % BLOCK_SIZE_N == 0 and K % BLOCK_SIZE_K == 0 are required when using TMA store.if USE_TMA_STORE:tl.static_assert(N % BLOCK_SIZE_N == 0, "N must be divisible by BLOCK_SIZE_N")tl.static_assert(K % BLOCK_SIZE_K == 0, "K must be divisible by BLOCK_SIZE_K")@@ -322,11 +313,9 @@ def _grouped_gemm_dW_kernel(accumulator = tl.zeros((BLOCK_SIZE_N, BLOCK_SIZE_K), dtype = acc_dtype)m_start = m_end- # The cast is needed or the compiler complains about mismatching types; reason unknown.m_size = tl.load(m_sizes_ptr + expert_idx).to(tl.int32)m_end = m_start + m_size- # Offset by n_start: the result goes into this expert's slice of the global [E, N, K] weight matrix.n_start = expert_idx.to(tl.int64) * Nstore_row_offs = n_start + n_offset + block_range_n@@ -368,11 +357,8 @@ def _grouped_gemm_dW_kernel(row_load_mask = gather_offsets < m_size- # Only (PERMUTE_X and not PERMUTE_Y) and (not PERMUTE_X and PERMUTE_Y) occur, so load/store offsets are flipped between the two cases, with the strides adjusted.if PERMUTE_X:- x_row_load_idx = (- (expert_token_offsets // TOPK) * K- ) # Permute on load: token to expert order, /TOPK for the original count.+ x_row_load_idx = (expert_token_offsets // TOPK) * KdY_row_load_idx = m_offsets[:, None] * Nelse:x_row_load_idx = indices_to_gather.to(tl.int64)[:, None] * K@@ -407,6 +393,22 @@ def _grouped_gemm_dW_kernel(x,)+ # ACCUMULATE: dW is an existing gradient. This program alone owns the (expert, n, k)+ # tile, so read-modify-write is race free. The old value is added after the reduction+ # (seeding the accumulator with it lets in-microbatch cancellation erase it); one rounding.+ if ACCUMULATE:+ if USE_TMA_STORE:+ prev = dW_desc.load([expert_idx, n_offset, k_offset])+ prev = tl.reshape(prev, (BLOCK_SIZE_N, BLOCK_SIZE_K))+ else:+ prev = tl.load(+ dW_ptr+ + store_row_offs[:, None] * K+ + (k_offset + block_range_k)[None, :],+ mask = nk_mask,+ other = 0.0,+ )+ accumulator += prev.to(acc_dtype)y = accumulator.to(output_dtype)if USE_TMA_STORE:y = tl.expand_dims(y, 0)@@ -422,6 +424,5 @@ def _grouped_gemm_dW_kernel(_autotuned_grouped_gemm_dW_kernel = triton.autotune(configs = get_dW_kernel_configs(),prune_configs_by = {"early_config_prune": prune_kernel_configs_backward_dW},- # NUM_TOKENS is left out of the key to avoid recompiling for every sequence length.key = ["NUM_EXPERTS", "N", "K", "PERMUTE_X", "PERMUTE_Y"],)(_grouped_gemm_dW_kernel)