Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

MoE grouped GEMM: accumulate expert dW into an existing param.grad (#14480)

Daniel Hand2f83d51d16b
diff --git a/tests/test_moe_dw_accumulate.py b/tests/test_moe_dw_accumulate.py
new file mode 100644
index 00000000000..c83d36ea95c
--- /dev/null
+++ b/tests/test_moe_dw_accumulate.py
@@ -0,0 +1,577 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""In-place grouped GEMM dW accumulation vs UNSLOTH_MOE_DW_ACCUMULATE=0 and an fp64 reference."""
+
+import os
+import re
+import socket
+import sys
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+import torch # noqa: E402
+
+pytest.importorskip("triton", reason = "the grouped GEMM is a Triton kernel")
+if not torch.cuda.is_available():
+ pytest.skip("grouped GEMM needs a CUDA device", allow_module_level = True)
+
+try:
+ from unsloth.kernels.moe.grouped_gemm import interface as gg
+ from unsloth.kernels.moe.grouped_gemm.kernels.tuning import (
+ KernelConfigBackward_dW,
+ KernelConfigBackward_dX,
+ KernelConfigForward,
+ )
+except Exception as exc: # pragma: no cover - depends on the installed stack
+ pytest.skip(f"grouped_gemm is unimportable here: {exc}", allow_module_level = True)
+
+DEV = "cuda"
+TMA_STORE = [False, True] if gg.supports_tma() else [False]
+
+
+def _cfgs(
+ bn = 64,
+ bk = 64,
+ tma_store = False,
+):
+ fwd = KernelConfigForward(BLOCK_SIZE_M = 64, BLOCK_SIZE_N = bn, BLOCK_SIZE_K = bk)
+ dx = KernelConfigBackward_dX(BLOCK_SIZE_M = 64, BLOCK_SIZE_N = bn, BLOCK_SIZE_K = bk)
+ dw = KernelConfigBackward_dW(
+ BLOCK_SIZE_M = 64, BLOCK_SIZE_N = bn, BLOCK_SIZE_K = bk, use_tma_store = tma_store
+ )
+ return fwd, dx, dw
+
+
+def _routing(
+ T,
+ E,
+ topk,
+ empty = (),
+):
+ logits = torch.randn(T, E, device = DEV)
+ for e in empty:
+ logits[:, e] = -float("inf")
+ idx = torch.topk(logits, topk).indices.view(-1)
+ m_sizes = torch.bincount(idx, minlength = E).to(torch.int32)
+ gather = torch.argsort(idx, stable = True).to(torch.int32)
+ return m_sizes, gather
+
+
+def _gemm(X, W, m_sizes, gather, topk, mode, cfgs):
+ fwd, dx, dw = cfgs
+ return gg.grouped_gemm(
+ X = X, W = W, m_sizes = m_sizes, topk = topk, gather_indices = gather,
+ permute_x = mode == "permute_x", permute_y = mode == "permute_y",
+ kernel_config_fwd = fwd, kernel_config_bwd_dX = dx, kernel_config_bwd_dW = dw,
+ is_first_gemm = mode != "permute_y",
+ ) # fmt: skip
+
+
+def _ref_dW(X, G, m_sizes, gather, topk, mode, E, N, K):
+ """fp64 dW = sum over the expert's rows of dY^T X, in the kernel's expert-grouped row order."""
+ X, G, gather = X.double(), G.double(), gather.long()
+ if mode == "permute_x":
+ xs, dys = X[gather // topk], G
+ elif mode == "permute_y":
+ xs, dys = X, G[gather]
+ else:
+ xs, dys = X, G
+ out = torch.zeros(E, N, K, dtype = torch.float64, device = X.device)
+ start = 0
+ for e, m in enumerate(m_sizes.tolist()):
+ out[e] = dys[start : start + m].T @ xs[start : start + m]
+ start += m
+ return out
+
+
+class _Case:
+ def __init__(
+ self,
+ E,
+ N,
+ K,
+ T,
+ topk,
+ mode,
+ dtype,
+ empty = (),
+ seed = 0,
+ ):
+ g = torch.Generator(device = DEV).manual_seed(seed)
+ torch.manual_seed(seed)
+ self.E, self.N, self.K, self.topk, self.mode = E, N, K, topk, mode
+ self.m_sizes, self.gather = _routing(T, E, topk, empty)
+ rows = T if mode == "permute_x" else T * topk
+ self.X = torch.randn(rows, K, device = DEV, generator = g).to(dtype)
+ self.G = torch.randn(T * topk, N, device = DEV, generator = g).to(dtype)
+ self.ref = _ref_dW(self.X, self.G, self.m_sizes, self.gather, topk, mode, E, N, K)
+ self.absref = _ref_dW(
+ self.X.abs(), self.G.abs(), self.m_sizes, self.gather, topk, mode, E, N, K
+ )
+
+ def backward(self, W, cfgs, **bw):
+ Y = _gemm(self.X, W, self.m_sizes, self.gather, self.topk, self.mode, cfgs)
+ Y.backward(self.G, **bw)
+
+
+def _param(
+ E,
+ N,
+ K,
+ dtype,
+ seed = 1,
+):
+ torch.manual_seed(seed)
+ return torch.nn.Parameter(torch.randn(E, N, K, device = DEV).to(dtype) * 0.05)
+
+
+def _run_steps(cases, W, cfgs, inplace, g0, monkeypatch):
+ monkeypatch.setenv("UNSLOTH_MOE_DW_ACCUMULATE", "1" if inplace else "0")
+ W.grad = None if g0 is None else g0.clone()
+ before = dict(gg.DW_ACCUMULATE_STATS)
+ steps = []
+ for c in cases:
+ prev = None if W.grad is None else W.grad.clone()
+ c.backward(W, cfgs)
+ steps.append((prev, W.grad.clone(), c.ref, c.absref))
+ used = gg.DW_ACCUMULATE_STATS["inplace"] - before["inplace"]
+ _run_steps.steps = steps
+ return W.grad.clone(), used
+
+
[email protected]("mode", ["permute_x", "permute_y", "none"])
[email protected]("dtype", [torch.bfloat16, torch.float16, torch.float32])
[email protected]("ga", [1, 4])
[email protected]("grad_start", ["none", "zeros", "random"])
[email protected]("tma_store", TMA_STORE)
+def test_matches_returned_gradient_path_and_is_closer_to_fp64(
+ mode, dtype, ga, grad_start, tma_store, monkeypatch
+):
+ E, N, K, T, topk = 8, 128, 192, 200, 2
+ cfgs = _cfgs(tma_store = tma_store)
+ cases = [_Case(E, N, K, T, topk, mode, dtype, seed = 10 + i) for i in range(ga)]
+ W = _param(E, N, K, dtype)
+ g0 = {
+ "none": None,
+ "zeros": torch.zeros_like(W),
+ "random": (torch.randn_like(W.float()) * 3).to(dtype),
+ }[grad_start]
+ stock, used_stock = _run_steps(cases, W, cfgs, False, g0, monkeypatch)
+ ours, used_ours = _run_steps(cases, W, cfgs, True, g0, monkeypatch)
+ ours_steps = _run_steps.steps
+ assert used_stock == 0
+ assert used_ours == (ga - 1 if grad_start == "none" else ga)
+
+ if ga == 1 and grad_start in ("none", "zeros"):
+ assert torch.equal(ours, stock)
+ return
+ ref = sum(c.ref for c in cases) + (0 if g0 is None else g0.double())
+ err_o, err_s = (ours.double() - ref).abs(), (stock.double() - ref).abs()
+ if dtype == torch.float32:
+ assert torch.equal(ours, stock)
+ return
+ assert err_o.mean() <= err_s.mean(), (err_o.mean().item(), err_s.mean().item())
+ closer = (err_o < err_s).sum().item()
+ farther = (err_o > err_s).sum().item()
+ assert closer >= farther, (closer, farther)
+ for prev, after, step_ref, absref in ours_steps:
+ if prev is None:
+ continue
+ r = prev.double() + step_ref
+ bound = _half_ulp(r, after, dtype) + _FP32_SLACK * (absref + prev.double().abs())
+ assert ((after.double() - r).abs() <= bound).all()
+
+
+_FP32_SLACK = 2.0**-20
+
+
+def _half_ulp(ref, out, dtype):
+ mant = {torch.bfloat16: 7, torch.float16: 10}[dtype]
+ mag = torch.maximum(ref.abs(), out.double().abs()).clamp_min(torch.finfo(dtype).tiny)
+ _, exp = torch.frexp(mag)
+ return torch.ldexp(torch.ones_like(mag), exp - 1 - mant) / 2
+
+
[email protected]("mode", ["permute_x", "permute_y"])
+def test_bf16_weight_with_fp32_activations(mode, monkeypatch):
+ E, N, K, T, topk = 8, 128, 192, 200, 2
+ cfgs = _cfgs()
+ cases = [_Case(E, N, K, T, topk, mode, torch.float32, seed = 20 + i) for i in range(3)]
+ kw = dict(BLOCK_SIZE_M = 64, BLOCK_SIZE_N = 64, BLOCK_SIZE_K = 64,
+ permute_x = mode == "permute_x", permute_y = mode == "permute_y") # fmt: skip
+ accs = [gg.grouped_gemm_dW(c.X, c.G, c.m_sizes, c.gather, topk, **kw) for c in cases]
+ assert accs[0].dtype == torch.float32
+ W = _param(E, N, K, torch.bfloat16)
+ out = {}
+ for inplace in (False, True):
+ _run_steps(cases, W, cfgs, inplace, None, monkeypatch)
+ out[inplace] = _run_steps.steps
+ for inplace in (False, True):
+ assert torch.equal(out[inplace][0][1], accs[0].to(torch.bfloat16)), inplace
+ assert gg.DW_ACCUMULATE_STATS["inplace"] > 0
+ errs = {True: [], False: []}
+ for k in (1, 2):
+ for inplace in (False, True):
+ prev, after, _, absref = out[inplace][k]
+ r = prev.double() + accs[k].double()
+ err = (after.double() - r).abs()
+ errs[inplace].append(err.mean().item())
+ if inplace:
+ bound = _half_ulp(r, after, torch.bfloat16) + _FP32_SLACK * (
+ absref + prev.double().abs()
+ )
+ assert (err <= bound).all()
+ assert sum(errs[True]) <= sum(errs[False]), errs
+
+
[email protected]("tma_store", TMA_STORE)
+def test_empty_experts_keep_their_gradient(tma_store, monkeypatch):
+ E, N, K, T, topk = 8, 128, 128, 64, 2
+ cfgs = _cfgs(tma_store = tma_store)
+ case = _Case(E, N, K, T, topk, "permute_x", torch.bfloat16, empty = (1, 5))
+ assert case.m_sizes[1].item() == 0 and case.m_sizes[5].item() == 0
+ W = _param(E, N, K, torch.bfloat16)
+ g0 = torch.randn_like(W)
+ g0[1, 0, 0] = -0.0
+ ours, used = _run_steps([case], W, cfgs, True, g0, monkeypatch)
+ assert used == 1
+ for e in (1, 5):
+ assert torch.equal(ours[e], g0[e]) and torch.signbit(ours[1, 0, 0])
+ fresh, _ = _run_steps([case], W, cfgs, True, None, monkeypatch)
+ assert fresh[1].abs().sum() == 0 and fresh[5].abs().sum() == 0
+
+
[email protected]("N, K, T", [(96, 80, 37), (40, 72, 1), (130, 48, 301)])
+def test_odd_shapes_direct_op(N, K, T):
+ E, topk = 5, 2
+ for mode in ("permute_x", "permute_y", "none"):
+ c = _Case(E, N, K, T, topk, mode, torch.bfloat16, seed = N + K + T)
+ g0 = torch.randn(E, N, K, device = DEV).bfloat16()
+ out = g0.clone()
+ kw = dict(BLOCK_SIZE_M = 32, BLOCK_SIZE_N = 32, BLOCK_SIZE_K = 32,
+ permute_x = mode == "permute_x", permute_y = mode == "permute_y") # fmt: skip
+ gg.grouped_gemm_dW_accumulate(c.X, c.G, c.m_sizes, c.gather, out, topk, **kw)
+ fresh = gg.grouped_gemm_dW(c.X, c.G, c.m_sizes, c.gather, topk, **kw)
+ ref = c.ref + g0.double()
+ err_o = (out.double() - ref).abs()
+ err_s = ((g0 + fresh).double() - ref).abs()
+ assert err_o.mean() <= err_s.mean(), mode
+ bound = _half_ulp(ref, out, torch.bfloat16) + _FP32_SLACK * (c.absref + g0.double().abs())
+ assert (err_o <= bound).all(), mode
+
+
+def _fallback_check(W, cfgs, case, fn):
+ g0 = torch.randn_like(W)
+ W.grad = g0.clone()
+ before = gg.DW_ACCUMULATE_STATS["inplace"]
+ out = fn()
+ assert gg.DW_ACCUMULATE_STATS["inplace"] == before, "must not write param.grad in place"
+ return g0, out
+
+
+def test_autograd_grad_and_backward_inputs_do_not_touch_param_grad(monkeypatch):
+ monkeypatch.setenv("UNSLOTH_MOE_DW_ACCUMULATE", "1")
+ E, N, K, T, topk = 4, 64, 64, 32, 2
+ cfgs = _cfgs()
+ c = _Case(E, N, K, T, topk, "permute_x", torch.bfloat16)
+ W = _param(E, N, K, torch.bfloat16)
+
+ def grad_call():
+ Y = _gemm(c.X, W, c.m_sizes, c.gather, topk, c.mode, cfgs)
+ return torch.autograd.grad(Y, [W], c.G)[0]
+
+ g0, dW = _fallback_check(W, cfgs, c, grad_call)
+ assert torch.equal(W.grad, g0)
+ torch.testing.assert_close(dW.double(), c.ref, rtol = 2e-2, atol = 2e-2 * c.ref.abs().max().item())
+
+ X = c.X.clone().requires_grad_(True)
+
+ def inputs_call():
+ _gemm(X, W, c.m_sizes, c.gather, topk, c.mode, cfgs).backward(c.G, inputs = [X])
+
+ g0, _ = _fallback_check(W, cfgs, c, inputs_call)
+ assert torch.equal(W.grad, g0) and X.grad is not None
+
+ W.grad = g0.clone()
+ before = gg.DW_ACCUMULATE_STATS["inplace"]
+ _gemm(X, W, c.m_sizes, c.gather, topk, c.mode, cfgs).backward(c.G, inputs = [W])
+ assert gg.DW_ACCUMULATE_STATS["inplace"] == before + 1
+ torch.testing.assert_close(
+ (W.grad.double() - g0.double()), c.ref, rtol = 3e-2, atol = 3e-2 * c.ref.abs().max().item()
+ )
+
+
+def test_hooks_create_graph_kill_switch_and_non_leaf_fall_back(monkeypatch):
+ monkeypatch.setenv("UNSLOTH_MOE_DW_ACCUMULATE", "1")
+ E, N, K, T, topk = 4, 64, 64, 32, 2
+ cfgs = _cfgs()
+ c = _Case(E, N, K, T, topk, "permute_x", torch.bfloat16)
+
+ W = _param(E, N, K, torch.bfloat16)
+ seen = []
+ h = W.register_hook(lambda g: seen.append(g is not None))
+ _fallback_check(W, cfgs, c, lambda: c.backward(W, cfgs))
+ assert seen == [True]
+ h.remove()
+
+ W = _param(E, N, K, torch.bfloat16)
+ W.register_post_accumulate_grad_hook(lambda p: seen.append("post"))
+ _fallback_check(W, cfgs, c, lambda: c.backward(W, cfgs))
+ assert seen[-1] == "post"
+
+ W = _param(E, N, K, torch.bfloat16)
+ _fallback_check(W, cfgs, c, lambda: c.backward(W, cfgs, create_graph = True))
+
+ W = _param(E, N, K, torch.bfloat16)
+ with torch.autograd.detect_anomaly(check_nan = True):
+ _fallback_check(W, cfgs, c, lambda: c.backward(W, cfgs))
+
+ monkeypatch.setenv("UNSLOTH_MOE_DW_ACCUMULATE", "0")
+ _fallback_check(W, cfgs, c, lambda: c.backward(W, cfgs))
+ monkeypatch.setenv("UNSLOTH_MOE_DW_ACCUMULATE", "1")
+
+ W = _param(E, N, K, torch.bfloat16)
+ g0, _ = _fallback_check(W, cfgs, c, lambda: c.backward(W * 1.0, cfgs))
+ torch.testing.assert_close(
+ (W.grad.double() - g0.double()), c.ref, rtol = 3e-2, atol = 3e-2 * c.ref.abs().max().item()
+ )
+
+ W = _param(E, N, K, torch.bfloat16).requires_grad_(False)
+ X = c.X.clone().requires_grad_(True)
+ before = dict(gg.DW_ACCUMULATE_STATS)
+ _gemm(X, W, c.m_sizes, c.gather, topk, c.mode, cfgs).backward(c.G)
+ assert gg.DW_ACCUMULATE_STATS == before and W.grad is None
+
+
+def test_retain_graph_tied_weight_and_inf_propagation(monkeypatch):
+ E, N, K, T, topk = 4, 64, 128, 48, 2
+ cfgs = _cfgs()
+ c = _Case(E, N, K, T, topk, "permute_x", torch.bfloat16)
+ c2 = _Case(E, N, K, T, topk, "permute_x", torch.bfloat16, seed = 7)
+ g0 = torch.randn(E, N, K, device = DEV).bfloat16()
+ results = {}
+ for inplace in (False, True):
+ monkeypatch.setenv("UNSLOTH_MOE_DW_ACCUMULATE", "1" if inplace else "0")
+ W = _param(E, N, K, torch.bfloat16)
+ W.grad = g0.clone()
+ Y = _gemm(c.X, W, c.m_sizes, c.gather, topk, c.mode, cfgs)
+ Y.backward(c.G, retain_graph = True)
+ Y.backward(c.G)
+ Y1 = _gemm(c.X, W, c.m_sizes, c.gather, topk, c.mode, cfgs)
+ Y2 = _gemm(c2.X, W, c2.m_sizes, c2.gather, topk, c2.mode, cfgs)
+ ((Y1 * c.G).sum() + (Y2 * c2.G).sum()).backward()
+ results[inplace] = W.grad.clone()
+ ref = g0.double() + 3 * c.ref + c2.ref
+ err_o = (results[True].double() - ref).abs()
+ err_s = (results[False].double() - ref).abs()
+ assert err_o.mean() <= err_s.mean()
+
+ monkeypatch.setenv("UNSLOTH_MOE_DW_ACCUMULATE", "1")
+ W = _param(E, N, K, torch.bfloat16)
+ W.grad = g0.clone()
+ G = c.G.clone()
+ G[3, 5] = float("inf")
+ _gemm(c.X, W, c.m_sizes, c.gather, topk, c.mode, cfgs).backward(G)
+ assert not torch.isfinite(W.grad).all()
+
+
+def test_compiled_fullgraph_zero_breaks_and_matches_eager(monkeypatch):
+ monkeypatch.setenv("UNSLOTH_MOE_DW_ACCUMULATE", "1")
+ from torch._dynamo.utils import counters
+
+ E, N, K, T, topk = 4, 128, 128, 64, 2
+ cfgs = _cfgs()
+ c = _Case(E, N, K, T, topk, "permute_x", torch.bfloat16)
+
+ def fn(X, W):
+ return _gemm(X, W, c.m_sizes, c.gather, topk, c.mode, cfgs)
+
+ torch._dynamo.reset()
+ counters.clear()
+ compiled = torch.compile(fn, fullgraph = True)
+ grads = []
+ for f in (fn, compiled):
+ W = _param(E, N, K, torch.bfloat16)
+ g0 = torch.randn_like(W)
+ W.grad = g0.clone()
+ X = c.X.clone().requires_grad_(True)
+ for _ in range(2):
+ f(X, W).backward(c.G)
+ grads.append(W.grad.double() - g0.double())
+ assert sum(counters["graph_break"].values()) == 0, dict(counters["graph_break"])
+ for g in grads:
+ torch.testing.assert_close(g, 2 * c.ref, rtol = 3e-2, atol = 3e-2 * c.ref.abs().max().item())
+
+ def acc(X, dY, buf):
+ gg.grouped_gemm_dW_accumulate(
+ X, dY, c.m_sizes, c.gather, buf, topk, BLOCK_SIZE_M = 64, BLOCK_SIZE_N = 64,
+ BLOCK_SIZE_K = 64, permute_x = True,
+ ) # fmt: skip
+ return buf
+
+ counters.clear()
+ buf_e = torch.randn(E, N, K, device = DEV).bfloat16()
+ buf_c = buf_e.clone()
+ acc(c.X, c.G, buf_e)
+ torch.compile(acc, fullgraph = True)(c.X, c.G, buf_c)
+ assert sum(counters["graph_break"].values()) == 0
+ assert torch.equal(buf_e, buf_c)
+ schema = str(torch.ops.unsloth.grouped_gemm_dW_accumulate.default._schema)
+ assert re.search(r"Tensor\(a\d*!\) dW", schema), schema
+
+
+def _free_port():
+ with socket.socket() as s:
+ s.bind(("127.0.0.1", 0))
+ return s.getsockname()[1]
+
+
+def _ddp_worker(rank, world, port, inplace, bucket_view, out_path):
+ import torch.distributed as dist
+
+ os.environ.update(MASTER_ADDR = "127.0.0.1", MASTER_PORT = str(port))
+ os.environ["UNSLOTH_MOE_DW_ACCUMULATE"] = "1" if inplace else "0"
+ dist.init_process_group("gloo", rank = rank, world_size = world)
+ E, N, K, T, topk = 4, 128, 128, 64, 2
+ cfgs = _cfgs()
+
+ class Block(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ torch.manual_seed(0)
+ self.w = torch.nn.Parameter(torch.randn(E, N, K, device = DEV).bfloat16() * 0.05)
+ self.head = torch.nn.Linear(N, 1, device = DEV, dtype = torch.bfloat16)
+
+ def forward(self, x, m_sizes, gather):
+ return self.head(_gemm(x, self.w, m_sizes, gather, topk, "permute_x", cfgs))
+
+ model = Block()
+ ddp = torch.nn.parallel.DistributedDataParallel(model, gradient_as_bucket_view = bucket_view)
+ cases = [
+ _Case(E, N, K, T, topk, "permute_x", torch.bfloat16, seed = 100 * rank + i) for i in range(4)
+ ]
+ for i, case in enumerate(cases):
+ loss = ddp(case.X, case.m_sizes, case.gather).float().pow(2).mean()
+ if i < 3:
+ with ddp.no_sync():
+ loss.backward()
+ else:
+ loss.backward()
+ torch.save(
+ {"grad": model.w.grad.float().cpu(), "inplace": gg.DW_ACCUMULATE_STATS["inplace"]},
+ f"{out_path}.{rank}",
+ )
+ dist.destroy_process_group()
+
+
[email protected]("bucket_view", [False, True])
+def test_ddp_gradient_accumulation_with_bucket_hooks(bucket_view, tmp_path):
+ import torch.multiprocessing as mp
+
+ got = {}
+ for inplace in (False, True):
+ out = str(tmp_path / f"g_{inplace}")
+ mp.spawn(_ddp_worker, args = (2, _free_port(), inplace, bucket_view, out), nprocs = 2)
+ got[inplace] = [torch.load(f"{out}.{r}") for r in range(2)]
+ for r in range(2):
+ assert got[False][r]["inplace"] == 0
+ assert got[True][r]["inplace"] == 3
+ assert torch.equal(got[True][0]["grad"], got[True][1]["grad"])
+ a, b = got[True][0]["grad"], got[False][0]["grad"]
+ rel = ((a - b).norm() / b.norm()).item()
+ assert rel < 1e-2, rel
+
+
+def _fsdp2_worker(rank, inplace, out_path):
+ import torch.distributed as dist
+
+ try:
+ from torch.distributed.fsdp import fully_shard
+ except ImportError:
+ from torch.distributed._composable.fsdp import fully_shard
+
+ os.environ.update(MASTER_ADDR = "127.0.0.1", MASTER_PORT = str(_free_port()))
+ os.environ["UNSLOTH_MOE_DW_ACCUMULATE"] = "1" if inplace else "0"
+ torch.cuda.set_device(0)
+ dist.init_process_group("nccl", rank = 0, world_size = 1)
+ E, N, K, T, topk = 4, 128, 128, 64, 2
+ cfgs = _cfgs()
+
+ class Block(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ torch.manual_seed(0)
+ self.w = torch.nn.Parameter(torch.randn(E, N, K, device = DEV).bfloat16() * 0.05)
+
+ def forward(self, x, m_sizes, gather):
+ return _gemm(x, self.w, m_sizes, gather, topk, "permute_x", cfgs)
+
+ model = Block()
+ fully_shard(model)
+ cases = [_Case(E, N, K, T, topk, "permute_x", torch.bfloat16, seed = i) for i in range(4)]
+ for i, case in enumerate(cases):
+ model.set_requires_gradient_sync(i == 3)
+ model(case.X, case.m_sizes, case.gather).float().pow(2).mean().backward()
+ grad = model.w.grad.full_tensor() if hasattr(model.w.grad, "full_tensor") else model.w.grad
+ torch.save({"grad": grad.float().cpu(), "inplace": gg.DW_ACCUMULATE_STATS["inplace"]}, out_path)
+ dist.destroy_process_group()
+
+
+def test_fsdp2_no_sync_gradient_accumulation(tmp_path):
+ import torch.multiprocessing as mp
+
+ if not torch.distributed.is_nccl_available():
+ pytest.skip("FSDP2 smoke needs NCCL")
+ got = {}
+ for inplace in (False, True):
+ out = str(tmp_path / f"f_{inplace}.pt")
+ mp.spawn(_fsdp2_worker, args = (inplace, out), nprocs = 1)
+ got[inplace] = torch.load(out)
+ assert got[False]["inplace"] == 0
+ a, b = got[True]["grad"], got[False]["grad"]
+ rel = ((a - b).norm() / b.norm()).item()
+ assert rel < 1e-2, rel
+ print(f"FSDP2 in-place launches: {got[True]['inplace']}, rel vs stock {rel:.2e}")
+
+
+def test_existing_gradient_survives_in_microbatch_cancellation(monkeypatch):
+ monkeypatch.setenv("UNSLOTH_MOE_DW_ACCUMULATE", "1")
+ E, N, K, T = 1, 64, 64, 128
+ dtype, big = torch.bfloat16, 1e4
+ X = torch.zeros(T, K, device = DEV, dtype = dtype)
+ G = torch.zeros(T, N, device = DEV, dtype = dtype)
+ X[0], X[64] = big, big
+ G[0], G[64] = big, -big
+ m_sizes = torch.tensor([T], device = DEV, dtype = torch.int32)
+ gather = torch.arange(T, device = DEV, dtype = torch.int32)
+ W = _param(E, N, K, dtype)
+ W.grad = torch.ones_like(W)
+ before = gg.DW_ACCUMULATE_STATS["inplace"]
+ _gemm(X, W, m_sizes, gather, 1, "none", _cfgs()).backward(G)
+ assert gg.DW_ACCUMULATE_STATS["inplace"] == before + 1
+ assert torch.equal(W.grad, torch.ones_like(W))
+
+
+def test_side_stream_use_keeps_the_returned_gradient_path(monkeypatch):
+ monkeypatch.setenv("UNSLOTH_MOE_DW_ACCUMULATE", "1")
+ E, N, K, T, topk = 4, 64, 64, 32, 2
+ cfgs = _cfgs()
+ c = _Case(E, N, K, T, topk, "permute_x", torch.bfloat16)
+ W = _param(E, N, K, torch.bfloat16)
+ side = torch.cuda.Stream()
+ side.wait_stream(torch.cuda.current_stream())
+
+ def side_call():
+ with torch.cuda.stream(side):
+ c.backward(W, cfgs)
+ torch.cuda.current_stream().wait_stream(side)
+
+ g0, _ = _fallback_check(W, cfgs, c, side_call)
+ torch.testing.assert_close(
+ (W.grad.double() - g0.double()), c.ref, rtol = 3e-2, atol = 3e-2 * c.ref.abs().max().item()
+ )
+ _fallback_check(W, cfgs, c, lambda: c.backward(W, cfgs))
diff --git a/unsloth/kernels/moe/grouped_gemm/interface.py b/unsloth/kernels/moe/grouped_gemm/interface.py
index 5e2870f7ca3..def12f79c72 100644
--- a/unsloth/kernels/moe/grouped_gemm/interface.py
+++ b/unsloth/kernels/moe/grouped_gemm/interface.py
@@ -5,6 +5,7 @@ import copy
import functools
import inspect
import logging
+import os
import warnings
from typing import Optional
from dataclasses import asdict
@@ -39,7 +40,6 @@ ch.setFormatter(formatter)
logger.addHandler(ch)
-# Precompute TMA support (NVIDIA capability >= 9 plus a triton carrying make_tensor_descriptor) to avoid graph breaks.
# Process-wide, so every visible card must be sm90+ (experts may sit on any); an unreadable card means False.
def _check_tma_support():
if DEVICE_TYPE in ("xpu", "hip"):
@@ -73,7 +73,6 @@ def _autotuner_for(kernel, device):
return _DEVICE_AUTOTUNERS[key]
-# triton.set_allocator is Triton 3.0+.
_HAS_SET_ALLOCATOR = hasattr(triton, "set_allocator")
@@ -132,7 +131,6 @@ def _grouped_gemm_forward_impl(
permute_y: bool = False,
fuse_mul_post: bool = False,
autotune: bool = False,
- # Kernel tuning params: must be tuned, else poor performance.
BLOCK_SIZE_M: int = 32,
BLOCK_SIZE_N: int = 32,
BLOCK_SIZE_K: int = 32,
@@ -141,7 +139,6 @@ def _grouped_gemm_forward_impl(
use_tma_load_w: bool = False,
use_tma_load_x: bool = False,
use_tma_store: bool = False,
- # Software pipelining; no effect until the loop is re-written.
flatten: bool = True,
debug: bool = False,
) -> torch.Tensor:
@@ -167,7 +164,6 @@ def _grouped_gemm_forward_impl(
assert not (permute_y and use_tma_store), "Cannot use both TMA store and permute_y"
if use_tma_load_x:
- # TMA load for activations; TMA gather is Blackwell+ only.
assert not permute_x, "Cannot use both use_tma_load_x and permute_x"
use_tma = use_tma_load_w or use_tma_load_x or use_tma_store
@@ -289,7 +285,6 @@ def _grouped_gemm_forward_impl(
is_fake = _is_tracing(X, W)
if not is_fake:
- # Triton launches on the current device: pin it to the operands' card.
with torch_gpu_device(X.device):
compiled_kernel: triton.compiler.CompiledKernel = kernel[grid](
**kernel_args, enable_fp_fusion = True
@@ -338,7 +333,6 @@ def _grouped_gemm_dX_impl(
assert m_sizes.ndim == 1
assert not (permute_x and permute_y), "Cannot permute both X and Y"
- # Flipped from the forward pass: a y permuted in the forward must be permuted on load in the backward.
assert not (permute_y and use_tma_load_dy), "Cannot use both TMA load and permute_y"
assert not (permute_x and use_tma_store), "Cannot use both TMA store and permute_x"
@@ -374,7 +368,6 @@ def _grouped_gemm_dX_impl(
assert M_total % topk == 0, f"M_total ({M_total}) must be divisible by topk ({topk})"
num_tokens = M_total // topk
- # The kernel only reads gather_indices under permute_x / permute_y, so it stays optional otherwise.
if permute_x or permute_y:
assert (
gather_indices is not None
@@ -382,7 +375,6 @@ def _grouped_gemm_dX_impl(
total_tokens = gather_indices.shape[0] if gather_indices is not None else M_total
assert total_tokens == M_total, f"Total tokens ({total_tokens}) must match M_total ({M_total})"
- # The output stays [NUM_TOKENS * TOPK, K] even under permute_x: gradients accumulate across every expert a token chose, reduced in a post-processing step.
output_shape = (total_tokens, K)
dX = torch.zeros(output_shape, device = dY.device, dtype = dY.dtype)
@@ -449,12 +441,15 @@ def _grouped_gemm_dX_impl(
return dX
-def _grouped_gemm_dW_impl(
+def _launch_grouped_gemm_dW(
X: torch.Tensor,
dY: torch.Tensor,
m_sizes: torch.Tensor,
gather_indices: torch.Tensor,
topk: int,
+ dW_out: Optional[torch.Tensor] = None,
+ accumulate: bool = False,
+ dW_dtype: Optional[torch.dtype] = None,
BLOCK_SIZE_M: int = 32,
BLOCK_SIZE_N: int = 32,
BLOCK_SIZE_K: int = 32,
@@ -523,7 +518,21 @@ def _grouped_gemm_dW_impl(
assert M_grad == total_tokens, f"dY M ({M_grad}) != total_tokens ({total_tokens})"
- dW = torch.zeros((num_experts, N, K), device = X.device, dtype = X.dtype)
+ if accumulate:
+ # Read-modify-write into an existing gradient: one program owns each (expert, n, k) tile.
+ assert not autotune, "autotune benchmarks re-run the kernel and would add dW several times"
+ assert dW_out is not None
+ assert dW_out.numel() == num_experts * N * K, f"{dW_out.shape} != {(num_experts, N, K)}"
+ assert dW_out.is_contiguous() and dW_out.device == X.device
+ dW = dW_out
+ else:
+ dW = torch.zeros(
+ (num_experts, N, K), device = X.device, dtype = X.dtype if dW_dtype is None else dW_dtype
+ )
+ if use_tma_store and (dW.element_size() > X.element_size() or dW.data_ptr() % 16 != 0):
+ # TMA needs a 16 byte aligned base (a DDP bucket view may not be), and the tuned tile
+ # sizes assume an output no wider than X.
+ use_tma_store = False
if not autotune:
pass
@@ -566,6 +575,8 @@ def _grouped_gemm_dW_impl(
"PERMUTE_X": permute_x,
"PERMUTE_Y": permute_y,
}
+ if accumulate:
+ kernel_args["ACCUMULATE"] = True
if not autotune:
kernel_args.update(
@@ -602,6 +613,58 @@ def _grouped_gemm_dW_impl(
return dW
+def _grouped_gemm_dW_impl(
+ X: torch.Tensor,
+ dY: torch.Tensor,
+ m_sizes: torch.Tensor,
+ gather_indices: torch.Tensor,
+ topk: int,
+ BLOCK_SIZE_M: int = 32,
+ BLOCK_SIZE_N: int = 32,
+ BLOCK_SIZE_K: int = 32,
+ permute_x: bool = False,
+ permute_y: bool = False,
+ use_tma_load_dy: bool = False,
+ use_tma_load_x: bool = False,
+ use_tma_store: bool = False,
+ fuse_mul_pre: bool = False,
+ fuse_mul_post: bool = False,
+ num_warps: int = 4,
+ num_stages: int = 2,
+ flatten: bool = True,
+ autotune: bool = False,
+ debug: bool = False,
+ dW_dtype: Optional[torch.dtype] = None,
+) -> torch.Tensor:
+ """dW into a fresh zeroed (E, N, K) buffer of dW_dtype (default X.dtype)."""
+ kwargs = dict(locals())
+ return _launch_grouped_gemm_dW(dW_out = None, accumulate = False, **kwargs)
+
+
+def _grouped_gemm_dW_accumulate_impl(
+ X: torch.Tensor,
+ dY: torch.Tensor,
+ m_sizes: torch.Tensor,
+ gather_indices: torch.Tensor,
+ dW: torch.Tensor,
+ topk: int,
+ BLOCK_SIZE_M: int = 32,
+ BLOCK_SIZE_N: int = 32,
+ BLOCK_SIZE_K: int = 32,
+ permute_x: bool = False,
+ permute_y: bool = False,
+ use_tma_load_dy: bool = False,
+ use_tma_load_x: bool = False,
+ use_tma_store: bool = False,
+ num_warps: int = 4,
+ num_stages: int = 2,
+) -> None:
+ """dW += X^T dY per expert, in place, added in fp32 and rounded once to dW.dtype."""
+ kwargs = dict(locals())
+ dW_out = kwargs.pop("dW")
+ _launch_grouped_gemm_dW(dW_out = dW_out, accumulate = True, **kwargs)
+
+
# Opaque ops: allow_in_graph let AOT trace the body on fake tensors, keeping torch.empty and dropping the launch.
# Not triton_op: torch.compile rejects prune_configs_by on triton.autotune.
def _fwd_fake(X, W, topk, m_sizes, gather_indices, topk_weights, permute_x, permute_y, *args):
@@ -618,20 +681,34 @@ def _dX_fake(dY, W, gather_indices, m_sizes, topk, *args):
def _dW_fake(X, dY, m_sizes, *args):
- return X.new_empty((m_sizes.shape[0], dY.shape[-1], X.shape[-1]))
+ dW_dtype = dict(zip(_DW_PARAM_NAMES[3:], args)).get("dW_dtype")
+ return X.new_empty(
+ (m_sizes.shape[0], dY.shape[-1], X.shape[-1]),
+ dtype = X.dtype if dW_dtype is None else dW_dtype,
+ )
+
+
+def _dW_accumulate_fake(X, dY, m_sizes, gather_indices, dW, *args):
+ return None
+_DW_PARAM_NAMES = tuple(inspect.signature(_grouped_gemm_dW_impl).parameters)
_OPTIONAL_TENSORS = ("gather_indices", "topk_weights")
-def _make_op(name, impl, fake):
+def _make_op(
+ name,
+ impl,
+ fake,
+ mutates_args = (),
+ returns = torch.Tensor,
+):
"""torch.library.custom_op (torch >= 2.4) over impl; torch.compiler.disable where it is missing."""
custom_op = getattr(getattr(torch, "library", None), "custom_op", None)
if custom_op is None:
return torch._dynamo.disable(impl)
existing = getattr(getattr(torch.ops, "unsloth", None), name, None)
if existing is not None:
- # grouped_gemm is also importable as a top-level package (glm4_moe, tests): reuse the registration.
return existing
params = [
p.replace(
@@ -645,9 +722,9 @@ def _make_op(name, impl, fake):
def op(*args):
return impl(**dict(zip(names, args)))
- op.__signature__ = inspect.Signature(params, return_annotation = torch.Tensor)
+ op.__signature__ = inspect.Signature(params, return_annotation = returns)
op.__name__ = name
- op = custom_op(f"unsloth::{name}", op, mutates_args = ())
+ op = custom_op(f"unsloth::{name}", op, mutates_args = tuple(mutates_args))
op.register_fake(fake)
return op
@@ -655,6 +732,13 @@ def _make_op(name, impl, fake):
_grouped_gemm_forward_op = _make_op("grouped_gemm_forward", _grouped_gemm_forward_impl, _fwd_fake)
_grouped_gemm_dX_op = _make_op("grouped_gemm_dX", _grouped_gemm_dX_impl, _dX_fake)
_grouped_gemm_dW_op = _make_op("grouped_gemm_dW", _grouped_gemm_dW_impl, _dW_fake)
+_grouped_gemm_dW_accumulate_op = _make_op(
+ "grouped_gemm_dW_accumulate",
+ _grouped_gemm_dW_accumulate_impl,
+ _dW_accumulate_fake,
+ mutates_args = ("dW",),
+ returns = None,
+)
def _signature(impl):
@@ -666,7 +750,12 @@ def _signature(impl):
_SIGNATURES = {
impl: _signature(impl)
- for impl in (_grouped_gemm_forward_impl, _grouped_gemm_dX_impl, _grouped_gemm_dW_impl)
+ for impl in (
+ _grouped_gemm_forward_impl,
+ _grouped_gemm_dX_impl,
+ _grouped_gemm_dW_impl,
+ _grouped_gemm_dW_accumulate_impl,
+ )
}
@@ -675,7 +764,6 @@ def _dispatch(impl, op, args, kwargs):
bound = dict(defaults)
bound.update(zip(names, args))
bound.update(kwargs)
- # Eager calls skip the custom-op dispatcher; traced calls (dynamo, or fake tensors under make_fx) need the op.
if torch.compiler.is_compiling() or _is_tracing(bound[names[0]], bound[names[1]]):
return op(*[bound[n] for n in names])
return impl(**bound)
@@ -696,6 +784,71 @@ def grouped_gemm_dW(*args, **kwargs) -> torch.Tensor:
return _dispatch(_grouped_gemm_dW_impl, _grouped_gemm_dW_op, args, kwargs)
[email protected](_grouped_gemm_dW_accumulate_impl)
+def grouped_gemm_dW_accumulate(*args, **kwargs) -> None:
+ return _dispatch(_grouped_gemm_dW_accumulate_impl, _grouped_gemm_dW_accumulate_op, args, kwargs)
+
+
+DW_ACCUMULATE_STATS = {"inplace": 0, "fallback": 0}
+
+
+_DW_DTYPES = (torch.float16, torch.bfloat16, torch.float32)
+
+
+def _dW_accumulate_enabled():
+ return os.environ.get("UNSLOTH_MOE_DW_ACCUMULATE", "1") != "0"
+
+
+def _dW_accumulate_target(ctx, X, autotune):
+ """The leaf parameter whose existing .grad the dW kernel may add into, else None.
+
+ Any case where AccumulateGrad would do more than `param.grad += dW` falls back to returning dW.
+ Hooks registered directly on autograd Nodes (Node.register_hook / register_prehook) cannot be
+ enumerated from Python and see None for W here; UNSLOTH_MOE_DW_ACCUMULATE=0 restores the
+ returned-gradient path."""
+ if torch.compiler.is_compiling() or _is_tracing(X) or torch.is_grad_enabled():
+ return None
+ # Anomaly mode checks the returned gradients for NaN; an in-place add returns None for W.
+ if torch.is_anomaly_enabled():
+ return None
+ if autotune or not _dW_accumulate_enabled():
+ return None
+ try:
+ acc = ctx.next_functions[1][0]
+ except (AttributeError, IndexError, TypeError):
+ return None
+ if acc is None or type(acc).__name__ != "AccumulateGrad":
+ return None
+ param = acc.variable
+ if type(param) not in (torch.Tensor, torch.nn.Parameter):
+ return None # tensor subclasses (DTensor, quantized) own their gradient layout
+ grad = param.grad
+ if grad is None or grad.layout != torch.strided or type(grad) is not torch.Tensor:
+ return None
+ if (
+ grad.dtype not in _DW_DTYPES
+ or grad.dtype != param.dtype
+ or grad.device != X.device
+ or grad.shape != param.shape
+ or not grad.is_contiguous()
+ ):
+ return None
+ if getattr(param, "_backward_hooks", None) or getattr(
+ param, "_post_accumulate_grad_hooks", None
+ ):
+ return None
+ if getattr(param, "_unsloth_dw_side_stream", False):
+ return None
+ if torch.cuda.current_stream(X.device) != torch.cuda.default_stream(X.device):
+ return None
+ try:
+ if not torch._C._will_engine_execute_node(acc):
+ return None
+ except RuntimeError:
+ return None # autograd.grad(): W's gradient is captured, never accumulated
+ return param
+
+
class GroupedGemm(torch.autograd.Function):
@staticmethod
def forward(
@@ -727,8 +880,12 @@ class GroupedGemm(torch.autograd.Function):
ctx.dX_only = dX_only
ctx.dW_only = dW_only
- # topk_weights is not saved for backward: training with fused_mul is unsupported.
ctx.save_for_backward(X, W, m_sizes, gather_indices)
+ if W.requires_grad and X.is_cuda and not torch.compiler.is_compiling():
+ if torch.cuda.current_stream(X.device) != torch.cuda.default_stream(X.device):
+ # A use on a side stream may run its backward concurrently with another use of W:
+ # in-place accumulation would race, so this weight keeps the returned-gradient path.
+ W._unsloth_dw_side_stream = True
fwd_config = {}
if kernel_config_fwd is not None:
@@ -768,7 +925,6 @@ class GroupedGemm(torch.autograd.Function):
assert (
not ctx.fuse_mul_post
), "fused_mul should only be used for inference, not for training"
- # Frozen W (LoRA, dequantized experts) or X: skip its kernel and buffer, as autograd does for _grouped_mm.
dX_only = ctx.dX_only or not ctx.needs_input_grad[1]
dW_only = ctx.dW_only or not ctx.needs_input_grad[0]
if dX_only and dW_only:
@@ -797,17 +953,38 @@ class GroupedGemm(torch.autograd.Function):
bwd_dW_config["num_warps"] = kernel_config_bwd_dW.num_warps
bwd_dW_config["num_stages"] = kernel_config_bwd_dW.num_stages
- dW = grouped_gemm_dW(
- X = X,
- dY = dY,
- m_sizes = m_sizes,
- gather_indices = gather_indices,
- topk = topk,
- permute_x = permute_x,
- permute_y = permute_y,
- autotune = autotune,
- **bwd_dW_config,
- )
+ target = _dW_accumulate_target(ctx, X, autotune)
+ if target is not None:
+ grouped_gemm_dW_accumulate(
+ X = X,
+ dY = dY,
+ m_sizes = m_sizes,
+ gather_indices = gather_indices,
+ dW = target.grad,
+ topk = topk,
+ permute_x = permute_x,
+ permute_y = permute_y,
+ **bwd_dW_config,
+ )
+ # Already in param.grad: returning it as well would add it twice.
+ dW = None
+ DW_ACCUMULATE_STATS["inplace"] += 1
+ else:
+ if W.dtype != X.dtype and W.dtype in _DW_DTYPES and _dW_accumulate_enabled():
+ bwd_dW_config["dW_dtype"] = W.dtype
+ dW = grouped_gemm_dW(
+ X = X,
+ dY = dY,
+ m_sizes = m_sizes,
+ gather_indices = gather_indices,
+ topk = topk,
+ permute_x = permute_x,
+ permute_y = permute_y,
+ autotune = autotune,
+ **bwd_dW_config,
+ )
+ if not torch.compiler.is_compiling():
+ DW_ACCUMULATE_STATS["fallback"] += 1
else:
dW = None
diff --git a/unsloth/kernels/moe/grouped_gemm/kernels/backward.py b/unsloth/kernels/moe/grouped_gemm/kernels/backward.py
index f19bcc4907f..45e2efd6ef8 100644
--- a/unsloth/kernels/moe/grouped_gemm/kernels/backward.py
+++ b/unsloth/kernels/moe/grouped_gemm/kernels/backward.py
@@ -71,11 +71,9 @@ def _grouped_gemm_dX_kernel(
output_dtype = dX_ptr.dtype.element_ty
tidx = tl.program_id(0)
- # Removes the need for predication along N in the GEMM main loop.
tl.static_assert(N % BLOCK_SIZE_N == 0, "N must be divisible by BLOCK_SIZE_N")
tl.static_assert(K % BLOCK_SIZE_K == 0, "K must be divisible by BLOCK_SIZE_K")
- # A single global TMA descriptor with one block shape; TMA load never permutes x, so the shape is [TOTAL_TOKENS, K]. Unverified across expert boundaries.
if USE_TMA_LOAD_dY:
dY_desc = tl.make_tensor_descriptor(
dY_ptr,
@@ -112,7 +110,6 @@ def _grouped_gemm_dX_kernel(
num_tiles_per_expert = num_m_tiles * num_k_tiles
if USE_TMA_STORE:
- # Define the descriptor inside the loop to predicate the store along M.
tl.static_assert(K % BLOCK_SIZE_K == 0, "K must be divisible by BLOCK_SIZE_K")
dX_desc = tl.make_tensor_descriptor(
dX_ptr,
@@ -121,7 +118,6 @@ def _grouped_gemm_dX_kernel(
block_shape = [BLOCK_SIZE_M, BLOCK_SIZE_K],
)
- # Bounds are relative to tiles processed so far, so only this expert's tiles are handled and the total across experts is never exceeded.
while tidx >= processed_tiles and tidx < (processed_tiles + num_tiles_per_expert):
group_index = tidx - processed_tiles
@@ -144,21 +140,15 @@ def _grouped_gemm_dX_kernel(
row_mask = gather_offsets < m_size
row_mask = row_mask[:, None]
- # Only (PERMUTE_X and not PERMUTE_Y) and (not PERMUTE_X and PERMUTE_Y) occur, so load/store offsets are simply flipped between the two cases, with the strides adjusted to match.
-
if PERMUTE_X:
- # Permuted on load in the forward pass (typically the first grouped GEMM in the MoE MLP), so load contiguous and permute on store.
load_a_idx = indices_to_gather_64[:, None] * N
store_idx = expert_token_offsets * K
else:
- # Permuted on store in the forward pass (typically the second grouped GEMM), so permute on load and store contiguous.
load_a_idx = expert_token_offsets * N
store_idx = indices_to_gather_64[:, None] * K
else:
- # Offsets relative to the CURRENT expert; m_start then advances to this expert's start token.
offs_am = tile_m_idx * BLOCK_SIZE_M + m_block_range
- # [M, N] @ [N, K] -> [M, K], so A strides by N and B by K, plus m_start for A's expert start token and n_start for B's slice of the [E, N, K] weight matrix.
row_offsets_a = (m_start + offs_am[:, None]).to(tl.int64)
load_a_idx = row_offsets_a * N
store_idx = row_offsets_a * K
@@ -193,7 +183,6 @@ def _grouped_gemm_dX_kernel(
if not USE_TMA_LOAD_dY:
dY_ptrs += BLOCK_SIZE_N
- # B is no longer advanced along the contiguous dimension: weights are [N, K], so stride by K to reach the next [N_BLOCK_SIZE, K_BLOCK_SIZE] tile.
if not USE_TMA_LOAD_W:
w_ptrs += BLOCK_SIZE_N * K
@@ -218,7 +207,6 @@ def _grouped_gemm_dX_kernel(
_autotuned_grouped_gemm_dX_kernel = triton.autotune(
configs = get_dX_kernel_configs(),
prune_configs_by = {"early_config_prune": prune_dX_configs},
- # NUM_TOKENS is left out of the key to avoid recompiling for every sequence length.
key = ["NUM_EXPERTS", "N", "K", "PERMUTE_X", "PERMUTE_Y"],
)(_grouped_gemm_dX_kernel)
@@ -265,6 +253,10 @@ def _grouped_gemm_dW_kernel(
USE_TMA_LOAD_X: tl.constexpr = False,
USE_TMA_STORE: tl.constexpr = False,
acc_dtype: tl.constexpr = tl.float32,
+ ACCUMULATE: tl.constexpr = False,
+ # Seed the fp32 accumulator with the existing tile (load overlaps the main loop) instead of adding
+ # it after the loop. Only for a bf16 / fp16 dW, whose final rounding dwarfs the fp32 order change;
+ # an fp32 dW adds at the end, the same single fp32 add autograd would do.
) -> None:
TOTAL_TOKENS = NUM_TOKENS * TOPK
TMA_LOAD_BOTH: tl.constexpr = USE_TMA_LOAD_X and USE_TMA_LOAD_dY
@@ -295,7 +287,6 @@ def _grouped_gemm_dW_kernel(
block_range_n = tl.arange(0, BLOCK_SIZE_N)
block_range_k = tl.arange(0, BLOCK_SIZE_K)
- # N % BLOCK_SIZE_N == 0 and K % BLOCK_SIZE_K == 0 are required when using TMA store.
if USE_TMA_STORE:
tl.static_assert(N % BLOCK_SIZE_N == 0, "N must be divisible by BLOCK_SIZE_N")
tl.static_assert(K % BLOCK_SIZE_K == 0, "K must be divisible by BLOCK_SIZE_K")
@@ -322,11 +313,9 @@ def _grouped_gemm_dW_kernel(
accumulator = tl.zeros((BLOCK_SIZE_N, BLOCK_SIZE_K), dtype = acc_dtype)
m_start = m_end
- # The cast is needed or the compiler complains about mismatching types; reason unknown.
m_size = tl.load(m_sizes_ptr + expert_idx).to(tl.int32)
m_end = m_start + m_size
- # Offset by n_start: the result goes into this expert's slice of the global [E, N, K] weight matrix.
n_start = expert_idx.to(tl.int64) * N
store_row_offs = n_start + n_offset + block_range_n
@@ -368,11 +357,8 @@ def _grouped_gemm_dW_kernel(
row_load_mask = gather_offsets < m_size
- # Only (PERMUTE_X and not PERMUTE_Y) and (not PERMUTE_X and PERMUTE_Y) occur, so load/store offsets are flipped between the two cases, with the strides adjusted.
if PERMUTE_X:
- x_row_load_idx = (
- (expert_token_offsets // TOPK) * K
- ) # Permute on load: token to expert order, /TOPK for the original count.
+ x_row_load_idx = (expert_token_offsets // TOPK) * K
dY_row_load_idx = m_offsets[:, None] * N
else:
x_row_load_idx = indices_to_gather.to(tl.int64)[:, None] * K
@@ -407,6 +393,22 @@ def _grouped_gemm_dW_kernel(
x,
)
+ # ACCUMULATE: dW is an existing gradient. This program alone owns the (expert, n, k)
+ # tile, so read-modify-write is race free. The old value is added after the reduction
+ # (seeding the accumulator with it lets in-microbatch cancellation erase it); one rounding.
+ if ACCUMULATE:
+ if USE_TMA_STORE:
+ prev = dW_desc.load([expert_idx, n_offset, k_offset])
+ prev = tl.reshape(prev, (BLOCK_SIZE_N, BLOCK_SIZE_K))
+ else:
+ prev = tl.load(
+ dW_ptr
+ + store_row_offs[:, None] * K
+ + (k_offset + block_range_k)[None, :],
+ mask = nk_mask,
+ other = 0.0,
+ )
+ accumulator += prev.to(acc_dtype)
y = accumulator.to(output_dtype)
if USE_TMA_STORE:
y = tl.expand_dims(y, 0)
@@ -422,6 +424,5 @@ def _grouped_gemm_dW_kernel(
_autotuned_grouped_gemm_dW_kernel = triton.autotune(
configs = get_dW_kernel_configs(),
prune_configs_by = {"early_config_prune": prune_kernel_configs_backward_dW},
- # NUM_TOKENS is left out of the key to avoid recompiling for every sequence length.
key = ["NUM_EXPERTS", "N", "K", "PERMUTE_X", "PERMUTE_Y"],
)(_grouped_gemm_dW_kernel)