Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Kernel follow-ups: norm launch on HIP and under compile, grouped GEMM frozen operands, no CE eviction hint on H100 (#13967)

Daniel Hanae99ca4b21f8
diff --git a/tests/test_cross_entropy_eviction_hints.py b/tests/test_cross_entropy_eviction_hints.py
index aaa9be81c64..5b6f29fa13e 100644
--- a/tests/test_cross_entropy_eviction_hints.py
+++ b/tests/test_cross_entropy_eviction_hints.py
@@ -15,6 +15,11 @@ triton = pytest.importorskip("triton")
from triton.backends.compiler import GPUTarget
from triton.compiler import ASTSource
+import inspect
+
+if "constexprs" not in inspect.signature(ASTSource.__init__).parameters:
+ pytest.skip("ASTSource takes constexprs from Triton 3.3", allow_module_level = True)
+
from unsloth.kernels import cross_entropy_loss as ce_mod
from unsloth.kernels import rope_embedding as rope_mod
@@ -60,14 +65,17 @@ def test_chunked_cross_entropy_assembles_before_volta(capability, monkeypatch):
assert _compile(evict, cc).asm["cubin"]
[email protected]("capability", [(10, 0), (10, 3)])
-def test_chunked_cross_entropy_skips_the_hint_on_sm100(capability, monkeypatch):
- # Measured 1.25% slower on B200 with the hint.
[email protected]("capability", [(9, 0), (10, 0), (10, 3)])
+def test_chunked_cross_entropy_skips_the_hint_on_sm90_and_sm100(capability, monkeypatch):
+ # Measured slower with the hint: H100 1.6% to 2.4%, B200 1.25%.
assert _policy(capability, monkeypatch) == ""
[email protected]("capability", [(7, 0), (7, 5), (8, 0), (9, 0), (12, 0)])
[email protected]("capability", [(7, 0), (7, 5), (8, 0), (8, 9), (12, 0)])
def test_chunked_cross_entropy_streams_logits_from_volta(capability, monkeypatch):
evict = _policy(capability, monkeypatch)
assert evict == "evict_first"
- assert "ld.global.L1::evict_first" in _compile(evict, 75).asm["ptx"]
+ assert (
+ "ld.global.L1::evict_first"
+ in _compile(evict, capability[0] * 10 + capability[1]).asm["ptx"]
+ )
diff --git a/tests/test_grouped_gemm_frozen_weight_grad.py b/tests/test_grouped_gemm_frozen_weight_grad.py
new file mode 100644
index 00000000000..f9f06498855
--- /dev/null
+++ b/tests/test_grouped_gemm_frozen_weight_grad.py
@@ -0,0 +1,97 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Backward runs only the kernels autograd asks for: a frozen W (LoRA, dequantized experts) gets no
+dW kernel or (E, N, K) buffer, a constant X no dX, and trainable operands keep both."""
+
+import sys
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+import torch # noqa: E402
+
+pytest.importorskip("triton", reason = "the grouped GEMM is a Triton kernel")
+if not torch.cuda.is_available():
+ pytest.skip("grouped GEMM needs a CUDA device", allow_module_level = True)
+
+try:
+ from unsloth.kernels.moe.grouped_gemm import interface
+except Exception as exc: # pragma: no cover - depends on the installed stack
+ pytest.skip(f"grouped_gemm is unimportable here: {exc}", allow_module_level = True)
+
+E, T, N, K = 2, 8, 256, 256
+
+
+def _run(x_grad, w_grad, monkeypatch):
+ calls = []
+ for name in ("grouped_gemm_dX", "grouped_gemm_dW"):
+ real = getattr(interface, name)
+ monkeypatch.setattr(
+ interface,
+ name,
+ lambda *a, real = real, name = name, **k: calls.append(name) or real(*a, **k),
+ )
+ torch.manual_seed(0)
+ X = torch.randn(T, K, device = "cuda", dtype = torch.float16, requires_grad = x_grad)
+ W = torch.randn(E, N, K, device = "cuda", dtype = torch.float16, requires_grad = w_grad)
+ m_sizes = torch.full((E,), T // E, device = "cuda", dtype = torch.int32)
+ bias = torch.zeros(
+ N, device = "cuda", dtype = torch.float16, requires_grad = True
+ ) # something always trains
+ out = interface.grouped_gemm(
+ X = X, W = W, m_sizes = m_sizes, topk = 1, permute_x = False, permute_y = False, autotune = True
+ )
+ (out + bias).float().square().sum().backward()
+ ref_X, ref_W = X.detach().float().requires_grad_(), W.detach().float().requires_grad_()
+ ref = torch.cat([ref_X[e * (T // E) : (e + 1) * (T // E)] @ ref_W[e].T for e in range(E)])
+ ref.square().sum().backward()
+ return calls, X, W, ref_X, ref_W
+
+
+ "x_grad, w_grad, want",
+ [
+ (True, False, ["grouped_gemm_dX"]),
+ (False, True, ["grouped_gemm_dW"]),
+ (True, True, ["grouped_gemm_dW", "grouped_gemm_dX"]),
+ (False, False, []),
+ ],
+)
+def test_backward_launches_only_the_needed_kernels(x_grad, w_grad, want, monkeypatch):
+ calls, X, W, ref_X, ref_W = _run(x_grad, w_grad, monkeypatch)
+ assert sorted(calls) == sorted(want)
+ for t, ref, needed in ((X, ref_X, x_grad), (W, ref_W, w_grad)):
+ if needed:
+ torch.testing.assert_close(
+ t.grad.float(), ref.grad, rtol = 2e-2, atol = 2e-2 * ref.grad.abs().max().item()
+ )
+ else:
+ assert t.grad is None
+
+
+def test_fused_mul_training_is_still_rejected_with_frozen_operands():
+ # The router weights train even when X and W do not: backward must refuse, not return None for them.
+ X = torch.randn(T, K, device = "cuda", dtype = torch.float16)
+ W = torch.randn(E, N, K, device = "cuda", dtype = torch.float16)
+ m_sizes = torch.full((E,), T // E, device = "cuda", dtype = torch.int32)
+ gather = torch.arange(T, device = "cuda", dtype = torch.int32)
+ topk_weights = torch.rand(T, device = "cuda", dtype = torch.float16, requires_grad = True)
+ out = interface.grouped_gemm(
+ X = X,
+ W = W,
+ m_sizes = m_sizes,
+ topk = 1,
+ gather_indices = gather,
+ permute_x = False,
+ permute_y = True,
+ topk_weights = topk_weights,
+ fuse_mul_post = True,
+ autotune = True,
+ )
+ if not out.requires_grad:
+ pytest.skip("fused mul output carries no graph here")
+ with pytest.raises(AssertionError, match = "fused_mul"):
+ out.float().sum().backward()
diff --git a/tests/test_norm_fast_launch.py b/tests/test_norm_fast_launch.py
index 070a0e61992..1abfb3d2447 100644
--- a/tests/test_norm_fast_launch.py
+++ b/tests/test_norm_fast_launch.py
@@ -129,8 +129,24 @@ def test_compiled_calls_never_use_fast_launch(monkeypatch, kind, width, gemma):
torch.compile(fn, fullgraph = True)(x).sum().backward()
assert calls == []
assert x.grad is not None and weight.grad is not None
- fn(x.detach().requires_grad_()).sum().backward() # eager: same module, launch() taken
- assert calls
+ fn(x.detach().requires_grad_()).sum().backward() # eager: launch() only when fast launch is on
+ assert bool(calls) == FAST
+
+
[email protected]("train", [False, True], ids = ["frozen", "trainable"])
[email protected]("kind,gemma", [("rms", False), ("rms", True), ("ln", False)])
+def test_launcher_off_calls_triton_directly(monkeypatch, kind, gemma, train):
+ calls = []
+ for module in (rms_layernorm, layernorm):
+ real = module.launch
+ monkeypatch.setattr(
+ module, "launch", lambda *a, real = real, **k: calls.append(a[0]) or real(*a, **k)
+ )
+ monkeypatch.setattr(triton_launch, "_ENABLED", False)
+ weight, bias = _module(kind, 256, train)
+ x, dy = _inputs(256)
+ _step(kind, gemma, x, weight, bias, dy)
+ assert calls == []
@pytest.mark.parametrize("train", [False, True], ids = ["frozen", "trainable"])
@@ -202,3 +218,29 @@ def test_second_device_with_first_current(kind, width, gemma, train):
torch.cuda.synchronize(1)
for a, b in zip(ref, got):
assert a.device == b.device and torch.equal(a, b)
+
+
[email protected]("kind", ["rms", "ln"])
+def test_flipping_the_launcher_flag_does_not_recompile(monkeypatch, kind):
+ # launch() disables the launcher on a failed launch; compiled graphs must not guard on that flag.
+ compiles = []
+
+ def backend(gm, example_inputs):
+ compiles.append(1)
+ return gm.forward
+
+ weight = torch.nn.Parameter(torch.randn(256, device = "cuda"))
+ norm = torch.nn.Module()
+ norm.weight, norm.bias, norm.eps, norm.variance_epsilon = weight, None, 1e-5, 1e-6
+ norm.elementwise_affine = True
+ if kind == "rms":
+ fn = lambda x: rms_layernorm.fast_rms_layernorm(norm, x)
+ else:
+ fn = lambda x: layernorm.fast_layernorm(norm, x)
+ torch._dynamo.reset()
+ compiled = torch.compile(fn, backend = backend)
+ x = torch.randn(4, 256, device = "cuda")
+ compiled(x)
+ monkeypatch.setattr(triton_launch, "_ENABLED", not triton_launch._ENABLED)
+ compiled(x)
+ assert len(compiles) == 1
diff --git a/unsloth/kernels/cross_entropy_loss.py b/unsloth/kernels/cross_entropy_loss.py
index 5d1cf3f1495..b1a18eb79d8 100644
--- a/unsloth/kernels/cross_entropy_loss.py
+++ b/unsloth/kernels/cross_entropy_loss.py
@@ -115,10 +115,10 @@ _cross_entropy_forward = triton.heuristics(
@functools.lru_cache(maxsize = None)
def _chunked_evict_at(device_type, index):
- # Chunks are read once: evict_first is faster except on B200 (sm_100); ROCm unmeasured.
+ # Read-once chunks: evict_first helps T4 / L4 / A100, hurts H100 and B200; ROCm unmeasured.
if device_type != "cuda" or torch.version.hip is not None:
return ""
- return "" if torch.cuda.get_device_capability(index)[0] == 10 else "evict_first"
+ return "" if torch.cuda.get_device_capability(index)[0] in (9, 10) else "evict_first"
def _chunked_evict_policy(device):
diff --git a/unsloth/kernels/layernorm.py b/unsloth/kernels/layernorm.py
index f5b34cd68ba..756a9e964b9 100644
--- a/unsloth/kernels/layernorm.py
+++ b/unsloth/kernels/layernorm.py
@@ -15,6 +15,7 @@ import triton
import triton.language as tl
import torch
from .utils import calculate_settings, long_indexing, torch_gpu_device
+from . import triton_launch
from .triton_launch import launch
from unsloth_zoo.patching_utils import (
patch_layernorm,
@@ -222,7 +223,8 @@ def _settings(n_cols):
def _launch(kernel, grid, args, n_tensors, constexprs, device, **options):
# Eager CUDA skips Triton's arg binding (triton_launch.py); compiling calls keep kernel[grid].
- if device.type == "cuda" and not torch.compiler.is_compiling():
+ # is_compiling first: Dynamo must not read (and guard on) the mutable _ENABLED.
+ if not torch.compiler.is_compiling() and triton_launch._ENABLED and device.type == "cuda":
launch(kernel, grid, args, n_tensors, constexprs, device.index, **options)
else:
kernel[grid](*args, **constexprs, **options)
diff --git a/unsloth/kernels/moe/grouped_gemm/interface.py b/unsloth/kernels/moe/grouped_gemm/interface.py
index 7f31b700c39..bcbc885c03b 100644
--- a/unsloth/kernels/moe/grouped_gemm/interface.py
+++ b/unsloth/kernels/moe/grouped_gemm/interface.py
@@ -724,12 +724,17 @@ class GroupedGemm(torch.autograd.Function):
topk = ctx.topk
permute_x = ctx.permute_x
permute_y = ctx.permute_y
- fuse_mul_post = ctx.fuse_mul_post
kernel_config_bwd_dX = ctx.kernel_config_bwd_dX
kernel_config_bwd_dW = ctx.kernel_config_bwd_dW
autotune = ctx.autotune
- dX_only = ctx.dX_only
- dW_only = ctx.dW_only
+ assert (
+ not ctx.fuse_mul_post
+ ), "fused_mul should only be used for inference, not for training"
+ # Frozen W (LoRA, dequantized experts) or X: skip its kernel and buffer, as autograd does for _grouped_mm.
+ dX_only = ctx.dX_only or not ctx.needs_input_grad[1]
+ dW_only = ctx.dW_only or not ctx.needs_input_grad[0]
+ if dX_only and dW_only:
+ return (None,) * 15
if not autotune:
if not dW_only:
@@ -741,8 +746,6 @@ class GroupedGemm(torch.autograd.Function):
kernel_config_bwd_dW is not None
), "kernel_config_bwd_dW must be provided if autotune is False"
- assert not fuse_mul_post, "fused_mul should only be used for inference, not for training"
-
if not dX_only:
bwd_dW_config = {}
diff --git a/unsloth/kernels/rms_layernorm.py b/unsloth/kernels/rms_layernorm.py
index c76092e00d9..102547cdd8c 100644
--- a/unsloth/kernels/rms_layernorm.py
+++ b/unsloth/kernels/rms_layernorm.py
@@ -18,6 +18,7 @@ import triton.language as tl
import torch
from typing import Tuple
from .utils import calculate_settings, long_indexing, torch_gpu_device
+from . import triton_launch
from .triton_launch import launch
@@ -482,7 +483,13 @@ def _settings(n_cols):
def _launch(wrap, kernel, grid, args, n_tensors, constexprs, device, **options):
# Eager CUDA skips Triton's arg binding (triton_launch.py); traced / non-Triton keep kernel[grid].
- if wrap is _eager_kernel and device.type == "cuda" and not torch.compiler.is_compiling():
+ # is_compiling first: Dynamo must not read (and guard on) the mutable _ENABLED.
+ if (
+ not torch.compiler.is_compiling()
+ and triton_launch._ENABLED
+ and wrap is _eager_kernel
+ and device.type == "cuda"
+ ):
fn = kernel
if isinstance(kernel, triton.runtime.autotuner.Heuristics):
# What Heuristics.run does; these heuristics only read their own constexpr.