Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Kernel follow-ups: norm launch on HIP and under compile, grouped GEMM frozen operands, no CE eviction hint on H100 (#13967)
Daniel Hanae99ca4b21f8
diff --git a/tests/test_cross_entropy_eviction_hints.py b/tests/test_cross_entropy_eviction_hints.pyindex aaa9be81c64..5b6f29fa13e 100644--- a/tests/test_cross_entropy_eviction_hints.py+++ b/tests/test_cross_entropy_eviction_hints.py@@ -15,6 +15,11 @@ triton = pytest.importorskip("triton")from triton.backends.compiler import GPUTargetfrom triton.compiler import ASTSource+import inspect++if "constexprs" not in inspect.signature(ASTSource.__init__).parameters:+ pytest.skip("ASTSource takes constexprs from Triton 3.3", allow_module_level = True)+from unsloth.kernels import cross_entropy_loss as ce_modfrom unsloth.kernels import rope_embedding as rope_mod@@ -60,14 +65,17 @@ def test_chunked_cross_entropy_assembles_before_volta(capability, monkeypatch):assert _compile(evict, cc).asm["cubin"][email protected]("capability", [(10, 0), (10, 3)])-def test_chunked_cross_entropy_skips_the_hint_on_sm100(capability, monkeypatch):- # Measured 1.25% slower on B200 with the hint.[email protected]("capability", [(9, 0), (10, 0), (10, 3)])+def test_chunked_cross_entropy_skips_the_hint_on_sm90_and_sm100(capability, monkeypatch):+ # Measured slower with the hint: H100 1.6% to 2.4%, B200 1.25%.assert _policy(capability, monkeypatch) == ""[email protected]("capability", [(7, 0), (7, 5), (8, 0), (9, 0), (12, 0)])[email protected]("capability", [(7, 0), (7, 5), (8, 0), (8, 9), (12, 0)])def test_chunked_cross_entropy_streams_logits_from_volta(capability, monkeypatch):evict = _policy(capability, monkeypatch)assert evict == "evict_first"- assert "ld.global.L1::evict_first" in _compile(evict, 75).asm["ptx"]+ assert (+ "ld.global.L1::evict_first"+ in _compile(evict, capability[0] * 10 + capability[1]).asm["ptx"]+ )diff --git a/tests/test_grouped_gemm_frozen_weight_grad.py b/tests/test_grouped_gemm_frozen_weight_grad.pynew file mode 100644index 00000000000..f9f06498855--- /dev/null+++ b/tests/test_grouped_gemm_frozen_weight_grad.py@@ -0,0 +1,97 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Backward runs only the kernels autograd asks for: a frozen W (LoRA, dequantized experts) gets no+dW kernel or (E, N, K) buffer, a constant X no dX, and trainable operands keep both."""++import sys+from pathlib import Path++import pytest++sys.path.insert(0, str(Path(__file__).resolve().parents[1]))++import torch # noqa: E402++pytest.importorskip("triton", reason = "the grouped GEMM is a Triton kernel")+if not torch.cuda.is_available():+ pytest.skip("grouped GEMM needs a CUDA device", allow_module_level = True)++try:+ from unsloth.kernels.moe.grouped_gemm import interface+except Exception as exc: # pragma: no cover - depends on the installed stack+ pytest.skip(f"grouped_gemm is unimportable here: {exc}", allow_module_level = True)++E, T, N, K = 2, 8, 256, 256+++def _run(x_grad, w_grad, monkeypatch):+ calls = []+ for name in ("grouped_gemm_dX", "grouped_gemm_dW"):+ real = getattr(interface, name)+ monkeypatch.setattr(+ interface,+ name,+ lambda *a, real = real, name = name, **k: calls.append(name) or real(*a, **k),+ )+ torch.manual_seed(0)+ X = torch.randn(T, K, device = "cuda", dtype = torch.float16, requires_grad = x_grad)+ W = torch.randn(E, N, K, device = "cuda", dtype = torch.float16, requires_grad = w_grad)+ m_sizes = torch.full((E,), T // E, device = "cuda", dtype = torch.int32)+ bias = torch.zeros(+ N, device = "cuda", dtype = torch.float16, requires_grad = True+ ) # something always trains+ out = interface.grouped_gemm(+ X = X, W = W, m_sizes = m_sizes, topk = 1, permute_x = False, permute_y = False, autotune = True+ )+ (out + bias).float().square().sum().backward()+ ref_X, ref_W = X.detach().float().requires_grad_(), W.detach().float().requires_grad_()+ ref = torch.cat([ref_X[e * (T // E) : (e + 1) * (T // E)] @ ref_W[e].T for e in range(E)])+ ref.square().sum().backward()+ return calls, X, W, ref_X, ref_W+++ "x_grad, w_grad, want",+ [+ (True, False, ["grouped_gemm_dX"]),+ (False, True, ["grouped_gemm_dW"]),+ (True, True, ["grouped_gemm_dW", "grouped_gemm_dX"]),+ (False, False, []),+ ],+)+def test_backward_launches_only_the_needed_kernels(x_grad, w_grad, want, monkeypatch):+ calls, X, W, ref_X, ref_W = _run(x_grad, w_grad, monkeypatch)+ assert sorted(calls) == sorted(want)+ for t, ref, needed in ((X, ref_X, x_grad), (W, ref_W, w_grad)):+ if needed:+ torch.testing.assert_close(+ t.grad.float(), ref.grad, rtol = 2e-2, atol = 2e-2 * ref.grad.abs().max().item()+ )+ else:+ assert t.grad is None+++def test_fused_mul_training_is_still_rejected_with_frozen_operands():+ # The router weights train even when X and W do not: backward must refuse, not return None for them.+ X = torch.randn(T, K, device = "cuda", dtype = torch.float16)+ W = torch.randn(E, N, K, device = "cuda", dtype = torch.float16)+ m_sizes = torch.full((E,), T // E, device = "cuda", dtype = torch.int32)+ gather = torch.arange(T, device = "cuda", dtype = torch.int32)+ topk_weights = torch.rand(T, device = "cuda", dtype = torch.float16, requires_grad = True)+ out = interface.grouped_gemm(+ X = X,+ W = W,+ m_sizes = m_sizes,+ topk = 1,+ gather_indices = gather,+ permute_x = False,+ permute_y = True,+ topk_weights = topk_weights,+ fuse_mul_post = True,+ autotune = True,+ )+ if not out.requires_grad:+ pytest.skip("fused mul output carries no graph here")+ with pytest.raises(AssertionError, match = "fused_mul"):+ out.float().sum().backward()diff --git a/tests/test_norm_fast_launch.py b/tests/test_norm_fast_launch.pyindex 070a0e61992..1abfb3d2447 100644--- a/tests/test_norm_fast_launch.py+++ b/tests/test_norm_fast_launch.py@@ -129,8 +129,24 @@ def test_compiled_calls_never_use_fast_launch(monkeypatch, kind, width, gemma):torch.compile(fn, fullgraph = True)(x).sum().backward()assert calls == []assert x.grad is not None and weight.grad is not None- fn(x.detach().requires_grad_()).sum().backward() # eager: same module, launch() taken- assert calls+ fn(x.detach().requires_grad_()).sum().backward() # eager: launch() only when fast launch is on+ assert bool(calls) == FAST++[email protected]("train", [False, True], ids = ["frozen", "trainable"])[email protected]("kind,gemma", [("rms", False), ("rms", True), ("ln", False)])+def test_launcher_off_calls_triton_directly(monkeypatch, kind, gemma, train):+ calls = []+ for module in (rms_layernorm, layernorm):+ real = module.launch+ monkeypatch.setattr(+ module, "launch", lambda *a, real = real, **k: calls.append(a[0]) or real(*a, **k)+ )+ monkeypatch.setattr(triton_launch, "_ENABLED", False)+ weight, bias = _module(kind, 256, train)+ x, dy = _inputs(256)+ _step(kind, gemma, x, weight, bias, dy)+ assert calls == []@pytest.mark.parametrize("train", [False, True], ids = ["frozen", "trainable"])@@ -202,3 +218,29 @@ def test_second_device_with_first_current(kind, width, gemma, train):torch.cuda.synchronize(1)for a, b in zip(ref, got):assert a.device == b.device and torch.equal(a, b)++[email protected]("kind", ["rms", "ln"])+def test_flipping_the_launcher_flag_does_not_recompile(monkeypatch, kind):+ # launch() disables the launcher on a failed launch; compiled graphs must not guard on that flag.+ compiles = []++ def backend(gm, example_inputs):+ compiles.append(1)+ return gm.forward++ weight = torch.nn.Parameter(torch.randn(256, device = "cuda"))+ norm = torch.nn.Module()+ norm.weight, norm.bias, norm.eps, norm.variance_epsilon = weight, None, 1e-5, 1e-6+ norm.elementwise_affine = True+ if kind == "rms":+ fn = lambda x: rms_layernorm.fast_rms_layernorm(norm, x)+ else:+ fn = lambda x: layernorm.fast_layernorm(norm, x)+ torch._dynamo.reset()+ compiled = torch.compile(fn, backend = backend)+ x = torch.randn(4, 256, device = "cuda")+ compiled(x)+ monkeypatch.setattr(triton_launch, "_ENABLED", not triton_launch._ENABLED)+ compiled(x)+ assert len(compiles) == 1diff --git a/unsloth/kernels/cross_entropy_loss.py b/unsloth/kernels/cross_entropy_loss.pyindex 5d1cf3f1495..b1a18eb79d8 100644--- a/unsloth/kernels/cross_entropy_loss.py+++ b/unsloth/kernels/cross_entropy_loss.py@@ -115,10 +115,10 @@ _cross_entropy_forward = triton.heuristics(@functools.lru_cache(maxsize = None)def _chunked_evict_at(device_type, index):- # Chunks are read once: evict_first is faster except on B200 (sm_100); ROCm unmeasured.+ # Read-once chunks: evict_first helps T4 / L4 / A100, hurts H100 and B200; ROCm unmeasured.if device_type != "cuda" or torch.version.hip is not None:return ""- return "" if torch.cuda.get_device_capability(index)[0] == 10 else "evict_first"+ return "" if torch.cuda.get_device_capability(index)[0] in (9, 10) else "evict_first"def _chunked_evict_policy(device):diff --git a/unsloth/kernels/layernorm.py b/unsloth/kernels/layernorm.pyindex f5b34cd68ba..756a9e964b9 100644--- a/unsloth/kernels/layernorm.py+++ b/unsloth/kernels/layernorm.py@@ -15,6 +15,7 @@ import tritonimport triton.language as tlimport torchfrom .utils import calculate_settings, long_indexing, torch_gpu_device+from . import triton_launchfrom .triton_launch import launchfrom unsloth_zoo.patching_utils import (patch_layernorm,@@ -222,7 +223,8 @@ def _settings(n_cols):def _launch(kernel, grid, args, n_tensors, constexprs, device, **options):# Eager CUDA skips Triton's arg binding (triton_launch.py); compiling calls keep kernel[grid].- if device.type == "cuda" and not torch.compiler.is_compiling():+ # is_compiling first: Dynamo must not read (and guard on) the mutable _ENABLED.+ if not torch.compiler.is_compiling() and triton_launch._ENABLED and device.type == "cuda":launch(kernel, grid, args, n_tensors, constexprs, device.index, **options)else:kernel[grid](*args, **constexprs, **options)diff --git a/unsloth/kernels/moe/grouped_gemm/interface.py b/unsloth/kernels/moe/grouped_gemm/interface.pyindex 7f31b700c39..bcbc885c03b 100644--- a/unsloth/kernels/moe/grouped_gemm/interface.py+++ b/unsloth/kernels/moe/grouped_gemm/interface.py@@ -724,12 +724,17 @@ class GroupedGemm(torch.autograd.Function):topk = ctx.topkpermute_x = ctx.permute_xpermute_y = ctx.permute_y- fuse_mul_post = ctx.fuse_mul_postkernel_config_bwd_dX = ctx.kernel_config_bwd_dXkernel_config_bwd_dW = ctx.kernel_config_bwd_dWautotune = ctx.autotune- dX_only = ctx.dX_only- dW_only = ctx.dW_only+ assert (+ not ctx.fuse_mul_post+ ), "fused_mul should only be used for inference, not for training"+ # Frozen W (LoRA, dequantized experts) or X: skip its kernel and buffer, as autograd does for _grouped_mm.+ dX_only = ctx.dX_only or not ctx.needs_input_grad[1]+ dW_only = ctx.dW_only or not ctx.needs_input_grad[0]+ if dX_only and dW_only:+ return (None,) * 15if not autotune:if not dW_only:@@ -741,8 +746,6 @@ class GroupedGemm(torch.autograd.Function):kernel_config_bwd_dW is not None), "kernel_config_bwd_dW must be provided if autotune is False"- assert not fuse_mul_post, "fused_mul should only be used for inference, not for training"-if not dX_only:bwd_dW_config = {}diff --git a/unsloth/kernels/rms_layernorm.py b/unsloth/kernels/rms_layernorm.pyindex c76092e00d9..102547cdd8c 100644--- a/unsloth/kernels/rms_layernorm.py+++ b/unsloth/kernels/rms_layernorm.py@@ -18,6 +18,7 @@ import triton.language as tlimport torchfrom typing import Tuplefrom .utils import calculate_settings, long_indexing, torch_gpu_device+from . import triton_launchfrom .triton_launch import launch@@ -482,7 +483,13 @@ def _settings(n_cols):def _launch(wrap, kernel, grid, args, n_tensors, constexprs, device, **options):# Eager CUDA skips Triton's arg binding (triton_launch.py); traced / non-Triton keep kernel[grid].- if wrap is _eager_kernel and device.type == "cuda" and not torch.compiler.is_compiling():+ # is_compiling first: Dynamo must not read (and guard on) the mutable _ENABLED.+ if (+ not torch.compiler.is_compiling()+ and triton_launch._ENABLED+ and wrap is _eager_kernel+ and device.type == "cuda"+ ):fn = kernelif isinstance(kernel, triton.runtime.autotuner.Heuristics):# What Heuristics.run does; these heuristics only read their own constexpr.