Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
RMSNorm: restore in-place dX for pre-norm decoder norms (#13984)
Daniel Han133489efc1d6
diff --git a/tests/test_norm_backward_grad_alias.py b/tests/test_norm_backward_grad_alias.pyindex 7e484a99299..a209938840c 100644--- a/tests/test_norm_backward_grad_alias.py+++ b/tests/test_norm_backward_grad_alias.py@@ -79,3 +79,104 @@ def test_layernorm_backward_leaves_shared_grad_alone(train_weight, dtype):b = (0.1 * torch.randn(256, device = "cuda")).to(dtype).requires_grad_(train_weight)norm = lambda x: Fast_Layernorm.apply(x, W, b, 1e-6)_check(norm, lambda x: _ln_fp64(x, W, b), dtype)++[email protected](not has_real_cuda(), reason = "CUDA Triton kernels required")[email protected]("gemma", [False, True])[email protected]("save_output", [False, True])[email protected]("dtype", [torch.float32, torch.float16, torch.bfloat16])+def test_rms_layernorm_reuse_grad_matches_and_saves_a_buffer(+ gemma, save_output, dtype, monkeypatch+):+ # Pre-norm decoder norms feed only layers that return a fresh grad, so they let dX overwrite+ # dY: same bits as the fresh buffer, and about one activation less at the backward peak.+ from unsloth.kernels import rms_layernorm+ from unsloth.kernels.rms_layernorm import Fast_RMS_Layernorm++ monkeypatch.setattr(rms_layernorm, "_SAVE_OUTPUT", save_output)+ torch.manual_seed(0)+ shape = (2, 512, 1024)+ W = (1.0 + 0.1 * torch.randn(shape[-1], device = "cuda")).to(dtype)+ proj = torch.randn(shape[-1], 64, device = "cuda", dtype = dtype) / 32+ x0 = torch.randn(shape, device = "cuda", dtype = dtype)++ def step(reuse_grad):+ x = x0.clone().requires_grad_()+ y = Fast_RMS_Layernorm.apply(x, W, 1e-6, gemma, save_output, reuse_grad)+ (y @ proj).float().square().mean().backward()+ return x.grad++ results = {}+ for reuse_grad in (False, True):+ step(reuse_grad)+ torch.cuda.synchronize()+ base = torch.cuda.memory_allocated()+ torch.cuda.reset_peak_memory_stats()+ grad = step(reuse_grad)+ torch.cuda.synchronize()+ results[reuse_grad] = (grad, torch.cuda.max_memory_allocated() - base)+ assert torch.equal(results[False][0], results[True][0])+ saved = results[False][1] - results[True][1]+ activation = x0.numel() * x0.element_size()+ # The peak moves to a neighbouring allocation, so the drop is most of an activation, not all.+ if gemma:+ assert saved == 0, saved+ else:+ assert activation * 7 // 8 <= saved <= activation, (saved, activation)++[email protected](not has_real_cuda(), reason = "CUDA Triton kernels required")+def test_final_norm_reuses_grad_only_under_the_causal_lm_head(monkeypatch):+ # A base model hands the final norm's output to the caller, so the caller's grad_outputs must+ # survive; only the CausalLM head, which consumes it itself, lets the norm write over its grad.+ from unsloth import FastLanguageModel+ from unsloth.kernels.rms_layernorm import Fast_RMS_Layernorm++ model, _ = FastLanguageModel.from_pretrained(+ "trl-internal-testing/tiny-Qwen3ForCausalLM",+ max_seq_length = 64,+ load_in_4bit = False,+ dtype = torch.bfloat16,+ )+ model.train()+ backbone = model.model+ torch.manual_seed(0)+ ids = torch.randint(0, model.config.vocab_size, (2, 32), device = "cuda")++ embeds = backbone.embed_tokens(ids).detach().requires_grad_()+ hidden = backbone(inputs_embeds = embeds, use_cache = False).last_hidden_state+ go = torch.randn_like(hidden)+ go_before = go.clone()+ torch.autograd.grad(hidden, embeds, go)+ assert torch.equal(go, go_before), "base model overwrote the caller's grad_outputs"++ calls = []+ apply = Fast_RMS_Layernorm.apply++ def recording_apply(*args):+ calls.append(bool(args[5]) if len(args) > 5 else False)+ return apply(*args)++ monkeypatch.setattr(Fast_RMS_Layernorm, "apply", recording_apply)+ model(input_ids = ids, labels = ids, use_cache = False)+ assert calls and calls[-1], "CausalLM head should let the final norm reuse its grad"+ calls.clear()+ model(input_ids = ids, labels = ids, use_cache = False, output_hidden_states = True)+ assert calls and not calls[-1], "returned hidden states need the final norm's own buffer"+ calls.clear()+ backbone(input_ids = ids, use_cache = False)+ assert calls and not calls[-1]+ # GRPO / DPO style: hidden states come back as logits and feed a chunked lm_head loss.+ calls.clear()+ monkeypatch.setenv("UNSLOTH_RETURN_HIDDEN_STATES", "1")+ embeds = backbone.embed_tokens(ids).detach().requires_grad_()+ hidden = model(inputs_embeds = embeds, use_cache = False).logits+ assert calls and calls[-1], "GRPO-style hidden states should let the final norm reuse its grad"+ assert hidden.shape[-1] == model.config.hidden_size+ logits = hidden.float() @ model.lm_head.weight.float().t()+ (grad,) = torch.autograd.grad(logits.logsumexp(-1).mean(), embeds)+ assert torch.isfinite(grad).all()+ monkeypatch.delenv("UNSLOTH_RETURN_HIDDEN_STATES")+ assert not getattr(backbone, "_unsloth_head_reads_final_norm", False)diff --git a/unsloth/kernels/rms_layernorm.py b/unsloth/kernels/rms_layernorm.pyindex 90a0ec22916..c76092e00d9 100644--- a/unsloth/kernels/rms_layernorm.py+++ b/unsloth/kernels/rms_layernorm.py@@ -856,6 +856,7 @@ class Fast_RMS_Layernorm(torch.autograd.Function):eps: float,gemma: bool = False,save_output: bool = False,+ reuse_grad: bool = False,):shape = X.shapedim: int = shape[-1]@@ -866,6 +867,7 @@ class Fast_RMS_Layernorm(torch.autograd.Function):Y, r = _rms_forward(X, W, eps, gemma, _eager_kernel)ctx.eps = epsctx.GEMMA = gemma+ ctx.reuse_grad = reuse_gradY = Y.view(*shape)# save_output: the caller guarantees nothing writes into Y (an untracked Triton write would# corrupt dX silently). Only frozen W: a trained W changes every step.@@ -893,8 +895,10 @@ class Fast_RMS_Layernorm(torch.autograd.Function):if ctx.FROM_Y:# The saved Y keeps the caller's shape; a saved X is already 2D.X = X.view(-1, dim)- # allocate dX because autograd may share dY with other nodes or the caller.- dX = torch.empty_like(dY)+ # Autograd may share dY with other nodes or the caller, so dX gets its own buffer.+ # reuse_grad is for call sites whose output only feeds layers that return a fresh grad+ # (pre-norm decoder norms): there dX overwrites dY and saves one activation.+ dX = dY if ctx.reuse_grad and not ctx.GEMMA else torch.empty_like(dY)with torch_gpu_device(dY.device):dW = _rms_backward(dY,@@ -909,7 +913,7 @@ class Fast_RMS_Layernorm(torch.autograd.Function):from_y = ctx.FROM_Y,)dX = dX.view(*shape)- return dX, dW, None, None, None+ return dX, dW, None, None, None, None# compiled graphs use these ops with the same kernels as the Function.@@ -1011,8 +1015,9 @@ def _fast_rms_layernorm_untraced(eps,gemma,save_output = False,+ reuse_grad = False,):- return Fast_RMS_Layernorm.apply(X, W, eps, gemma, save_output)+ return Fast_RMS_Layernorm.apply(X, W, eps, gemma, save_output, reuse_grad)def fast_rms_layernorm(@@ -1020,19 +1025,20 @@ def fast_rms_layernorm(X: torch.Tensor,gemma: bool = False,save_output: bool = False,+ reuse_grad: bool = False,):W: torch.Tensor = layernorm.weighteps: float = (layernorm.variance_epsilon if hasattr(layernorm, "variance_epsilon") else layernorm.eps)if not torch.compiler.is_compiling():- return Fast_RMS_Layernorm.apply(X, W, eps, gemma, save_output)+ return Fast_RMS_Layernorm.apply(X, W, eps, gemma, save_output, reuse_grad)if (not _TRACEABLEor X.device.type != "cuda"or (X.dtype == torch.bfloat16 and not _BF16_TRACEABLE)):- return _fast_rms_layernorm_untraced(X, W, eps, gemma, save_output)+ return _fast_rms_layernorm_untraced(X, W, eps, gemma, save_output, reuse_grad)shape = X.shapeY, _ = torch.ops.unsloth.rms_layernorm(X.reshape(-1, shape[-1]).contiguous(), W.contiguous(), eps, gemmadiff --git a/unsloth/models/falcon_h1.py b/unsloth/models/falcon_h1.pyindex d93f0a6cb23..e7eeb7311b1 100644--- a/unsloth/models/falcon_h1.py+++ b/unsloth/models/falcon_h1.py@@ -421,7 +421,7 @@ def FalconH1DecoderLayer_fast_forward(hidden_states += residualelse:residual = hidden_states- hidden_states = fast_rms_layernorm(self.input_layernorm, hidden_states)+ hidden_states = fast_rms_layernorm(self.input_layernorm, hidden_states, reuse_grad = True)mamba_hidden_states = self.mamba(hidden_states = hidden_states,@@ -451,7 +451,7 @@ def FalconH1DecoderLayer_fast_forward(hidden_states = residual + hidden_statesresidual = hidden_states- hidden_states = fast_rms_layernorm(self.pre_ff_layernorm, hidden_states)+ hidden_states = fast_rms_layernorm(self.pre_ff_layernorm, hidden_states, reuse_grad = True)hidden_states = self.feed_forward(hidden_states)hidden_states = residual + hidden_statesdiff --git a/unsloth/models/glm4_moe.py b/unsloth/models/glm4_moe.pyindex 12b5b962d18..133e5271423 100644--- a/unsloth/models/glm4_moe.py+++ b/unsloth/models/glm4_moe.py@@ -315,7 +315,7 @@ def Glm4MoeLiteDecoderLayer_fast_forward(hidden_states = residual + hidden_stateselse:residual = hidden_states- hidden_states = fast_rms_layernorm(self.input_layernorm, hidden_states)+ hidden_states = fast_rms_layernorm(self.input_layernorm, hidden_states, reuse_grad = True)hidden_states, _ = self.self_attn(hidden_states = hidden_states,attention_mask = attention_mask,@@ -329,7 +329,9 @@ def Glm4MoeLiteDecoderLayer_fast_forward(hidden_states = residual + hidden_statesresidual = hidden_states- hidden_states = fast_rms_layernorm(self.post_attention_layernorm, hidden_states)+ hidden_states = fast_rms_layernorm(+ self.post_attention_layernorm, hidden_states, reuse_grad = True+ )hidden_states = self.mlp(hidden_states)hidden_states = residual + hidden_statesdiff --git a/unsloth/models/granite.py b/unsloth/models/granite.pyindex 1f10894605e..9be2a7a85dc 100644--- a/unsloth/models/granite.py+++ b/unsloth/models/granite.py@@ -242,7 +242,7 @@ def GraniteDecoderLayer_fast_forward(hidden_states = torch.add(residual, hidden_states, alpha = residual_multiplier)else:residual = hidden_states- hidden_states = fast_rms_layernorm(self.input_layernorm, hidden_states)+ hidden_states = fast_rms_layernorm(self.input_layernorm, hidden_states, reuse_grad = True)hidden_states, self_attn_weights, present_key_value = self.self_attn(hidden_states = hidden_states,causal_mask = causal_mask,@@ -258,7 +258,9 @@ def GraniteDecoderLayer_fast_forward(hidden_states = torch.add(residual, hidden_states, alpha = residual_multiplier)residual = hidden_states- hidden_states = fast_rms_layernorm(self.post_attention_layernorm, hidden_states)+ hidden_states = fast_rms_layernorm(+ self.post_attention_layernorm, hidden_states, reuse_grad = True+ )hidden_states = self.mlp(hidden_states)hidden_states = torch.add(residual, hidden_states, alpha = residual_multiplier)diff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex 54078768963..193cb1eee09 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -1072,7 +1072,9 @@ def LlamaDecoderLayer_fast_forward(hidden_states += residualelse:residual = hidden_states- hidden_states = fast_rms_layernorm(self.input_layernorm, hidden_states, save_output = True)+ hidden_states = fast_rms_layernorm(+ self.input_layernorm, hidden_states, save_output = True, reuse_grad = True+ )hidden_states, self_attn_weights, present_key_value = self.self_attn(hidden_states = hidden_states,causal_mask = causal_mask,@@ -1089,7 +1091,7 @@ def LlamaDecoderLayer_fast_forward(residual = hidden_stateshidden_states = fast_rms_layernorm(- self.post_attention_layernorm, hidden_states, save_output = True+ self.post_attention_layernorm, hidden_states, save_output = True, reuse_grad = True)hidden_states = self.mlp(hidden_states)hidden_states = residual + hidden_states@@ -1518,10 +1520,16 @@ def LlamaModel_fast_forward()(self.norm, hidden_states)elif IS_COHERE:hidden_states = self.norm(hidden_states)- elif IS_FALCON_H1:- hidden_states = fast_rms_layernorm(self.final_layernorm, hidden_states, gemma = IS_GEMMA)else:- hidden_states = fast_rms_layernorm(self.norm, hidden_states, gemma = IS_GEMMA)+ # Only the CausalLM head may let the final norm overwrite its incoming grad: a base model+ # hands this output to the caller, whose grad_outputs must stay intact.+ reuse_grad = getattr(self, "_unsloth_head_reads_final_norm", False)+ hidden_states = fast_rms_layernorm(+ self.final_layernorm if IS_FALCON_H1 else self.norm,+ hidden_states,+ gemma = IS_GEMMA,+ reuse_grad = reuse_grad,+ )if output_hidden_states:all_hidden_states += (hidden_states,)@@ -1745,6 +1753,19 @@ def apply_logit_transforms(logits, logit_softcapping, logit_scaling):return logits+def _run_backbone(self, **kwargs):+ # The CausalLM forward feeds the final norm's output to lm_head, the fused CE loss, or (with+ # UNSLOTH_RETURN_HIDDEN_STATES, set only by Unsloth's GRPO / DPO / GKD trainers) their chunked+ # lm_head loss; all return a fresh grad, so the norm may write dX over it. Returned+ # hidden_states would let a caller own that grad, so they keep the norm's own buffer.+ model = self.model+ model._unsloth_head_reads_final_norm = not kwargs.get("output_hidden_states")+ try:+ return model(**kwargs)+ finally:+ model._unsloth_head_reads_final_norm = False++def CausalLM_fast_forward(fast_forward_inference):def _CausalLM_fast_forward(self,@@ -1801,7 +1822,8 @@ def CausalLM_fast_forward(fast_forward_inference):return_dict if return_dict is not None else config_return_dict(self.config))self.model._has_no_labels = labels is None- outputs = self.model(+ outputs = _run_backbone(+ self,input_ids = input_ids,causal_mask = causal_mask,attention_mask = attention_mask,diff --git a/unsloth/models/mistral.py b/unsloth/models/mistral.pyindex 7b7eda96098..9147dd53d2c 100644--- a/unsloth/models/mistral.py+++ b/unsloth/models/mistral.py@@ -38,6 +38,7 @@ from .llama import (resolve_logit_transforms,_cache_as_legacy_tuple,_cached_prefill_defaults,+ _run_backbone,unsloth_fsdp_unshard,)from transformers.models.mistral.modeling_mistral import (@@ -287,7 +288,8 @@ def MistralForCausalLM_fast_forward(attention_mask = attention_mask,)else:- outputs = self.model(+ outputs = _run_backbone(+ self,input_ids = input_ids,causal_mask = causal_mask,attention_mask = attention_mask,diff --git a/unsloth/models/qwen3.py b/unsloth/models/qwen3.pyindex 55f3e70c408..3f1bee48e51 100644--- a/unsloth/models/qwen3.py+++ b/unsloth/models/qwen3.py@@ -121,8 +121,8 @@ def Qwen3Attention_fast_forward(else:# Qwen3 adds QKNorm, the only difference from Qwen2. A compiled norm mismatches Transformers'# numbers, so use fast_rms_layernorm.- Q = fast_rms_layernorm(self.q_norm, Q)- K = fast_rms_layernorm(self.k_norm, K)+ Q = fast_rms_layernorm(self.q_norm, Q, reuse_grad = True)+ K = fast_rms_layernorm(self.k_norm, K, reuse_grad = True)Q = Q.transpose(1, 2)K = K.transpose(1, 2)Q, K = fast_rope_embedding(Q, K, cos, sin, rope_position_ids)diff --git a/unsloth/models/qwen3_moe.py b/unsloth/models/qwen3_moe.pyindex ab923702cc8..56e717cb4f4 100644--- a/unsloth/models/qwen3_moe.py+++ b/unsloth/models/qwen3_moe.py@@ -122,7 +122,7 @@ def Qwen3MoeDecoderLayer_fast_forward(hidden_states += residualelse:residual = hidden_states- hidden_states = fast_rms_layernorm(self.input_layernorm, hidden_states)+ hidden_states = fast_rms_layernorm(self.input_layernorm, hidden_states, reuse_grad = True)hidden_states, self_attn_weights, present_key_value = self.self_attn(hidden_states = hidden_states,causal_mask = causal_mask,@@ -138,7 +138,9 @@ def Qwen3MoeDecoderLayer_fast_forward(# MoE Router MLP.residual = hidden_states- hidden_states = fast_rms_layernorm(self.post_attention_layernorm, hidden_states)+ hidden_states = fast_rms_layernorm(+ self.post_attention_layernorm, hidden_states, reuse_grad = True+ )hidden_states, router_logits = self.mlp(hidden_states)hidden_states = residual + hidden_states