Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

RMSNorm: restore in-place dX for pre-norm decoder norms (#13984)

Daniel Han133489efc1d6
diff --git a/tests/test_norm_backward_grad_alias.py b/tests/test_norm_backward_grad_alias.py
index 7e484a99299..a209938840c 100644
--- a/tests/test_norm_backward_grad_alias.py
+++ b/tests/test_norm_backward_grad_alias.py
@@ -79,3 +79,104 @@ def test_layernorm_backward_leaves_shared_grad_alone(train_weight, dtype):
b = (0.1 * torch.randn(256, device = "cuda")).to(dtype).requires_grad_(train_weight)
norm = lambda x: Fast_Layernorm.apply(x, W, b, 1e-6)
_check(norm, lambda x: _ln_fp64(x, W, b), dtype)
+
+
[email protected](not has_real_cuda(), reason = "CUDA Triton kernels required")
[email protected]("gemma", [False, True])
[email protected]("save_output", [False, True])
[email protected]("dtype", [torch.float32, torch.float16, torch.bfloat16])
+def test_rms_layernorm_reuse_grad_matches_and_saves_a_buffer(
+ gemma, save_output, dtype, monkeypatch
+):
+ # Pre-norm decoder norms feed only layers that return a fresh grad, so they let dX overwrite
+ # dY: same bits as the fresh buffer, and about one activation less at the backward peak.
+ from unsloth.kernels import rms_layernorm
+ from unsloth.kernels.rms_layernorm import Fast_RMS_Layernorm
+
+ monkeypatch.setattr(rms_layernorm, "_SAVE_OUTPUT", save_output)
+ torch.manual_seed(0)
+ shape = (2, 512, 1024)
+ W = (1.0 + 0.1 * torch.randn(shape[-1], device = "cuda")).to(dtype)
+ proj = torch.randn(shape[-1], 64, device = "cuda", dtype = dtype) / 32
+ x0 = torch.randn(shape, device = "cuda", dtype = dtype)
+
+ def step(reuse_grad):
+ x = x0.clone().requires_grad_()
+ y = Fast_RMS_Layernorm.apply(x, W, 1e-6, gemma, save_output, reuse_grad)
+ (y @ proj).float().square().mean().backward()
+ return x.grad
+
+ results = {}
+ for reuse_grad in (False, True):
+ step(reuse_grad)
+ torch.cuda.synchronize()
+ base = torch.cuda.memory_allocated()
+ torch.cuda.reset_peak_memory_stats()
+ grad = step(reuse_grad)
+ torch.cuda.synchronize()
+ results[reuse_grad] = (grad, torch.cuda.max_memory_allocated() - base)
+ assert torch.equal(results[False][0], results[True][0])
+ saved = results[False][1] - results[True][1]
+ activation = x0.numel() * x0.element_size()
+ # The peak moves to a neighbouring allocation, so the drop is most of an activation, not all.
+ if gemma:
+ assert saved == 0, saved
+ else:
+ assert activation * 7 // 8 <= saved <= activation, (saved, activation)
+
+
[email protected](not has_real_cuda(), reason = "CUDA Triton kernels required")
+def test_final_norm_reuses_grad_only_under_the_causal_lm_head(monkeypatch):
+ # A base model hands the final norm's output to the caller, so the caller's grad_outputs must
+ # survive; only the CausalLM head, which consumes it itself, lets the norm write over its grad.
+ from unsloth import FastLanguageModel
+ from unsloth.kernels.rms_layernorm import Fast_RMS_Layernorm
+
+ model, _ = FastLanguageModel.from_pretrained(
+ "trl-internal-testing/tiny-Qwen3ForCausalLM",
+ max_seq_length = 64,
+ load_in_4bit = False,
+ dtype = torch.bfloat16,
+ )
+ model.train()
+ backbone = model.model
+ torch.manual_seed(0)
+ ids = torch.randint(0, model.config.vocab_size, (2, 32), device = "cuda")
+
+ embeds = backbone.embed_tokens(ids).detach().requires_grad_()
+ hidden = backbone(inputs_embeds = embeds, use_cache = False).last_hidden_state
+ go = torch.randn_like(hidden)
+ go_before = go.clone()
+ torch.autograd.grad(hidden, embeds, go)
+ assert torch.equal(go, go_before), "base model overwrote the caller's grad_outputs"
+
+ calls = []
+ apply = Fast_RMS_Layernorm.apply
+
+ def recording_apply(*args):
+ calls.append(bool(args[5]) if len(args) > 5 else False)
+ return apply(*args)
+
+ monkeypatch.setattr(Fast_RMS_Layernorm, "apply", recording_apply)
+ model(input_ids = ids, labels = ids, use_cache = False)
+ assert calls and calls[-1], "CausalLM head should let the final norm reuse its grad"
+ calls.clear()
+ model(input_ids = ids, labels = ids, use_cache = False, output_hidden_states = True)
+ assert calls and not calls[-1], "returned hidden states need the final norm's own buffer"
+ calls.clear()
+ backbone(input_ids = ids, use_cache = False)
+ assert calls and not calls[-1]
+ # GRPO / DPO style: hidden states come back as logits and feed a chunked lm_head loss.
+ calls.clear()
+ monkeypatch.setenv("UNSLOTH_RETURN_HIDDEN_STATES", "1")
+ embeds = backbone.embed_tokens(ids).detach().requires_grad_()
+ hidden = model(inputs_embeds = embeds, use_cache = False).logits
+ assert calls and calls[-1], "GRPO-style hidden states should let the final norm reuse its grad"
+ assert hidden.shape[-1] == model.config.hidden_size
+ logits = hidden.float() @ model.lm_head.weight.float().t()
+ (grad,) = torch.autograd.grad(logits.logsumexp(-1).mean(), embeds)
+ assert torch.isfinite(grad).all()
+ monkeypatch.delenv("UNSLOTH_RETURN_HIDDEN_STATES")
+ assert not getattr(backbone, "_unsloth_head_reads_final_norm", False)
diff --git a/unsloth/kernels/rms_layernorm.py b/unsloth/kernels/rms_layernorm.py
index 90a0ec22916..c76092e00d9 100644
--- a/unsloth/kernels/rms_layernorm.py
+++ b/unsloth/kernels/rms_layernorm.py
@@ -856,6 +856,7 @@ class Fast_RMS_Layernorm(torch.autograd.Function):
eps: float,
gemma: bool = False,
save_output: bool = False,
+ reuse_grad: bool = False,
):
shape = X.shape
dim: int = shape[-1]
@@ -866,6 +867,7 @@ class Fast_RMS_Layernorm(torch.autograd.Function):
Y, r = _rms_forward(X, W, eps, gemma, _eager_kernel)
ctx.eps = eps
ctx.GEMMA = gemma
+ ctx.reuse_grad = reuse_grad
Y = Y.view(*shape)
# save_output: the caller guarantees nothing writes into Y (an untracked Triton write would
# corrupt dX silently). Only frozen W: a trained W changes every step.
@@ -893,8 +895,10 @@ class Fast_RMS_Layernorm(torch.autograd.Function):
if ctx.FROM_Y:
# The saved Y keeps the caller's shape; a saved X is already 2D.
X = X.view(-1, dim)
- # allocate dX because autograd may share dY with other nodes or the caller.
- dX = torch.empty_like(dY)
+ # Autograd may share dY with other nodes or the caller, so dX gets its own buffer.
+ # reuse_grad is for call sites whose output only feeds layers that return a fresh grad
+ # (pre-norm decoder norms): there dX overwrites dY and saves one activation.
+ dX = dY if ctx.reuse_grad and not ctx.GEMMA else torch.empty_like(dY)
with torch_gpu_device(dY.device):
dW = _rms_backward(
dY,
@@ -909,7 +913,7 @@ class Fast_RMS_Layernorm(torch.autograd.Function):
from_y = ctx.FROM_Y,
)
dX = dX.view(*shape)
- return dX, dW, None, None, None
+ return dX, dW, None, None, None, None
# compiled graphs use these ops with the same kernels as the Function.
@@ -1011,8 +1015,9 @@ def _fast_rms_layernorm_untraced(
eps,
gemma,
save_output = False,
+ reuse_grad = False,
):
- return Fast_RMS_Layernorm.apply(X, W, eps, gemma, save_output)
+ return Fast_RMS_Layernorm.apply(X, W, eps, gemma, save_output, reuse_grad)
def fast_rms_layernorm(
@@ -1020,19 +1025,20 @@ def fast_rms_layernorm(
X: torch.Tensor,
gemma: bool = False,
save_output: bool = False,
+ reuse_grad: bool = False,
):
W: torch.Tensor = layernorm.weight
eps: float = (
layernorm.variance_epsilon if hasattr(layernorm, "variance_epsilon") else layernorm.eps
)
if not torch.compiler.is_compiling():
- return Fast_RMS_Layernorm.apply(X, W, eps, gemma, save_output)
+ return Fast_RMS_Layernorm.apply(X, W, eps, gemma, save_output, reuse_grad)
if (
not _TRACEABLE
or X.device.type != "cuda"
or (X.dtype == torch.bfloat16 and not _BF16_TRACEABLE)
):
- return _fast_rms_layernorm_untraced(X, W, eps, gemma, save_output)
+ return _fast_rms_layernorm_untraced(X, W, eps, gemma, save_output, reuse_grad)
shape = X.shape
Y, _ = torch.ops.unsloth.rms_layernorm(
X.reshape(-1, shape[-1]).contiguous(), W.contiguous(), eps, gemma
diff --git a/unsloth/models/falcon_h1.py b/unsloth/models/falcon_h1.py
index d93f0a6cb23..e7eeb7311b1 100644
--- a/unsloth/models/falcon_h1.py
+++ b/unsloth/models/falcon_h1.py
@@ -421,7 +421,7 @@ def FalconH1DecoderLayer_fast_forward(
hidden_states += residual
else:
residual = hidden_states
- hidden_states = fast_rms_layernorm(self.input_layernorm, hidden_states)
+ hidden_states = fast_rms_layernorm(self.input_layernorm, hidden_states, reuse_grad = True)
mamba_hidden_states = self.mamba(
hidden_states = hidden_states,
@@ -451,7 +451,7 @@ def FalconH1DecoderLayer_fast_forward(
hidden_states = residual + hidden_states
residual = hidden_states
- hidden_states = fast_rms_layernorm(self.pre_ff_layernorm, hidden_states)
+ hidden_states = fast_rms_layernorm(self.pre_ff_layernorm, hidden_states, reuse_grad = True)
hidden_states = self.feed_forward(hidden_states)
hidden_states = residual + hidden_states
diff --git a/unsloth/models/glm4_moe.py b/unsloth/models/glm4_moe.py
index 12b5b962d18..133e5271423 100644
--- a/unsloth/models/glm4_moe.py
+++ b/unsloth/models/glm4_moe.py
@@ -315,7 +315,7 @@ def Glm4MoeLiteDecoderLayer_fast_forward(
hidden_states = residual + hidden_states
else:
residual = hidden_states
- hidden_states = fast_rms_layernorm(self.input_layernorm, hidden_states)
+ hidden_states = fast_rms_layernorm(self.input_layernorm, hidden_states, reuse_grad = True)
hidden_states, _ = self.self_attn(
hidden_states = hidden_states,
attention_mask = attention_mask,
@@ -329,7 +329,9 @@ def Glm4MoeLiteDecoderLayer_fast_forward(
hidden_states = residual + hidden_states
residual = hidden_states
- hidden_states = fast_rms_layernorm(self.post_attention_layernorm, hidden_states)
+ hidden_states = fast_rms_layernorm(
+ self.post_attention_layernorm, hidden_states, reuse_grad = True
+ )
hidden_states = self.mlp(hidden_states)
hidden_states = residual + hidden_states
diff --git a/unsloth/models/granite.py b/unsloth/models/granite.py
index 1f10894605e..9be2a7a85dc 100644
--- a/unsloth/models/granite.py
+++ b/unsloth/models/granite.py
@@ -242,7 +242,7 @@ def GraniteDecoderLayer_fast_forward(
hidden_states = torch.add(residual, hidden_states, alpha = residual_multiplier)
else:
residual = hidden_states
- hidden_states = fast_rms_layernorm(self.input_layernorm, hidden_states)
+ hidden_states = fast_rms_layernorm(self.input_layernorm, hidden_states, reuse_grad = True)
hidden_states, self_attn_weights, present_key_value = self.self_attn(
hidden_states = hidden_states,
causal_mask = causal_mask,
@@ -258,7 +258,9 @@ def GraniteDecoderLayer_fast_forward(
hidden_states = torch.add(residual, hidden_states, alpha = residual_multiplier)
residual = hidden_states
- hidden_states = fast_rms_layernorm(self.post_attention_layernorm, hidden_states)
+ hidden_states = fast_rms_layernorm(
+ self.post_attention_layernorm, hidden_states, reuse_grad = True
+ )
hidden_states = self.mlp(hidden_states)
hidden_states = torch.add(residual, hidden_states, alpha = residual_multiplier)
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index 54078768963..193cb1eee09 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -1072,7 +1072,9 @@ def LlamaDecoderLayer_fast_forward(
hidden_states += residual
else:
residual = hidden_states
- hidden_states = fast_rms_layernorm(self.input_layernorm, hidden_states, save_output = True)
+ hidden_states = fast_rms_layernorm(
+ self.input_layernorm, hidden_states, save_output = True, reuse_grad = True
+ )
hidden_states, self_attn_weights, present_key_value = self.self_attn(
hidden_states = hidden_states,
causal_mask = causal_mask,
@@ -1089,7 +1091,7 @@ def LlamaDecoderLayer_fast_forward(
residual = hidden_states
hidden_states = fast_rms_layernorm(
- self.post_attention_layernorm, hidden_states, save_output = True
+ self.post_attention_layernorm, hidden_states, save_output = True, reuse_grad = True
)
hidden_states = self.mlp(hidden_states)
hidden_states = residual + hidden_states
@@ -1518,10 +1520,16 @@ def LlamaModel_fast_forward(
)(self.norm, hidden_states)
elif IS_COHERE:
hidden_states = self.norm(hidden_states)
- elif IS_FALCON_H1:
- hidden_states = fast_rms_layernorm(self.final_layernorm, hidden_states, gemma = IS_GEMMA)
else:
- hidden_states = fast_rms_layernorm(self.norm, hidden_states, gemma = IS_GEMMA)
+ # Only the CausalLM head may let the final norm overwrite its incoming grad: a base model
+ # hands this output to the caller, whose grad_outputs must stay intact.
+ reuse_grad = getattr(self, "_unsloth_head_reads_final_norm", False)
+ hidden_states = fast_rms_layernorm(
+ self.final_layernorm if IS_FALCON_H1 else self.norm,
+ hidden_states,
+ gemma = IS_GEMMA,
+ reuse_grad = reuse_grad,
+ )
if output_hidden_states:
all_hidden_states += (hidden_states,)
@@ -1745,6 +1753,19 @@ def apply_logit_transforms(logits, logit_softcapping, logit_scaling):
return logits
+def _run_backbone(self, **kwargs):
+ # The CausalLM forward feeds the final norm's output to lm_head, the fused CE loss, or (with
+ # UNSLOTH_RETURN_HIDDEN_STATES, set only by Unsloth's GRPO / DPO / GKD trainers) their chunked
+ # lm_head loss; all return a fresh grad, so the norm may write dX over it. Returned
+ # hidden_states would let a caller own that grad, so they keep the norm's own buffer.
+ model = self.model
+ model._unsloth_head_reads_final_norm = not kwargs.get("output_hidden_states")
+ try:
+ return model(**kwargs)
+ finally:
+ model._unsloth_head_reads_final_norm = False
+
+
def CausalLM_fast_forward(fast_forward_inference):
def _CausalLM_fast_forward(
self,
@@ -1801,7 +1822,8 @@ def CausalLM_fast_forward(fast_forward_inference):
return_dict if return_dict is not None else config_return_dict(self.config)
)
self.model._has_no_labels = labels is None
- outputs = self.model(
+ outputs = _run_backbone(
+ self,
input_ids = input_ids,
causal_mask = causal_mask,
attention_mask = attention_mask,
diff --git a/unsloth/models/mistral.py b/unsloth/models/mistral.py
index 7b7eda96098..9147dd53d2c 100644
--- a/unsloth/models/mistral.py
+++ b/unsloth/models/mistral.py
@@ -38,6 +38,7 @@ from .llama import (
resolve_logit_transforms,
_cache_as_legacy_tuple,
_cached_prefill_defaults,
+ _run_backbone,
unsloth_fsdp_unshard,
)
from transformers.models.mistral.modeling_mistral import (
@@ -287,7 +288,8 @@ def MistralForCausalLM_fast_forward(
attention_mask = attention_mask,
)
else:
- outputs = self.model(
+ outputs = _run_backbone(
+ self,
input_ids = input_ids,
causal_mask = causal_mask,
attention_mask = attention_mask,
diff --git a/unsloth/models/qwen3.py b/unsloth/models/qwen3.py
index 55f3e70c408..3f1bee48e51 100644
--- a/unsloth/models/qwen3.py
+++ b/unsloth/models/qwen3.py
@@ -121,8 +121,8 @@ def Qwen3Attention_fast_forward(
else:
# Qwen3 adds QKNorm, the only difference from Qwen2. A compiled norm mismatches Transformers'
# numbers, so use fast_rms_layernorm.
- Q = fast_rms_layernorm(self.q_norm, Q)
- K = fast_rms_layernorm(self.k_norm, K)
+ Q = fast_rms_layernorm(self.q_norm, Q, reuse_grad = True)
+ K = fast_rms_layernorm(self.k_norm, K, reuse_grad = True)
Q = Q.transpose(1, 2)
K = K.transpose(1, 2)
Q, K = fast_rope_embedding(Q, K, cos, sin, rope_position_ids)
diff --git a/unsloth/models/qwen3_moe.py b/unsloth/models/qwen3_moe.py
index ab923702cc8..56e717cb4f4 100644
--- a/unsloth/models/qwen3_moe.py
+++ b/unsloth/models/qwen3_moe.py
@@ -122,7 +122,7 @@ def Qwen3MoeDecoderLayer_fast_forward(
hidden_states += residual
else:
residual = hidden_states
- hidden_states = fast_rms_layernorm(self.input_layernorm, hidden_states)
+ hidden_states = fast_rms_layernorm(self.input_layernorm, hidden_states, reuse_grad = True)
hidden_states, self_attn_weights, present_key_value = self.self_attn(
hidden_states = hidden_states,
causal_mask = causal_mask,
@@ -138,7 +138,9 @@ def Qwen3MoeDecoderLayer_fast_forward(
# MoE Router MLP.
residual = hidden_states
- hidden_states = fast_rms_layernorm(self.post_attention_layernorm, hidden_states)
+ hidden_states = fast_rms_layernorm(
+ self.post_attention_layernorm, hidden_states, reuse_grad = True
+ )
hidden_states, router_logits = self.mlp(hidden_states)
hidden_states = residual + hidden_states