Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

RMSNorm save output: skip inference tensor weights, keep the default path free of opt-in checks (#13981)

Daniel Hanadebbfac2617
diff --git a/tests/test_rmsnorm_save_output.py b/tests/test_rmsnorm_save_output.py
index 429b0dfa313..e9f0b094b7a 100644
--- a/tests/test_rmsnorm_save_output.py
+++ b/tests/test_rmsnorm_save_output.py
@@ -162,3 +162,22 @@ def test_guard_does_not_sync_inside_cuda_graph_capture():
# Inside capture the check would need .item(): fall back to saving X, cache nothing.
assert not rms_layernorm._save_output_ok(w, False, torch.bfloat16)
assert rms_layernorm._save_output_ok(w, False, torch.bfloat16)
+
+
+def test_guard_skips_inference_tensor_weights():
+ # Inference tensors have no version counter; reading W._version would raise.
+ with torch.inference_mode():
+ w = torch.full((8,), 0.5)
+ assert torch.is_inference(w)
+ assert not rms_layernorm._save_output_ok(w, False, torch.bfloat16)
+
+
+@cuda
+def test_save_output_with_inference_created_weights(monkeypatch):
+ monkeypatch.setattr(rms_layernorm, "_SAVE_OUTPUT", True)
+ with torch.inference_mode():
+ x = torch.randn(1, 8, 128, device = "cuda", dtype = torch.float16)
+ w = torch.ones(128, device = "cuda", dtype = torch.float16)
+ y = rms_layernorm.Fast_RMS_Layernorm.apply(x, w, 1e-6, False, True)
+ assert torch.isfinite(y).all()
+ assert not getattr(y, "_unsloth_rmsnorm_saved", False)
diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.py
index b7ec6e119c6..433f03515b4 100644
--- a/unsloth/kernels/fast_lora.py
+++ b/unsloth/kernels/fast_lora.py
@@ -25,6 +25,7 @@ from .utils import (
torch_amp_custom_fwd,
torch_amp_custom_bwd,
)
+from . import rms_layernorm as _rms_layernorm
_is_compiling = torch.compiler.is_compiling
@@ -168,7 +169,8 @@ class LoRA_MLP(torch.autograd.Function):
_backward_function,
)
ctx.save_for_backward(gateA, gateB, upA, upB, downA, downB, X, e, g)
- ctx.inplace = inplace and _may_overwrite_input(X)
+ # Only an opt-in UNSLOTH_RMSNORM_SAVE_OUTPUT norm marks its output; skip the check otherwise.
+ ctx.inplace = inplace and (not _rms_layernorm._SAVE_OUTPUT or _may_overwrite_input(X))
return i
@staticmethod
@@ -496,7 +498,8 @@ class LoRA_QKV(torch.autograd.Function):
VA,
VB,
)
- ctx.inplace = inplace and _may_overwrite_input(X)
+ # Only an opt-in UNSLOTH_RMSNORM_SAVE_OUTPUT norm marks its output; skip the check otherwise.
+ ctx.inplace = inplace and (not _rms_layernorm._SAVE_OUTPUT or _may_overwrite_input(X))
return Q, K, V
@staticmethod
diff --git a/unsloth/kernels/rms_layernorm.py b/unsloth/kernels/rms_layernorm.py
index 6348975dae4..90a0ec22916 100644
--- a/unsloth/kernels/rms_layernorm.py
+++ b/unsloth/kernels/rms_layernorm.py
@@ -827,6 +827,9 @@ _SAVE_OUTPUT = os.environ.get("UNSLOTH_RMSNORM_SAVE_OUTPUT", "0") == "1"
def _save_output_ok(W, gemma, dtype):
# Y / W needs |W| (|1 + W| for Gemma) clear of 0 and fp16 Y not subnormal. Cached per weight version.
+ if torch.is_inference(W):
+ # Inference tensors have no version counter to key the cache on: save X.
+ return False
key = (W._version, W.data_ptr(), gemma, dtype)
cached = getattr(W, "_unsloth_save_output", None)
if cached is not None and cached[0] == key:
@@ -869,6 +872,7 @@ class Fast_RMS_Layernorm(torch.autograd.Function):
ctx.FROM_Y = (
save_output
and _SAVE_OUTPUT
+ and ctx.needs_input_grad[0]
and not ctx.needs_input_grad[1]
and _save_output_ok(W, gemma, Y.dtype)
)
@@ -886,7 +890,9 @@ class Fast_RMS_Layernorm(torch.autograd.Function):
dim: int = shape[-1]
dY = dY.reshape(-1, dim).contiguous()
X, W, r = ctx.saved_tensors
- X = X.view(-1, dim)
+ if ctx.FROM_Y:
+ # The saved Y keeps the caller's shape; a saved X is already 2D.
+ X = X.view(-1, dim)
# allocate dX because autograd may share dY with other nodes or the caller.
dX = torch.empty_like(dY)
with torch_gpu_device(dY.device):