Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
RMSNorm save output: skip inference tensor weights, keep the default path free of opt-in checks (#13981)
Daniel Hanadebbfac2617
diff --git a/tests/test_rmsnorm_save_output.py b/tests/test_rmsnorm_save_output.pyindex 429b0dfa313..e9f0b094b7a 100644--- a/tests/test_rmsnorm_save_output.py+++ b/tests/test_rmsnorm_save_output.py@@ -162,3 +162,22 @@ def test_guard_does_not_sync_inside_cuda_graph_capture():# Inside capture the check would need .item(): fall back to saving X, cache nothing.assert not rms_layernorm._save_output_ok(w, False, torch.bfloat16)assert rms_layernorm._save_output_ok(w, False, torch.bfloat16)+++def test_guard_skips_inference_tensor_weights():+ # Inference tensors have no version counter; reading W._version would raise.+ with torch.inference_mode():+ w = torch.full((8,), 0.5)+ assert torch.is_inference(w)+ assert not rms_layernorm._save_output_ok(w, False, torch.bfloat16)+++@cuda+def test_save_output_with_inference_created_weights(monkeypatch):+ monkeypatch.setattr(rms_layernorm, "_SAVE_OUTPUT", True)+ with torch.inference_mode():+ x = torch.randn(1, 8, 128, device = "cuda", dtype = torch.float16)+ w = torch.ones(128, device = "cuda", dtype = torch.float16)+ y = rms_layernorm.Fast_RMS_Layernorm.apply(x, w, 1e-6, False, True)+ assert torch.isfinite(y).all()+ assert not getattr(y, "_unsloth_rmsnorm_saved", False)diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.pyindex b7ec6e119c6..433f03515b4 100644--- a/unsloth/kernels/fast_lora.py+++ b/unsloth/kernels/fast_lora.py@@ -25,6 +25,7 @@ from .utils import (torch_amp_custom_fwd,torch_amp_custom_bwd,)+from . import rms_layernorm as _rms_layernorm_is_compiling = torch.compiler.is_compiling@@ -168,7 +169,8 @@ class LoRA_MLP(torch.autograd.Function):_backward_function,)ctx.save_for_backward(gateA, gateB, upA, upB, downA, downB, X, e, g)- ctx.inplace = inplace and _may_overwrite_input(X)+ # Only an opt-in UNSLOTH_RMSNORM_SAVE_OUTPUT norm marks its output; skip the check otherwise.+ ctx.inplace = inplace and (not _rms_layernorm._SAVE_OUTPUT or _may_overwrite_input(X))return i@staticmethod@@ -496,7 +498,8 @@ class LoRA_QKV(torch.autograd.Function):VA,VB,)- ctx.inplace = inplace and _may_overwrite_input(X)+ # Only an opt-in UNSLOTH_RMSNORM_SAVE_OUTPUT norm marks its output; skip the check otherwise.+ ctx.inplace = inplace and (not _rms_layernorm._SAVE_OUTPUT or _may_overwrite_input(X))return Q, K, V@staticmethoddiff --git a/unsloth/kernels/rms_layernorm.py b/unsloth/kernels/rms_layernorm.pyindex 6348975dae4..90a0ec22916 100644--- a/unsloth/kernels/rms_layernorm.py+++ b/unsloth/kernels/rms_layernorm.py@@ -827,6 +827,9 @@ _SAVE_OUTPUT = os.environ.get("UNSLOTH_RMSNORM_SAVE_OUTPUT", "0") == "1"def _save_output_ok(W, gemma, dtype):# Y / W needs |W| (|1 + W| for Gemma) clear of 0 and fp16 Y not subnormal. Cached per weight version.+ if torch.is_inference(W):+ # Inference tensors have no version counter to key the cache on: save X.+ return Falsekey = (W._version, W.data_ptr(), gemma, dtype)cached = getattr(W, "_unsloth_save_output", None)if cached is not None and cached[0] == key:@@ -869,6 +872,7 @@ class Fast_RMS_Layernorm(torch.autograd.Function):ctx.FROM_Y = (save_outputand _SAVE_OUTPUT+ and ctx.needs_input_grad[0]and not ctx.needs_input_grad[1]and _save_output_ok(W, gemma, Y.dtype))@@ -886,7 +890,9 @@ class Fast_RMS_Layernorm(torch.autograd.Function):dim: int = shape[-1]dY = dY.reshape(-1, dim).contiguous()X, W, r = ctx.saved_tensors- X = X.view(-1, dim)+ if ctx.FROM_Y:+ # The saved Y keeps the caller's shape; a saved X is already 2D.+ X = X.view(-1, dim)# allocate dX because autograd may share dY with other nodes or the caller.dX = torch.empty_like(dY)with torch_gpu_device(dY.device):