Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Copy the input in the Gemma inference RMSNorm so float32 decode keeps its residual (#13643)
Vineeth Sai Varikuntlaae842f7635fd
diff --git a/tests/test_gemma_fast_decode_float32.py b/tests/test_gemma_fast_decode_float32.pynew file mode 100644index 00000000000..8c25335b060--- /dev/null+++ b/tests/test_gemma_fast_decode_float32.py@@ -0,0 +1,89 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Gemma / Gemma2 fast decode must match transformers in float32.++fast_rms_layernorm_inference_gemma normalized a float32 input in place (.to(torch.float32) returns+the same tensor), and the decode loops keep that input as the residual.+"""++import types++import pytest+import torch+import unsloth # noqa: F401++from unsloth.models import gemma as gemma_module+from unsloth.models import gemma2 as gemma2_module+from unsloth_zoo.hf_utils import set_dtype_in_config++[email protected]("arch", ["gemma", "gemma2"])+def test_gemma_fast_decode_matches_transformers_in_float32(arch, monkeypatch):+ from transformers import Gemma2Config, Gemma2ForCausalLM, GemmaConfig, GemmaForCausalLM++ module = gemma_module if arch == "gemma" else gemma2_module+ norm = module.fast_rms_layernorm_inference_gemma+ # No per-device float32 buffers on CPU: run the real norm without one.+ monkeypatch.setattr(module, "DEVICE_COUNT", 0)+ monkeypatch.setattr(+ module, "per_layer_device", lambda layer: (torch.device("cpu"), slice(None))+ )+ monkeypatch.setattr(module, "fast_rms_layernorm_inference_gemma", lambda ln, x, w: norm(ln, x))++ sizes = dict(+ vocab_size = 64,+ hidden_size = 64,+ intermediate_size = 128,+ num_hidden_layers = 2,+ num_attention_heads = 4,+ num_key_value_heads = 2,+ head_dim = 16,+ max_position_embeddings = 64,+ )+ torch.manual_seed(0)+ if arch == "gemma":+ model = GemmaForCausalLM(GemmaConfig(**sizes))+ fast_forward = gemma_module.GemmaModel_fast_forward_inference+ else:+ # Eager: transformers' SDPA path drops Gemma2's attention softcapping.+ config = Gemma2Config(+ **sizes,+ sliding_window = 4,+ query_pre_attn_scalar = 16,+ attn_implementation = "eager",+ )+ model = Gemma2ForCausalLM(config)+ fast_forward = gemma2_module.Gemma2Model_fast_forward_inference+ model = model.float().eval()+ set_dtype_in_config(model.config, torch.float32)+ model.max_seq_length = 64+ for module_ in model.modules():+ if hasattr(module_, "eps") and not hasattr(module_, "variance_epsilon"):+ module_.variance_epsilon = module_.eps+ with torch.no_grad():+ cos, sin = model.model.rotary_emb(torch.zeros(1), torch.arange(64)[None])+ rotary = types.SimpleNamespace(+ get_cached = lambda seq_len, device_index = None: (cos[0], sin[0]),+ extend_rope_embedding = lambda *args: None,+ )+ for layer in model.model.layers:+ layer.self_attn.rotary_emb = rotary++ prompt = torch.randint(1, 64, (1, 6))+ new_tokens = torch.randint(1, 64, (1, 3))+ with torch.no_grad():+ cache = model.model(input_ids = prompt, use_cache = True).past_key_values+ past_key_values = [(layer.keys.clone(), layer.values.clone()) for layer in cache.layers]+ for step in range(new_tokens.shape[1]):+ token = new_tokens[:, step : step + 1]+ position_ids = torch.tensor([[prompt.shape[1] + step]])+ expected = model.model(+ input_ids = token,+ position_ids = position_ids,+ past_key_values = cache,+ use_cache = True,+ ).last_hidden_state+ out = fast_forward(model, token, past_key_values, position_ids)+ past_key_values = out.past_key_values+ torch.testing.assert_close(out.last_hidden_state, expected, atol = 1e-5, rtol = 1e-5)diff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex c4f5efe83ca..77dc277ed80 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -852,7 +852,8 @@ def fast_rms_layernorm_inference_gemma(X,out_weight = None,):- XX = X.to(torch.float32)+ # copy: for a float32 X, .to() returns X itself, and the decode loops keep X as the residual.+ XX = X.to(torch.float32, copy = True)variance = XX.square().mean(-1, keepdim = True)variance += self.variance_epsilonXX *= variance.rsqrt_()