Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Copy the residual before the in-place norm in Granite fast decode (#13454)
Vineeth Sai Varikuntlae7252fffe081
diff --git a/tests/test_granite_fast_decode_residual.py b/tests/test_granite_fast_decode_residual.pynew file mode 100644index 00000000000..9e3595c1b9a--- /dev/null+++ b/tests/test_granite_fast_decode_residual.py@@ -0,0 +1,55 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""GraniteModel_fast_forward_inference must match transformers' Granite decode."""++import torch+import unsloth # noqa: F401++from transformers import GraniteConfig, GraniteForCausalLM+from unsloth.models import granite as granite_module+++def test_granite_fast_decode_matches_transformers(monkeypatch):+ monkeypatch.setattr(+ granite_module, "per_layer_device", lambda module, *args, **kwargs: (torch.device("cpu"), 0)+ )+ torch.manual_seed(0)+ config = GraniteConfig(+ vocab_size = 64,+ hidden_size = 64,+ intermediate_size = 128,+ num_hidden_layers = 2,+ num_attention_heads = 4,+ num_key_value_heads = 2,+ max_position_embeddings = 64,+ attention_multiplier = 0.25,+ embedding_multiplier = 1.0,+ residual_multiplier = 1.0,+ logits_scaling = 1.0,+ )+ model = GraniteForCausalLM(config).eval()+ model.max_seq_length = 64+ with torch.no_grad():+ cos, sin = model.model.rotary_emb(torch.zeros(1), torch.arange(64)[None])+ model.model.rotary_emb.get_cached = lambda seq_len, device_index = None: (cos[0], sin[0])++ prompt = torch.randint(1, 64, (1, 6))+ new_tokens = torch.randint(1, 64, (1, 3))+ with torch.no_grad():+ cache = model.model(input_ids = prompt, use_cache = True).past_key_values+ past_key_values = [(layer.keys.clone(), layer.values.clone()) for layer in cache.layers]+ for step in range(new_tokens.shape[1]):+ token = new_tokens[:, step : step + 1]+ position_ids = torch.tensor([[prompt.shape[1] + step]])+ expected = model.model(+ input_ids = token,+ position_ids = position_ids,+ past_key_values = cache,+ use_cache = True,+ ).last_hidden_state+ out = granite_module.GraniteModel_fast_forward_inference(+ model, token, past_key_values, position_ids, next_sequence_length = 1+ )+ past_key_values = out.past_key_values+ torch.testing.assert_close(out.last_hidden_state, expected, atol = 1e-5, rtol = 1e-5)diff --git a/unsloth/models/granite.py b/unsloth/models/granite.pyindex 9be2a7a85dc..811e2ee3cf4 100644--- a/unsloth/models/granite.py+++ b/unsloth/models/granite.py@@ -428,6 +428,7 @@ def GraniteModel_fast_forward_inference(past_key_values,position_ids,attention_mask = None,+ **kwargs,):input_ids = input_ids[:, : self.max_seq_length]hidden_states = self.model.embed_tokens(input_ids)@@ -467,7 +468,8 @@ def GraniteModel_fast_forward_inference(layer_device, _ = per_layer_device(decoder_layer)hidden_states, position_ids = move_to_device(layer_device, hidden_states, position_ids)- residual = hidden_states+ # fast_rms_layernorm_inference normalizes in place, so keep a copy for the residual.+ residual = hidden_states.clone()hidden_states = fast_rms_layernorm_inference(decoder_layer.input_layernorm, hidden_states)hidden_states, present_key_value = GraniteAttention_fast_forward_inference(decoder_layer.self_attn,@@ -481,7 +483,7 @@ def GraniteModel_fast_forward_inference(hidden_states = torch.add(residual, hidden_states, alpha = residual_multiplier)- residual = hidden_states+ residual = hidden_states.clone()hidden_states = fast_rms_layernorm_inference(decoder_layer.post_attention_layernorm, hidden_states)