Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Copy the residual before the in-place norm in Granite fast decode (#13454)

Vineeth Sai Varikuntlae7252fffe081
diff --git a/tests/test_granite_fast_decode_residual.py b/tests/test_granite_fast_decode_residual.py
new file mode 100644
index 00000000000..9e3595c1b9a
--- /dev/null
+++ b/tests/test_granite_fast_decode_residual.py
@@ -0,0 +1,55 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""GraniteModel_fast_forward_inference must match transformers' Granite decode."""
+
+import torch
+import unsloth # noqa: F401
+
+from transformers import GraniteConfig, GraniteForCausalLM
+from unsloth.models import granite as granite_module
+
+
+def test_granite_fast_decode_matches_transformers(monkeypatch):
+ monkeypatch.setattr(
+ granite_module, "per_layer_device", lambda module, *args, **kwargs: (torch.device("cpu"), 0)
+ )
+ torch.manual_seed(0)
+ config = GraniteConfig(
+ vocab_size = 64,
+ hidden_size = 64,
+ intermediate_size = 128,
+ num_hidden_layers = 2,
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ max_position_embeddings = 64,
+ attention_multiplier = 0.25,
+ embedding_multiplier = 1.0,
+ residual_multiplier = 1.0,
+ logits_scaling = 1.0,
+ )
+ model = GraniteForCausalLM(config).eval()
+ model.max_seq_length = 64
+ with torch.no_grad():
+ cos, sin = model.model.rotary_emb(torch.zeros(1), torch.arange(64)[None])
+ model.model.rotary_emb.get_cached = lambda seq_len, device_index = None: (cos[0], sin[0])
+
+ prompt = torch.randint(1, 64, (1, 6))
+ new_tokens = torch.randint(1, 64, (1, 3))
+ with torch.no_grad():
+ cache = model.model(input_ids = prompt, use_cache = True).past_key_values
+ past_key_values = [(layer.keys.clone(), layer.values.clone()) for layer in cache.layers]
+ for step in range(new_tokens.shape[1]):
+ token = new_tokens[:, step : step + 1]
+ position_ids = torch.tensor([[prompt.shape[1] + step]])
+ expected = model.model(
+ input_ids = token,
+ position_ids = position_ids,
+ past_key_values = cache,
+ use_cache = True,
+ ).last_hidden_state
+ out = granite_module.GraniteModel_fast_forward_inference(
+ model, token, past_key_values, position_ids, next_sequence_length = 1
+ )
+ past_key_values = out.past_key_values
+ torch.testing.assert_close(out.last_hidden_state, expected, atol = 1e-5, rtol = 1e-5)
diff --git a/unsloth/models/granite.py b/unsloth/models/granite.py
index 9be2a7a85dc..811e2ee3cf4 100644
--- a/unsloth/models/granite.py
+++ b/unsloth/models/granite.py
@@ -428,6 +428,7 @@ def GraniteModel_fast_forward_inference(
past_key_values,
position_ids,
attention_mask = None,
+ **kwargs,
):
input_ids = input_ids[:, : self.max_seq_length]
hidden_states = self.model.embed_tokens(input_ids)
@@ -467,7 +468,8 @@ def GraniteModel_fast_forward_inference(
layer_device, _ = per_layer_device(decoder_layer)
hidden_states, position_ids = move_to_device(layer_device, hidden_states, position_ids)
- residual = hidden_states
+ # fast_rms_layernorm_inference normalizes in place, so keep a copy for the residual.
+ residual = hidden_states.clone()
hidden_states = fast_rms_layernorm_inference(decoder_layer.input_layernorm, hidden_states)
hidden_states, present_key_value = GraniteAttention_fast_forward_inference(
decoder_layer.self_attn,
@@ -481,7 +483,7 @@ def GraniteModel_fast_forward_inference(
hidden_states = torch.add(residual, hidden_states, alpha = residual_multiplier)
- residual = hidden_states
+ residual = hidden_states.clone()
hidden_states = fast_rms_layernorm_inference(
decoder_layer.post_attention_layernorm, hidden_states
)