Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Decode: cover explicit positions past the lazy RoPE cache in Gemma2, Cohere, Granite and FalconH1 (#14199)
Daniel Han7e3e0d90c0f0
diff --git a/tests/test_decode_offset_rope_model_loops.py b/tests/test_decode_offset_rope_model_loops.pynew file mode 100644index 00000000000..9f4ff014eff--- /dev/null+++ b/tests/test_decode_offset_rope_model_loops.py@@ -0,0 +1,165 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.+"""Decode with an explicit position past the rows the prefill built, for the models with their own+decode loop (Gemma2, Cohere, Granite, FalconH1).++The RoPE cache only builds the rows a forward asks for, so a prefill without position_ids on a+model whose max_position_embeddings exceeds the 32768-row starting cache leaves later rows unbuilt.+A decode step at such a position must still read a valid row, matching the decode after a prefill+that built every row. Runs in a subprocess because an out-of-range index is a CUDA device assert.+"""++import os+import subprocess+import sys+import textwrap++import pytest++torch = pytest.importorskip("torch")+needs_cuda = pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs CUDA")++TOKENIZER = "trl-internal-testing/tiny-Qwen3ForCausalLM"+MODELS = {+ "gemma2": "trl-internal-testing/tiny-Gemma2ForCausalLM",+ "cohere": "trl-internal-testing/tiny-CohereForCausalLM",+ "granite": "hf-internal-testing/tiny-random-GraniteForCausalLM",+ "falcon_h1": None, # No tiny hub repo: built from a shrunk config below.+}++SCRIPT = textwrap.dedent(+ """+ import sys+ import unsloth+ from unsloth import FastLanguageModel+ import torch+ from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer++ arch, repo, out_dir, tok_repo, pos = sys.argv[1:6]+ pos = int(pos)+ torch.manual_seed(0)+ if repo == "None":+ config = AutoConfig.for_model(+ "falcon_h1", hidden_size = 64, intermediate_size = 128, num_hidden_layers = 2,+ num_attention_heads = 4, num_key_value_heads = 2, head_dim = 16, mamba_d_ssm = 128,+ mamba_n_heads = 8, mamba_d_head = 16, mamba_d_state = 16, mamba_chunk_size = 16,+ vocab_size = 32784,+ )+ model = AutoModelForCausalLM.from_config(config)+ elif arch == "cohere":+ # The hub tiny has head_dim 2, which no xformers prefill kernel takes.+ config = AutoConfig.from_pretrained(repo)+ config.update(dict(+ hidden_size = 128, intermediate_size = 256, num_attention_heads = 4, num_key_value_heads = 2+ ))+ model = AutoModelForCausalLM.from_config(config)+ else:+ model = AutoModelForCausalLM.from_pretrained(repo, dtype = torch.float32)+ # Past the 32768-row starting cache, so the prefill below leaves the later rows unbuilt.+ model.config.max_position_embeddings = 40960+ model.save_pretrained(out_dir)+ AutoTokenizer.from_pretrained(tok_repo if repo == "None" else repo).save_pretrained(out_dir)+ del model++ load = dict(max_seq_length = 2048, load_in_4bit = False, dtype = torch.float16)+ if arch in ("cohere", "granite", "falcon_h1"):+ # FastLanguageModel routes these to the generic path; their own decode loop is+ # reached through their patcher.+ from unsloth.models.llama import FastLlamaModel+ from unsloth.models.cohere import FastCohereModel+ from unsloth.models.granite import FastGraniteModel+ from unsloth.models.falcon_h1 import FastFalconH1Model++ patcher = {+ "cohere": FastCohereModel,+ "granite": FastGraniteModel,+ "falcon_h1": FastFalconH1Model,+ }[arch]+ model, _ = FastLlamaModel.from_pretrained(out_dir, model_patcher = patcher, **load)+ else:+ model, _ = FastLanguageModel.from_pretrained(out_dir, **load)+ FastLanguageModel.for_inference(model)+ if arch == "falcon_h1":+ # The decode loop hands Mamba the legacy tuple cache, which it cannot read; Mamba does not+ # touch RoPE, so a zero branch keeps the check on the attention rows.+ for layer in model.model.layers:+ layer.mamba.forward = lambda hidden_states, **kwargs: torch.zeros_like(hidden_states)+ ids = torch.tensor([[1, 2, 3, 4]], device = "cuda")++ def logits(out):+ return out.logits if hasattr(out, "logits") else out[0]++ def decode(prefill_position_ids):+ for layer in model.model.layers:+ if hasattr(layer.self_attn, "paged_attention"):+ del layer.self_attn.paged_attention+ with torch.no_grad():+ out = model(input_ids = ids, position_ids = prefill_position_ids, use_cache = True)+ out = model(+ input_ids = ids[:, :1],+ past_key_values = out.past_key_values,+ position_ids = torch.tensor([[pos]], device = "cuda"),+ use_cache = True,+ )+ torch.cuda.synchronize()+ return logits(out).float()++ lazy = decode(None)+ full = decode(torch.arange(4, device = "cuda")[None])+ assert torch.isfinite(lazy).all()+ torch.testing.assert_close(lazy, full, rtol = 0, atol = 0)+ print("OK")+ """+)+++@needs_cuda[email protected]("arch", list(MODELS))+def test_decode_reads_rows_past_the_lazy_prefill_cache(arch, tmp_path):+ script = tmp_path / "decode_offset.py"+ script.write_text(SCRIPT)+ env = dict(os.environ)+ repo_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+ env["PYTHONPATH"] = repo_root + os.pathsep + env.get("PYTHONPATH", "")+ env.setdefault("UNSLOTH_COMPILE_LOCATION", str(tmp_path / "unsloth_compiled_cache"))+ result = subprocess.run(+ [+ sys.executable,+ str(script),+ arch,+ str(MODELS[arch]),+ str(tmp_path / arch),+ TOKENIZER,+ "35000",+ ],+ capture_output = True,+ text = True,+ env = env,+ timeout = 900,+ )+ assert result.returncode == 0 and "OK" in result.stdout, (+ result.stdout[-2000:] + result.stderr[-4000:]+ )+++class _NoSync:+ def max(self):+ raise AssertionError("host sync on a cache that already holds every position")+++class _Rope:+ def __init__(self, floor, max_position_embeddings):+ self._rope_cache_floor = floor+ self.max_position_embeddings = max_position_embeddings+++def test_decode_rotary_seq_len_syncs_only_when_rows_can_be_missing():+ from unsloth.models.llama import _decode_rotary_seq_len++ # Every supported position is in the eager cache (max_position_embeddings <= 32768): no sync.+ assert _decode_rotary_seq_len(_Rope(8192, 8192), 9, _NoSync()) == 9+ assert _decode_rotary_seq_len(_Rope(8192, 8192), 9, None) == 9+ # Lazy rows past the floor: one sync, covering the explicit position.+ position_ids = torch.tensor([[35000]])+ assert _decode_rotary_seq_len(_Rope(32768, 40960), 9, position_ids) == 35001+ assert _decode_rotary_seq_len(_Rope(32768, 40960), 9, torch.tensor([[3]])) == 9diff --git a/tests/test_rope_cache_current_device.py b/tests/test_rope_cache_current_device.pyindex 1e2579e88ff..32e8c509669 100644--- a/tests/test_rope_cache_current_device.py+++ b/tests/test_rope_cache_current_device.py@@ -244,6 +244,7 @@ def test_granite_decode_reads_rows_past_max_seq_length():"self": types.SimpleNamespace(max_seq_length = max_seq_length),"seq_len": seq_len,"q_len": q_len,+ "rotary_seq_len": seq_len + q_len,},)rope = _llama(1 << 17)diff --git a/unsloth/models/cohere.py b/unsloth/models/cohere.pyindex f0519c0dd19..00461d6d327 100644--- a/unsloth/models/cohere.py+++ b/unsloth/models/cohere.py@@ -11,6 +11,7 @@from .llama import *from .llama import (+ _decode_rotary_seq_len,original_apply_qkv,original_apply_o,_gqa_decode_scores,@@ -353,7 +354,13 @@ def CohereAttention_fast_forward_inference(Qn = fast_layernorm_inference(self.q_norm, Qn, self.q_norm_out_weight)Kn = fast_layernorm_inference(self.k_norm, Kn, self.k_norm_out_weight)- cos, sin = self.rotary_emb.get_cached(kv_seq_len, Qn.device.index)+ # rotary_seq_len (once per step from the decode loop) covers explicit positions past the KV length.+ rotary_seq_len = kwargs.get("rotary_seq_len")+ if rotary_seq_len is not None and rotary_seq_len > kv_seq_len:+ self.rotary_emb.extend_rope_embedding(Vn, rotary_seq_len + 1)+ cos, sin = self.rotary_emb.get_cached(rotary_seq_len, Qn.device.index)+ else:+ cos, sin = self.rotary_emb.get_cached(kv_seq_len, Qn.device.index)# Transformers 5.x: position_ids may be [batch, full_seq_len]; slice to last.if position_ids.dim() >= 2 and position_ids.shape[-1] > 1:position_ids = position_ids[:, -1:]@@ -446,6 +453,12 @@ def CohereModel_fast_forward_inference(else:attention_mask = None+ # Once per step for every layer, never per layer (one host sync at most).+ rotary_seq_len = _decode_rotary_seq_len(+ getattr(getattr(self.model.layers[0], "self_attn", None), "rotary_emb", None),+ seq_len + 1,+ position_ids,+ )next_decoder_cache = []block_swap = getattr(self.model.layers, "_unsloth_block_swap", None)for idx, decoder_layer in enumerate(self.model.layers):@@ -464,6 +477,7 @@ def CohereModel_fast_forward_inference(position_ids = position_ids,attention_mask = attention_mask,do_prefill = not hasattr(decoder_layer.self_attn, "paged_attention"),+ rotary_seq_len = rotary_seq_len,)hidden_states_mlp = fast_swiglu_inference(decoder_layer.mlp, hidden_states)diff --git a/unsloth/models/falcon_h1.py b/unsloth/models/falcon_h1.pyindex e7eeb7311b1..338140aac8b 100644--- a/unsloth/models/falcon_h1.py+++ b/unsloth/models/falcon_h1.py@@ -24,6 +24,7 @@ from ..utils.attention_dispatch import (SDPA,)from .llama import (+ _decode_rotary_seq_len,LlamaRotaryEmbedding,LlamaLinearScalingRotaryEmbedding,_gqa_decode_scores,@@ -287,8 +288,14 @@ def FalconH1Attention_fast_forward_inference(Kn = Kn.transpose(1, 2)# Must happen 2 steps before hitting full on a short KV cache, or it errors.- self.rotary_emb.extend_rope_embedding(Vn, seq_len + 2)- cos, sin = self.rotary_emb.get_cached(kv_seq_len, Qn.device.index)+ # rotary_seq_len (once per step from the decode loop) covers explicit positions past the KV length.+ rotary_seq_len = kwargs.get("rotary_seq_len")+ if rotary_seq_len is not None and rotary_seq_len > kv_seq_len:+ self.rotary_emb.extend_rope_embedding(Vn, max(seq_len + 2, rotary_seq_len + 1))+ cos, sin = self.rotary_emb.get_cached(rotary_seq_len, Qn.device.index)+ else:+ self.rotary_emb.extend_rope_embedding(Vn, seq_len + 2)+ cos, sin = self.rotary_emb.get_cached(kv_seq_len, Qn.device.index)# Transformers 5.x: position_ids may be [batch, full_seq_len]; slice to last.if position_ids.dim() >= 2 and position_ids.shape[-1] > 1:position_ids = position_ids[:, -1:]@@ -514,6 +521,12 @@ def _FalconH1_fast_forward_inference(else:attention_mask = None+ # Once per step for every layer, never per layer (one host sync at most).+ rotary_seq_len = _decode_rotary_seq_len(+ getattr(getattr(self.model.layers[0], "self_attn", None), "rotary_emb", None),+ seq_len + 1,+ position_ids,+ )next_decoder_cache = []block_swap = getattr(self.model.layers, "_unsloth_block_swap", None)@@ -535,6 +548,7 @@ def _FalconH1_fast_forward_inference(position_ids = position_ids,attention_mask = attention_mask,do_prefill = not hasattr(decoder_layer.self_attn, "paged_attention"),+ rotary_seq_len = rotary_seq_len,)attention_hidden_states = attention_hidden_states * decoder_layer.attn_out_multipliermamba_hidden_states = decoder_layer.mamba(diff --git a/unsloth/models/gemma2.py b/unsloth/models/gemma2.pyindex 2f0c3a57ce6..fa1ef3a8605 100644--- a/unsloth/models/gemma2.py+++ b/unsloth/models/gemma2.py@@ -11,6 +11,7 @@from .llama import *from .llama import (+ _decode_rotary_seq_len,original_apply_qkv,original_apply_o,_gqa_decode_scores,@@ -433,7 +434,13 @@ def Gemma2Attention_fast_forward_inference(Kn = Kn.view(bsz, 1, n_kv_heads, head_dim).transpose(1, 2)Vn = Vn.view(bsz, 1, n_kv_heads, head_dim).transpose(1, 2)- cos, sin = self.rotary_emb.get_cached(kv_seq_len, Qn.device.index)+ # rotary_seq_len (once per step from the decode loop) covers explicit positions past the KV length.+ rotary_seq_len = kwargs.get("rotary_seq_len")+ if rotary_seq_len is not None and rotary_seq_len > kv_seq_len:+ self.rotary_emb.extend_rope_embedding(Vn, rotary_seq_len + 1)+ cos, sin = self.rotary_emb.get_cached(rotary_seq_len, Qn.device.index)+ else:+ cos, sin = self.rotary_emb.get_cached(kv_seq_len, Qn.device.index)# Transformers 5.x: position_ids may be [batch, full_seq_len]; slice to last.if position_ids.dim() >= 2 and position_ids.shape[-1] > 1:position_ids = position_ids[:, -1:]@@ -568,6 +575,12 @@ def Gemma2Model_fast_forward_inference(# One row has no padding; the raw 2D mask cannot be sliced to the sliding window.SWA = NoneGA = None+ # Once per step for every layer, never per layer (one host sync at most).+ rotary_seq_len = _decode_rotary_seq_len(+ getattr(getattr(self.model.layers[0], "self_attn", None), "rotary_emb", None),+ seq_len + 1,+ position_ids,+ )next_decoder_cache = []block_swap = getattr(self.model.layers, "_unsloth_block_swap", None)for idx, decoder_layer in enumerate(self.model.layers):@@ -599,6 +612,7 @@ def Gemma2Model_fast_forward_inference(use_sliding_window = use_sliding_window,flash_decode = flash_decode,leftpad = leftpad,+ rotary_seq_len = rotary_seq_len,)hidden_states = fast_rms_layernorm_inference_gemma(decoder_layer.post_attention_layernorm,diff --git a/unsloth/models/granite.py b/unsloth/models/granite.pyindex 13b2891f594..e980bd84298 100644--- a/unsloth/models/granite.py+++ b/unsloth/models/granite.py@@ -25,6 +25,7 @@ from ..utils.attention_dispatch import (SDPA,)from .llama import (+ _decode_rotary_seq_len,LlamaRotaryEmbedding,LlamaLinearScalingRotaryEmbedding,_gqa_decode_scores,@@ -458,8 +459,12 @@ def GraniteModel_fast_forward_inference(attention_mask = None# Decode positions run past max_seq_length; the cache is only built for the rows asked for.+ # Explicit positions can run past both: one host sync at most, here and not per layer.+ rotary_seq_len = _decode_rotary_seq_len(self.model.rotary_emb, seq_len + q_len, position_ids)+ if rotary_seq_len > max(self.max_seq_length, seq_len + q_len):+ self.model.rotary_emb.extend_rope_embedding(hidden_states, rotary_seq_len + 1)position_embeddings = self.model.rotary_emb.get_cached(- max(self.max_seq_length, seq_len + q_len), hidden_states.device.index+ max(self.max_seq_length, seq_len + q_len, rotary_seq_len), hidden_states.device.index)next_decoder_cache = []diff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex 22c73cc9528..bf1406beb9c 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -2233,6 +2233,25 @@ def _round_rope_size(seq_len):return ((seq_len // 8192) + ((seq_len % 8192) != 0)) * 8192+def _decode_rotary_seq_len(rotary_emb, kv_seq_len, position_ids):+ """RoPE rows one decode step reads, for decode loops that pass it to every layer.++ Explicit positions can run past the KV length, and the lazy cache only builds the rows asked+ for. Costs one host sync per step (as LlamaModel_fast_forward_inference), skipped when the+ cache always holds every position the model supports."""+ if position_ids is None:+ return kv_seq_len+ max_position_embeddings = getattr(rotary_emb, "max_position_embeddings", None)+ floor = getattr(rotary_emb, "_rope_cache_floor", None)+ if (+ max_position_embeddings is not None+ and floor is not None+ and floor >= max_position_embeddings+ ):+ return kv_seq_len+ return max(kv_seq_len, int(position_ids.max().item()) + 1)++class _PerDeviceRopeCache:"""cos/sin built lazily per device with only the rows asked for, so torchrun ranks do notallocate on their peers. A row depends only on its position, so a shorter cache matches the