Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Decode: cover explicit positions past the lazy RoPE cache in Gemma2, Cohere, Granite and FalconH1 (#14199)

Daniel Han7e3e0d90c0f0
diff --git a/tests/test_decode_offset_rope_model_loops.py b/tests/test_decode_offset_rope_model_loops.py
new file mode 100644
index 00000000000..9f4ff014eff
--- /dev/null
+++ b/tests/test_decode_offset_rope_model_loops.py
@@ -0,0 +1,165 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+"""Decode with an explicit position past the rows the prefill built, for the models with their own
+decode loop (Gemma2, Cohere, Granite, FalconH1).
+
+The RoPE cache only builds the rows a forward asks for, so a prefill without position_ids on a
+model whose max_position_embeddings exceeds the 32768-row starting cache leaves later rows unbuilt.
+A decode step at such a position must still read a valid row, matching the decode after a prefill
+that built every row. Runs in a subprocess because an out-of-range index is a CUDA device assert.
+"""
+
+import os
+import subprocess
+import sys
+import textwrap
+
+import pytest
+
+torch = pytest.importorskip("torch")
+needs_cuda = pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs CUDA")
+
+TOKENIZER = "trl-internal-testing/tiny-Qwen3ForCausalLM"
+MODELS = {
+ "gemma2": "trl-internal-testing/tiny-Gemma2ForCausalLM",
+ "cohere": "trl-internal-testing/tiny-CohereForCausalLM",
+ "granite": "hf-internal-testing/tiny-random-GraniteForCausalLM",
+ "falcon_h1": None, # No tiny hub repo: built from a shrunk config below.
+}
+
+SCRIPT = textwrap.dedent(
+ """
+ import sys
+ import unsloth
+ from unsloth import FastLanguageModel
+ import torch
+ from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
+
+ arch, repo, out_dir, tok_repo, pos = sys.argv[1:6]
+ pos = int(pos)
+ torch.manual_seed(0)
+ if repo == "None":
+ config = AutoConfig.for_model(
+ "falcon_h1", hidden_size = 64, intermediate_size = 128, num_hidden_layers = 2,
+ num_attention_heads = 4, num_key_value_heads = 2, head_dim = 16, mamba_d_ssm = 128,
+ mamba_n_heads = 8, mamba_d_head = 16, mamba_d_state = 16, mamba_chunk_size = 16,
+ vocab_size = 32784,
+ )
+ model = AutoModelForCausalLM.from_config(config)
+ elif arch == "cohere":
+ # The hub tiny has head_dim 2, which no xformers prefill kernel takes.
+ config = AutoConfig.from_pretrained(repo)
+ config.update(dict(
+ hidden_size = 128, intermediate_size = 256, num_attention_heads = 4, num_key_value_heads = 2
+ ))
+ model = AutoModelForCausalLM.from_config(config)
+ else:
+ model = AutoModelForCausalLM.from_pretrained(repo, dtype = torch.float32)
+ # Past the 32768-row starting cache, so the prefill below leaves the later rows unbuilt.
+ model.config.max_position_embeddings = 40960
+ model.save_pretrained(out_dir)
+ AutoTokenizer.from_pretrained(tok_repo if repo == "None" else repo).save_pretrained(out_dir)
+ del model
+
+ load = dict(max_seq_length = 2048, load_in_4bit = False, dtype = torch.float16)
+ if arch in ("cohere", "granite", "falcon_h1"):
+ # FastLanguageModel routes these to the generic path; their own decode loop is
+ # reached through their patcher.
+ from unsloth.models.llama import FastLlamaModel
+ from unsloth.models.cohere import FastCohereModel
+ from unsloth.models.granite import FastGraniteModel
+ from unsloth.models.falcon_h1 import FastFalconH1Model
+
+ patcher = {
+ "cohere": FastCohereModel,
+ "granite": FastGraniteModel,
+ "falcon_h1": FastFalconH1Model,
+ }[arch]
+ model, _ = FastLlamaModel.from_pretrained(out_dir, model_patcher = patcher, **load)
+ else:
+ model, _ = FastLanguageModel.from_pretrained(out_dir, **load)
+ FastLanguageModel.for_inference(model)
+ if arch == "falcon_h1":
+ # The decode loop hands Mamba the legacy tuple cache, which it cannot read; Mamba does not
+ # touch RoPE, so a zero branch keeps the check on the attention rows.
+ for layer in model.model.layers:
+ layer.mamba.forward = lambda hidden_states, **kwargs: torch.zeros_like(hidden_states)
+ ids = torch.tensor([[1, 2, 3, 4]], device = "cuda")
+
+ def logits(out):
+ return out.logits if hasattr(out, "logits") else out[0]
+
+ def decode(prefill_position_ids):
+ for layer in model.model.layers:
+ if hasattr(layer.self_attn, "paged_attention"):
+ del layer.self_attn.paged_attention
+ with torch.no_grad():
+ out = model(input_ids = ids, position_ids = prefill_position_ids, use_cache = True)
+ out = model(
+ input_ids = ids[:, :1],
+ past_key_values = out.past_key_values,
+ position_ids = torch.tensor([[pos]], device = "cuda"),
+ use_cache = True,
+ )
+ torch.cuda.synchronize()
+ return logits(out).float()
+
+ lazy = decode(None)
+ full = decode(torch.arange(4, device = "cuda")[None])
+ assert torch.isfinite(lazy).all()
+ torch.testing.assert_close(lazy, full, rtol = 0, atol = 0)
+ print("OK")
+ """
+)
+
+
+@needs_cuda
[email protected]("arch", list(MODELS))
+def test_decode_reads_rows_past_the_lazy_prefill_cache(arch, tmp_path):
+ script = tmp_path / "decode_offset.py"
+ script.write_text(SCRIPT)
+ env = dict(os.environ)
+ repo_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+ env["PYTHONPATH"] = repo_root + os.pathsep + env.get("PYTHONPATH", "")
+ env.setdefault("UNSLOTH_COMPILE_LOCATION", str(tmp_path / "unsloth_compiled_cache"))
+ result = subprocess.run(
+ [
+ sys.executable,
+ str(script),
+ arch,
+ str(MODELS[arch]),
+ str(tmp_path / arch),
+ TOKENIZER,
+ "35000",
+ ],
+ capture_output = True,
+ text = True,
+ env = env,
+ timeout = 900,
+ )
+ assert result.returncode == 0 and "OK" in result.stdout, (
+ result.stdout[-2000:] + result.stderr[-4000:]
+ )
+
+
+class _NoSync:
+ def max(self):
+ raise AssertionError("host sync on a cache that already holds every position")
+
+
+class _Rope:
+ def __init__(self, floor, max_position_embeddings):
+ self._rope_cache_floor = floor
+ self.max_position_embeddings = max_position_embeddings
+
+
+def test_decode_rotary_seq_len_syncs_only_when_rows_can_be_missing():
+ from unsloth.models.llama import _decode_rotary_seq_len
+
+ # Every supported position is in the eager cache (max_position_embeddings <= 32768): no sync.
+ assert _decode_rotary_seq_len(_Rope(8192, 8192), 9, _NoSync()) == 9
+ assert _decode_rotary_seq_len(_Rope(8192, 8192), 9, None) == 9
+ # Lazy rows past the floor: one sync, covering the explicit position.
+ position_ids = torch.tensor([[35000]])
+ assert _decode_rotary_seq_len(_Rope(32768, 40960), 9, position_ids) == 35001
+ assert _decode_rotary_seq_len(_Rope(32768, 40960), 9, torch.tensor([[3]])) == 9
diff --git a/tests/test_rope_cache_current_device.py b/tests/test_rope_cache_current_device.py
index 1e2579e88ff..32e8c509669 100644
--- a/tests/test_rope_cache_current_device.py
+++ b/tests/test_rope_cache_current_device.py
@@ -244,6 +244,7 @@ def test_granite_decode_reads_rows_past_max_seq_length():
"self": types.SimpleNamespace(max_seq_length = max_seq_length),
"seq_len": seq_len,
"q_len": q_len,
+ "rotary_seq_len": seq_len + q_len,
},
)
rope = _llama(1 << 17)
diff --git a/unsloth/models/cohere.py b/unsloth/models/cohere.py
index f0519c0dd19..00461d6d327 100644
--- a/unsloth/models/cohere.py
+++ b/unsloth/models/cohere.py
@@ -11,6 +11,7 @@
from .llama import *
from .llama import (
+ _decode_rotary_seq_len,
original_apply_qkv,
original_apply_o,
_gqa_decode_scores,
@@ -353,7 +354,13 @@ def CohereAttention_fast_forward_inference(
Qn = fast_layernorm_inference(self.q_norm, Qn, self.q_norm_out_weight)
Kn = fast_layernorm_inference(self.k_norm, Kn, self.k_norm_out_weight)
- cos, sin = self.rotary_emb.get_cached(kv_seq_len, Qn.device.index)
+ # rotary_seq_len (once per step from the decode loop) covers explicit positions past the KV length.
+ rotary_seq_len = kwargs.get("rotary_seq_len")
+ if rotary_seq_len is not None and rotary_seq_len > kv_seq_len:
+ self.rotary_emb.extend_rope_embedding(Vn, rotary_seq_len + 1)
+ cos, sin = self.rotary_emb.get_cached(rotary_seq_len, Qn.device.index)
+ else:
+ cos, sin = self.rotary_emb.get_cached(kv_seq_len, Qn.device.index)
# Transformers 5.x: position_ids may be [batch, full_seq_len]; slice to last.
if position_ids.dim() >= 2 and position_ids.shape[-1] > 1:
position_ids = position_ids[:, -1:]
@@ -446,6 +453,12 @@ def CohereModel_fast_forward_inference(
else:
attention_mask = None
+ # Once per step for every layer, never per layer (one host sync at most).
+ rotary_seq_len = _decode_rotary_seq_len(
+ getattr(getattr(self.model.layers[0], "self_attn", None), "rotary_emb", None),
+ seq_len + 1,
+ position_ids,
+ )
next_decoder_cache = []
block_swap = getattr(self.model.layers, "_unsloth_block_swap", None)
for idx, decoder_layer in enumerate(self.model.layers):
@@ -464,6 +477,7 @@ def CohereModel_fast_forward_inference(
position_ids = position_ids,
attention_mask = attention_mask,
do_prefill = not hasattr(decoder_layer.self_attn, "paged_attention"),
+ rotary_seq_len = rotary_seq_len,
)
hidden_states_mlp = fast_swiglu_inference(decoder_layer.mlp, hidden_states)
diff --git a/unsloth/models/falcon_h1.py b/unsloth/models/falcon_h1.py
index e7eeb7311b1..338140aac8b 100644
--- a/unsloth/models/falcon_h1.py
+++ b/unsloth/models/falcon_h1.py
@@ -24,6 +24,7 @@ from ..utils.attention_dispatch import (
SDPA,
)
from .llama import (
+ _decode_rotary_seq_len,
LlamaRotaryEmbedding,
LlamaLinearScalingRotaryEmbedding,
_gqa_decode_scores,
@@ -287,8 +288,14 @@ def FalconH1Attention_fast_forward_inference(
Kn = Kn.transpose(1, 2)
# Must happen 2 steps before hitting full on a short KV cache, or it errors.
- self.rotary_emb.extend_rope_embedding(Vn, seq_len + 2)
- cos, sin = self.rotary_emb.get_cached(kv_seq_len, Qn.device.index)
+ # rotary_seq_len (once per step from the decode loop) covers explicit positions past the KV length.
+ rotary_seq_len = kwargs.get("rotary_seq_len")
+ if rotary_seq_len is not None and rotary_seq_len > kv_seq_len:
+ self.rotary_emb.extend_rope_embedding(Vn, max(seq_len + 2, rotary_seq_len + 1))
+ cos, sin = self.rotary_emb.get_cached(rotary_seq_len, Qn.device.index)
+ else:
+ self.rotary_emb.extend_rope_embedding(Vn, seq_len + 2)
+ cos, sin = self.rotary_emb.get_cached(kv_seq_len, Qn.device.index)
# Transformers 5.x: position_ids may be [batch, full_seq_len]; slice to last.
if position_ids.dim() >= 2 and position_ids.shape[-1] > 1:
position_ids = position_ids[:, -1:]
@@ -514,6 +521,12 @@ def _FalconH1_fast_forward_inference(
else:
attention_mask = None
+ # Once per step for every layer, never per layer (one host sync at most).
+ rotary_seq_len = _decode_rotary_seq_len(
+ getattr(getattr(self.model.layers[0], "self_attn", None), "rotary_emb", None),
+ seq_len + 1,
+ position_ids,
+ )
next_decoder_cache = []
block_swap = getattr(self.model.layers, "_unsloth_block_swap", None)
@@ -535,6 +548,7 @@ def _FalconH1_fast_forward_inference(
position_ids = position_ids,
attention_mask = attention_mask,
do_prefill = not hasattr(decoder_layer.self_attn, "paged_attention"),
+ rotary_seq_len = rotary_seq_len,
)
attention_hidden_states = attention_hidden_states * decoder_layer.attn_out_multiplier
mamba_hidden_states = decoder_layer.mamba(
diff --git a/unsloth/models/gemma2.py b/unsloth/models/gemma2.py
index 2f0c3a57ce6..fa1ef3a8605 100644
--- a/unsloth/models/gemma2.py
+++ b/unsloth/models/gemma2.py
@@ -11,6 +11,7 @@
from .llama import *
from .llama import (
+ _decode_rotary_seq_len,
original_apply_qkv,
original_apply_o,
_gqa_decode_scores,
@@ -433,7 +434,13 @@ def Gemma2Attention_fast_forward_inference(
Kn = Kn.view(bsz, 1, n_kv_heads, head_dim).transpose(1, 2)
Vn = Vn.view(bsz, 1, n_kv_heads, head_dim).transpose(1, 2)
- cos, sin = self.rotary_emb.get_cached(kv_seq_len, Qn.device.index)
+ # rotary_seq_len (once per step from the decode loop) covers explicit positions past the KV length.
+ rotary_seq_len = kwargs.get("rotary_seq_len")
+ if rotary_seq_len is not None and rotary_seq_len > kv_seq_len:
+ self.rotary_emb.extend_rope_embedding(Vn, rotary_seq_len + 1)
+ cos, sin = self.rotary_emb.get_cached(rotary_seq_len, Qn.device.index)
+ else:
+ cos, sin = self.rotary_emb.get_cached(kv_seq_len, Qn.device.index)
# Transformers 5.x: position_ids may be [batch, full_seq_len]; slice to last.
if position_ids.dim() >= 2 and position_ids.shape[-1] > 1:
position_ids = position_ids[:, -1:]
@@ -568,6 +575,12 @@ def Gemma2Model_fast_forward_inference(
# One row has no padding; the raw 2D mask cannot be sliced to the sliding window.
SWA = None
GA = None
+ # Once per step for every layer, never per layer (one host sync at most).
+ rotary_seq_len = _decode_rotary_seq_len(
+ getattr(getattr(self.model.layers[0], "self_attn", None), "rotary_emb", None),
+ seq_len + 1,
+ position_ids,
+ )
next_decoder_cache = []
block_swap = getattr(self.model.layers, "_unsloth_block_swap", None)
for idx, decoder_layer in enumerate(self.model.layers):
@@ -599,6 +612,7 @@ def Gemma2Model_fast_forward_inference(
use_sliding_window = use_sliding_window,
flash_decode = flash_decode,
leftpad = leftpad,
+ rotary_seq_len = rotary_seq_len,
)
hidden_states = fast_rms_layernorm_inference_gemma(
decoder_layer.post_attention_layernorm,
diff --git a/unsloth/models/granite.py b/unsloth/models/granite.py
index 13b2891f594..e980bd84298 100644
--- a/unsloth/models/granite.py
+++ b/unsloth/models/granite.py
@@ -25,6 +25,7 @@ from ..utils.attention_dispatch import (
SDPA,
)
from .llama import (
+ _decode_rotary_seq_len,
LlamaRotaryEmbedding,
LlamaLinearScalingRotaryEmbedding,
_gqa_decode_scores,
@@ -458,8 +459,12 @@ def GraniteModel_fast_forward_inference(
attention_mask = None
# Decode positions run past max_seq_length; the cache is only built for the rows asked for.
+ # Explicit positions can run past both: one host sync at most, here and not per layer.
+ rotary_seq_len = _decode_rotary_seq_len(self.model.rotary_emb, seq_len + q_len, position_ids)
+ if rotary_seq_len > max(self.max_seq_length, seq_len + q_len):
+ self.model.rotary_emb.extend_rope_embedding(hidden_states, rotary_seq_len + 1)
position_embeddings = self.model.rotary_emb.get_cached(
- max(self.max_seq_length, seq_len + q_len), hidden_states.device.index
+ max(self.max_seq_length, seq_len + q_len, rotary_seq_len), hidden_states.device.index
)
next_decoder_cache = []
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index 22c73cc9528..bf1406beb9c 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -2233,6 +2233,25 @@ def _round_rope_size(seq_len):
return ((seq_len // 8192) + ((seq_len % 8192) != 0)) * 8192
+def _decode_rotary_seq_len(rotary_emb, kv_seq_len, position_ids):
+ """RoPE rows one decode step reads, for decode loops that pass it to every layer.
+
+ Explicit positions can run past the KV length, and the lazy cache only builds the rows asked
+ for. Costs one host sync per step (as LlamaModel_fast_forward_inference), skipped when the
+ cache always holds every position the model supports."""
+ if position_ids is None:
+ return kv_seq_len
+ max_position_embeddings = getattr(rotary_emb, "max_position_embeddings", None)
+ floor = getattr(rotary_emb, "_rope_cache_floor", None)
+ if (
+ max_position_embeddings is not None
+ and floor is not None
+ and floor >= max_position_embeddings
+ ):
+ return kv_seq_len
+ return max(kv_seq_len, int(position_ids.max().item()) + 1)
+
+
class _PerDeviceRopeCache:
"""cos/sin built lazily per device with only the rows asked for, so torchrun ranks do not
allocate on their peers. A row depends only on its position, so a shorter cache matches the