Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Qwen3 decode: size the RoPE read by the explicit position, not only the KV length (#14156)
Daniel Han3697c7d62084
diff --git a/tests/test_qwen3_decode_offset_rope.py b/tests/test_qwen3_decode_offset_rope.pynew file mode 100644index 00000000000..80cc2d2047b--- /dev/null+++ b/tests/test_qwen3_decode_offset_rope.py@@ -0,0 +1,77 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.+"""Qwen3 decode with an explicit position past the rows the prefill built.++The RoPE cache only builds the rows a forward asks for, so a prefill without position_ids on a+model whose max_position_embeddings exceeds the 32768-row starting cache leaves later rows unbuilt.+A decode step at such a position must still read a valid row, matching the decode after a prefill+that built every row. Runs in a subprocess because an out-of-range index is a CUDA device assert.+"""++import os+import subprocess+import sys+import textwrap++import pytest++torch = pytest.importorskip("torch")+pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs CUDA")++MODEL = "trl-internal-testing/tiny-Qwen3ForCausalLM" # max_position_embeddings 40960++SCRIPT = textwrap.dedent(+ """+ import sys+ import unsloth+ from unsloth import FastLanguageModel+ import torch++ pos = int(sys.argv[2])+ model, _ = FastLanguageModel.from_pretrained(+ sys.argv[1], max_seq_length = 2048, load_in_4bit = False, dtype = torch.float16+ )+ FastLanguageModel.for_inference(model)+ ids = torch.tensor([[1, 2, 3, 4]], device = "cuda")++ def logits(out):+ return out.logits if hasattr(out, "logits") else out[0]++ def decode(prefill_position_ids):+ with torch.no_grad():+ out = model(input_ids = ids, position_ids = prefill_position_ids, use_cache = True)+ out = model(+ input_ids = ids[:, :1],+ past_key_values = out.past_key_values,+ position_ids = torch.tensor([[pos]], device = "cuda"),+ use_cache = True,+ )+ torch.cuda.synchronize()+ return logits(out).float()++ lazy = decode(None)+ full = decode(torch.arange(4, device = "cuda")[None])+ assert torch.isfinite(lazy).all()+ torch.testing.assert_close(lazy, full, rtol = 0, atol = 0)+ print("OK")+ """+)+++def test_qwen3_decode_reads_rows_past_the_lazy_prefill_cache(tmp_path):+ script = tmp_path / "decode_offset.py"+ script.write_text(SCRIPT)+ env = dict(os.environ)+ repo_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+ env["PYTHONPATH"] = repo_root + os.pathsep + env.get("PYTHONPATH", "")+ env.setdefault("UNSLOTH_COMPILE_LOCATION", str(tmp_path / "unsloth_compiled_cache"))+ result = subprocess.run(+ [sys.executable, str(script), MODEL, "35000"],+ capture_output = True,+ text = True,+ env = env,+ timeout = 900,+ )+ assert result.returncode == 0 and "OK" in result.stdout, (+ result.stdout[-2000:] + result.stderr[-4000:]+ )diff --git a/unsloth/models/qwen3.py b/unsloth/models/qwen3.pyindex 3f1bee48e51..c9e57eef21b 100644--- a/unsloth/models/qwen3.py+++ b/unsloth/models/qwen3.py@@ -261,8 +261,12 @@ def Qwen3Attention_fast_forward_inference(Kn = Kn.transpose(1, 2)# Must be done 2 steps before hitting full on a short KV cache, or it errors.- self.rotary_emb.extend_rope_embedding(Vn, seq_len + 2)- cos, sin = self.rotary_emb.get_cached(kv_seq_len, Qn.device.index)+ # rotary_seq_len (from the model-level decode loop, no extra sync) covers explicit positions+ # past the KV length, since the cache only builds the rows asked for.+ rotary_seq_len = kwargs.get("rotary_seq_len")+ rope_len = kv_seq_len if rotary_seq_len is None else max(kv_seq_len, rotary_seq_len)+ self.rotary_emb.extend_rope_embedding(Vn, max(seq_len + 2, rope_len + 1))+ cos, sin = self.rotary_emb.get_cached(rope_len, Qn.device.index)# Transformers 5.x: position_ids may be [batch, full_seq_len]; slice to last.if position_ids.dim() >= 2 and position_ids.shape[-1] > 1:position_ids = position_ids[:, -1:]