Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Cover explicit RoPE position ids and compute the starting cache on CPU (#13976)
Daniel Han815829f73b36
diff --git a/tests/test_rope_cache_current_device.py b/tests/test_rope_cache_current_device.pyindex 1dd8b6b483f..dc34228dd44 100644--- a/tests/test_rope_cache_current_device.py+++ b/tests/test_rope_cache_current_device.py@@ -266,3 +266,151 @@ def test_covered_reads_still_rebuild_a_device_left_stale_by_growth():assert torch.equal(cos, ref_cos)rope.extend_rope_embedding(_x(1, torch.float16), 11)assert rope.multi_gpu_cos_cached[1] is cos+++def _model_forward_rope(+ rope,+ seq_length_with_past,+ position_ids,+ max_position,+ max_seq_length = 2048,+):+ # Runs the rotary block of LlamaModel_fast_forward as written in llama.py.+ tree = ast.parse(LLAMA.read_text(encoding = "utf-8"))+ fn = next(+ n+ for n in ast.walk(tree)+ if isinstance(n, ast.FunctionDef) and n.name == "LlamaModel_fast_forward"+ )+ block = next(+ n+ for n in fn.body+ if isinstance(n, ast.If) and "IS_ATTENTION_REFACTOR" in ast.unparse(n.test)+ )+ scope = {+ "self": types.SimpleNamespace(+ rotary_emb = rope,+ max_seq_length = max_seq_length,+ config = types.SimpleNamespace(max_position_embeddings = max_position),+ ),+ "hidden_states": _x(0, torch.float16),+ "seq_length_with_past": seq_length_with_past,+ "position_ids": position_ids,+ "IS_ATTENTION_REFACTOR": True,+ "IS_GRANITE": False,+ "torch": torch,+ # isinstance needs the class this rope was built from; other ropes never match.+ "LongRopeRotaryEmbedding": type(rope)+ if type(rope).__name__ == "LongRopeRotaryEmbedding"+ else type("LongRopeRotaryEmbedding", (), {}),+ "max": max,+ "getattr": getattr,+ }+ exec(compile(ast.Module(body = [block], type_ignores = []), str(LLAMA), "exec"), scope)+ return scope["position_embeddings"]+++def test_model_forward_covers_explicit_offset_position_ids():+ # Valid positions past the batch length (offsets, generate, packing) must index real rows.+ max_position = 1 << 17+ position_ids = torch.tensor([[40000, 40001]])+ cos, sin = _model_forward_rope(_llama(max_position), 2, position_ids, max_position)+ assert cos.shape[0] >= int(position_ids.max()) + 1+ ref_cos, ref_sin = _reference(cos.shape[0], torch.float16)+ assert torch.equal(cos[position_ids], ref_cos[position_ids])+ assert torch.equal(sin[position_ids], ref_sin[position_ids])+++def test_model_forward_without_position_ids_stays_short():+ # Implicit positions are 0..seq_length_with_past-1, so the short cache is enough.+ max_position = 1 << 20+ rope = _llama(max_position)+ cos, _ = _model_forward_rope(rope, 2, None, max_position)+ assert cos.shape[0] == 4 * 8192+ assert _built(rope.multi_gpu_cos_cached) == [0]+++def test_starting_cache_is_computed_on_cpu():+ # The eager __init__ computed the starting cache on CPU even when loading later moved inv_freq to+ # an accelerator; computing it there instead raises the first forward's peak memory.+ rope = _llama(4096)+ rope.inv_freq = rope.inv_freq.to("meta")+ cos, sin = rope.get_cached(10, 0)+ ref_cos, ref_sin = _reference(4096)+ assert torch.equal(cos, ref_cos) and torch.equal(sin, ref_sin)+++def test_model_forward_position_ids_keep_longrope_short_factors():+ # Building every row for explicit positions must not switch a short input to the long factors.+ rope = _llama_ns()["LongRopeRotaryEmbedding"](+ dim = DIM,+ max_position_embeddings = 128,+ original_max_position_embeddings = 16,+ short_factor = [1.0] * (DIM // 2),+ long_factor = [2.0] * (DIM // 2),+ )+ cos, sin = _model_forward_rope(rope, 4, torch.tensor([[0, 1, 2, 3]]), 128, max_seq_length = 8)+ assert cos is rope.multi_gpu_short_cos_cached[0]+ assert sin is rope.multi_gpu_short_sin_cached[0]+++def _longrope(max_position, original_max_position):+ return _llama_ns()["LongRopeRotaryEmbedding"](+ dim = DIM,+ max_position_embeddings = max_position,+ original_max_position_embeddings = original_max_position,+ short_factor = [1.0] * (DIM // 2),+ long_factor = [2.0] * (DIM // 2),+ )+++def test_model_forward_longrope_offset_positions_match_transformers():+ # transformers picks LongRoPE's factors from max(position_ids) + 1, so 4 tokens at 40000 with a 4096+ # pretrained window use the long factors; the short cache has no row 40000.+ from transformers import Phi3Config+ from transformers.models.phi3.modeling_phi3 import Phi3RotaryEmbedding++ max_position, original_max_position = 1 << 17, 4096+ position_ids = torch.tensor([[40000, 40001, 40002, 40003]])+ rope = _longrope(max_position, original_max_position)+ cos, sin = _model_forward_rope(rope, 4, position_ids, max_position)+ assert cos.shape[0] > int(position_ids.max())++ config = Phi3Config(+ hidden_size = DIM,+ num_attention_heads = 1,+ num_key_value_heads = 1,+ max_position_embeddings = max_position,+ original_max_position_embeddings = original_max_position,+ rope_scaling = {+ "type": "longrope",+ "short_factor": [1.0] * (DIM // 2),+ "long_factor": [2.0] * (DIM // 2),+ },+ )+ ref_cos, ref_sin = Phi3RotaryEmbedding(config)(torch.zeros(1, 4, DIM), position_ids)+ torch.testing.assert_close(cos[position_ids[0]].float(), ref_cos[0], atol = 2e-3, rtol = 0)+ torch.testing.assert_close(sin[position_ids[0]].float(), ref_sin[0], atol = 2e-3, rtol = 0)++[email protected]("start", [0, 40000])+def test_model_forward_longrope_under_compile_matches_eager(monkeypatch, start):+ # While tracing, the choice still follows max(position_ids) + 1, so compiled runs match eager.+ position_ids = torch.arange(start, start + 4)[None]+ eager_cos, eager_sin = _model_forward_rope(_longrope(1 << 17, 4096), 4, position_ids, 1 << 17)+ monkeypatch.setattr(torch.compiler, "is_compiling", lambda: True)+ cos, sin = _model_forward_rope(_longrope(1 << 17, 4096), 4, position_ids, 1 << 17)+ assert torch.equal(cos[position_ids[0]], eager_cos[position_ids[0]])+ assert torch.equal(sin[position_ids[0]], eager_sin[position_ids[0]])+++def test_first_grown_cache_is_computed_on_cpu():+ # A configured max above the 4 * 8192 floor grows the size on the first forward, before any cache+ # exists; that first build must still run on CPU, while later growth uses inv_freq's device.+ rope = _llama(1 << 17)+ rope.inv_freq = rope.inv_freq.to("meta")+ cos, sin = _model_forward_rope(rope, 2, None, 1 << 17)+ ref_cos, ref_sin = _reference(cos.shape[0], torch.float16)+ assert torch.equal(cos, ref_cos) and torch.equal(sin, ref_sin)+ with pytest.raises(NotImplementedError):+ rope.extend_rope_embedding(_x(0, torch.float16), 1 << 18)diff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex 433d55148a6..ccca75cfd83 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -1419,13 +1419,31 @@ def LlamaModel_fast_forward(# had the attention refactor, so it always takes this path.# max_position_embeddings still sets the cache size and dtype as before, but only the rows# this batch (or the declared max_seq_length) can index are built, on this device only.- rope_rows = max(seq_length_with_past, getattr(self, "max_seq_length", None) or 0)+ rope_len = max(seq_length_with_past, getattr(self, "max_seq_length", None) or 0)+ rope_rows = rope_len+ if position_ids is not None:+ # Explicit positions (offsets, generate, packing) can index any row below+ # max_position_embeddings; bounding them needs a host sync, so build them all as before.+ rope_rows = max(rope_rows, self.config.max_position_embeddings)self.rotary_emb.extend_rope_embedding(hidden_states,self.config.max_position_embeddings,materialize_len = rope_rows,)- position_embeddings = self.rotary_emb.get_cached(rope_rows, hidden_states.device.index)+ device_index = hidden_states.device.index+ original = getattr(self.rotary_emb, "original_max_position_embeddings", None)+ if (+ position_ids is not None+ and isinstance(self.rotary_emb, LongRopeRotaryEmbedding)+ and rope_len <= original < rope_rows+ ):+ # transformers' longrope_frequency_update picks the factors from max(position_ids) + 1, a host+ # sync it pays too; other models and LongRoPE without position_ids stay sync-free.+ # Also under compile (one graph break): an in-graph choice would rebuild full-size hybrid+ # tables every forward, and this forward is not compiled by Unsloth.+ rope_len = max(rope_len, int(position_ids.max()) + 1)+ # rope_rows only sizes the build; LongRoPE picks its short or long factors from rope_len.+ position_embeddings = self.rotary_emb.get_cached(rope_len, device_index)else:position_embeddings = None@@ -2338,15 +2356,22 @@ class LlamaRotaryEmbedding(_PerDeviceRopeCache, torch.nn.Module):# Rebuild inv_freq instead of reading the buffer: transformers v5 refills it with uninitialized# memory on models _fix_rope_inv_freq never sees, e.g. a PPO value / reward model loaded after# Unsloth patched the classes, so extend_rope_embedding built NaN or garbage cos/sin (#884).- self.inv_freq = self._unsloth_recompute_inv_freq().to(self.inv_freq.device)+ inv_freq = self._unsloth_recompute_inv_freq()+ self.inv_freq = inv_freq.to(self.inv_freq.device)+ # The first cache built (and every device's copy of that size) is computed on CPU like the eager+ # __init__ build was, even when the first forward already grew the size, so the first forward's+ # peak memory does not rise; later growth computes on inv_freq's device as before.+ version = getattr(self, "_rope_cache_version", 0)+ if getattr(self, "_rope_cache_cpu_version", None) is None:+ self._rope_cache_cpu_version = version+ if version != self._rope_cache_cpu_version:+ inv_freq = self.inv_freq# The original Llama codebase creates these on the target device in FP32 and multiplies in FP32.self.current_rope_size = seq_len- t = torch.arange(- self.current_rope_size, device = self.inv_freq.device, dtype = torch.int64- ).float()+ t = torch.arange(self.current_rope_size, device = inv_freq.device, dtype = torch.int64).float()t = self._apply_time_scaling(t)- freqs = torch.outer(t, self.inv_freq)+ freqs = torch.outer(t, inv_freq)# Different from the paper, but it uses a different permutation to obtain the same calculation.emb = torch.cat((freqs, freqs), dim = -1)# Applied here so attention_scaling survives extend_rope_embedding rebuilds; the 1.0 default