Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Cover explicit RoPE position ids and compute the starting cache on CPU (#13976)

Daniel Han815829f73b36
diff --git a/tests/test_rope_cache_current_device.py b/tests/test_rope_cache_current_device.py
index 1dd8b6b483f..dc34228dd44 100644
--- a/tests/test_rope_cache_current_device.py
+++ b/tests/test_rope_cache_current_device.py
@@ -266,3 +266,151 @@ def test_covered_reads_still_rebuild_a_device_left_stale_by_growth():
assert torch.equal(cos, ref_cos)
rope.extend_rope_embedding(_x(1, torch.float16), 11)
assert rope.multi_gpu_cos_cached[1] is cos
+
+
+def _model_forward_rope(
+ rope,
+ seq_length_with_past,
+ position_ids,
+ max_position,
+ max_seq_length = 2048,
+):
+ # Runs the rotary block of LlamaModel_fast_forward as written in llama.py.
+ tree = ast.parse(LLAMA.read_text(encoding = "utf-8"))
+ fn = next(
+ n
+ for n in ast.walk(tree)
+ if isinstance(n, ast.FunctionDef) and n.name == "LlamaModel_fast_forward"
+ )
+ block = next(
+ n
+ for n in fn.body
+ if isinstance(n, ast.If) and "IS_ATTENTION_REFACTOR" in ast.unparse(n.test)
+ )
+ scope = {
+ "self": types.SimpleNamespace(
+ rotary_emb = rope,
+ max_seq_length = max_seq_length,
+ config = types.SimpleNamespace(max_position_embeddings = max_position),
+ ),
+ "hidden_states": _x(0, torch.float16),
+ "seq_length_with_past": seq_length_with_past,
+ "position_ids": position_ids,
+ "IS_ATTENTION_REFACTOR": True,
+ "IS_GRANITE": False,
+ "torch": torch,
+ # isinstance needs the class this rope was built from; other ropes never match.
+ "LongRopeRotaryEmbedding": type(rope)
+ if type(rope).__name__ == "LongRopeRotaryEmbedding"
+ else type("LongRopeRotaryEmbedding", (), {}),
+ "max": max,
+ "getattr": getattr,
+ }
+ exec(compile(ast.Module(body = [block], type_ignores = []), str(LLAMA), "exec"), scope)
+ return scope["position_embeddings"]
+
+
+def test_model_forward_covers_explicit_offset_position_ids():
+ # Valid positions past the batch length (offsets, generate, packing) must index real rows.
+ max_position = 1 << 17
+ position_ids = torch.tensor([[40000, 40001]])
+ cos, sin = _model_forward_rope(_llama(max_position), 2, position_ids, max_position)
+ assert cos.shape[0] >= int(position_ids.max()) + 1
+ ref_cos, ref_sin = _reference(cos.shape[0], torch.float16)
+ assert torch.equal(cos[position_ids], ref_cos[position_ids])
+ assert torch.equal(sin[position_ids], ref_sin[position_ids])
+
+
+def test_model_forward_without_position_ids_stays_short():
+ # Implicit positions are 0..seq_length_with_past-1, so the short cache is enough.
+ max_position = 1 << 20
+ rope = _llama(max_position)
+ cos, _ = _model_forward_rope(rope, 2, None, max_position)
+ assert cos.shape[0] == 4 * 8192
+ assert _built(rope.multi_gpu_cos_cached) == [0]
+
+
+def test_starting_cache_is_computed_on_cpu():
+ # The eager __init__ computed the starting cache on CPU even when loading later moved inv_freq to
+ # an accelerator; computing it there instead raises the first forward's peak memory.
+ rope = _llama(4096)
+ rope.inv_freq = rope.inv_freq.to("meta")
+ cos, sin = rope.get_cached(10, 0)
+ ref_cos, ref_sin = _reference(4096)
+ assert torch.equal(cos, ref_cos) and torch.equal(sin, ref_sin)
+
+
+def test_model_forward_position_ids_keep_longrope_short_factors():
+ # Building every row for explicit positions must not switch a short input to the long factors.
+ rope = _llama_ns()["LongRopeRotaryEmbedding"](
+ dim = DIM,
+ max_position_embeddings = 128,
+ original_max_position_embeddings = 16,
+ short_factor = [1.0] * (DIM // 2),
+ long_factor = [2.0] * (DIM // 2),
+ )
+ cos, sin = _model_forward_rope(rope, 4, torch.tensor([[0, 1, 2, 3]]), 128, max_seq_length = 8)
+ assert cos is rope.multi_gpu_short_cos_cached[0]
+ assert sin is rope.multi_gpu_short_sin_cached[0]
+
+
+def _longrope(max_position, original_max_position):
+ return _llama_ns()["LongRopeRotaryEmbedding"](
+ dim = DIM,
+ max_position_embeddings = max_position,
+ original_max_position_embeddings = original_max_position,
+ short_factor = [1.0] * (DIM // 2),
+ long_factor = [2.0] * (DIM // 2),
+ )
+
+
+def test_model_forward_longrope_offset_positions_match_transformers():
+ # transformers picks LongRoPE's factors from max(position_ids) + 1, so 4 tokens at 40000 with a 4096
+ # pretrained window use the long factors; the short cache has no row 40000.
+ from transformers import Phi3Config
+ from transformers.models.phi3.modeling_phi3 import Phi3RotaryEmbedding
+
+ max_position, original_max_position = 1 << 17, 4096
+ position_ids = torch.tensor([[40000, 40001, 40002, 40003]])
+ rope = _longrope(max_position, original_max_position)
+ cos, sin = _model_forward_rope(rope, 4, position_ids, max_position)
+ assert cos.shape[0] > int(position_ids.max())
+
+ config = Phi3Config(
+ hidden_size = DIM,
+ num_attention_heads = 1,
+ num_key_value_heads = 1,
+ max_position_embeddings = max_position,
+ original_max_position_embeddings = original_max_position,
+ rope_scaling = {
+ "type": "longrope",
+ "short_factor": [1.0] * (DIM // 2),
+ "long_factor": [2.0] * (DIM // 2),
+ },
+ )
+ ref_cos, ref_sin = Phi3RotaryEmbedding(config)(torch.zeros(1, 4, DIM), position_ids)
+ torch.testing.assert_close(cos[position_ids[0]].float(), ref_cos[0], atol = 2e-3, rtol = 0)
+ torch.testing.assert_close(sin[position_ids[0]].float(), ref_sin[0], atol = 2e-3, rtol = 0)
+
+
[email protected]("start", [0, 40000])
+def test_model_forward_longrope_under_compile_matches_eager(monkeypatch, start):
+ # While tracing, the choice still follows max(position_ids) + 1, so compiled runs match eager.
+ position_ids = torch.arange(start, start + 4)[None]
+ eager_cos, eager_sin = _model_forward_rope(_longrope(1 << 17, 4096), 4, position_ids, 1 << 17)
+ monkeypatch.setattr(torch.compiler, "is_compiling", lambda: True)
+ cos, sin = _model_forward_rope(_longrope(1 << 17, 4096), 4, position_ids, 1 << 17)
+ assert torch.equal(cos[position_ids[0]], eager_cos[position_ids[0]])
+ assert torch.equal(sin[position_ids[0]], eager_sin[position_ids[0]])
+
+
+def test_first_grown_cache_is_computed_on_cpu():
+ # A configured max above the 4 * 8192 floor grows the size on the first forward, before any cache
+ # exists; that first build must still run on CPU, while later growth uses inv_freq's device.
+ rope = _llama(1 << 17)
+ rope.inv_freq = rope.inv_freq.to("meta")
+ cos, sin = _model_forward_rope(rope, 2, None, 1 << 17)
+ ref_cos, ref_sin = _reference(cos.shape[0], torch.float16)
+ assert torch.equal(cos, ref_cos) and torch.equal(sin, ref_sin)
+ with pytest.raises(NotImplementedError):
+ rope.extend_rope_embedding(_x(0, torch.float16), 1 << 18)
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index 433d55148a6..ccca75cfd83 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -1419,13 +1419,31 @@ def LlamaModel_fast_forward(
# had the attention refactor, so it always takes this path.
# max_position_embeddings still sets the cache size and dtype as before, but only the rows
# this batch (or the declared max_seq_length) can index are built, on this device only.
- rope_rows = max(seq_length_with_past, getattr(self, "max_seq_length", None) or 0)
+ rope_len = max(seq_length_with_past, getattr(self, "max_seq_length", None) or 0)
+ rope_rows = rope_len
+ if position_ids is not None:
+ # Explicit positions (offsets, generate, packing) can index any row below
+ # max_position_embeddings; bounding them needs a host sync, so build them all as before.
+ rope_rows = max(rope_rows, self.config.max_position_embeddings)
self.rotary_emb.extend_rope_embedding(
hidden_states,
self.config.max_position_embeddings,
materialize_len = rope_rows,
)
- position_embeddings = self.rotary_emb.get_cached(rope_rows, hidden_states.device.index)
+ device_index = hidden_states.device.index
+ original = getattr(self.rotary_emb, "original_max_position_embeddings", None)
+ if (
+ position_ids is not None
+ and isinstance(self.rotary_emb, LongRopeRotaryEmbedding)
+ and rope_len <= original < rope_rows
+ ):
+ # transformers' longrope_frequency_update picks the factors from max(position_ids) + 1, a host
+ # sync it pays too; other models and LongRoPE without position_ids stay sync-free.
+ # Also under compile (one graph break): an in-graph choice would rebuild full-size hybrid
+ # tables every forward, and this forward is not compiled by Unsloth.
+ rope_len = max(rope_len, int(position_ids.max()) + 1)
+ # rope_rows only sizes the build; LongRoPE picks its short or long factors from rope_len.
+ position_embeddings = self.rotary_emb.get_cached(rope_len, device_index)
else:
position_embeddings = None
@@ -2338,15 +2356,22 @@ class LlamaRotaryEmbedding(_PerDeviceRopeCache, torch.nn.Module):
# Rebuild inv_freq instead of reading the buffer: transformers v5 refills it with uninitialized
# memory on models _fix_rope_inv_freq never sees, e.g. a PPO value / reward model loaded after
# Unsloth patched the classes, so extend_rope_embedding built NaN or garbage cos/sin (#884).
- self.inv_freq = self._unsloth_recompute_inv_freq().to(self.inv_freq.device)
+ inv_freq = self._unsloth_recompute_inv_freq()
+ self.inv_freq = inv_freq.to(self.inv_freq.device)
+ # The first cache built (and every device's copy of that size) is computed on CPU like the eager
+ # __init__ build was, even when the first forward already grew the size, so the first forward's
+ # peak memory does not rise; later growth computes on inv_freq's device as before.
+ version = getattr(self, "_rope_cache_version", 0)
+ if getattr(self, "_rope_cache_cpu_version", None) is None:
+ self._rope_cache_cpu_version = version
+ if version != self._rope_cache_cpu_version:
+ inv_freq = self.inv_freq
# The original Llama codebase creates these on the target device in FP32 and multiplies in FP32.
self.current_rope_size = seq_len
- t = torch.arange(
- self.current_rope_size, device = self.inv_freq.device, dtype = torch.int64
- ).float()
+ t = torch.arange(self.current_rope_size, device = inv_freq.device, dtype = torch.int64).float()
t = self._apply_time_scaling(t)
- freqs = torch.outer(t, self.inv_freq)
+ freqs = torch.outer(t, inv_freq)
# Different from the paper, but it uses a different permutation to obtain the same calculation.
emb = torch.cat((freqs, freqs), dim = -1)
# Applied here so attention_scaling survives extend_rope_embedding rebuilds; the 1.0 default