Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

RoPE: build grown caches on the GPU again, 32768 rows at a time (#14206)

Daniel Han0cf4929de047
diff --git a/tests/test_rope_cache_current_device.py b/tests/test_rope_cache_current_device.py
index dc34228dd44..1e2579e88ff 100644
--- a/tests/test_rope_cache_current_device.py
+++ b/tests/test_rope_cache_current_device.py
@@ -404,13 +404,23 @@ def test_model_forward_longrope_under_compile_matches_eager(monkeypatch, start):
assert torch.equal(sin[position_ids[0]], eager_sin[position_ids[0]])
-def test_first_grown_cache_is_computed_on_cpu():
- # A configured max above the 4 * 8192 floor grows the size on the first forward, before any cache
- # exists; that first build must still run on CPU, while later growth uses inv_freq's device.
+def test_first_grown_cache_is_computed_on_inv_freq_device():
+ # A configured max above the 4 * 8192 floor grows the size on the first forward. The eager code then
+ # rebuilt it on inv_freq's device, so the lazy build must too: on CPU it took 0.3-1.6 s for 128K-1M
+ # rows and rounded cos/sin differently from every earlier release.
rope = _llama(1 << 17)
rope.inv_freq = rope.inv_freq.to("meta")
- cos, sin = _model_forward_rope(rope, 2, None, 1 << 17)
- ref_cos, ref_sin = _reference(cos.shape[0], torch.float16)
- assert torch.equal(cos, ref_cos) and torch.equal(sin, ref_sin)
with pytest.raises(NotImplementedError):
- rope.extend_rope_embedding(_x(0, torch.float16), 1 << 18)
+ _model_forward_rope(rope, 2, None, 1 << 17)
+
+
[email protected]("rows", [40, 64])
[email protected]("dtype", [torch.float32, torch.float16, torch.bfloat16])
+def test_cache_built_in_chunks_matches_one_shot(rows, dtype):
+ # Building row chunks into the output must keep every row bit-identical to the one-shot build.
+ rope = _llama(rows)
+ rope._rope_build_chunk_rows = 16
+ cos, sin = rope._set_cos_sin_cache(rows, torch.device("cpu", 0), dtype)
+ ref_cos, ref_sin = _reference(rows, dtype)
+ assert cos.shape == ref_cos.shape and cos.dtype == dtype
+ assert torch.equal(cos, ref_cos) and torch.equal(sin, ref_sin)
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index b55a12578c7..1c623178703 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -2419,32 +2419,50 @@ class LlamaRotaryEmbedding(_PerDeviceRopeCache, torch.nn.Module):
"""Override to apply custom time scaling (e.g., linear scaling)."""
return t
+ _rope_build_chunk_rows = 32768
+
def _set_cos_sin_cache(self, seq_len, device, dtype):
# Rebuild inv_freq instead of reading the buffer: transformers v5 refills it with uninitialized
# memory on models _fix_rope_inv_freq never sees, e.g. a PPO value / reward model loaded after
# Unsloth patched the classes, so extend_rope_embedding built NaN or garbage cos/sin (#884).
inv_freq = self._unsloth_recompute_inv_freq()
self.inv_freq = inv_freq.to(self.inv_freq.device)
- # The first cache built (and every device's copy of that size) is computed on CPU like the eager
- # __init__ build was, even when the first forward already grew the size, so the first forward's
- # peak memory does not rise; later growth computes on inv_freq's device as before.
- version = getattr(self, "_rope_cache_version", 0)
- if getattr(self, "_rope_cache_cpu_version", None) is None:
- self._rope_cache_cpu_version = version
- if version != self._rope_cache_cpu_version:
+ # Same device as before lazy building: the starting size (every device's copy) is computed on
+ # CPU like the eager __init__ build, any grown size on inv_freq's device like the eager
+ # extend_rope_embedding. Computing a grown cache (e.g. 131072 rows when max_position_embeddings
+ # exceeds 4 * 8192) on CPU instead takes 100s of ms to seconds and changes cos/sin rounding.
+ if getattr(self, "_rope_cache_version", 0) != 0:
inv_freq = self.inv_freq
# The original Llama codebase creates these on the target device in FP32 and multiplies in FP32.
self.current_rope_size = seq_len
t = torch.arange(self.current_rope_size, device = inv_freq.device, dtype = torch.int64).float()
t = self._apply_time_scaling(t)
- freqs = torch.outer(t, inv_freq)
- # Different from the paper, but it uses a different permutation to obtain the same calculation.
- emb = torch.cat((freqs, freqs), dim = -1)
- # Applied here so attention_scaling survives extend_rope_embedding rebuilds; the 1.0 default
- # keeps unscaled paths bit-identical.
- cos = (emb.cos() * self.attention_scaling).to(dtype = dtype, device = device, non_blocking = True)
- sin = (emb.sin() * self.attention_scaling).to(dtype = dtype, device = device, non_blocking = True)
+ # Rows are independent, so the build goes 32768 rows at a time into the output (bit-identical,
+ # the same elementwise ops): the fp32 temporaries no longer scale with the cache size (about
+ # 2 GiB at 1M rows in one shot).
+ rows = self._rope_build_chunk_rows
+ cos = sin = None
+ for start in range(0, max(seq_len, 1), rows):
+ freqs = torch.outer(t[start : start + rows], inv_freq)
+ # Different from the paper, but it uses a different permutation to obtain the same calculation.
+ emb = torch.cat((freqs, freqs), dim = -1)
+ # Applied here so attention_scaling survives extend_rope_embedding rebuilds; the 1.0 default
+ # keeps unscaled paths bit-identical.
+ cos_rows = (emb.cos() * self.attention_scaling).to(
+ dtype = dtype, device = device, non_blocking = True
+ )
+ sin_rows = (emb.sin() * self.attention_scaling).to(
+ dtype = dtype, device = device, non_blocking = True
+ )
+ if rows >= seq_len:
+ cos, sin = cos_rows, sin_rows
+ break
+ if cos is None:
+ cos = torch.empty((seq_len, emb.shape[-1]), dtype = dtype, device = device)
+ sin = torch.empty_like(cos)
+ cos[start : start + rows] = cos_rows
+ sin[start : start + rows] = sin_rows
self.multi_gpu_cos_cached[device.index] = cos
self.multi_gpu_sin_cached[device.index] = sin
return cos, sin