Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Build RoPE cos/sin caches only on the device that uses them (#13855)

Daniel Hanaa0b22b2ca9f
diff --git a/tests/test_device_helpers.py b/tests/test_device_helpers.py
index 4341e3da231..282432079ec 100644
--- a/tests/test_device_helpers.py
+++ b/tests/test_device_helpers.py
@@ -197,7 +197,7 @@ def test_model_call_sites_use_shared_cache_dispatch():
assert "device_context" not in vision_source
assert 'if DEVICE_TYPE == "xpu":\n vllm_version = ""' in vision_source
assert "torch.cuda.current_device()" not in gemma_source
- assert gemma_source.count("get_current_device()") >= 3
+ assert "get_current_device()" in gemma_source
assert "torch.cuda.empty_cache()" not in gemma_source
assert "clean_gpu_cache()" in gemma_source
assert "torch.cuda.empty_cache()" not in gemma2_source
diff --git a/tests/test_longrope_short_long_boundary.py b/tests/test_longrope_short_long_boundary.py
index f81e8b3594e..110887de366 100644
--- a/tests/test_longrope_short_long_boundary.py
+++ b/tests/test_longrope_short_long_boundary.py
@@ -57,6 +57,13 @@ def _load_longrope():
"LongRopeRotaryEmbedding is gone from unsloth/models/llama.py. If it moved or was "
"renamed, repoint this test rather than deleting it."
)
+ # The per-device cache helpers it inherits from, when this checkout has them.
+ helpers = [
+ n
+ for n in tree.body
+ if isinstance(n, (ast.ClassDef, ast.FunctionDef))
+ and n.name in ("_round_rope_size", "_PerDeviceRopeCache")
+ ]
ns = {
"torch": _CPUTorch(),
"math": __import__("math"),
@@ -68,7 +75,7 @@ def _load_longrope():
"_get_rope_theta": lambda config, default = 10000: default,
}
try:
- exec(compile(ast.Module(body = [cls], type_ignores = []), str(LLAMA), "exec"), ns)
+ exec(compile(ast.Module(body = helpers + [cls], type_ignores = []), str(LLAMA), "exec"), ns)
except NameError as e:
pytest.fail(
f"LongRopeRotaryEmbedding grew a module-level dependency this test does not stub "
diff --git a/tests/test_plain_load_after_unsloth_rope.py b/tests/test_plain_load_after_unsloth_rope.py
index 94b97bd5bdc..ace7a23c8ca 100644
--- a/tests/test_plain_load_after_unsloth_rope.py
+++ b/tests/test_plain_load_after_unsloth_rope.py
@@ -57,7 +57,7 @@ rotary = model.model.rotary_emb
assert isinstance(rotary, LlamaRotaryEmbedding), type(rotary)
expected = rotary._unsloth_recompute_inv_freq().float()
torch.testing.assert_close(rotary.inv_freq.detach().cpu().float(), expected)
-cos = rotary.multi_gpu_cos_cached[torch.cuda.current_device()].float().cpu()
+cos = rotary.get_cached(device_index = torch.cuda.current_device())[0].float().cpu()
freqs = torch.outer(torch.arange(cos.shape[0], dtype = torch.float32), expected)
torch.testing.assert_close(cos, torch.cat((freqs, freqs), -1).cos(), atol = 1e-3, rtol = 0)
"""
diff --git a/tests/test_rope_cache_current_device.py b/tests/test_rope_cache_current_device.py
new file mode 100644
index 00000000000..1dd8b6b483f
--- /dev/null
+++ b/tests/test_rope_cache_current_device.py
@@ -0,0 +1,268 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+"""Unsloth's rotary caches must be built only on the device that reads them.
+
+llama.py needs an accelerator to import, so the classes are `ast`-extracted and run against a
+CPU-pinned prelude in which device index i is cpu:i.
+"""
+
+from __future__ import annotations
+
+import ast
+import math
+import pathlib
+import types
+
+import pytest
+import torch
+
+REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent
+LLAMA = REPO_ROOT / "unsloth" / "models" / "llama.py"
+GEMMA = REPO_ROOT / "unsloth" / "models" / "gemma.py"
+
+DEVICES = 4
+DIM = 8
+BASE = 10000
+
+
+class _CPUTorch:
+ """`torch`, with a bare-int device pinned to CPU so the test runs anywhere."""
+
+ def __getattr__(self, name):
+ return getattr(torch, name)
+
+ def device(
+ self,
+ spec,
+ index = None,
+ ):
+ if isinstance(spec, int) and not isinstance(spec, bool):
+ return torch.device("cpu", spec)
+ return torch.device(spec) if index is None else torch.device(spec, index)
+
+ def empty(self, *args, **kwargs):
+ kwargs["device"] = "cpu"
+ return torch.empty(*args, **kwargs)
+
+
+_HELPERS = ("_round_rope_size", "_PerDeviceRopeCache")
+
+
+def _load(
+ path,
+ names,
+ extra_ns = None,
+):
+ tree = ast.parse(path.read_text(encoding = "utf-8"))
+ nodes = []
+ for name in names:
+ node = next(
+ (
+ n
+ for n in ast.walk(tree)
+ if isinstance(n, (ast.ClassDef, ast.FunctionDef)) and n.name == name
+ ),
+ None,
+ )
+ if node is None:
+ # Helpers only exist once the cache is per device; the classes must always exist.
+ if name in _HELPERS:
+ continue
+ pytest.fail(
+ f"{name} is gone from {path.name}; repoint this test rather than deleting it."
+ )
+ nodes.append(node)
+ ns = {
+ "torch": _CPUTorch(),
+ "math": math,
+ "DEVICE_COUNT": DEVICES,
+ "DEVICE_TYPE_TORCH": "cpu",
+ "get_current_device": lambda: 0,
+ "is_bfloat16_supported": lambda: False,
+ "_get_rope_theta": lambda config, default = 10000: default,
+ "_compute_config_rope_inv_freq": lambda config, rope_scaling: (None, 1.0),
+ "_rope_scaling_as_dict": lambda value: dict(value or {}),
+ }
+ ns.update(extra_ns or {})
+ exec(compile(ast.Module(body = nodes, type_ignores = []), str(path), "exec"), ns)
+ return ns
+
+
+def _llama_ns():
+ return _load(
+ LLAMA,
+ list(_HELPERS) + ["LlamaRotaryEmbedding", "LongRopeRotaryEmbedding"],
+ )
+
+
+def _built(cache):
+ return [i for i, c in enumerate(cache) if c is not None]
+
+
+def _x(device_index, dtype = torch.float32):
+ # extend_rope_embedding / forward only read .device and .dtype.
+ return types.SimpleNamespace(device = torch.device("cpu", device_index), dtype = dtype)
+
+
+def _reference(rows, dtype = torch.float32):
+ inv_freq = 1.0 / (BASE ** (torch.arange(0, DIM, 2, dtype = torch.int64).float() / DIM))
+ t = torch.arange(rows, dtype = torch.int64).float()
+ emb = torch.cat((torch.outer(t, inv_freq),) * 2, dim = -1)
+ return emb.cos().to(dtype), emb.sin().to(dtype)
+
+
+def _llama(max_position):
+ return _llama_ns()["LlamaRotaryEmbedding"](
+ dim = DIM, max_position_embeddings = max_position, base = BASE
+ )
+
+
+def test_init_builds_no_device():
+ rope = _llama(1 << 20)
+ assert _built(rope.multi_gpu_cos_cached) == []
+ assert _built(rope.multi_gpu_sin_cached) == []
+
+
+def test_get_cached_builds_only_the_requested_device():
+ rope = _llama(1 << 20)
+ cos, sin = rope.get_cached(100, 2)
+ assert _built(rope.multi_gpu_cos_cached) == [2]
+ ref_cos, ref_sin = _reference(cos.shape[0])
+ assert cos.shape[0] == 4 * 8192
+ assert torch.equal(cos, ref_cos) and torch.equal(sin, ref_sin)
+
+
+def test_model_forward_extend_builds_needed_rows_on_one_device():
+ # LlamaModel_fast_forward: extend to max_position_embeddings, materialise this batch's rows.
+ max_position = 1 << 20
+ rope = _llama(max_position)
+ x = _x(1, torch.float16)
+ rope.extend_rope_embedding(x, max_position, materialize_len = 40000)
+ # Size and dtype move exactly as the eager extend left them.
+ assert rope.current_rope_size == max_position
+ assert _built(rope.multi_gpu_cos_cached) == [1]
+ cos, sin = rope.get_cached(40000, 1)
+ # 40000 rounded up to the 8192 growth step, not 1M rows.
+ assert cos.shape[0] == 40960
+ assert cos.dtype == torch.float16
+ ref_cos, ref_sin = _reference(40960, torch.float16)
+ assert torch.equal(cos, ref_cos) and torch.equal(sin, ref_sin)
+
+
+def test_extend_grows_used_device_and_stale_peers_rebuild_lazily():
+ rope = _llama(1 << 17)
+ rope.get_cached(10, 0)
+ rope.get_cached(10, 3)
+ rope.extend_rope_embedding(_x(3, torch.float16), 50000)
+ assert rope.current_rope_size == 57344
+ assert rope.multi_gpu_cos_cached[3].shape[0] == 57344
+ assert rope.multi_gpu_cos_cached[3].dtype == torch.float16
+ # Device 0 is untouched until it is read, then matches what the eager rebuild produced.
+ assert rope.multi_gpu_cos_cached[0].shape[0] == 4 * 8192
+ cos, sin = rope.get_cached(50000, 0)
+ ref_cos, ref_sin = _reference(57344, torch.float16)
+ assert torch.equal(cos, ref_cos) and torch.equal(sin, ref_sin)
+ assert _built(rope.multi_gpu_cos_cached) == [0, 3]
+
+
+def test_get_cached_materialises_the_length_it_is_asked_for():
+ # Gemma2 / Cohere decode steps read get_cached(kv_seq_len) with no extend first, past the rows
+ # the model forward built for max_seq_length.
+ rope = _llama(1 << 17)
+ rope.extend_rope_embedding(_x(0), 1 << 17, materialize_len = 2048)
+ assert rope.multi_gpu_cos_cached[0].shape[0] == 4 * 8192
+ cos, sin = rope.get_cached(40000, 0)
+ assert cos.shape[0] >= 40000
+ ref_cos, ref_sin = _reference(cos.shape[0])
+ assert torch.equal(cos, ref_cos) and torch.equal(sin, ref_sin)
+
+
+def test_forward_builds_only_input_device():
+ rope = _llama(4096)
+ cos, sin = rope(_x(2), seq_len = 100)
+ assert _built(rope.multi_gpu_cos_cached) == [2]
+ ref_cos, ref_sin = _reference(100)
+ assert torch.equal(cos, ref_cos) and torch.equal(sin, ref_sin)
+
+
+def test_longrope_builds_only_the_requested_device():
+ rope = _llama_ns()["LongRopeRotaryEmbedding"](
+ dim = DIM,
+ max_position_embeddings = 128,
+ original_max_position_embeddings = 16,
+ short_factor = [1.0] * (DIM // 2),
+ long_factor = [2.0] * (DIM // 2),
+ )
+ assert _built(rope.multi_gpu_short_cos_cached) == []
+ rope.get_cached(8, 1)
+ assert _built(rope.multi_gpu_short_cos_cached) == [1]
+ assert _built(rope.multi_gpu_long_cos_cached) == []
+ rope.extend_rope_embedding(_x(1, torch.float16), 100)
+ assert _built(rope.multi_gpu_long_cos_cached) == [1]
+ assert _built(rope.multi_gpu_short_cos_cached) == [1]
+
+
+def test_gemma_builds_only_the_requested_device():
+ llama_ns = _llama_ns()
+ if "_PerDeviceRopeCache" not in llama_ns:
+ pytest.fail("Gemma's rotary cache is built for every device")
+ ns = _load(
+ GEMMA,
+ ["GemmaFixedRotaryEmbedding"],
+ {"_PerDeviceRopeCache": llama_ns["_PerDeviceRopeCache"]},
+ )
+ rope = ns["GemmaFixedRotaryEmbedding"](dim = DIM, max_position_embeddings = 1 << 20, base = BASE)
+ assert _built(rope.multi_gpu_cos_cached) == []
+ rope.extend_rope_embedding(_x(3, torch.bfloat16), 70000)
+ assert _built(rope.multi_gpu_cos_cached) == [3]
+ assert rope.multi_gpu_cos_cached[3].shape[0] == 73728
+ # Gemma keeps its cache in float32 whatever the input dtype.
+ assert rope.multi_gpu_cos_cached[3].dtype == torch.float32
+
+
+def test_granite_decode_reads_rows_past_max_seq_length():
+ # GraniteModel_fast_forward_inference indexes the cache it fetches once per step at the decode
+ # position, which runs past max_seq_length; the eager cache held max_position_embeddings rows.
+ tree = ast.parse((REPO_ROOT / "unsloth" / "models" / "granite.py").read_text(encoding = "utf-8"))
+ fn = next(
+ n
+ for n in ast.walk(tree)
+ if isinstance(n, ast.FunctionDef) and n.name == "GraniteModel_fast_forward_inference"
+ )
+ call = next(
+ n
+ for n in ast.walk(fn)
+ if isinstance(n, ast.Call)
+ and isinstance(n.func, ast.Attribute)
+ and n.func.attr == "get_cached"
+ )
+ max_seq_length, seq_len, q_len = 32768, 32790, 1
+ rows = eval(
+ compile(ast.Expression(call.args[0]), "granite.py", "eval"),
+ {"max": max},
+ {
+ "self": types.SimpleNamespace(max_seq_length = max_seq_length),
+ "seq_len": seq_len,
+ "q_len": q_len,
+ },
+ )
+ rope = _llama(1 << 17)
+ # Prefill of a 32760 token prompt through LlamaModel_fast_forward.
+ rope.extend_rope_embedding(_x(0), 1 << 17, materialize_len = max(32760, max_seq_length))
+ cos, sin = rope.get_cached(rows, 0)
+ assert cos.shape[0] > seq_len
+ ref_cos, _ = _reference(cos.shape[0])
+ assert torch.equal(cos[seq_len], ref_cos[seq_len])
+
+
+def test_covered_reads_still_rebuild_a_device_left_stale_by_growth():
+ rope = _llama(1 << 17)
+ rope.get_cached(10, 1)
+ rope.extend_rope_embedding(_x(0, torch.float16), 50000)
+ # Device 1 already holds 32768 rows covering 10, but growth changed the dtype.
+ cos, _ = rope.get_cached(10, 1)
+ assert cos.dtype == torch.float16 and cos.shape[0] == 57344
+ ref_cos, _ = _reference(57344, torch.float16)
+ assert torch.equal(cos, ref_cos)
+ rope.extend_rope_embedding(_x(1, torch.float16), 11)
+ assert rope.multi_gpu_cos_cached[1] is cos
diff --git a/unsloth/models/gemma.py b/unsloth/models/gemma.py
index 849caa74b6c..7d804406a74 100644
--- a/unsloth/models/gemma.py
+++ b/unsloth/models/gemma.py
@@ -10,7 +10,7 @@
# limitations under the License.
from .llama import *
-from .llama import _get_rope_theta, _LazyDeviceScratch
+from .llama import _get_rope_theta, _LazyDeviceScratch, _PerDeviceRopeCache
from ._utils import __version__
from ._utils import move_to_device, per_layer_device, embedding_applies_scale
from unsloth_zoo.utils import _get_dtype, Version
@@ -20,7 +20,6 @@ from ..utils.packing import (
build_xformers_block_causal_mask,
get_packed_info_from_kwargs,
)
-import math
try:
from transformers.models.gemma.modeling_gemma import (
@@ -241,7 +240,7 @@ def GemmaModel_fast_forward_inference(
# Follows google-deepmind/gemma positional_embeddings.py#L45 line by line, which formulates cos and
# sin differently from Llama.
-class GemmaFixedRotaryEmbedding(torch.nn.Module):
+class GemmaFixedRotaryEmbedding(_PerDeviceRopeCache, torch.nn.Module):
# RoPE buffer precision is wrong, so cast to int64; see huggingface/transformers#28837 and microsoft/DeepSpeed#4932.
def __init__(
self,
@@ -273,14 +272,8 @@ class GemmaFixedRotaryEmbedding(torch.nn.Module):
self.current_rope_size = min(4 * 8192, self.max_position_embeddings)
self.multi_gpu_cos_cached = [None] * DEVICE_COUNT
self.multi_gpu_sin_cached = [None] * DEVICE_COUNT
-
- # Build here to make `torch.jit.trace` work.
- for device in range(DEVICE_COUNT):
- self._set_cos_sin_cache(
- seq_len = self.current_rope_size,
- device = torch.device(device),
- dtype = torch.get_default_dtype(),
- )
+ # Built per device on first use, not for every visible GPU here.
+ self._init_rope_cache_state(torch.get_default_dtype())
self.cos_cached = torch.empty(
1, device = get_current_device(), dtype = torch.get_default_dtype()
@@ -311,41 +304,6 @@ class GemmaFixedRotaryEmbedding(torch.nn.Module):
self.multi_gpu_sin_cached[device.index] = sin
return cos, sin
- def forward(
- self,
- x,
- position_ids = None,
- seq_len = None,
- ):
- if seq_len is not None and seq_len > self.current_rope_size:
- self._set_cos_sin_cache(seq_len = seq_len, device = x.device, dtype = x.dtype)
-
- device_index = x.device.index
-
- return (
- self.multi_gpu_cos_cached[device_index][:seq_len],
- self.multi_gpu_sin_cached[device_index][:seq_len],
- )
-
- def get_cached(
- self,
- seq_len = None,
- device_index = None,
- ):
- if device_index is None:
- device_index = get_current_device()
- return self.multi_gpu_cos_cached[device_index], self.multi_gpu_sin_cached[device_index]
-
- def extend_rope_embedding(self, x, seq_len):
- if seq_len <= self.current_rope_size:
- return
- # Iteratively grow by increments of 8192
- self.current_rope_size = math.ceil(seq_len / 8192) * 8192
- for device in range(DEVICE_COUNT):
- self._set_cos_sin_cache(
- self.current_rope_size, device = torch.device(device), dtype = x.dtype
- )
-
class GemmaFixedLinearScalingRotaryEmbedding(GemmaFixedRotaryEmbedding):
"""LlamaRotaryEmbedding extended with linear scaling. Credits to the Reddit user /u/kaiokendev"""
diff --git a/unsloth/models/gemma2.py b/unsloth/models/gemma2.py
index 6b142951cc3..2f0c3a57ce6 100644
--- a/unsloth/models/gemma2.py
+++ b/unsloth/models/gemma2.py
@@ -173,8 +173,8 @@ def Gemma2Attention_fast_forward(
kv_seq_len += past_key_value[0].shape[-2]
device_index = Q.device.index
- cos = self.rotary_emb.multi_gpu_cos_cached[device_index]
- sin = self.rotary_emb.multi_gpu_sin_cached[device_index]
+ # get_cached builds this device's cache on first use.
+ cos, sin = self.rotary_emb.get_cached(kv_seq_len, device_index)
rope_position_ids = position_ids if position_ids is not None else kwargs.get("position_ids")
if rope_position_ids is not None:
diff --git a/unsloth/models/granite.py b/unsloth/models/granite.py
index 8b82ed25860..1f10894605e 100644
--- a/unsloth/models/granite.py
+++ b/unsloth/models/granite.py
@@ -452,8 +452,9 @@ def GraniteModel_fast_forward_inference(
else:
attention_mask = None
+ # Decode positions run past max_seq_length; the cache is only built for the rows asked for.
position_embeddings = self.model.rotary_emb.get_cached(
- self.max_seq_length, hidden_states.device.index
+ max(self.max_seq_length, seq_len + q_len), hidden_states.device.index
)
next_decoder_cache = []
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index 7ee1001a42e..7713d0abb81 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -1414,10 +1414,15 @@ def LlamaModel_fast_forward(
) or IS_GRANITE:
# position_embeddings is mandatory on main (huggingface/transformers#34858), and granite always
# had the attention refactor, so it always takes this path.
- self.rotary_emb.extend_rope_embedding(hidden_states, self.config.max_position_embeddings)
- position_embeddings = self.rotary_emb.get_cached(
- self.config.max_position_embeddings, hidden_states.device.index
+ # max_position_embeddings still sets the cache size and dtype as before, but only the rows
+ # this batch (or the declared max_seq_length) can index are built, on this device only.
+ rope_rows = max(seq_length_with_past, getattr(self, "max_seq_length", None) or 0)
+ self.rotary_emb.extend_rope_embedding(
+ hidden_states,
+ self.config.max_position_embeddings,
+ materialize_len = rope_rows,
)
+ position_embeddings = self.rotary_emb.get_cached(rope_rows, hidden_states.device.index)
else:
position_embeddings = None
@@ -2136,9 +2141,120 @@ def _compute_config_rope_inv_freq(config, rope_scaling):
return None, 1.0
+def _round_rope_size(seq_len):
+ return ((seq_len // 8192) + ((seq_len % 8192) != 0)) * 8192
+
+
+class _PerDeviceRopeCache:
+ """cos/sin built lazily per device with only the rows asked for, so torchrun ranks do not
+ allocate on their peers. A row depends only on its position, so a shorter cache matches the
+ full one row for row."""
+
+ def _init_rope_cache_state(self, dtype):
+ self._rope_cache_dtype = dtype
+ # Bumped whenever the eager code rebuilt every device (growth), so stale devices rebuild.
+ self._rope_cache_version = 0
+ self._rope_cache_versions = [None] * DEVICE_COUNT
+ # Rows each device can serve without bookkeeping; reset on growth. Read per layer per decode step.
+ self._rope_rows_ready = [0] * DEVICE_COUNT
+ # Never fewer rows than the eager cache started with, so those positions stay valid.
+ self._rope_cache_floor = self.current_rope_size
+ self._rope_cache_needed = self.current_rope_size
+
+ def _rope_cache_rows(self):
+ rows = max(self._rope_cache_floor, _round_rope_size(self._rope_cache_needed))
+ return min(self.current_rope_size, rows)
+
+ def _build_rope_cache(self, device_index):
+ # _set_cos_sin_cache records its length as current_rope_size; a lazy build may be shorter.
+ current_rope_size = self.current_rope_size
+ self._set_cos_sin_cache(
+ seq_len = self._rope_cache_rows(),
+ device = torch.device(device_index),
+ dtype = self._rope_cache_dtype,
+ )
+ self.current_rope_size = current_rope_size
+ self._rope_cache_versions[device_index] = self._rope_cache_version
+
+ def _rope_cache_stale(self, cached, device_index):
+ return (
+ cached is None
+ or cached.shape[0] < self._rope_cache_rows()
+ or self._rope_cache_versions[device_index] != self._rope_cache_version
+ )
+
+ def _ensure_rope_cache(self, device_index):
+ if device_index is None:
+ device_index = get_current_device()
+ if self._rope_cache_stale(self.multi_gpu_cos_cached[device_index], device_index):
+ self._build_rope_cache(device_index)
+ self._rope_rows_ready[device_index] = self.multi_gpu_cos_cached[device_index].shape[0]
+ return device_index
+
+ def _grow_rope_cache(self, seq_len, dtype):
+ self.current_rope_size = seq_len
+ self._rope_cache_dtype = dtype
+ self._rope_cache_version += 1
+ self._rope_rows_ready = [0] * DEVICE_COUNT
+
+ def _need_rope_rows(self, seq_len):
+ if seq_len > self._rope_cache_needed:
+ self._rope_cache_needed = seq_len
+
+ def forward(
+ self,
+ x,
+ position_ids = None,
+ seq_len = None,
+ ):
+ if seq_len is not None and seq_len > self.current_rope_size:
+ self._grow_rope_cache(seq_len, x.dtype)
+ self._need_rope_rows(self.current_rope_size if seq_len is None else seq_len)
+ device_index = self._ensure_rope_cache(x.device.index)
+ return self._slice_rope_cache(device_index, seq_len)
+
+ def _slice_rope_cache(self, device_index, seq_len):
+ return (
+ self.multi_gpu_cos_cached[device_index][:seq_len],
+ self.multi_gpu_sin_cached[device_index][:seq_len],
+ )
+
+ def get_cached(
+ self,
+ seq_len = None,
+ device_index = None,
+ ):
+ if seq_len is not None:
+ if device_index is not None and seq_len <= self._rope_rows_ready[device_index]:
+ return self.multi_gpu_cos_cached[device_index], self.multi_gpu_sin_cached[
+ device_index
+ ]
+ self._need_rope_rows(seq_len)
+ device_index = self._ensure_rope_cache(device_index)
+ return self.multi_gpu_cos_cached[device_index], self.multi_gpu_sin_cached[device_index]
+
+ def extend_rope_embedding(
+ self,
+ x,
+ seq_len,
+ materialize_len = None,
+ ):
+ # seq_len moves current_rope_size and the dtype as before; materialize_len (default seq_len)
+ # is how many rows x's device needs now.
+ rows = seq_len if materialize_len is None else materialize_len
+ if seq_len <= self.current_rope_size:
+ device_index = x.device.index
+ if device_index is not None and rows <= self._rope_rows_ready[device_index]:
+ return
+ self._need_rope_rows(rows)
+ if seq_len > self.current_rope_size:
+ self._grow_rope_cache(_round_rope_size(seq_len), x.dtype)
+ self._ensure_rope_cache(x.device.index)
+
+
# Static KV Cache landed in 4.38.0 and made training much slower (#168,
# huggingface/transformers#27931), so the old rotary embeddings are retained.
-class LlamaRotaryEmbedding(torch.nn.Module):
+class LlamaRotaryEmbedding(_PerDeviceRopeCache, torch.nn.Module):
# RoPE buffer precision is wrong, so cast to int64; see huggingface/transformers#28837 and microsoft/DeepSpeed#4932.
def __init__(
self,
@@ -2178,12 +2294,8 @@ class LlamaRotaryEmbedding(torch.nn.Module):
inv_freq = self._unsloth_recompute_inv_freq()
self.register_buffer("inv_freq", inv_freq, persistent = False)
- for device_idx in range(DEVICE_COUNT):
- self._set_cos_sin_cache(
- seq_len = self.current_rope_size,
- device = torch.device(device_idx),
- dtype = torch.get_default_dtype(),
- )
+ # Built per device on first use, not for every visible GPU here.
+ self._init_rope_cache_state(torch.get_default_dtype())
self.cos_cached = torch.empty(
1, device = get_current_device(), dtype = torch.get_default_dtype()
@@ -2242,40 +2354,6 @@ class LlamaRotaryEmbedding(torch.nn.Module):
self.multi_gpu_sin_cached[device.index] = sin
return cos, sin
- def forward(
- self,
- x,
- position_ids = None,
- seq_len = None,
- ):
- if seq_len is not None and seq_len > self.current_rope_size:
- self._set_cos_sin_cache(seq_len = seq_len, device = x.device, dtype = x.dtype)
-
- device_index = x.device.index
- return (
- self.multi_gpu_cos_cached[device_index][:seq_len],
- self.multi_gpu_sin_cached[device_index][:seq_len],
- )
-
- def get_cached(
- self,
- seq_len = None,
- device_index = None,
- ):
- if device_index is None:
- device_index = get_current_device()
- return self.multi_gpu_cos_cached[device_index], self.multi_gpu_sin_cached[device_index]
-
- def extend_rope_embedding(self, x, seq_len):
- if seq_len <= self.current_rope_size:
- return
- # Iteratively grow by increments of 8192.
- self.current_rope_size = ((seq_len // 8192) + ((seq_len % 8192) != 0)) * 8192
- for device_idx in range(DEVICE_COUNT):
- self._set_cos_sin_cache(
- self.current_rope_size, device = torch.device(device_idx), dtype = x.dtype
- )
-
class LlamaLinearScalingRotaryEmbedding(LlamaRotaryEmbedding):
"""LlamaRotaryEmbedding extended with linear scaling. Credits to the Reddit user /u/kaiokendev"""
@@ -2353,7 +2431,7 @@ class LlamaExtendedRotaryEmbedding(LlamaRotaryEmbedding):
return torch.tensor(new_freqs, dtype = freqs.dtype, device = freqs.device)
-class LongRopeRotaryEmbedding(torch.nn.Module):
+class LongRopeRotaryEmbedding(_PerDeviceRopeCache, torch.nn.Module):
# For Phi 3.5 128K; see microsoft/Phi-3.5-mini-instruct modeling_phi3.py
def __init__(
self,
@@ -2413,7 +2491,8 @@ class LongRopeRotaryEmbedding(torch.nn.Module):
self.register_buffer("short_inv_freq", short_inv_freq, persistent = False)
self.register_buffer("long_inv_freq", long_inv_freq, persistent = False)
- # Build here to make torch.jit.trace work, initializing the short-sequence cache for all devices.
+ # The short-sequence cache is computed here, on CPU, and copied to a device on first use. Kept
+ # off the buffers so a later load refilling short_inv_freq cannot reach it (#884).
dtype = torch.bfloat16 if is_bfloat16_supported() else torch.float16
t = torch.arange(
original_max_position_embeddings,
@@ -2422,17 +2501,10 @@ class LongRopeRotaryEmbedding(torch.nn.Module):
).float()
freqs = torch.outer(t, self.short_inv_freq)
emb = torch.cat((freqs, freqs), dim = -1)
-
- for device_idx in range(DEVICE_COUNT):
- device_obj = torch.device(device_idx)
- cos_cached = (emb.cos() * self.scaling_factor).to(
- dtype = dtype, device = device_obj, non_blocking = True
- )
- sin_cached = (emb.sin() * self.scaling_factor).to(
- dtype = dtype, device = device_obj, non_blocking = True
- )
- self.multi_gpu_short_cos_cached[device_idx] = cos_cached
- self.multi_gpu_short_sin_cached[device_idx] = sin_cached
+ self._unsloth_short_cos_cpu = (emb.cos() * self.scaling_factor).to(dtype = dtype)
+ self._unsloth_short_sin_cpu = (emb.sin() * self.scaling_factor).to(dtype = dtype)
+ # The long cache only exists once something asks for more than original_max_position_embeddings.
+ self._init_rope_cache_state(dtype)
self.short_cos_cached = torch.empty(
1, device = get_current_device(), dtype = torch.get_default_dtype()
@@ -2466,17 +2538,24 @@ class LongRopeRotaryEmbedding(torch.nn.Module):
self.multi_gpu_long_sin_cached[device.index] = sin_cached
return cos_cached, sin_cached
- def forward(
- self,
- x,
- position_ids = None,
- seq_len = None,
- ):
- if seq_len is not None and seq_len > self.current_rope_size:
- self._set_cos_sin_cache(seq_len = seq_len, device = x.device, dtype = x.dtype)
-
- device_index = x.device.index
+ def _ensure_rope_cache(self, device_index):
+ if device_index is None:
+ device_index = get_current_device()
+ if self.multi_gpu_short_cos_cached[device_index] is None:
+ device = torch.device(device_index)
+ self.multi_gpu_short_cos_cached[device_index] = self._unsloth_short_cos_cpu.to(
+ device = device, non_blocking = True
+ )
+ self.multi_gpu_short_sin_cached[device_index] = self._unsloth_short_sin_cpu.to(
+ device = device, non_blocking = True
+ )
+ if self._rope_cache_version > 0 and self._rope_cache_stale(
+ self.multi_gpu_long_cos_cached[device_index], device_index
+ ):
+ self._build_rope_cache(device_index)
+ return device_index
+ def _slice_rope_cache(self, device_index, seq_len):
# transformers' _compute_longrope_parameters takes the long factor only on
# `seq_len and seq_len > original_max_position_embeddings`, so an unknown length is
# short too. Long stays None until something asks for a longer sequence, so routing
@@ -2497,8 +2576,7 @@ class LongRopeRotaryEmbedding(torch.nn.Module):
seq_len = None,
device_index = None,
):
- if device_index is None:
- device_index = get_current_device()
+ device_index = self._ensure_rope_cache(device_index)
if seq_len is None or seq_len <= self.original_max_position_embeddings:
return self.multi_gpu_short_cos_cached[device_index], self.multi_gpu_short_sin_cached[
device_index
@@ -2507,16 +2585,6 @@ class LongRopeRotaryEmbedding(torch.nn.Module):
device_index
]
- def extend_rope_embedding(self, x, seq_len):
- if seq_len <= self.current_rope_size:
- return
- # Iteratively grow by increments of 8192.
- self.current_rope_size = ((seq_len // 8192) + ((seq_len % 8192) != 0)) * 8192
- for device_idx in range(DEVICE_COUNT):
- self._set_cos_sin_cache(
- self.current_rope_size, device = torch.device(device_idx), dtype = x.dtype
- )
-
def unsloth_fast_generate(self, *args, **kwargs):
restore_training_mode = self.training
diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py
index 33302ae8cd4..0321be561bd 100644
--- a/unsloth/models/loader.py
+++ b/unsloth/models/loader.py
@@ -943,6 +943,14 @@ def _fix_rope_inv_freq(model):
).float()
freqs = torch.outer(t, module.short_inv_freq)
emb = torch.cat((freqs, freqs), dim = -1)
+ if hasattr(module, "_unsloth_short_cos_cpu"):
+ # Devices not built yet copy the short cache from these on first use.
+ module._unsloth_short_cos_cpu = (emb.cos() * module.scaling_factor).to(
+ dtype = dtype
+ )
+ module._unsloth_short_sin_cpu = (emb.sin() * module.scaling_factor).to(
+ dtype = dtype
+ )
for device_idx in range(len(module.multi_gpu_short_cos_cached)):
if module.multi_gpu_short_cos_cached[device_idx] is not None:
device_obj = torch.device(device_idx)