Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Build RoPE cos/sin caches only on the device that uses them (#13855)
Daniel Hanaa0b22b2ca9f
diff --git a/tests/test_device_helpers.py b/tests/test_device_helpers.pyindex 4341e3da231..282432079ec 100644--- a/tests/test_device_helpers.py+++ b/tests/test_device_helpers.py@@ -197,7 +197,7 @@ def test_model_call_sites_use_shared_cache_dispatch():assert "device_context" not in vision_sourceassert 'if DEVICE_TYPE == "xpu":\n vllm_version = ""' in vision_sourceassert "torch.cuda.current_device()" not in gemma_source- assert gemma_source.count("get_current_device()") >= 3+ assert "get_current_device()" in gemma_sourceassert "torch.cuda.empty_cache()" not in gemma_sourceassert "clean_gpu_cache()" in gemma_sourceassert "torch.cuda.empty_cache()" not in gemma2_sourcediff --git a/tests/test_longrope_short_long_boundary.py b/tests/test_longrope_short_long_boundary.pyindex f81e8b3594e..110887de366 100644--- a/tests/test_longrope_short_long_boundary.py+++ b/tests/test_longrope_short_long_boundary.py@@ -57,6 +57,13 @@ def _load_longrope():"LongRopeRotaryEmbedding is gone from unsloth/models/llama.py. If it moved or was ""renamed, repoint this test rather than deleting it.")+ # The per-device cache helpers it inherits from, when this checkout has them.+ helpers = [+ n+ for n in tree.body+ if isinstance(n, (ast.ClassDef, ast.FunctionDef))+ and n.name in ("_round_rope_size", "_PerDeviceRopeCache")+ ]ns = {"torch": _CPUTorch(),"math": __import__("math"),@@ -68,7 +75,7 @@ def _load_longrope():"_get_rope_theta": lambda config, default = 10000: default,}try:- exec(compile(ast.Module(body = [cls], type_ignores = []), str(LLAMA), "exec"), ns)+ exec(compile(ast.Module(body = helpers + [cls], type_ignores = []), str(LLAMA), "exec"), ns)except NameError as e:pytest.fail(f"LongRopeRotaryEmbedding grew a module-level dependency this test does not stub "diff --git a/tests/test_plain_load_after_unsloth_rope.py b/tests/test_plain_load_after_unsloth_rope.pyindex 94b97bd5bdc..ace7a23c8ca 100644--- a/tests/test_plain_load_after_unsloth_rope.py+++ b/tests/test_plain_load_after_unsloth_rope.py@@ -57,7 +57,7 @@ rotary = model.model.rotary_embassert isinstance(rotary, LlamaRotaryEmbedding), type(rotary)expected = rotary._unsloth_recompute_inv_freq().float()torch.testing.assert_close(rotary.inv_freq.detach().cpu().float(), expected)-cos = rotary.multi_gpu_cos_cached[torch.cuda.current_device()].float().cpu()+cos = rotary.get_cached(device_index = torch.cuda.current_device())[0].float().cpu()freqs = torch.outer(torch.arange(cos.shape[0], dtype = torch.float32), expected)torch.testing.assert_close(cos, torch.cat((freqs, freqs), -1).cos(), atol = 1e-3, rtol = 0)"""diff --git a/tests/test_rope_cache_current_device.py b/tests/test_rope_cache_current_device.pynew file mode 100644index 00000000000..1dd8b6b483f--- /dev/null+++ b/tests/test_rope_cache_current_device.py@@ -0,0 +1,268 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.+"""Unsloth's rotary caches must be built only on the device that reads them.++llama.py needs an accelerator to import, so the classes are `ast`-extracted and run against a+CPU-pinned prelude in which device index i is cpu:i.+"""++from __future__ import annotations++import ast+import math+import pathlib+import types++import pytest+import torch++REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent+LLAMA = REPO_ROOT / "unsloth" / "models" / "llama.py"+GEMMA = REPO_ROOT / "unsloth" / "models" / "gemma.py"++DEVICES = 4+DIM = 8+BASE = 10000+++class _CPUTorch:+ """`torch`, with a bare-int device pinned to CPU so the test runs anywhere."""++ def __getattr__(self, name):+ return getattr(torch, name)++ def device(+ self,+ spec,+ index = None,+ ):+ if isinstance(spec, int) and not isinstance(spec, bool):+ return torch.device("cpu", spec)+ return torch.device(spec) if index is None else torch.device(spec, index)++ def empty(self, *args, **kwargs):+ kwargs["device"] = "cpu"+ return torch.empty(*args, **kwargs)+++_HELPERS = ("_round_rope_size", "_PerDeviceRopeCache")+++def _load(+ path,+ names,+ extra_ns = None,+):+ tree = ast.parse(path.read_text(encoding = "utf-8"))+ nodes = []+ for name in names:+ node = next(+ (+ n+ for n in ast.walk(tree)+ if isinstance(n, (ast.ClassDef, ast.FunctionDef)) and n.name == name+ ),+ None,+ )+ if node is None:+ # Helpers only exist once the cache is per device; the classes must always exist.+ if name in _HELPERS:+ continue+ pytest.fail(+ f"{name} is gone from {path.name}; repoint this test rather than deleting it."+ )+ nodes.append(node)+ ns = {+ "torch": _CPUTorch(),+ "math": math,+ "DEVICE_COUNT": DEVICES,+ "DEVICE_TYPE_TORCH": "cpu",+ "get_current_device": lambda: 0,+ "is_bfloat16_supported": lambda: False,+ "_get_rope_theta": lambda config, default = 10000: default,+ "_compute_config_rope_inv_freq": lambda config, rope_scaling: (None, 1.0),+ "_rope_scaling_as_dict": lambda value: dict(value or {}),+ }+ ns.update(extra_ns or {})+ exec(compile(ast.Module(body = nodes, type_ignores = []), str(path), "exec"), ns)+ return ns+++def _llama_ns():+ return _load(+ LLAMA,+ list(_HELPERS) + ["LlamaRotaryEmbedding", "LongRopeRotaryEmbedding"],+ )+++def _built(cache):+ return [i for i, c in enumerate(cache) if c is not None]+++def _x(device_index, dtype = torch.float32):+ # extend_rope_embedding / forward only read .device and .dtype.+ return types.SimpleNamespace(device = torch.device("cpu", device_index), dtype = dtype)+++def _reference(rows, dtype = torch.float32):+ inv_freq = 1.0 / (BASE ** (torch.arange(0, DIM, 2, dtype = torch.int64).float() / DIM))+ t = torch.arange(rows, dtype = torch.int64).float()+ emb = torch.cat((torch.outer(t, inv_freq),) * 2, dim = -1)+ return emb.cos().to(dtype), emb.sin().to(dtype)+++def _llama(max_position):+ return _llama_ns()["LlamaRotaryEmbedding"](+ dim = DIM, max_position_embeddings = max_position, base = BASE+ )+++def test_init_builds_no_device():+ rope = _llama(1 << 20)+ assert _built(rope.multi_gpu_cos_cached) == []+ assert _built(rope.multi_gpu_sin_cached) == []+++def test_get_cached_builds_only_the_requested_device():+ rope = _llama(1 << 20)+ cos, sin = rope.get_cached(100, 2)+ assert _built(rope.multi_gpu_cos_cached) == [2]+ ref_cos, ref_sin = _reference(cos.shape[0])+ assert cos.shape[0] == 4 * 8192+ assert torch.equal(cos, ref_cos) and torch.equal(sin, ref_sin)+++def test_model_forward_extend_builds_needed_rows_on_one_device():+ # LlamaModel_fast_forward: extend to max_position_embeddings, materialise this batch's rows.+ max_position = 1 << 20+ rope = _llama(max_position)+ x = _x(1, torch.float16)+ rope.extend_rope_embedding(x, max_position, materialize_len = 40000)+ # Size and dtype move exactly as the eager extend left them.+ assert rope.current_rope_size == max_position+ assert _built(rope.multi_gpu_cos_cached) == [1]+ cos, sin = rope.get_cached(40000, 1)+ # 40000 rounded up to the 8192 growth step, not 1M rows.+ assert cos.shape[0] == 40960+ assert cos.dtype == torch.float16+ ref_cos, ref_sin = _reference(40960, torch.float16)+ assert torch.equal(cos, ref_cos) and torch.equal(sin, ref_sin)+++def test_extend_grows_used_device_and_stale_peers_rebuild_lazily():+ rope = _llama(1 << 17)+ rope.get_cached(10, 0)+ rope.get_cached(10, 3)+ rope.extend_rope_embedding(_x(3, torch.float16), 50000)+ assert rope.current_rope_size == 57344+ assert rope.multi_gpu_cos_cached[3].shape[0] == 57344+ assert rope.multi_gpu_cos_cached[3].dtype == torch.float16+ # Device 0 is untouched until it is read, then matches what the eager rebuild produced.+ assert rope.multi_gpu_cos_cached[0].shape[0] == 4 * 8192+ cos, sin = rope.get_cached(50000, 0)+ ref_cos, ref_sin = _reference(57344, torch.float16)+ assert torch.equal(cos, ref_cos) and torch.equal(sin, ref_sin)+ assert _built(rope.multi_gpu_cos_cached) == [0, 3]+++def test_get_cached_materialises_the_length_it_is_asked_for():+ # Gemma2 / Cohere decode steps read get_cached(kv_seq_len) with no extend first, past the rows+ # the model forward built for max_seq_length.+ rope = _llama(1 << 17)+ rope.extend_rope_embedding(_x(0), 1 << 17, materialize_len = 2048)+ assert rope.multi_gpu_cos_cached[0].shape[0] == 4 * 8192+ cos, sin = rope.get_cached(40000, 0)+ assert cos.shape[0] >= 40000+ ref_cos, ref_sin = _reference(cos.shape[0])+ assert torch.equal(cos, ref_cos) and torch.equal(sin, ref_sin)+++def test_forward_builds_only_input_device():+ rope = _llama(4096)+ cos, sin = rope(_x(2), seq_len = 100)+ assert _built(rope.multi_gpu_cos_cached) == [2]+ ref_cos, ref_sin = _reference(100)+ assert torch.equal(cos, ref_cos) and torch.equal(sin, ref_sin)+++def test_longrope_builds_only_the_requested_device():+ rope = _llama_ns()["LongRopeRotaryEmbedding"](+ dim = DIM,+ max_position_embeddings = 128,+ original_max_position_embeddings = 16,+ short_factor = [1.0] * (DIM // 2),+ long_factor = [2.0] * (DIM // 2),+ )+ assert _built(rope.multi_gpu_short_cos_cached) == []+ rope.get_cached(8, 1)+ assert _built(rope.multi_gpu_short_cos_cached) == [1]+ assert _built(rope.multi_gpu_long_cos_cached) == []+ rope.extend_rope_embedding(_x(1, torch.float16), 100)+ assert _built(rope.multi_gpu_long_cos_cached) == [1]+ assert _built(rope.multi_gpu_short_cos_cached) == [1]+++def test_gemma_builds_only_the_requested_device():+ llama_ns = _llama_ns()+ if "_PerDeviceRopeCache" not in llama_ns:+ pytest.fail("Gemma's rotary cache is built for every device")+ ns = _load(+ GEMMA,+ ["GemmaFixedRotaryEmbedding"],+ {"_PerDeviceRopeCache": llama_ns["_PerDeviceRopeCache"]},+ )+ rope = ns["GemmaFixedRotaryEmbedding"](dim = DIM, max_position_embeddings = 1 << 20, base = BASE)+ assert _built(rope.multi_gpu_cos_cached) == []+ rope.extend_rope_embedding(_x(3, torch.bfloat16), 70000)+ assert _built(rope.multi_gpu_cos_cached) == [3]+ assert rope.multi_gpu_cos_cached[3].shape[0] == 73728+ # Gemma keeps its cache in float32 whatever the input dtype.+ assert rope.multi_gpu_cos_cached[3].dtype == torch.float32+++def test_granite_decode_reads_rows_past_max_seq_length():+ # GraniteModel_fast_forward_inference indexes the cache it fetches once per step at the decode+ # position, which runs past max_seq_length; the eager cache held max_position_embeddings rows.+ tree = ast.parse((REPO_ROOT / "unsloth" / "models" / "granite.py").read_text(encoding = "utf-8"))+ fn = next(+ n+ for n in ast.walk(tree)+ if isinstance(n, ast.FunctionDef) and n.name == "GraniteModel_fast_forward_inference"+ )+ call = next(+ n+ for n in ast.walk(fn)+ if isinstance(n, ast.Call)+ and isinstance(n.func, ast.Attribute)+ and n.func.attr == "get_cached"+ )+ max_seq_length, seq_len, q_len = 32768, 32790, 1+ rows = eval(+ compile(ast.Expression(call.args[0]), "granite.py", "eval"),+ {"max": max},+ {+ "self": types.SimpleNamespace(max_seq_length = max_seq_length),+ "seq_len": seq_len,+ "q_len": q_len,+ },+ )+ rope = _llama(1 << 17)+ # Prefill of a 32760 token prompt through LlamaModel_fast_forward.+ rope.extend_rope_embedding(_x(0), 1 << 17, materialize_len = max(32760, max_seq_length))+ cos, sin = rope.get_cached(rows, 0)+ assert cos.shape[0] > seq_len+ ref_cos, _ = _reference(cos.shape[0])+ assert torch.equal(cos[seq_len], ref_cos[seq_len])+++def test_covered_reads_still_rebuild_a_device_left_stale_by_growth():+ rope = _llama(1 << 17)+ rope.get_cached(10, 1)+ rope.extend_rope_embedding(_x(0, torch.float16), 50000)+ # Device 1 already holds 32768 rows covering 10, but growth changed the dtype.+ cos, _ = rope.get_cached(10, 1)+ assert cos.dtype == torch.float16 and cos.shape[0] == 57344+ ref_cos, _ = _reference(57344, torch.float16)+ assert torch.equal(cos, ref_cos)+ rope.extend_rope_embedding(_x(1, torch.float16), 11)+ assert rope.multi_gpu_cos_cached[1] is cosdiff --git a/unsloth/models/gemma.py b/unsloth/models/gemma.pyindex 849caa74b6c..7d804406a74 100644--- a/unsloth/models/gemma.py+++ b/unsloth/models/gemma.py@@ -10,7 +10,7 @@# limitations under the License.from .llama import *-from .llama import _get_rope_theta, _LazyDeviceScratch+from .llama import _get_rope_theta, _LazyDeviceScratch, _PerDeviceRopeCachefrom ._utils import __version__from ._utils import move_to_device, per_layer_device, embedding_applies_scalefrom unsloth_zoo.utils import _get_dtype, Version@@ -20,7 +20,6 @@ from ..utils.packing import (build_xformers_block_causal_mask,get_packed_info_from_kwargs,)-import mathtry:from transformers.models.gemma.modeling_gemma import (@@ -241,7 +240,7 @@ def GemmaModel_fast_forward_inference(# Follows google-deepmind/gemma positional_embeddings.py#L45 line by line, which formulates cos and# sin differently from Llama.-class GemmaFixedRotaryEmbedding(torch.nn.Module):+class GemmaFixedRotaryEmbedding(_PerDeviceRopeCache, torch.nn.Module):# RoPE buffer precision is wrong, so cast to int64; see huggingface/transformers#28837 and microsoft/DeepSpeed#4932.def __init__(self,@@ -273,14 +272,8 @@ class GemmaFixedRotaryEmbedding(torch.nn.Module):self.current_rope_size = min(4 * 8192, self.max_position_embeddings)self.multi_gpu_cos_cached = [None] * DEVICE_COUNTself.multi_gpu_sin_cached = [None] * DEVICE_COUNT-- # Build here to make `torch.jit.trace` work.- for device in range(DEVICE_COUNT):- self._set_cos_sin_cache(- seq_len = self.current_rope_size,- device = torch.device(device),- dtype = torch.get_default_dtype(),- )+ # Built per device on first use, not for every visible GPU here.+ self._init_rope_cache_state(torch.get_default_dtype())self.cos_cached = torch.empty(1, device = get_current_device(), dtype = torch.get_default_dtype()@@ -311,41 +304,6 @@ class GemmaFixedRotaryEmbedding(torch.nn.Module):self.multi_gpu_sin_cached[device.index] = sinreturn cos, sin- def forward(- self,- x,- position_ids = None,- seq_len = None,- ):- if seq_len is not None and seq_len > self.current_rope_size:- self._set_cos_sin_cache(seq_len = seq_len, device = x.device, dtype = x.dtype)-- device_index = x.device.index-- return (- self.multi_gpu_cos_cached[device_index][:seq_len],- self.multi_gpu_sin_cached[device_index][:seq_len],- )-- def get_cached(- self,- seq_len = None,- device_index = None,- ):- if device_index is None:- device_index = get_current_device()- return self.multi_gpu_cos_cached[device_index], self.multi_gpu_sin_cached[device_index]-- def extend_rope_embedding(self, x, seq_len):- if seq_len <= self.current_rope_size:- return- # Iteratively grow by increments of 8192- self.current_rope_size = math.ceil(seq_len / 8192) * 8192- for device in range(DEVICE_COUNT):- self._set_cos_sin_cache(- self.current_rope_size, device = torch.device(device), dtype = x.dtype- )-class GemmaFixedLinearScalingRotaryEmbedding(GemmaFixedRotaryEmbedding):"""LlamaRotaryEmbedding extended with linear scaling. Credits to the Reddit user /u/kaiokendev"""diff --git a/unsloth/models/gemma2.py b/unsloth/models/gemma2.pyindex 6b142951cc3..2f0c3a57ce6 100644--- a/unsloth/models/gemma2.py+++ b/unsloth/models/gemma2.py@@ -173,8 +173,8 @@ def Gemma2Attention_fast_forward(kv_seq_len += past_key_value[0].shape[-2]device_index = Q.device.index- cos = self.rotary_emb.multi_gpu_cos_cached[device_index]- sin = self.rotary_emb.multi_gpu_sin_cached[device_index]+ # get_cached builds this device's cache on first use.+ cos, sin = self.rotary_emb.get_cached(kv_seq_len, device_index)rope_position_ids = position_ids if position_ids is not None else kwargs.get("position_ids")if rope_position_ids is not None:diff --git a/unsloth/models/granite.py b/unsloth/models/granite.pyindex 8b82ed25860..1f10894605e 100644--- a/unsloth/models/granite.py+++ b/unsloth/models/granite.py@@ -452,8 +452,9 @@ def GraniteModel_fast_forward_inference(else:attention_mask = None+ # Decode positions run past max_seq_length; the cache is only built for the rows asked for.position_embeddings = self.model.rotary_emb.get_cached(- self.max_seq_length, hidden_states.device.index+ max(self.max_seq_length, seq_len + q_len), hidden_states.device.index)next_decoder_cache = []diff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex 7ee1001a42e..7713d0abb81 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -1414,10 +1414,15 @@ def LlamaModel_fast_forward() or IS_GRANITE:# position_embeddings is mandatory on main (huggingface/transformers#34858), and granite always# had the attention refactor, so it always takes this path.- self.rotary_emb.extend_rope_embedding(hidden_states, self.config.max_position_embeddings)- position_embeddings = self.rotary_emb.get_cached(- self.config.max_position_embeddings, hidden_states.device.index+ # max_position_embeddings still sets the cache size and dtype as before, but only the rows+ # this batch (or the declared max_seq_length) can index are built, on this device only.+ rope_rows = max(seq_length_with_past, getattr(self, "max_seq_length", None) or 0)+ self.rotary_emb.extend_rope_embedding(+ hidden_states,+ self.config.max_position_embeddings,+ materialize_len = rope_rows,)+ position_embeddings = self.rotary_emb.get_cached(rope_rows, hidden_states.device.index)else:position_embeddings = None@@ -2136,9 +2141,120 @@ def _compute_config_rope_inv_freq(config, rope_scaling):return None, 1.0+def _round_rope_size(seq_len):+ return ((seq_len // 8192) + ((seq_len % 8192) != 0)) * 8192+++class _PerDeviceRopeCache:+ """cos/sin built lazily per device with only the rows asked for, so torchrun ranks do not+ allocate on their peers. A row depends only on its position, so a shorter cache matches the+ full one row for row."""++ def _init_rope_cache_state(self, dtype):+ self._rope_cache_dtype = dtype+ # Bumped whenever the eager code rebuilt every device (growth), so stale devices rebuild.+ self._rope_cache_version = 0+ self._rope_cache_versions = [None] * DEVICE_COUNT+ # Rows each device can serve without bookkeeping; reset on growth. Read per layer per decode step.+ self._rope_rows_ready = [0] * DEVICE_COUNT+ # Never fewer rows than the eager cache started with, so those positions stay valid.+ self._rope_cache_floor = self.current_rope_size+ self._rope_cache_needed = self.current_rope_size++ def _rope_cache_rows(self):+ rows = max(self._rope_cache_floor, _round_rope_size(self._rope_cache_needed))+ return min(self.current_rope_size, rows)++ def _build_rope_cache(self, device_index):+ # _set_cos_sin_cache records its length as current_rope_size; a lazy build may be shorter.+ current_rope_size = self.current_rope_size+ self._set_cos_sin_cache(+ seq_len = self._rope_cache_rows(),+ device = torch.device(device_index),+ dtype = self._rope_cache_dtype,+ )+ self.current_rope_size = current_rope_size+ self._rope_cache_versions[device_index] = self._rope_cache_version++ def _rope_cache_stale(self, cached, device_index):+ return (+ cached is None+ or cached.shape[0] < self._rope_cache_rows()+ or self._rope_cache_versions[device_index] != self._rope_cache_version+ )++ def _ensure_rope_cache(self, device_index):+ if device_index is None:+ device_index = get_current_device()+ if self._rope_cache_stale(self.multi_gpu_cos_cached[device_index], device_index):+ self._build_rope_cache(device_index)+ self._rope_rows_ready[device_index] = self.multi_gpu_cos_cached[device_index].shape[0]+ return device_index++ def _grow_rope_cache(self, seq_len, dtype):+ self.current_rope_size = seq_len+ self._rope_cache_dtype = dtype+ self._rope_cache_version += 1+ self._rope_rows_ready = [0] * DEVICE_COUNT++ def _need_rope_rows(self, seq_len):+ if seq_len > self._rope_cache_needed:+ self._rope_cache_needed = seq_len++ def forward(+ self,+ x,+ position_ids = None,+ seq_len = None,+ ):+ if seq_len is not None and seq_len > self.current_rope_size:+ self._grow_rope_cache(seq_len, x.dtype)+ self._need_rope_rows(self.current_rope_size if seq_len is None else seq_len)+ device_index = self._ensure_rope_cache(x.device.index)+ return self._slice_rope_cache(device_index, seq_len)++ def _slice_rope_cache(self, device_index, seq_len):+ return (+ self.multi_gpu_cos_cached[device_index][:seq_len],+ self.multi_gpu_sin_cached[device_index][:seq_len],+ )++ def get_cached(+ self,+ seq_len = None,+ device_index = None,+ ):+ if seq_len is not None:+ if device_index is not None and seq_len <= self._rope_rows_ready[device_index]:+ return self.multi_gpu_cos_cached[device_index], self.multi_gpu_sin_cached[+ device_index+ ]+ self._need_rope_rows(seq_len)+ device_index = self._ensure_rope_cache(device_index)+ return self.multi_gpu_cos_cached[device_index], self.multi_gpu_sin_cached[device_index]++ def extend_rope_embedding(+ self,+ x,+ seq_len,+ materialize_len = None,+ ):+ # seq_len moves current_rope_size and the dtype as before; materialize_len (default seq_len)+ # is how many rows x's device needs now.+ rows = seq_len if materialize_len is None else materialize_len+ if seq_len <= self.current_rope_size:+ device_index = x.device.index+ if device_index is not None and rows <= self._rope_rows_ready[device_index]:+ return+ self._need_rope_rows(rows)+ if seq_len > self.current_rope_size:+ self._grow_rope_cache(_round_rope_size(seq_len), x.dtype)+ self._ensure_rope_cache(x.device.index)++# Static KV Cache landed in 4.38.0 and made training much slower (#168,# huggingface/transformers#27931), so the old rotary embeddings are retained.-class LlamaRotaryEmbedding(torch.nn.Module):+class LlamaRotaryEmbedding(_PerDeviceRopeCache, torch.nn.Module):# RoPE buffer precision is wrong, so cast to int64; see huggingface/transformers#28837 and microsoft/DeepSpeed#4932.def __init__(self,@@ -2178,12 +2294,8 @@ class LlamaRotaryEmbedding(torch.nn.Module):inv_freq = self._unsloth_recompute_inv_freq()self.register_buffer("inv_freq", inv_freq, persistent = False)- for device_idx in range(DEVICE_COUNT):- self._set_cos_sin_cache(- seq_len = self.current_rope_size,- device = torch.device(device_idx),- dtype = torch.get_default_dtype(),- )+ # Built per device on first use, not for every visible GPU here.+ self._init_rope_cache_state(torch.get_default_dtype())self.cos_cached = torch.empty(1, device = get_current_device(), dtype = torch.get_default_dtype()@@ -2242,40 +2354,6 @@ class LlamaRotaryEmbedding(torch.nn.Module):self.multi_gpu_sin_cached[device.index] = sinreturn cos, sin- def forward(- self,- x,- position_ids = None,- seq_len = None,- ):- if seq_len is not None and seq_len > self.current_rope_size:- self._set_cos_sin_cache(seq_len = seq_len, device = x.device, dtype = x.dtype)-- device_index = x.device.index- return (- self.multi_gpu_cos_cached[device_index][:seq_len],- self.multi_gpu_sin_cached[device_index][:seq_len],- )-- def get_cached(- self,- seq_len = None,- device_index = None,- ):- if device_index is None:- device_index = get_current_device()- return self.multi_gpu_cos_cached[device_index], self.multi_gpu_sin_cached[device_index]-- def extend_rope_embedding(self, x, seq_len):- if seq_len <= self.current_rope_size:- return- # Iteratively grow by increments of 8192.- self.current_rope_size = ((seq_len // 8192) + ((seq_len % 8192) != 0)) * 8192- for device_idx in range(DEVICE_COUNT):- self._set_cos_sin_cache(- self.current_rope_size, device = torch.device(device_idx), dtype = x.dtype- )-class LlamaLinearScalingRotaryEmbedding(LlamaRotaryEmbedding):"""LlamaRotaryEmbedding extended with linear scaling. Credits to the Reddit user /u/kaiokendev"""@@ -2353,7 +2431,7 @@ class LlamaExtendedRotaryEmbedding(LlamaRotaryEmbedding):return torch.tensor(new_freqs, dtype = freqs.dtype, device = freqs.device)-class LongRopeRotaryEmbedding(torch.nn.Module):+class LongRopeRotaryEmbedding(_PerDeviceRopeCache, torch.nn.Module):# For Phi 3.5 128K; see microsoft/Phi-3.5-mini-instruct modeling_phi3.pydef __init__(self,@@ -2413,7 +2491,8 @@ class LongRopeRotaryEmbedding(torch.nn.Module):self.register_buffer("short_inv_freq", short_inv_freq, persistent = False)self.register_buffer("long_inv_freq", long_inv_freq, persistent = False)- # Build here to make torch.jit.trace work, initializing the short-sequence cache for all devices.+ # The short-sequence cache is computed here, on CPU, and copied to a device on first use. Kept+ # off the buffers so a later load refilling short_inv_freq cannot reach it (#884).dtype = torch.bfloat16 if is_bfloat16_supported() else torch.float16t = torch.arange(original_max_position_embeddings,@@ -2422,17 +2501,10 @@ class LongRopeRotaryEmbedding(torch.nn.Module):).float()freqs = torch.outer(t, self.short_inv_freq)emb = torch.cat((freqs, freqs), dim = -1)-- for device_idx in range(DEVICE_COUNT):- device_obj = torch.device(device_idx)- cos_cached = (emb.cos() * self.scaling_factor).to(- dtype = dtype, device = device_obj, non_blocking = True- )- sin_cached = (emb.sin() * self.scaling_factor).to(- dtype = dtype, device = device_obj, non_blocking = True- )- self.multi_gpu_short_cos_cached[device_idx] = cos_cached- self.multi_gpu_short_sin_cached[device_idx] = sin_cached+ self._unsloth_short_cos_cpu = (emb.cos() * self.scaling_factor).to(dtype = dtype)+ self._unsloth_short_sin_cpu = (emb.sin() * self.scaling_factor).to(dtype = dtype)+ # The long cache only exists once something asks for more than original_max_position_embeddings.+ self._init_rope_cache_state(dtype)self.short_cos_cached = torch.empty(1, device = get_current_device(), dtype = torch.get_default_dtype()@@ -2466,17 +2538,24 @@ class LongRopeRotaryEmbedding(torch.nn.Module):self.multi_gpu_long_sin_cached[device.index] = sin_cachedreturn cos_cached, sin_cached- def forward(- self,- x,- position_ids = None,- seq_len = None,- ):- if seq_len is not None and seq_len > self.current_rope_size:- self._set_cos_sin_cache(seq_len = seq_len, device = x.device, dtype = x.dtype)-- device_index = x.device.index+ def _ensure_rope_cache(self, device_index):+ if device_index is None:+ device_index = get_current_device()+ if self.multi_gpu_short_cos_cached[device_index] is None:+ device = torch.device(device_index)+ self.multi_gpu_short_cos_cached[device_index] = self._unsloth_short_cos_cpu.to(+ device = device, non_blocking = True+ )+ self.multi_gpu_short_sin_cached[device_index] = self._unsloth_short_sin_cpu.to(+ device = device, non_blocking = True+ )+ if self._rope_cache_version > 0 and self._rope_cache_stale(+ self.multi_gpu_long_cos_cached[device_index], device_index+ ):+ self._build_rope_cache(device_index)+ return device_index+ def _slice_rope_cache(self, device_index, seq_len):# transformers' _compute_longrope_parameters takes the long factor only on# `seq_len and seq_len > original_max_position_embeddings`, so an unknown length is# short too. Long stays None until something asks for a longer sequence, so routing@@ -2497,8 +2576,7 @@ class LongRopeRotaryEmbedding(torch.nn.Module):seq_len = None,device_index = None,):- if device_index is None:- device_index = get_current_device()+ device_index = self._ensure_rope_cache(device_index)if seq_len is None or seq_len <= self.original_max_position_embeddings:return self.multi_gpu_short_cos_cached[device_index], self.multi_gpu_short_sin_cached[device_index@@ -2507,16 +2585,6 @@ class LongRopeRotaryEmbedding(torch.nn.Module):device_index]- def extend_rope_embedding(self, x, seq_len):- if seq_len <= self.current_rope_size:- return- # Iteratively grow by increments of 8192.- self.current_rope_size = ((seq_len // 8192) + ((seq_len % 8192) != 0)) * 8192- for device_idx in range(DEVICE_COUNT):- self._set_cos_sin_cache(- self.current_rope_size, device = torch.device(device_idx), dtype = x.dtype- )-def unsloth_fast_generate(self, *args, **kwargs):restore_training_mode = self.trainingdiff --git a/unsloth/models/loader.py b/unsloth/models/loader.pyindex 33302ae8cd4..0321be561bd 100644--- a/unsloth/models/loader.py+++ b/unsloth/models/loader.py@@ -943,6 +943,14 @@ def _fix_rope_inv_freq(model):).float()freqs = torch.outer(t, module.short_inv_freq)emb = torch.cat((freqs, freqs), dim = -1)+ if hasattr(module, "_unsloth_short_cos_cpu"):+ # Devices not built yet copy the short cache from these on first use.+ module._unsloth_short_cos_cpu = (emb.cos() * module.scaling_factor).to(+ dtype = dtype+ )+ module._unsloth_short_sin_cpu = (emb.sin() * module.scaling_factor).to(+ dtype = dtype+ )for device_idx in range(len(module.multi_gpu_short_cos_cached)):if module.multi_gpu_short_cos_cached[device_idx] is not None:device_obj = torch.device(device_idx)