Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
RoPE cache: build cos/sin outside inference_mode so GRPO keeps the compiled decoder block (#14584)
Daniel Han77e742c8e2f2
diff --git a/tests/test_rope_cache_current_device.py b/tests/test_rope_cache_current_device.pyindex 32e8c509669..81f0cc57a61 100644--- a/tests/test_rope_cache_current_device.py+++ b/tests/test_rope_cache_current_device.py@@ -425,3 +425,71 @@ def test_cache_built_in_chunks_matches_one_shot(rows, dtype):ref_cos, ref_sin = _reference(rows, dtype)assert cos.shape == ref_cos.shape and cos.dtype == dtypeassert torch.equal(cos, ref_cos) and torch.equal(sin, ref_sin)+++def _assert_trainable(*tensors):+ for t in tensors:+ assert not t.is_inference()+ cos = tensors[0]+ x = torch.ones(cos.shape, dtype = cos.dtype, requires_grad = True)+ (x * cos).sum().backward()+ assert torch.equal(x.grad, cos)+++def _gemma(max_position):+ llama_ns = _llama_ns()+ ns = _load(+ GEMMA,+ ["GemmaFixedRotaryEmbedding"],+ {"_PerDeviceRopeCache": llama_ns["_PerDeviceRopeCache"]},+ )+ return ns["GemmaFixedRotaryEmbedding"](dim = DIM, max_position_embeddings = max_position, base = BASE)++[email protected]("kind", ["llama", "gemma"])+def test_cache_first_built_in_inference_mode_is_trainable(kind):+ rope = _llama(1 << 17) if kind == "llama" else _gemma(1 << 17)+ reference = _llama(1 << 17) if kind == "llama" else _gemma(1 << 17)+ with torch.inference_mode():+ cos, sin = rope.get_cached(100, 1)+ assert torch.is_inference_mode_enabled()+ _assert_trainable(cos, sin, *([rope.inv_freq] if kind == "llama" else []))+ ref_cos, ref_sin = reference.get_cached(100, 1)+ assert torch.equal(cos, ref_cos) and torch.equal(sin, ref_sin)+++def test_cache_grown_in_inference_mode_is_trainable():+ rope = _llama(1 << 17)+ rope.get_cached(10, 0)+ with torch.inference_mode():+ rope.extend_rope_embedding(_x(0, torch.float16), 50000)+ cos, sin = rope(_x(0, torch.float16), seq_len = 50000)+ _assert_trainable(cos, sin, rope.inv_freq)+ ref_cos, ref_sin = _reference(57344, torch.float16)+ assert torch.equal(rope.multi_gpu_cos_cached[0], ref_cos)+ assert torch.equal(rope.multi_gpu_sin_cached[0], ref_sin)+++def test_longrope_cache_built_in_inference_mode_is_trainable():+ rope = _llama_ns()["LongRopeRotaryEmbedding"](+ dim = DIM,+ max_position_embeddings = 128,+ original_max_position_embeddings = 16,+ short_factor = [1.0] * (DIM // 2),+ long_factor = [2.0] * (DIM // 2),+ )+ with torch.inference_mode():+ rope.get_cached(8, 1)+ rope.extend_rope_embedding(_x(1, torch.float16), 100)+ _assert_trainable(rope.multi_gpu_short_cos_cached[1], rope.multi_gpu_short_sin_cached[1])+ _assert_trainable(rope.multi_gpu_long_cos_cached[1], rope.multi_gpu_long_sin_cached[1])+++def test_cache_build_keeps_grad_mode():+ rope = _llama(1 << 17)+ with torch.inference_mode():+ rope.get_cached(10, 0)+ assert torch.is_inference_mode_enabled() and not torch.is_grad_enabled()+ rope = _llama(1 << 17)+ rope.get_cached(10, 0)+ assert torch.is_grad_enabled() and not rope.multi_gpu_cos_cached[0].requires_graddiff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex 7f8590ee5ed..e39aa7da9d4 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -2314,14 +2314,28 @@ class _PerDeviceRopeCache:rows = max(self._rope_cache_floor, _round_rope_size(self._rope_cache_needed))return min(self.current_rope_size, rows)+ @staticmethod+ def _rope_build_mode():+ # The first build often runs inside generate's inference_mode; inference tensors cannot be+ # saved for backward, so the compiled training block failed and fell back to eager.+ import contextlib++ if not torch.is_inference_mode_enabled():+ return contextlib.nullcontext()+ stack = contextlib.ExitStack()+ stack.enter_context(torch.inference_mode(False))+ stack.enter_context(torch.no_grad())+ return stack+def _build_rope_cache(self, device_index):# _set_cos_sin_cache records its length as current_rope_size; a lazy build may be shorter.current_rope_size = self.current_rope_size- self._set_cos_sin_cache(- seq_len = self._rope_cache_rows(),- device = torch.device(device_index),- dtype = self._rope_cache_dtype,- )+ with self._rope_build_mode():+ self._set_cos_sin_cache(+ seq_len = self._rope_cache_rows(),+ device = torch.device(device_index),+ dtype = self._rope_cache_dtype,+ )self.current_rope_size = current_rope_sizeself._rope_cache_versions[device_index] = self._rope_cache_version@@ -2717,12 +2731,13 @@ class LongRopeRotaryEmbedding(_PerDeviceRopeCache, torch.nn.Module):device_index = get_current_device()if self.multi_gpu_short_cos_cached[device_index] is None:device = torch.device(device_index)- self.multi_gpu_short_cos_cached[device_index] = self._unsloth_short_cos_cpu.to(- device = device, non_blocking = True- )- self.multi_gpu_short_sin_cached[device_index] = self._unsloth_short_sin_cpu.to(- device = device, non_blocking = True- )+ with self._rope_build_mode():+ self.multi_gpu_short_cos_cached[device_index] = self._unsloth_short_cos_cpu.to(+ device = device, non_blocking = True+ )+ self.multi_gpu_short_sin_cached[device_index] = self._unsloth_short_sin_cpu.to(+ device = device, non_blocking = True+ )if self._rope_cache_version > 0 and self._rope_cache_stale(self.multi_gpu_long_cos_cached[device_index], device_index):