Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

RoPE cache: build cos/sin outside inference_mode so GRPO keeps the compiled decoder block (#14584)

Daniel Han77e742c8e2f2
diff --git a/tests/test_rope_cache_current_device.py b/tests/test_rope_cache_current_device.py
index 32e8c509669..81f0cc57a61 100644
--- a/tests/test_rope_cache_current_device.py
+++ b/tests/test_rope_cache_current_device.py
@@ -425,3 +425,71 @@ def test_cache_built_in_chunks_matches_one_shot(rows, dtype):
ref_cos, ref_sin = _reference(rows, dtype)
assert cos.shape == ref_cos.shape and cos.dtype == dtype
assert torch.equal(cos, ref_cos) and torch.equal(sin, ref_sin)
+
+
+def _assert_trainable(*tensors):
+ for t in tensors:
+ assert not t.is_inference()
+ cos = tensors[0]
+ x = torch.ones(cos.shape, dtype = cos.dtype, requires_grad = True)
+ (x * cos).sum().backward()
+ assert torch.equal(x.grad, cos)
+
+
+def _gemma(max_position):
+ llama_ns = _llama_ns()
+ ns = _load(
+ GEMMA,
+ ["GemmaFixedRotaryEmbedding"],
+ {"_PerDeviceRopeCache": llama_ns["_PerDeviceRopeCache"]},
+ )
+ return ns["GemmaFixedRotaryEmbedding"](dim = DIM, max_position_embeddings = max_position, base = BASE)
+
+
[email protected]("kind", ["llama", "gemma"])
+def test_cache_first_built_in_inference_mode_is_trainable(kind):
+ rope = _llama(1 << 17) if kind == "llama" else _gemma(1 << 17)
+ reference = _llama(1 << 17) if kind == "llama" else _gemma(1 << 17)
+ with torch.inference_mode():
+ cos, sin = rope.get_cached(100, 1)
+ assert torch.is_inference_mode_enabled()
+ _assert_trainable(cos, sin, *([rope.inv_freq] if kind == "llama" else []))
+ ref_cos, ref_sin = reference.get_cached(100, 1)
+ assert torch.equal(cos, ref_cos) and torch.equal(sin, ref_sin)
+
+
+def test_cache_grown_in_inference_mode_is_trainable():
+ rope = _llama(1 << 17)
+ rope.get_cached(10, 0)
+ with torch.inference_mode():
+ rope.extend_rope_embedding(_x(0, torch.float16), 50000)
+ cos, sin = rope(_x(0, torch.float16), seq_len = 50000)
+ _assert_trainable(cos, sin, rope.inv_freq)
+ ref_cos, ref_sin = _reference(57344, torch.float16)
+ assert torch.equal(rope.multi_gpu_cos_cached[0], ref_cos)
+ assert torch.equal(rope.multi_gpu_sin_cached[0], ref_sin)
+
+
+def test_longrope_cache_built_in_inference_mode_is_trainable():
+ rope = _llama_ns()["LongRopeRotaryEmbedding"](
+ dim = DIM,
+ max_position_embeddings = 128,
+ original_max_position_embeddings = 16,
+ short_factor = [1.0] * (DIM // 2),
+ long_factor = [2.0] * (DIM // 2),
+ )
+ with torch.inference_mode():
+ rope.get_cached(8, 1)
+ rope.extend_rope_embedding(_x(1, torch.float16), 100)
+ _assert_trainable(rope.multi_gpu_short_cos_cached[1], rope.multi_gpu_short_sin_cached[1])
+ _assert_trainable(rope.multi_gpu_long_cos_cached[1], rope.multi_gpu_long_sin_cached[1])
+
+
+def test_cache_build_keeps_grad_mode():
+ rope = _llama(1 << 17)
+ with torch.inference_mode():
+ rope.get_cached(10, 0)
+ assert torch.is_inference_mode_enabled() and not torch.is_grad_enabled()
+ rope = _llama(1 << 17)
+ rope.get_cached(10, 0)
+ assert torch.is_grad_enabled() and not rope.multi_gpu_cos_cached[0].requires_grad
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index 7f8590ee5ed..e39aa7da9d4 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -2314,14 +2314,28 @@ class _PerDeviceRopeCache:
rows = max(self._rope_cache_floor, _round_rope_size(self._rope_cache_needed))
return min(self.current_rope_size, rows)
+ @staticmethod
+ def _rope_build_mode():
+ # The first build often runs inside generate's inference_mode; inference tensors cannot be
+ # saved for backward, so the compiled training block failed and fell back to eager.
+ import contextlib
+
+ if not torch.is_inference_mode_enabled():
+ return contextlib.nullcontext()
+ stack = contextlib.ExitStack()
+ stack.enter_context(torch.inference_mode(False))
+ stack.enter_context(torch.no_grad())
+ return stack
+
def _build_rope_cache(self, device_index):
# _set_cos_sin_cache records its length as current_rope_size; a lazy build may be shorter.
current_rope_size = self.current_rope_size
- self._set_cos_sin_cache(
- seq_len = self._rope_cache_rows(),
- device = torch.device(device_index),
- dtype = self._rope_cache_dtype,
- )
+ with self._rope_build_mode():
+ self._set_cos_sin_cache(
+ seq_len = self._rope_cache_rows(),
+ device = torch.device(device_index),
+ dtype = self._rope_cache_dtype,
+ )
self.current_rope_size = current_rope_size
self._rope_cache_versions[device_index] = self._rope_cache_version
@@ -2717,12 +2731,13 @@ class LongRopeRotaryEmbedding(_PerDeviceRopeCache, torch.nn.Module):
device_index = get_current_device()
if self.multi_gpu_short_cos_cached[device_index] is None:
device = torch.device(device_index)
- self.multi_gpu_short_cos_cached[device_index] = self._unsloth_short_cos_cpu.to(
- device = device, non_blocking = True
- )
- self.multi_gpu_short_sin_cached[device_index] = self._unsloth_short_sin_cpu.to(
- device = device, non_blocking = True
- )
+ with self._rope_build_mode():
+ self.multi_gpu_short_cos_cached[device_index] = self._unsloth_short_cos_cpu.to(
+ device = device, non_blocking = True
+ )
+ self.multi_gpu_short_sin_cached[device_index] = self._unsloth_short_sin_cpu.to(
+ device = device, non_blocking = True
+ )
if self._rope_cache_version > 0 and self._rope_cache_stale(
self.multi_gpu_long_cos_cached[device_index], device_index
):