Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

RoPE cache: a cold build inside a compiled block traces again (no is_inference_mode_enabled under compile) (#14728)

Daniel Han1fe53a8c5a9b
diff --git a/tests/test_rms_rope_compile.py b/tests/test_rms_rope_compile.py
index e0b0e24e968..5e6e38e02b0 100644
--- a/tests/test_rms_rope_compile.py
+++ b/tests/test_rms_rope_compile.py
@@ -267,6 +267,31 @@ def test_bf16_without_native_support_stays_untraced(monkeypatch):
assert _graph_breaks() > 0
[email protected]("fullgraph", [False, True], ids = ["default", "fullgraph"])
+def test_rope_cache_cold_build_traces_without_graph_breaks(fullgraph):
+ """A compiled block that builds the per-device RoPE cache first (a device_map split, or an
+ attention that reads the cache itself) traces it with no graph break, as before lazy caches."""
+ from transformers import LlamaConfig
+ from unsloth.models.llama import LlamaRotaryEmbedding
+
+ config = LlamaConfig(hidden_size = 256, num_attention_heads = 4, max_position_embeddings = 4096)
+ rope = LlamaRotaryEmbedding(config = config)
+
+ def attention(V):
+ seq = V.shape[-2]
+ rope.extend_rope_embedding(V, seq_len = seq)
+ cos, sin = rope.get_cached(seq, V.device.index)
+ return V * cos[:seq] + sin[:seq]
+
+ torch._dynamo.reset()
+ dynamo_utils.counters.clear()
+ compiled = torch.compile(attention, dynamic = True, fullgraph = fullgraph, backend = "eager")
+ for seq in (64, 80, 96):
+ V = torch.randn(1, seq, 4, 64, device = "cuda").transpose(1, 2).contiguous()
+ torch.testing.assert_close(compiled(V), attention(V))
+ assert _graph_breaks() == 0
+
+
@pytest.mark.parametrize("gemma", [False, True], ids = ["llama", "gemma"])
@pytest.mark.parametrize("dim", [64, 4096])
def test_rms_layernorm_dynamic_shapes(dim, gemma):
diff --git a/tests/test_rope_cache_current_device.py b/tests/test_rope_cache_current_device.py
index 81f0cc57a61..fb677ab2c57 100644
--- a/tests/test_rope_cache_current_device.py
+++ b/tests/test_rope_cache_current_device.py
@@ -493,3 +493,122 @@ def test_cache_build_keeps_grad_mode():
rope = _llama(1 << 17)
rope.get_cached(10, 0)
assert torch.is_grad_enabled() and not rope.multi_gpu_cos_cached[0].requires_grad
+
+
[email protected]("kind", ["llama", "gemma", "longrope"])
+def test_cold_build_under_compile_never_asks_for_inference_mode(kind, monkeypatch):
+ # A compiled block that builds a device's cache first (a device_map split) must not reach
+ # torch.is_inference_mode_enabled: Dynamo cannot trace it (graph break; fullgraph error).
+ if kind == "longrope":
+ rope = _llama_ns()["LongRopeRotaryEmbedding"](
+ dim = DIM,
+ max_position_embeddings = 128,
+ original_max_position_embeddings = 16,
+ short_factor = [1.0] * (DIM // 2),
+ long_factor = [2.0] * (DIM // 2),
+ )
+ else:
+ rope = _llama(1 << 17) if kind == "llama" else _gemma(1 << 17)
+
+ def untraceable():
+ raise AssertionError("is_inference_mode_enabled reached while compiling")
+
+ monkeypatch.setattr(torch.compiler, "is_compiling", lambda: True)
+ monkeypatch.setattr(torch, "is_inference_mode_enabled", untraceable)
+ cos, sin = rope.get_cached(8, 1)
+ assert cos is not None and sin is not None
+ if kind == "longrope":
+ rope.extend_rope_embedding(_x(1), 100)
+ else:
+ rope.extend_rope_embedding(_x(2), 50000)
+
+
+def _rope_kind(kind):
+ if kind == "longrope":
+ return _longrope(128, 8)
+ return _llama(1 << 12) if kind == "llama" else _gemma(1 << 12)
+
+
+def _rope_cos_sin(rope, kind, device_index):
+ if kind == "longrope":
+ return rope.multi_gpu_long_cos_cached[device_index], rope.multi_gpu_long_sin_cached[
+ device_index
+ ]
+ return rope.multi_gpu_cos_cached[device_index], rope.multi_gpu_sin_cached[device_index]
+
+
[email protected]("kind", ["llama", "gemma", "longrope"])
+def test_compiled_inference_build_rebuilds_for_training(kind, monkeypatch):
+ # A compiled block under inference_mode cannot leave it inside the graph, so its cold build
+ # holds inference tensors; the next grad-enabled read must rebuild instead of serving them.
+ rope = _rope_kind(kind)
+ monkeypatch.setattr(torch.compiler, "is_compiling", lambda: True)
+ with torch.inference_mode():
+ # Stand-in for the compiled graph, which runs its build in inference mode.
+ monkeypatch.setattr(rope, "_rope_build_mode", torch.inference_mode)
+ rope.extend_rope_embedding(_x(1), 16)
+ cos, sin = rope.get_cached(16, 1)
+ assert cos.is_inference() and _rope_cos_sin(rope, kind, 1)[0].is_inference()
+ # Inference reads keep the cache.
+ assert rope.get_cached(16, 1)[0] is cos
+ monkeypatch.undo()
+ for compiling in (True, False):
+ rope = _rope_kind(kind)
+ monkeypatch.setattr(torch.compiler, "is_compiling", lambda: True)
+ with torch.inference_mode():
+ monkeypatch.setattr(rope, "_rope_build_mode", torch.inference_mode)
+ rope.extend_rope_embedding(_x(1), 16)
+ rope.get_cached(16, 1)
+ monkeypatch.undo()
+ monkeypatch.setattr(torch.compiler, "is_compiling", lambda: compiling)
+ rope.extend_rope_embedding(_x(1), 16)
+ cos, sin = rope.get_cached(16, 1)
+ _assert_trainable(cos, sin, *_rope_cos_sin(rope, kind, 1))
+ monkeypatch.undo()
+
+
[email protected]("kind", ["llama", "gemma", "longrope"])
+def test_compiled_inference_build_then_training_under_dynamo(kind):
+ # Compiled generation (inference_mode) builds the cache cold, then a compiled training block
+ # and an eager one run backward through it; fullgraph with no graph break throughout.
+ from torch._dynamo.utils import counters
+
+ rope = _rope_kind(kind)
+
+ def block(V):
+ seq = V.shape[-2]
+ rope.extend_rope_embedding(V, seq)
+ cos, sin = rope.get_cached(seq, 0)
+ # The CPU prelude names device 0 cpu:0; V is plain cpu.
+ return V * cos[:seq].to(V.device) + sin[:seq].to(V.device)
+
+ torch._dynamo.reset()
+ counters.clear()
+ compiled = torch.compile(block, backend = "aot_eager", fullgraph = True)
+ with torch.inference_mode():
+ compiled(torch.randn(2, 16, DIM))
+ for fn in (compiled, block):
+ V = torch.randn(2, 16, DIM, requires_grad = True)
+ fn(V).sum().backward()
+ cos = _rope_cos_sin(rope, kind, 0)[0][:16]
+ torch.testing.assert_close(V.grad, cos.expand_as(V))
+ assert sum(counters["graph_break"].values()) == 0
+ torch._dynamo.reset()
+
+
+def test_longrope_long_rebuild_keeps_short_marked_for_training(monkeypatch):
+ # Compiled inference builds only the short cache (inference tensors); an eager inference request
+ # then grows the long one. Rebuilding long must not clear the short cache's grad-rebuild mark.
+ rope = _longrope(128, 8)
+ monkeypatch.setattr(torch.compiler, "is_compiling", lambda: True)
+ with torch.inference_mode():
+ monkeypatch.setattr(rope, "_rope_build_mode", torch.inference_mode)
+ rope.get_cached(4, 1)
+ assert rope.multi_gpu_short_cos_cached[1].is_inference()
+ monkeypatch.undo()
+ with torch.inference_mode():
+ rope.extend_rope_embedding(_x(1), 100)
+ assert not rope.multi_gpu_long_cos_cached[1].is_inference()
+ cos, sin = rope.get_cached(4, 1)
+ _assert_trainable(cos, sin)
+ _assert_trainable(rope.multi_gpu_long_cos_cached[1], rope.multi_gpu_long_sin_cached[1])
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index f7f872e6d60..922b063ee14 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -2453,6 +2453,9 @@ class _PerDeviceRopeCache:
self._rope_cache_versions = [None] * DEVICE_COUNT
# Rows each device can serve without bookkeeping; reset on growth. Read per layer per decode step.
self._rope_rows_ready = [0] * DEVICE_COUNT
+ # A compiled grad-off build may hold inference tensors (AOTAutograd drops
+ # inference_mode(False)); rebuilt on the next grad-enabled read, off the fast path.
+ self._rope_rebuild_for_grad = [False] * DEVICE_COUNT
# Never fewer rows than the eager cache started with, so those positions stay valid.
self._rope_cache_floor = self.current_rope_size
self._rope_cache_needed = self.current_rope_size
@@ -2467,13 +2470,22 @@ class _PerDeviceRopeCache:
# saved for backward, so the compiled training block failed and fell back to eager.
import contextlib
- if not torch.is_inference_mode_enabled():
+ # Dynamo cannot trace is_inference_mode_enabled; _mark_rope_build covers compiled builds.
+ if torch.compiler.is_compiling() or not torch.is_inference_mode_enabled():
return contextlib.nullcontext()
stack = contextlib.ExitStack()
stack.enter_context(torch.inference_mode(False))
stack.enter_context(torch.no_grad())
return stack
+ def _mark_rope_build(self, device_index):
+ self._rope_rebuild_for_grad[device_index] = (
+ torch.compiler.is_compiling() and not torch.is_grad_enabled()
+ )
+
+ def _needs_grad_rebuild(self, device_index):
+ return self._rope_rebuild_for_grad[device_index] and torch.is_grad_enabled()
+
def _build_rope_cache(self, device_index):
# _set_cos_sin_cache records its length as current_rope_size; a lazy build may be shorter.
current_rope_size = self.current_rope_size
@@ -2485,12 +2497,14 @@ class _PerDeviceRopeCache:
)
self.current_rope_size = current_rope_size
self._rope_cache_versions[device_index] = self._rope_cache_version
+ self._mark_rope_build(device_index)
def _rope_cache_stale(self, cached, device_index):
return (
cached is None
or cached.shape[0] < self._rope_cache_rows()
or self._rope_cache_versions[device_index] != self._rope_cache_version
+ or self._needs_grad_rebuild(device_index)
)
def _ensure_rope_cache(self, device_index):
@@ -2498,7 +2512,11 @@ class _PerDeviceRopeCache:
device_index = get_current_device()
if self._rope_cache_stale(self.multi_gpu_cos_cached[device_index], device_index):
self._build_rope_cache(device_index)
- self._rope_rows_ready[device_index] = self.multi_gpu_cos_cached[device_index].shape[0]
+ self._rope_rows_ready[device_index] = (
+ 0
+ if self._rope_rebuild_for_grad[device_index]
+ else self.multi_gpu_cos_cached[device_index].shape[0]
+ )
return device_index
def _grow_rope_cache(self, seq_len, dtype):
@@ -2850,6 +2868,8 @@ class LongRopeRotaryEmbedding(_PerDeviceRopeCache, torch.nn.Module):
self._unsloth_short_sin_cpu = (emb.sin() * self.scaling_factor).to(dtype = dtype)
# The long cache only exists once something asks for more than original_max_position_embeddings.
self._init_rope_cache_state(dtype)
+ # Own mark for the short cache: a long rebuild must not clear it.
+ self._short_rope_rebuild_for_grad = [False] * DEVICE_COUNT
self.short_cos_cached = torch.empty(
1, device = get_current_device(), dtype = torch.get_default_dtype()
@@ -2886,7 +2906,10 @@ class LongRopeRotaryEmbedding(_PerDeviceRopeCache, torch.nn.Module):
def _ensure_rope_cache(self, device_index):
if device_index is None:
device_index = get_current_device()
- if self.multi_gpu_short_cos_cached[device_index] is None:
+ short_rebuild_for_grad = (
+ self._short_rope_rebuild_for_grad[device_index] and torch.is_grad_enabled()
+ )
+ if self.multi_gpu_short_cos_cached[device_index] is None or short_rebuild_for_grad:
device = torch.device(device_index)
with self._rope_build_mode():
self.multi_gpu_short_cos_cached[device_index] = self._unsloth_short_cos_cpu.to(
@@ -2895,6 +2918,9 @@ class LongRopeRotaryEmbedding(_PerDeviceRopeCache, torch.nn.Module):
self.multi_gpu_short_sin_cached[device_index] = self._unsloth_short_sin_cpu.to(
device = device, non_blocking = True
)
+ self._short_rope_rebuild_for_grad[device_index] = (
+ torch.compiler.is_compiling() and not torch.is_grad_enabled()
+ )
if self._rope_cache_version > 0 and self._rope_cache_stale(
self.multi_gpu_long_cos_cached[device_index], device_index
):