Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
RoPE cache: a cold build inside a compiled block traces again (no is_inference_mode_enabled under compile) (#14728)
Daniel Han1fe53a8c5a9b
diff --git a/tests/test_rms_rope_compile.py b/tests/test_rms_rope_compile.pyindex e0b0e24e968..5e6e38e02b0 100644--- a/tests/test_rms_rope_compile.py+++ b/tests/test_rms_rope_compile.py@@ -267,6 +267,31 @@ def test_bf16_without_native_support_stays_untraced(monkeypatch):assert _graph_breaks() > 0[email protected]("fullgraph", [False, True], ids = ["default", "fullgraph"])+def test_rope_cache_cold_build_traces_without_graph_breaks(fullgraph):+ """A compiled block that builds the per-device RoPE cache first (a device_map split, or an+ attention that reads the cache itself) traces it with no graph break, as before lazy caches."""+ from transformers import LlamaConfig+ from unsloth.models.llama import LlamaRotaryEmbedding++ config = LlamaConfig(hidden_size = 256, num_attention_heads = 4, max_position_embeddings = 4096)+ rope = LlamaRotaryEmbedding(config = config)++ def attention(V):+ seq = V.shape[-2]+ rope.extend_rope_embedding(V, seq_len = seq)+ cos, sin = rope.get_cached(seq, V.device.index)+ return V * cos[:seq] + sin[:seq]++ torch._dynamo.reset()+ dynamo_utils.counters.clear()+ compiled = torch.compile(attention, dynamic = True, fullgraph = fullgraph, backend = "eager")+ for seq in (64, 80, 96):+ V = torch.randn(1, seq, 4, 64, device = "cuda").transpose(1, 2).contiguous()+ torch.testing.assert_close(compiled(V), attention(V))+ assert _graph_breaks() == 0++@pytest.mark.parametrize("gemma", [False, True], ids = ["llama", "gemma"])@pytest.mark.parametrize("dim", [64, 4096])def test_rms_layernorm_dynamic_shapes(dim, gemma):diff --git a/tests/test_rope_cache_current_device.py b/tests/test_rope_cache_current_device.pyindex 81f0cc57a61..fb677ab2c57 100644--- a/tests/test_rope_cache_current_device.py+++ b/tests/test_rope_cache_current_device.py@@ -493,3 +493,122 @@ def test_cache_build_keeps_grad_mode():rope = _llama(1 << 17)rope.get_cached(10, 0)assert torch.is_grad_enabled() and not rope.multi_gpu_cos_cached[0].requires_grad++[email protected]("kind", ["llama", "gemma", "longrope"])+def test_cold_build_under_compile_never_asks_for_inference_mode(kind, monkeypatch):+ # A compiled block that builds a device's cache first (a device_map split) must not reach+ # torch.is_inference_mode_enabled: Dynamo cannot trace it (graph break; fullgraph error).+ if kind == "longrope":+ rope = _llama_ns()["LongRopeRotaryEmbedding"](+ dim = DIM,+ max_position_embeddings = 128,+ original_max_position_embeddings = 16,+ short_factor = [1.0] * (DIM // 2),+ long_factor = [2.0] * (DIM // 2),+ )+ else:+ rope = _llama(1 << 17) if kind == "llama" else _gemma(1 << 17)++ def untraceable():+ raise AssertionError("is_inference_mode_enabled reached while compiling")++ monkeypatch.setattr(torch.compiler, "is_compiling", lambda: True)+ monkeypatch.setattr(torch, "is_inference_mode_enabled", untraceable)+ cos, sin = rope.get_cached(8, 1)+ assert cos is not None and sin is not None+ if kind == "longrope":+ rope.extend_rope_embedding(_x(1), 100)+ else:+ rope.extend_rope_embedding(_x(2), 50000)+++def _rope_kind(kind):+ if kind == "longrope":+ return _longrope(128, 8)+ return _llama(1 << 12) if kind == "llama" else _gemma(1 << 12)+++def _rope_cos_sin(rope, kind, device_index):+ if kind == "longrope":+ return rope.multi_gpu_long_cos_cached[device_index], rope.multi_gpu_long_sin_cached[+ device_index+ ]+ return rope.multi_gpu_cos_cached[device_index], rope.multi_gpu_sin_cached[device_index]++[email protected]("kind", ["llama", "gemma", "longrope"])+def test_compiled_inference_build_rebuilds_for_training(kind, monkeypatch):+ # A compiled block under inference_mode cannot leave it inside the graph, so its cold build+ # holds inference tensors; the next grad-enabled read must rebuild instead of serving them.+ rope = _rope_kind(kind)+ monkeypatch.setattr(torch.compiler, "is_compiling", lambda: True)+ with torch.inference_mode():+ # Stand-in for the compiled graph, which runs its build in inference mode.+ monkeypatch.setattr(rope, "_rope_build_mode", torch.inference_mode)+ rope.extend_rope_embedding(_x(1), 16)+ cos, sin = rope.get_cached(16, 1)+ assert cos.is_inference() and _rope_cos_sin(rope, kind, 1)[0].is_inference()+ # Inference reads keep the cache.+ assert rope.get_cached(16, 1)[0] is cos+ monkeypatch.undo()+ for compiling in (True, False):+ rope = _rope_kind(kind)+ monkeypatch.setattr(torch.compiler, "is_compiling", lambda: True)+ with torch.inference_mode():+ monkeypatch.setattr(rope, "_rope_build_mode", torch.inference_mode)+ rope.extend_rope_embedding(_x(1), 16)+ rope.get_cached(16, 1)+ monkeypatch.undo()+ monkeypatch.setattr(torch.compiler, "is_compiling", lambda: compiling)+ rope.extend_rope_embedding(_x(1), 16)+ cos, sin = rope.get_cached(16, 1)+ _assert_trainable(cos, sin, *_rope_cos_sin(rope, kind, 1))+ monkeypatch.undo()++[email protected]("kind", ["llama", "gemma", "longrope"])+def test_compiled_inference_build_then_training_under_dynamo(kind):+ # Compiled generation (inference_mode) builds the cache cold, then a compiled training block+ # and an eager one run backward through it; fullgraph with no graph break throughout.+ from torch._dynamo.utils import counters++ rope = _rope_kind(kind)++ def block(V):+ seq = V.shape[-2]+ rope.extend_rope_embedding(V, seq)+ cos, sin = rope.get_cached(seq, 0)+ # The CPU prelude names device 0 cpu:0; V is plain cpu.+ return V * cos[:seq].to(V.device) + sin[:seq].to(V.device)++ torch._dynamo.reset()+ counters.clear()+ compiled = torch.compile(block, backend = "aot_eager", fullgraph = True)+ with torch.inference_mode():+ compiled(torch.randn(2, 16, DIM))+ for fn in (compiled, block):+ V = torch.randn(2, 16, DIM, requires_grad = True)+ fn(V).sum().backward()+ cos = _rope_cos_sin(rope, kind, 0)[0][:16]+ torch.testing.assert_close(V.grad, cos.expand_as(V))+ assert sum(counters["graph_break"].values()) == 0+ torch._dynamo.reset()+++def test_longrope_long_rebuild_keeps_short_marked_for_training(monkeypatch):+ # Compiled inference builds only the short cache (inference tensors); an eager inference request+ # then grows the long one. Rebuilding long must not clear the short cache's grad-rebuild mark.+ rope = _longrope(128, 8)+ monkeypatch.setattr(torch.compiler, "is_compiling", lambda: True)+ with torch.inference_mode():+ monkeypatch.setattr(rope, "_rope_build_mode", torch.inference_mode)+ rope.get_cached(4, 1)+ assert rope.multi_gpu_short_cos_cached[1].is_inference()+ monkeypatch.undo()+ with torch.inference_mode():+ rope.extend_rope_embedding(_x(1), 100)+ assert not rope.multi_gpu_long_cos_cached[1].is_inference()+ cos, sin = rope.get_cached(4, 1)+ _assert_trainable(cos, sin)+ _assert_trainable(rope.multi_gpu_long_cos_cached[1], rope.multi_gpu_long_sin_cached[1])diff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex f7f872e6d60..922b063ee14 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -2453,6 +2453,9 @@ class _PerDeviceRopeCache:self._rope_cache_versions = [None] * DEVICE_COUNT# Rows each device can serve without bookkeeping; reset on growth. Read per layer per decode step.self._rope_rows_ready = [0] * DEVICE_COUNT+ # A compiled grad-off build may hold inference tensors (AOTAutograd drops+ # inference_mode(False)); rebuilt on the next grad-enabled read, off the fast path.+ self._rope_rebuild_for_grad = [False] * DEVICE_COUNT# Never fewer rows than the eager cache started with, so those positions stay valid.self._rope_cache_floor = self.current_rope_sizeself._rope_cache_needed = self.current_rope_size@@ -2467,13 +2470,22 @@ class _PerDeviceRopeCache:# saved for backward, so the compiled training block failed and fell back to eager.import contextlib- if not torch.is_inference_mode_enabled():+ # Dynamo cannot trace is_inference_mode_enabled; _mark_rope_build covers compiled builds.+ if torch.compiler.is_compiling() or not torch.is_inference_mode_enabled():return contextlib.nullcontext()stack = contextlib.ExitStack()stack.enter_context(torch.inference_mode(False))stack.enter_context(torch.no_grad())return stack+ def _mark_rope_build(self, device_index):+ self._rope_rebuild_for_grad[device_index] = (+ torch.compiler.is_compiling() and not torch.is_grad_enabled()+ )++ def _needs_grad_rebuild(self, device_index):+ return self._rope_rebuild_for_grad[device_index] and torch.is_grad_enabled()+def _build_rope_cache(self, device_index):# _set_cos_sin_cache records its length as current_rope_size; a lazy build may be shorter.current_rope_size = self.current_rope_size@@ -2485,12 +2497,14 @@ class _PerDeviceRopeCache:)self.current_rope_size = current_rope_sizeself._rope_cache_versions[device_index] = self._rope_cache_version+ self._mark_rope_build(device_index)def _rope_cache_stale(self, cached, device_index):return (cached is Noneor cached.shape[0] < self._rope_cache_rows()or self._rope_cache_versions[device_index] != self._rope_cache_version+ or self._needs_grad_rebuild(device_index))def _ensure_rope_cache(self, device_index):@@ -2498,7 +2512,11 @@ class _PerDeviceRopeCache:device_index = get_current_device()if self._rope_cache_stale(self.multi_gpu_cos_cached[device_index], device_index):self._build_rope_cache(device_index)- self._rope_rows_ready[device_index] = self.multi_gpu_cos_cached[device_index].shape[0]+ self._rope_rows_ready[device_index] = (+ 0+ if self._rope_rebuild_for_grad[device_index]+ else self.multi_gpu_cos_cached[device_index].shape[0]+ )return device_indexdef _grow_rope_cache(self, seq_len, dtype):@@ -2850,6 +2868,8 @@ class LongRopeRotaryEmbedding(_PerDeviceRopeCache, torch.nn.Module):self._unsloth_short_sin_cpu = (emb.sin() * self.scaling_factor).to(dtype = dtype)# The long cache only exists once something asks for more than original_max_position_embeddings.self._init_rope_cache_state(dtype)+ # Own mark for the short cache: a long rebuild must not clear it.+ self._short_rope_rebuild_for_grad = [False] * DEVICE_COUNTself.short_cos_cached = torch.empty(1, device = get_current_device(), dtype = torch.get_default_dtype()@@ -2886,7 +2906,10 @@ class LongRopeRotaryEmbedding(_PerDeviceRopeCache, torch.nn.Module):def _ensure_rope_cache(self, device_index):if device_index is None:device_index = get_current_device()- if self.multi_gpu_short_cos_cached[device_index] is None:+ short_rebuild_for_grad = (+ self._short_rope_rebuild_for_grad[device_index] and torch.is_grad_enabled()+ )+ if self.multi_gpu_short_cos_cached[device_index] is None or short_rebuild_for_grad:device = torch.device(device_index)with self._rope_build_mode():self.multi_gpu_short_cos_cached[device_index] = self._unsloth_short_cos_cpu.to(@@ -2895,6 +2918,9 @@ class LongRopeRotaryEmbedding(_PerDeviceRopeCache, torch.nn.Module):self.multi_gpu_short_sin_cached[device_index] = self._unsloth_short_sin_cpu.to(device = device, non_blocking = True)+ self._short_rope_rebuild_for_grad[device_index] = (+ torch.compiler.is_compiling() and not torch.is_grad_enabled()+ )if self._rope_cache_version > 0 and self._rope_cache_stale(self.multi_gpu_long_cos_cached[device_index], device_index):