Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
RoPE: compile stock positions without full-context caches (#16182)
Daniel Han6b01508401df
diff --git a/tests/test_rope_cache_current_device.py b/tests/test_rope_cache_current_device.pyindex c527c9c6981..cde5dd8beac 100644--- a/tests/test_rope_cache_current_device.py+++ b/tests/test_rope_cache_current_device.py@@ -15,6 +15,7 @@ import typesimport pytestimport torch+from real_accelerator import has_real_cudaREPO_ROOT = pathlib.Path(__file__).resolve().parent.parentLLAMA = REPO_ROOT / "unsloth" / "models" / "llama.py"@@ -174,7 +175,9 @@ def test_get_cached_materialises_the_length_it_is_asked_for():cos, sin = rope.get_cached(40000, 0)assert cos.shape[0] >= 40000ref_cos, ref_sin = _reference(cos.shape[0])- assert torch.equal(cos, ref_cos) and torch.equal(sin, ref_sin)+ # ARM trig vectorization can round differently across cache chunk sizes.+ torch.testing.assert_close(cos, ref_cos, rtol = 0, atol = 2 * torch.finfo(cos.dtype).eps)+ torch.testing.assert_close(sin, ref_sin, rtol = 0, atol = 2 * torch.finfo(sin.dtype).eps)def test_forward_builds_only_input_device():@@ -611,3 +614,175 @@ def test_longrope_long_rebuild_keeps_short_marked_for_training(monkeypatch):cos, sin = rope.get_cached(4, 1)_assert_trainable(cos, sin)_assert_trainable(rope.multi_gpu_long_cos_cached[1], rope.multi_gpu_long_sin_cached[1])+++def test_stock_call_compiles_as_one_graph():+ rope = _llama(1 << 17)+ x = torch.zeros(1, 4, DIM)+ position_ids = torch.tensor([[3, 4, 5, 6]])+ eager_cos, eager_sin = rope(x, position_ids)+ torch._dynamo.reset()+ explained = torch._dynamo.explain(lambda x, p: rope(x, p))(x, position_ids)+ assert explained.graph_break_count == 0, explained.break_reasons+ cos, sin = torch.compile(rope, backend = "eager", fullgraph = True)(x, position_ids + 1)+ ref_cos, ref_sin = _reference(8)+ assert torch.equal(eager_cos[0], ref_cos[3:7]) and torch.equal(eager_sin[0], ref_sin[3:7])+ assert torch.equal(cos[0], ref_cos[4:8]) and torch.equal(sin[0], ref_sin[4:8])++[email protected]("backend", ["eager", "aot_eager"])+def test_compiled_stock_call_serves_every_position_below_max_position(backend):+ max_position = 1 << 16+ rope = _llama(max_position)+ torch._dynamo.reset()+ compiled = torch.compile(rope, backend = backend, fullgraph = True)+ ref_cos, ref_sin = _reference(max_position)+ x = torch.zeros(1, 4, DIM)+ for start in (0, 32766, 32768, max_position - 4):+ position_ids = torch.arange(start, start + 4)[None]+ cos, sin = compiled(x, position_ids)+ torch.testing.assert_close(+ cos[0], ref_cos[start : start + 4], rtol = 0, atol = 2 * torch.finfo(cos.dtype).eps+ )+ torch.testing.assert_close(+ sin[0], ref_sin[start : start + 4], rtol = 0, atol = 2 * torch.finfo(sin.dtype).eps+ )+ explained = torch._dynamo.explain(lambda x, p: rope(x, p))(x, torch.tensor([[40000, 40001]]))+ assert explained.graph_break_count == 0, explained.break_reasons+++def test_compiled_stock_call_preserves_eager_cache_dtype():+ max_position = 1 << 16+ rope = _llama(max_position)+ torch._dynamo.reset()+ compiled = torch.compile(rope, backend = "eager", fullgraph = True)+ position_ids = torch.arange(3, 7)[None]+ compiled(torch.zeros(1, 4, DIM, dtype = torch.bfloat16), position_ids)+ torch._dynamo.reset()+ cos, sin = rope(torch.zeros(1, 4, DIM), position_ids)+ ref_cos, ref_sin = _reference(max_position)+ assert cos.dtype == torch.float32+ assert torch.equal(cos[0], ref_cos[3:7])+ assert torch.equal(sin[0], ref_sin[3:7])++[email protected]("backend", ["eager", "aot_eager"])[email protected]("kind", ["llama", "linear", "extended", "gemma", "gemma_linear"])+def test_compiled_stock_positions_cross_configured_extent(kind, backend):+ ns = _load(+ LLAMA,+ list(_HELPERS)+ + [+ "LlamaRotaryEmbedding",+ "LlamaLinearScalingRotaryEmbedding",+ "LlamaExtendedRotaryEmbedding",+ ],+ )+ if kind.startswith("gemma"):+ ns = _load(+ GEMMA,+ ["GemmaFixedRotaryEmbedding", "GemmaFixedLinearScalingRotaryEmbedding"],+ {"_PerDeviceRopeCache": ns["_PerDeviceRopeCache"]},+ )+ name = {+ "llama": "LlamaRotaryEmbedding",+ "linear": "LlamaLinearScalingRotaryEmbedding",+ "extended": "LlamaExtendedRotaryEmbedding",+ "gemma": "GemmaFixedRotaryEmbedding",+ "gemma_linear": "GemmaFixedLinearScalingRotaryEmbedding",+ }[kind]+ kwargs = {"dim": DIM, "max_position_embeddings": 16, "base": BASE}+ if kind.endswith("linear"):+ kwargs["scaling_factor"] = 1.3 if kind == "gemma_linear" else 2.0+ rope = ns[name](**kwargs)+ reference = ns[name](**kwargs)+ torch._dynamo.reset()+ compiled = torch.compile(rope, backend = backend, fullgraph = True, dynamic = True)+ for start in (0, 15, 32768):+ positions = torch.arange(start, start + 4)[None]+ x = torch.ones(1, 4, DIM, requires_grad = True)+ cos, sin = compiled(x, positions)+ ref_cos, ref_sin = reference(x, positions)+ assert torch.equal(cos, ref_cos)+ assert torch.equal(sin, ref_sin)+ (x * cos + x * sin).sum().backward()+ assert torch.equal(x.grad, cos + sin)+ assert _built(rope.multi_gpu_cos_cached) == []++[email protected]("rope_type", ["llama3", "yarn", "linear", "dynamic"])+def test_scaled_config_compiles_without_tracing_config_initialization(rope_type):+ import contextlib+ from transformers import LlamaConfig++ names = list(_HELPERS) + [+ "_get_rope_theta",+ "_rope_scaling_as_dict",+ "_llama3_inv_freq_from_config",+ "_vanilla_inv_freq_from_config",+ "_compute_config_rope_inv_freq",+ "LlamaRotaryEmbedding",+ ]+ ns = _load(LLAMA, names, {"contextlib": contextlib})+ scaling = {"rope_type": rope_type, "factor": 8.0}+ if rope_type == "llama3":+ scaling.update(+ low_freq_factor = 1.0, high_freq_factor = 4.0, original_max_position_embeddings = 8192+ )+ config = LlamaConfig(+ hidden_size = DIM,+ num_attention_heads = 1,+ head_dim = DIM,+ max_position_embeddings = 1 << 17,+ rope_scaling = scaling,+ )+ rope = ns["LlamaRotaryEmbedding"](config = config)+ positions = torch.tensor([[0, 32768, 131071, 131072]])+ inv_freq = rope.inv_freq.clone()+ freqs = positions[..., None].float() * inv_freq+ emb = torch.cat((freqs, freqs), dim = -1)+ expected_cos = emb.cos() * rope.attention_scaling+ expected_sin = emb.sin() * rope.attention_scaling+ rope.inv_freq.fill_(float("nan"))+ torch._dynamo.reset()+ compiled = torch.compile(rope, backend = "aot_eager", fullgraph = True)+ x = torch.ones(1, 4, DIM, requires_grad = True)+ cos, sin = compiled(x, positions)+ assert torch.equal(cos, expected_cos)+ assert torch.equal(sin, expected_sin)+ (x * cos + x * sin).sum().backward()+ assert torch.equal(x.grad, cos + sin)++[email protected](not has_real_cuda(), reason = "CUDA/ROCm accelerator required")+ "kind", ["GemmaFixedRotaryEmbedding", "GemmaFixedLinearScalingRotaryEmbedding"]+)[email protected]("dtype", [torch.float32, torch.float16])+def test_compiled_gemma_stock_cuda_preserves_cpu_timescale(kind, dtype):+ ns = _load(LLAMA, list(_HELPERS), {"torch": torch, "get_current_device": lambda: 0})+ ns = _load(+ GEMMA,+ ["GemmaFixedRotaryEmbedding", "GemmaFixedLinearScalingRotaryEmbedding"],+ {+ "torch": torch,+ "get_current_device": lambda: 0,+ "_PerDeviceRopeCache": ns["_PerDeviceRopeCache"],+ },+ )+ kwargs = {"dim": 128, "max_position_embeddings": 16}+ if kind == "GemmaFixedLinearScalingRotaryEmbedding":+ kwargs["scaling_factor"] = 1.3+ rope = ns[kind](**kwargs)+ reference = ns[kind](**kwargs)+ positions = torch.tensor([[0, 15, 32767, 131071]], device = "cuda")+ x = torch.ones(1, 4, 128, device = "cuda", dtype = dtype, requires_grad = True)+ torch._dynamo.reset()+ cos, sin = torch.compile(rope, fullgraph = True, dynamic = True)(x, positions)+ ref_cos, ref_sin = reference(x, positions)+ tolerance = 1e-5 if dtype == torch.float32 else 2e-3+ torch.testing.assert_close(cos, ref_cos, rtol = tolerance, atol = tolerance)+ torch.testing.assert_close(sin, ref_sin, rtol = tolerance, atol = tolerance)+ ((cos + sin) * x).sum().backward()+ assert torch.isfinite(x.grad).all()diff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex 4ad30b03dab..d4398e02117 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -2463,8 +2463,22 @@ class _PerDeviceRopeCache:allocate on their peers. A row depends only on its position, so a shorter cache matches thefull one row for row."""+ _rope_rows_pick_factors = False+def _init_rope_cache_state(self, dtype):self._rope_cache_dtype = dtype+ if not hasattr(self, "inv_freq") and not self._rope_rows_pick_factors:+ # Keep Gemma's CPU FP32 constants before widening the compiled arithmetic.+ exponents = (2.0 / self.dim) * torch.arange(+ self.dim // 2, dtype = torch.int64, device = "cpu"+ ).float()+ self._rope_timescale_cpu = self.base**exponents+ self._rope_reciprocal_cpu = self._rope_timescale_cpu.double().reciprocal()+ self._rope_scaling_factor = float(+ torch.tensor(+ getattr(self, "scaling_factor", 1.0), dtype = torch.float32, device = "cpu"+ )+ )# Bumped whenever the eager code rebuilt every device (growth), so stale devices rebuild.self._rope_cache_version = 0self._rope_cache_versions = [None] * DEVICE_COUNT@@ -2554,6 +2568,8 @@ class _PerDeviceRopeCache:):if position_ids is not None and seq_len is None:# Stock transformers call (a plain model built after the patch): per-position cos/sin.+ if torch.compiler.is_compiling() and not self._rope_rows_pick_factors:+ return self._compiled_position_embeddings(x, position_ids)rows = int(position_ids.max()) + 1if rows > self.current_rope_size:self._grow_rope_cache(rows, x.dtype)@@ -2568,6 +2584,20 @@ class _PerDeviceRopeCache:device_index = self._ensure_rope_cache(x.device.index)return self._slice_rope_cache(device_index, seq_len)+ def _compiled_position_embeddings(self, x, position_ids):+ positions = position_ids.to(x.device).float()+ # Arc and Metal do not support FP64 tensors.+ compute_dtype = torch.float64 if x.device.type in ("cpu", "cuda") else torch.float32+ positions = (positions.to(compute_dtype) / self._rope_scaling_factor).float()+ if compute_dtype == torch.float64:+ # FP64 reciprocals preserve FP32 division rounding without slow GPU divides.+ reciprocal = self._rope_reciprocal_cpu.to(x.device)+ freqs = (positions[..., None].double() * reciprocal).float()+ else:+ freqs = positions[..., None] / self._rope_timescale_cpu.to(x.device)+ emb = torch.cat((freqs, freqs), dim = -1)+ return emb.cos().to(x.dtype), emb.sin().to(x.dtype)+def _slice_rope_cache(self, device_index, seq_len):return (self.multi_gpu_cos_cached[device_index][:seq_len],@@ -2646,6 +2676,8 @@ class LlamaRotaryEmbedding(_PerDeviceRopeCache, torch.nn.Module):self.multi_gpu_sin_cached = [None] * DEVICE_COUNTinv_freq = self._unsloth_recompute_inv_freq()+ # Transformers may replace nonpersistent buffers while loading a stock model.+ self._unsloth_inv_freq_cpu = inv_freq.detach().clone()self.register_buffer("inv_freq", inv_freq, persistent = False)# Built per device on first use, not for every visible GPU here.@@ -2658,6 +2690,15 @@ class LlamaRotaryEmbedding(_PerDeviceRopeCache, torch.nn.Module):1, device = get_current_device(), dtype = torch.get_default_dtype())+ def _compiled_position_embeddings(self, x, position_ids):+ inv_freq = self._unsloth_inv_freq_cpu.to(device = x.device)+ positions = self._apply_time_scaling(position_ids.to(x.device).float())+ freqs = positions[..., None] * inv_freq+ emb = torch.cat((freqs, freqs), dim = -1)+ cos = (emb.cos() * self.attention_scaling).to(self._rope_cache_dtype)+ sin = (emb.sin() * self.attention_scaling).to(self._rope_cache_dtype)+ return cos.to(x.dtype), sin.to(x.dtype)+def _apply_inv_freq_scaling(self, inv_freq):"""Override to apply custom inv_freq scaling (e.g., extended RoPE)."""return inv_freq@@ -2667,6 +2708,7 @@ class LlamaRotaryEmbedding(_PerDeviceRopeCache, torch.nn.Module):config = getattr(self, "_unsloth_rope_config", None)config_inv_freq = Nonerope_scaling = getattr(config, "rope_scaling", None) if config is not None else None+ # Scaling subclasses here would apply their factor twice.if rope_scaling is not None and type(self) is LlamaRotaryEmbedding:config_inv_freq, self.attention_scaling = _compute_config_rope_inv_freq(config,@@ -2809,6 +2851,8 @@ class LlamaExtendedRotaryEmbedding(LlamaRotaryEmbedding):class LongRopeRotaryEmbedding(_PerDeviceRopeCache, torch.nn.Module):# For Phi 3.5 128K; see microsoft/Phi-3.5-mini-instruct modeling_phi3.py+ _rope_rows_pick_factors = True+def __init__(self,dim = None,