Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Make RMSNorm, RoPE and the input-embedding hook traceable by torch.compile (#12171)
Daniel Han5d75cdeb8b8a
diff --git a/tests/test_rms_rope_compile.py b/tests/test_rms_rope_compile.pynew file mode 100644index 00000000000..3dc049cd5f1--- /dev/null+++ b/tests/test_rms_rope_compile.py@@ -0,0 +1,506 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""fast_rms_layernorm / fast_rope_embedding trace under torch.compile with eager's bytes."""++import hashlib+import math+import os++import pytest+import torch++if not torch.cuda.is_available():+ pytest.skip("needs a CUDA device", allow_module_level = True)++pytest.importorskip("triton")++import unsloth # noqa: F401 (patches first)+import torch._dynamo.utils as dynamo_utils+from unsloth.kernels import fast_lora, rms_layernorm, rope_embedding+from unsloth.kernels.rms_layernorm import Fast_RMS_Layernorm, fast_rms_layernorm+from unsloth.kernels.rope_embedding import (+ Fast_RoPE_Embedding,+ Fast_RoPE_Embedding_QK,+ fast_rope_embedding,+)++# Without triton_op the implementation may keep a graph break; results must still be right.+TRACEABLE = hasattr(torch.library, "triton_op") and hasattr(torch.library, "wrap_triton")+# Pre-Ampere GPUs (T4) keep bf16 on the untraced path, so bf16 byte / graph-break checks need bf16 there.+BF16 = pytest.param(+ torch.bfloat16,+ marks = pytest.mark.skipif(+ not rms_layernorm._BF16_TRACEABLE, reason = "no native bf16: bf16 stays untraced"+ ),+)+DTYPES = [BF16, torch.float16, torch.float32]+CDTYPE = torch.bfloat16 if rms_layernorm._BF16_TRACEABLE else torch.float16++[email protected](autouse = True)+def _fresh_dynamo():+ torch._dynamo.reset()+ dynamo_utils.counters.clear()+ yield+ torch._dynamo.reset()+++def _compile(fn):+ return torch.compile(fn, fullgraph = TRACEABLE)+++def _graph_breaks():+ return sum(dynamo_utils.counters["graph_break"].values())+++def _bytes_equal(a, b):+ if a is None or b is None:+ return a is None and b is None+ assert a.shape == b.shape and a.dtype == b.dtype+ iv = {2: torch.int16, 4: torch.int32}[a.element_size()]+ return torch.equal(a.contiguous().view(iv), b.contiguous().view(iv))+++class _Norm(torch.nn.Module):+ def __init__(self, dim, dtype, gemma):+ super().__init__()+ g = torch.Generator(device = "cuda").manual_seed(1)+ w = torch.randn(dim, device = "cuda", generator = g, dtype = torch.float32) * 0.1+ self.weight = torch.nn.Parameter((w if gemma else w + 1).to(dtype))+ self.variance_epsilon = 1e-6+++def _rms_case(fn, shape, dtype, gemma, non_contiguous):+ g = torch.Generator(device = "cuda").manual_seed(0)+ norm = _Norm(shape[-1], dtype, gemma)+ if non_contiguous:+ base = torch.randn(*shape[:-1], 2 * shape[-1], device = "cuda", generator = g, dtype = dtype)+ X = base[..., ::2].detach().requires_grad_(True)+ else:+ X = torch.randn(shape, device = "cuda", generator = g, dtype = dtype).requires_grad_(True)+ dY = torch.randn(shape, device = "cuda", generator = g, dtype = dtype)+ before = X.detach().clone()+ Y = fn(norm, X, gemma)+ Y.backward(dY)+ assert _bytes_equal(X.detach(), before), "the input was mutated"+ return Y.detach(), X.grad, norm.weight.grad++[email protected]("dtype", DTYPES, ids = str)[email protected]("gemma", [False, True], ids = ["llama", "gemma"])+ "shape,non_contiguous",+ [((64, 256), False), ((2, 33, 384), False), ((2, 33, 384), True)],+ ids = ["2d", "3d", "3d_strided"],+)+def test_rms_layernorm_compiled_matches_eager(shape, non_contiguous, gemma, dtype):+ """Compiled forward and backward are byte-identical to eager, with no graph break."""+ eager = _rms_case(fast_rms_layernorm, shape, dtype, gemma, non_contiguous)+ compiled = _rms_case(+ _compile(lambda n, x, gm: fast_rms_layernorm(n, x, gm)), shape, dtype, gemma, non_contiguous+ )+ for e, c, name in zip(eager, compiled, ("Y", "dX", "dW")):+ assert _bytes_equal(e, c), name+ if TRACEABLE:+ assert _graph_breaks() == 0++[email protected]("gemma", [False, True], ids = ["llama", "gemma"])+def test_rms_layernorm_eager_is_the_autograd_function(gemma):+ """Outside torch.compile the public function is still Fast_RMS_Layernorm, bytes for bytes."""+ got = _rms_case(fast_rms_layernorm, (2, 17, 256), torch.bfloat16, gemma, False)+ ref = _rms_case(+ lambda n, x, gm: Fast_RMS_Layernorm.apply(x, n.weight, n.variance_epsilon, gm),+ (2, 17, 256),+ torch.bfloat16,+ gemma,+ False,+ )+ for e, c, name in zip(ref, got, ("Y", "dX", "dW")):+ assert _bytes_equal(e, c), name+++def _cos_sin(rope_size, head_dim, dtype):+ # As LlamaRotaryEmbedding builds it: fp32 outer product, cat, cast to Q's dtype.+ inv_freq = 1.0 / (10000 ** (torch.arange(0, head_dim, 2, dtype = torch.int64).float() / head_dim))+ freqs = torch.outer(torch.arange(rope_size).float(), inv_freq)+ emb = torch.cat((freqs, freqs), dim = -1).cuda()+ return emb.cos().to(dtype), emb.sin().to(dtype)+++def _rope_case(+ fn,+ bsz,+ seq,+ n_heads,+ n_kv,+ head_dim,+ dtype,+ with_indices,+ pure = False,+):+ g = torch.Generator(device = "cuda").manual_seed(0)+ q_lin = torch.randn(bsz, seq, n_heads * head_dim, device = "cuda", generator = g, dtype = dtype)+ k_lin = torch.randn(bsz, seq, n_kv * head_dim, device = "cuda", generator = g, dtype = dtype)+ q_lin.requires_grad_(True)+ k_lin.requires_grad_(True)+ Q = q_lin.view(bsz, seq, n_heads, head_dim).transpose(1, 2)+ K = k_lin.view(bsz, seq, n_kv, head_dim).transpose(1, 2)+ cos, sin = _cos_sin(max(seq, 64) + 16, head_dim, dtype)+ indices = None+ if with_indices:+ # TRL-style packed position ids: every packed sequence restarts at 0, int32.+ lens = [seq // 3, seq // 3, seq - 2 * (seq // 3)]+ row = torch.cat([torch.arange(n, dtype = torch.int32) for n in lens])+ indices = row.repeat(bsz, 1).cuda()+ dQ = torch.randn(bsz, n_heads, seq, head_dim, device = "cuda", generator = g, dtype = dtype)+ dK = torch.randn(bsz, n_kv, seq, head_dim, device = "cuda", generator = g, dtype = dtype)+ q_before, k_before = q_lin.detach().clone(), k_lin.detach().clone()+ Q_out, K_out = fn(Q, K, cos, sin, indices)+ torch.autograd.backward((Q_out, K_out), (dQ, dK))+ if pure:+ assert _bytes_equal(q_lin.detach(), q_before), "the q projection was mutated"+ assert _bytes_equal(k_lin.detach(), k_before), "the k projection was mutated"+ elif with_indices:+ if n_heads > 1:+ assert _bytes_equal(q_lin.detach(), q_before), "the q projection was mutated"+ if n_kv > 1:+ assert _bytes_equal(k_lin.detach(), k_before), "the k projection was mutated"+ return Q_out.detach(), K_out.detach(), q_lin.grad, k_lin.grad+++ROPE_SHAPES = [(2, 37, 8, 8, 64), (1, 64, 32, 8, 128), (3, 20, 4, 1, 64)]++[email protected]("dtype", [BF16, torch.float16], ids = str)[email protected]("with_indices", [False, True], ids = ["positions", "rope_indices"])+ "shape", ROPE_SHAPES, ids = lambda s: f"b{s[0]}_s{s[1]}_h{s[2]}_kv{s[3]}_d{s[4]}"+)+def test_rope_compiled_matches_eager(shape, with_indices, dtype):+ """Compiled forward and backward (GQA included) are byte-identical to eager, no graph break."""+ eager = _rope_case(fast_rope_embedding, *shape, dtype, with_indices)+ compiled = _rope_case(+ _compile(lambda q, k, c, s, i: fast_rope_embedding(q, k, c, s, i)),+ *shape,+ dtype,+ with_indices,+ pure = TRACEABLE,+ )+ for e, c, name in zip(eager, compiled, ("Q", "K", "dQ", "dK")):+ assert _bytes_equal(e, c), name+ if TRACEABLE:+ assert _graph_breaks() == 0++[email protected]("with_indices", [False, True], ids = ["positions", "rope_indices"])+def test_rope_eager_is_the_autograd_function(with_indices):+ """Outside torch.compile the public function is still the Fast_RoPE autograd Functions."""++ def direct(Q, K, cos, sin, indices):+ if indices is not None:+ return Fast_RoPE_Embedding_QK.apply(Q, K, cos, sin, indices)+ Q_out = Fast_RoPE_Embedding.apply(Q.transpose(1, 2).contiguous(), cos, sin).transpose(1, 2)+ K_out = Fast_RoPE_Embedding.apply(K.transpose(1, 2).contiguous(), cos, sin).transpose(1, 2)+ return Q_out, K_out++ ref = _rope_case(direct, 2, 37, 8, 4, 64, torch.bfloat16, with_indices)+ got = _rope_case(fast_rope_embedding, 2, 37, 8, 4, 64, torch.bfloat16, with_indices)+ for e, c, name in zip(ref, got, ("Q", "K", "dQ", "dK")):+ assert _bytes_equal(e, c), name+++def test_rope_matches_the_rotation_formula():+ """Control for the byte tests: the kernels compute Q * cos + rotate_half(Q) * sin."""+ Q_out, K_out, _, _ = _rope_case(fast_rope_embedding, 2, 16, 4, 2, 64, torch.float32, False)+ g = torch.Generator(device = "cuda").manual_seed(0)+ q = torch.randn(2, 16, 4 * 64, device = "cuda", generator = g).view(2, 16, 4, 64).transpose(1, 2)+ cos, sin = _cos_sin(80, 64, torch.float32)+ rot = torch.cat((-q[..., 32:], q[..., :32]), dim = -1)+ torch.testing.assert_close(Q_out, q * cos[:16] + rot * sin[:16], rtol = 1e-5, atol = 1e-5)+++def test_bf16_tracing_needs_native_bf16_on_every_gpu(monkeypatch):+ """A pre-Ampere GPU anywhere in the process keeps bf16 untraced (a layer can land on it)."""+ monkeypatch.setattr(torch.version, "hip", None)+ monkeypatch.setattr(torch.cuda, "device_count", lambda: 2)+ for caps, expected in (({0: (8, 0), 1: (8, 9)}, True), ({0: (8, 0), 1: (7, 5)}, False)):+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda i, caps = caps: caps[i])+ assert rms_layernorm._bf16_traceable() is expected, caps+ monkeypatch.setattr(torch.version, "hip", "6.4")+ assert rms_layernorm._bf16_traceable() is True+++def test_bf16_without_native_support_stays_untraced(monkeypatch):+ """Where bf16 cannot be traced, bf16 inputs keep the old graph break and match eager."""+ if not TRACEABLE:+ pytest.skip("this torch has no torch.library.triton_op")+ monkeypatch.setattr(rms_layernorm, "_BF16_TRACEABLE", False)+ monkeypatch.setattr(rope_embedding, "_BF16_TRACEABLE", False)+ eager = _rms_case(fast_rms_layernorm, (64, 256), torch.float16, False, False)+ compiled = _rms_case(+ torch.compile(lambda n, x, gm: fast_rms_layernorm(n, x, gm)),+ (64, 256),+ torch.float16,+ False,+ False,+ )+ assert all(_bytes_equal(e, c) for e, c in zip(eager, compiled)) and _graph_breaks() == 0+ torch._dynamo.reset()+ dynamo_utils.counters.clear()+ eager = _rms_case(fast_rms_layernorm, (64, 256), torch.bfloat16, False, False)+ compiled = _rms_case(+ torch.compile(lambda n, x, gm: fast_rms_layernorm(n, x, gm)),+ (64, 256),+ torch.bfloat16,+ False,+ False,+ )+ assert all(_bytes_equal(e, c) for e, c in zip(eager, compiled)) and _graph_breaks() > 0+++def test_rope_dynamic_shapes():+ """dynamic=True makes the launch grid SymInts (divmod rejected them)."""+ if not TRACEABLE:+ pytest.skip("this torch has no torch.library.triton_op")+ cos, sin = _cos_sin(64, 64, CDTYPE)++ def f(q_lin, k_lin, bsz, seq, n_heads, n_kv):+ Q = q_lin.view(bsz, seq, n_heads, 64).transpose(1, 2)+ K = k_lin.view(bsz, seq, n_kv, 64).transpose(1, 2)+ return fast_rope_embedding(Q, K, cos, sin, None)++ compiled = torch.compile(f, fullgraph = True, dynamic = True)+ for seq in (16, 24, 40):+ g = torch.Generator(device = "cuda").manual_seed(seq)+ q_lin = torch.randn(2, seq, 8 * 64, device = "cuda", generator = g, dtype = CDTYPE)+ k_lin = torch.randn(2, seq, 2 * 64, device = "cuda", generator = g, dtype = CDTYPE)+ for e, c in zip(+ f(q_lin.clone(), k_lin.clone(), 2, seq, 8, 2), compiled(q_lin, k_lin, 2, seq, 8, 2)+ ):+ assert _bytes_equal(e, c), seq++[email protected]("compiled", [False, True], ids = ["eager", "compiled"])[email protected]("with_indices", [False, True], ids = ["positions", "rope_indices"])+def test_rope_expanded_gradient(with_indices, compiled):+ """Q.sum() gives a stride-0 gradient; rotating it in place gave wrong dQ / dK."""+ g = torch.Generator(device = "cuda").manual_seed(0)+ q_lin = torch.randn(2, 16, 4 * 64, device = "cuda", generator = g).requires_grad_(True)+ k_lin = torch.randn(2, 16, 2 * 64, device = "cuda", generator = g).requires_grad_(True)+ cos, sin = _cos_sin(80, 64, torch.float32)+ indices = torch.arange(16, dtype = torch.int32).repeat(2, 1).cuda() if with_indices else None+ fn = _compile(fast_rope_embedding) if compiled else fast_rope_embedding+ Q_out, K_out = fn(+ q_lin.view(2, 16, 4, 64).transpose(1, 2),+ k_lin.view(2, 16, 2, 64).transpose(1, 2),+ cos,+ sin,+ indices,+ )+ (Q_out.sum() + K_out.sum()).backward()+ # d/dx of sum(x * cos + rotate_half(x) * sin): cos + rotate_half^T(ones) * sin, per position.+ ones = torch.ones(16, 64, device = "cuda")+ rot_t = torch.cat((ones[..., 32:], -ones[..., :32]), dim = -1)+ per_pos = cos[:16] + rot_t * sin[:16]+ torch.testing.assert_close(+ q_lin.grad.view(2, 16, 4, 64),+ per_pos[None, :, None].expand(2, 16, 4, 64),+ rtol = 1e-5,+ atol = 1e-5,+ )+ torch.testing.assert_close(+ k_lin.grad.view(2, 16, 2, 64),+ per_pos[None, :, None].expand(2, 16, 2, 64),+ rtol = 1e-5,+ atol = 1e-5,+ )+++def test_tiny_llama_decoder_layers_compile_fullgraph():+ """Decoder layers compile inside Unsloth's gradient checkpointing and match eager."""+ if not TRACEABLE:+ pytest.skip("this torch has no torch.library.triton_op")+ from unsloth import FastLanguageModel++ model, _ = FastLanguageModel.from_pretrained(+ "hf-internal-testing/tiny-random-LlamaForCausalLM",+ max_seq_length = 64,+ load_in_4bit = False,+ dtype = torch.bfloat16,+ )+ model = FastLanguageModel.get_peft_model(+ model,+ r = 8,+ lora_alpha = 16,+ lora_dropout = 0,+ target_modules = [+ "q_proj",+ "k_proj",+ "v_proj",+ "o_proj",+ "gate_proj",+ "up_proj",+ "down_proj",+ ],+ use_gradient_checkpointing = "unsloth",+ random_state = 3407,+ )+ model.train()+ vocab = model.config.vocab_size+ g = torch.Generator().manual_seed(0)+ # With several GPUs visible Unsloth may place the model on the emptiest one, not cuda:0.+ ids = torch.randint(0, vocab, (2, 48), generator = g).to(+ model.get_input_embeddings().weight.device+ )++ params = [p for p in model.parameters() if p.requires_grad]+ eager_loss = model(input_ids = ids, labels = ids).loss+ eager_loss.backward()+ eager_grads = [p.grad.clone() for p in params]+ model.zero_grad(set_to_none = True)++ # Before torch 2.11 fast_lora's Functions stay opaque: the only graph breaks allowed.+ trace_lora = fast_lora.TRACE_LORA_FUNCTIONS+ layers = model.base_model.model.model.layers+ for layer in layers:+ layer.forward = torch.compile(layer.forward, fullgraph = trace_lora)+ loss = model(input_ids = ids, labels = ids).loss+ loss.backward()+ reasons = [" ".join(str(k).split()) for k in dynamo_utils.counters["graph_break"]]+ assert all("_apply" in r for r in reasons), reasons+ assert math.isfinite(loss.item())+ assert abs(loss.item() - eager_loss.item()) <= 1e-3 * abs(eager_loss.item())+ grads = [p.grad for p in params]+ assert grads and all(gr is not None and torch.isfinite(gr).all() for gr in grads)+ scale = max(g.abs().max().item() for g in eager_grads)+ worst = max((a.float() - b.float()).abs().max().item() for a, b in zip(grads, eager_grads))+ # Two bf16 rounding steps of the largest gradient: compiled kernels sum in another order (one+ # step over 1% on an RTX PRO 6000).+ assert worst <= 2 * torch.finfo(torch.bfloat16).eps * scale, (worst, scale)+++def test_tiny_llama_causal_lm_compiles_fullgraph():+ """The whole causal LM compiles with no requires-grad hook or use_return_dict graph break,+ and its LoRA gradients match eager (torch 2.10 still breaks in the fused loss)."""+ if not TRACEABLE:+ pytest.skip("this torch has no torch.library.triton_op")+ from unsloth import FastLanguageModel+ from unsloth_zoo.utils import Version+ import unsloth_zoo.fused_losses.cross_entropy_loss as fused_ce++ model, _ = FastLanguageModel.from_pretrained(+ "hf-internal-testing/tiny-random-LlamaForCausalLM",+ max_seq_length = 64,+ load_in_4bit = False,+ dtype = torch.bfloat16,+ )+ model = FastLanguageModel.get_peft_model(+ model,+ r = 8,+ lora_alpha = 16,+ lora_dropout = 0,+ target_modules = [+ "q_proj",+ "k_proj",+ "v_proj",+ "o_proj",+ "gate_proj",+ "up_proj",+ "down_proj",+ ],+ use_gradient_checkpointing = False,+ random_state = 3407,+ )+ model.train()+ embeddings = model.get_input_embeddings()+ assert embeddings._forward_hooks, "enable_input_require_grads registered no hook"+ g = torch.Generator().manual_seed(0)+ ids = torch.randint(0, model.config.vocab_size, (2, 48), generator = g).to(+ embeddings.weight.device+ )+ params = [p for p in model.parameters() if p.requires_grad]++ eager_loss = model(input_ids = ids, labels = ids).loss+ eager_loss.backward()+ eager_grads = [p.grad.detach().float().clone() for p in params]+ model.zero_grad(set_to_none = True)++ loss_opaque = getattr(fused_ce, "_FUSED_LOSS_OPAQUE", False)+ fullgraph = (+ fast_lora.TRACE_LORA_FUNCTIONS+ and Version(torch.__version__) >= Version("2.11.0")+ and not loss_opaque+ )+ causal_lm = model.base_model.model+ causal_lm.forward = torch.compile(causal_lm.forward, fullgraph = fullgraph)+ loss = model(input_ids = ids, labels = ids).loss+ loss.backward()+ reasons = [" ".join(str(k).split()) for k in dynamo_utils.counters["graph_break"]]+ assert not any("requires_grad_()" in r or "logging.Logger" in r for r in reasons), reasons+ if loss_opaque and fast_lora.TRACE_LORA_FUNCTIONS:+ assert all("_fused_loss_opaque" in r for r in reasons), reasons+ assert abs(loss.item() - eager_loss.item()) <= 1e-3 * abs(eager_loss.item())+ total = sum(float(e.norm()) for e in eager_grads)+ assert total > 0+ for p, e in zip(params, eager_grads):+ assert p.grad is not None+ err = float((p.grad.float() - e).norm())+ assert err <= 0.05 * float(e.norm()) + 1e-6, (err, float(e.norm()))+++def test_input_require_grads_hook_is_exact():+ """Compiled, the hook returns x + -0.0 (exact) needing grad; eager flips requires_grad."""+ from transformers import AutoModelForCausalLM++ model = AutoModelForCausalLM.from_pretrained(+ "hf-internal-testing/tiny-random-LlamaForCausalLM"+ ).cuda()+ model.requires_grad_(False)+ model.enable_input_require_grads()+ emb = model.get_input_embeddings()+ with torch.no_grad():+ emb.weight[0].fill_(-0.0)+ ids = torch.tensor([[0, 1, 2]], device = "cuda")+ eager = emb(ids)+ assert eager.requires_grad+ compiled = torch.compile(lambda i: emb(i), fullgraph = TRACEABLE)(ids)+ assert compiled.requires_grad+ assert _bytes_equal(compiled.detach(), eager.detach())+ assert torch.signbit(compiled[0, 0]).all()+++def test_input_require_grads_hook_is_a_no_op_in_compiled_no_grad_decode():+ """A compiled decode step runs without grad: the hook leaves the embedding output alone."""+ from transformers import AutoModelForCausalLM++ model = AutoModelForCausalLM.from_pretrained(+ "hf-internal-testing/tiny-random-LlamaForCausalLM"+ ).cuda()+ model.requires_grad_(False)+ model.enable_input_require_grads()+ emb = model.get_input_embeddings()+ ids = torch.tensor([[0, 1, 2]], device = "cuda")+ dynamo_utils.counters.clear()+ with torch.no_grad():+ compiled = torch.compile(lambda i: emb(i), fullgraph = TRACEABLE)(ids)+ eager = emb(ids)+ assert not compiled.requires_grad+ assert _bytes_equal(compiled, eager)+ assert not dynamo_utils.counters["graph_break"]+++def test_compile_cache_key_covers_the_kernel_source():+ """A warm FX graph cache served an edited kernel as the old one; key it on the files."""+ from unsloth.kernels import rms_layernorm, rope_embedding++ if not hasattr(getattr(torch.compiler, "config", None), "cache_key_tag"):+ pytest.skip("torch has no compile cache key tag")+ if not rms_layernorm._TRACEABLE:+ pytest.skip("no triton_op, nothing is cached by op name")+ tags = torch.compiler.config.cache_key_tag.split(",")+ for mod in (rms_layernorm, rope_embedding):+ with open(mod.__file__, "rb") as file:+ digest = hashlib.sha256(file.read()).hexdigest()[:16]+ assert f"unsloth/{os.path.basename(mod.__file__)}:{digest}" in tagsdiff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.pyindex aca464a67dd..ed1e6b04b2e 100644--- a/unsloth/import_fixes.py+++ b/unsloth/import_fixes.py@@ -4568,28 +4568,54 @@ def check_fbgemm_gpu_version():def patch_enable_input_require_grads():"""Patch PreTrainedModel.enable_input_require_grads to tolerate vision models- that raise NotImplementedError from get_input_embeddings()."""+ that raise NotImplementedError from get_input_embeddings(), and so that its hook traces+ under torch.compile."""import inspect+ import torchfrom transformers import PreTrainedModel- # Only patch the new variant that iterates over self.modules(); see huggingface/transformers#41993.try:original_source = inspect.getsource(PreTrainedModel.enable_input_require_grads)except:return+ class _RequireGrad(torch.autograd.Function):+ # x + -0.0 is x exactly (signed zeros too); the anchor only makes the output need grad.+ @staticmethod+ def forward(ctx, x, anchor):+ return x + anchor++ @staticmethod+ def backward(ctx, grad):+ return grad, None++ # Created here: torch.compile cannot create a tensor that requires grad inside a graph.+ anchor = torch.tensor(-0.0, requires_grad = True)++ def make_inputs_require_grads(module, input, output):+ # requires_grad_() on an intermediate is a graph break under torch.compile.+ if torch.compiler.is_compiling():+ if not torch.is_grad_enabled():+ return # a compiled decode step records no grad+ return _RequireGrad.apply(output, anchor)+ output.requires_grad_(True)++ # Older transformers hooks a single embedding (huggingface/transformers#41993 added the loop).+ # wraps keeps inspect.getsource on transformers' source for later source checks.+ original = PreTrainedModel.enable_input_require_gradsif "for module in self.modules()" not in original_source:- return- def _patched_enable_input_require_grads(self):- import torch+ @functools.wraps(original)+ def _patched_single_enable_input_require_grads(self):+ self._require_grads_hook = self.get_input_embeddings().register_forward_hook(+ make_inputs_require_grads+ )- def make_inputs_require_grads(module, input, output):- # Dynamo graph-breaks on requires_grad_(); a compiled decode step records no grad anyway.- if torch.compiler.is_compiling() and not torch.is_grad_enabled():- return- output.requires_grad_(True)+ PreTrainedModel.enable_input_require_grads = _patched_single_enable_input_require_grads+ return+ @functools.wraps(original)+ def _patched_enable_input_require_grads(self):hooks = []seen_modules = set()diff --git a/unsloth/kernels/rms_layernorm.py b/unsloth/kernels/rms_layernorm.pyindex c9976b24361..fa4940c5f49 100644--- a/unsloth/kernels/rms_layernorm.py+++ b/unsloth/kernels/rms_layernorm.py@@ -9,9 +9,12 @@# See the License for the specific language governing permissions and# limitations under the License.+import hashlib+import osimport tritonimport triton.language as tlimport torch+from typing import Tuplefrom .utils import calculate_settings, torch_gpu_device@@ -154,6 +157,55 @@ def _gemma_rms_layernorm_forward(tl.store(Y + col_offsets, output, mask = mask)+def _rms_forward(X, W, eps, gemma, wrap):+ n_rows, n_cols = X.shape+ BLOCK_SIZE, num_warps = calculate_settings(n_cols)+ Y = torch.empty((n_rows, n_cols), dtype = X.dtype, device = X.device)+ r = torch.empty(n_rows, dtype = torch.float32, device = X.device)+ fx = _gemma_rms_layernorm_forward if gemma else _rms_layernorm_forward+ wrap(fx)[(n_rows,)](+ Y,+ Y.stride(0),+ X,+ X.stride(0),+ W,+ W.stride(0),+ r,+ r.stride(0),+ n_cols,+ eps,+ BLOCK_SIZE = BLOCK_SIZE,+ num_warps = num_warps,+ )+ return Y, r+++def _rms_backward(dY, dX, X, W, r, eps, gemma, wrap):+ # Non-Gemma writes dX over dY (the kernel ignores dX); Gemma writes into dX.+ n_rows, n_cols = dY.shape+ BLOCK_SIZE, num_warps = calculate_settings(n_cols)+ wrap(_rms_layernorm_backward)[(n_rows,)](+ dY,+ dY.stride(0),+ dX,+ dX.stride(0),+ X,+ X.stride(0),+ W,+ W.stride(0),+ r,+ r.stride(0),+ n_cols,+ eps,+ GEMMA = gemma,+ BLOCK_SIZE = BLOCK_SIZE,+ num_warps = num_warps,+ )+++_eager_kernel = lambda kernel: kernel++class Fast_RMS_Layernorm(torch.autograd.Function):@staticmethoddef forward(@@ -168,36 +220,9 @@ class Fast_RMS_Layernorm(torch.autograd.Function):X = X.reshape(-1, dim).contiguous()# kernels read W at unit stride, and this W is the one saved for backward.W = W.contiguous()- n_rows: int- n_cols: int- n_rows, n_cols = X.shape- BLOCK_SIZE: int- num_warps: int- BLOCK_SIZE, num_warps = calculate_settings(n_cols)- device = X.device-- Y = torch.empty((n_rows, n_cols), dtype = X.dtype, device = device)- r = torch.empty(n_rows, dtype = torch.float32, device = device)-- fx = _gemma_rms_layernorm_forward if gemma else _rms_layernorm_forward- with torch_gpu_device(device):- fx[(n_rows,)](- Y,- Y.stride(0),- X,- X.stride(0),- W,- W.stride(0),- r,- r.stride(0),- n_cols,- eps,- BLOCK_SIZE = BLOCK_SIZE,- num_warps = num_warps,- )+ with torch_gpu_device(X.device):+ Y, r = _rms_forward(X, W, eps, gemma, _eager_kernel)ctx.eps = eps- ctx.BLOCK_SIZE = BLOCK_SIZE- ctx.num_warps = num_warpsctx.GEMMA = gemmactx.save_for_backward(X, W, r)return Y.view(*shape)@@ -208,35 +233,88 @@ class Fast_RMS_Layernorm(torch.autograd.Function):dim: int = shape[-1]dY = dY.reshape(-1, dim).contiguous()X, W, r = ctx.saved_tensors- n_rows: int- n_cols: int- n_rows, n_cols = dY.shapedX = torch.empty_like(dY) if ctx.GEMMA else dY-with torch_gpu_device(dY.device):- _rms_layernorm_backward[(n_rows,)](- dY,- dY.stride(0),- dX,- dX.stride(0),- X,- X.stride(0),- W,- W.stride(0),- r,- r.stride(0),- n_cols,- ctx.eps,- GEMMA = ctx.GEMMA,- BLOCK_SIZE = ctx.BLOCK_SIZE,- num_warps = ctx.num_warps,- )+ _rms_backward(dY, dX, X, W, r, ctx.eps, ctx.GEMMA, _eager_kernel)dX = dX.view(*shape)return dX, None, None, None-# RMS Layernorm does not torch.compile properly; reason unknown.+# Compiled graphs take these ops: same kernels, fresh outputs (the Function writes dX over dY).+_TRACEABLE = hasattr(torch.library, "triton_op") and hasattr(torch.library, "wrap_triton")+++def _bf16_traceable():+ # Dynamo cannot trace bf16 Triton without native bf16 (T4); any visible GPU may hold a layer.+ try:+ if torch.version.hip:+ return True+ n = torch.cuda.device_count()+ return n > 0 and all(torch.cuda.get_device_capability(i)[0] >= 8 for i in range(n))+ except Exception:+ return False+++_BF16_TRACEABLE = _TRACEABLE and _bf16_traceable()+++def _tag_compile_cache(path):+ # Inductor's FX cache keys a triton_op without its source; key on the file so upgrades miss.+ config = getattr(torch.compiler, "config", None)+ if config is None or not hasattr(config, "cache_key_tag"):+ return+ with open(path, "rb") as file:+ tag = f"unsloth/{os.path.basename(path)}:{hashlib.sha256(file.read()).hexdigest()[:16]}"+ tags = [t for t in config.cache_key_tag.split(",") if t]+ if tag not in tags:+ config.cache_key_tag = ",".join(tags + [tag])+++def _traced_kernel(kernel):+ # wrap_triton needs the JITFunction under triton.heuristics; callers pass its constexprs.+ if isinstance(kernel, triton.runtime.autotuner.Heuristics):+ kernel = kernel.fn+ return torch.library.wrap_triton(kernel)+++if _TRACEABLE:++ @torch.library.triton_op("unsloth::rms_layernorm", mutates_args = ())+ def _rms_layernorm_op(+ X: torch.Tensor, W: torch.Tensor, eps: float, gemma: bool+ ) -> Tuple[torch.Tensor, torch.Tensor]:+ return _rms_forward(X, W, eps, gemma, _traced_kernel)++ @torch.library.triton_op("unsloth::rms_layernorm_backward", mutates_args = ())+ def _rms_layernorm_backward_op(+ dY: torch.Tensor, X: torch.Tensor, W: torch.Tensor, r: torch.Tensor, eps: float, gemma: bool+ ) -> torch.Tensor:+ dX = torch.empty_like(dY)+ # Non-Gemma rewrites its dY argument, so it gets a copy.+ _rms_backward(dY if gemma else dX.copy_(dY), dX, X, W, r, eps, gemma, _traced_kernel)+ return dX++ def _rms_setup_context(ctx, inputs, output):+ X, W, eps, gemma = inputs+ ctx.eps, ctx.gemma = eps, gemma+ ctx.save_for_backward(X, W, output[1])++ def _rms_layernorm_op_backward(ctx, dY, dr):+ X, W, r = ctx.saved_tensors+ dX = torch.ops.unsloth.rms_layernorm_backward(dY.contiguous(), X, W, r, ctx.eps, ctx.gemma)+ return dX, None, None, None++ _rms_layernorm_op.register_autograd(+ _rms_layernorm_op_backward, setup_context = _rms_setup_context+ )+ _tag_compile_cache(__file__)++@torch.compiler.disable+def _fast_rms_layernorm_untraced(X, W, eps, gemma):+ return Fast_RMS_Layernorm.apply(X, W, eps, gemma)++def fast_rms_layernorm(layernorm,X: torch.Tensor,@@ -246,8 +324,19 @@ def fast_rms_layernorm(eps: float = (layernorm.variance_epsilon if hasattr(layernorm, "variance_epsilon") else layernorm.eps)- out = Fast_RMS_Layernorm.apply(X, W, eps, gemma)- return out+ if not torch.compiler.is_compiling():+ return Fast_RMS_Layernorm.apply(X, W, eps, gemma)+ if (+ not _TRACEABLE+ or X.device.type != "cuda"+ or (X.dtype == torch.bfloat16 and not _BF16_TRACEABLE)+ ):+ return _fast_rms_layernorm_untraced(X, W, eps, gemma)+ shape = X.shape+ Y, _ = torch.ops.unsloth.rms_layernorm(+ X.reshape(-1, shape[-1]).contiguous(), W.contiguous(), eps, gemma+ )+ return Y.view(shape)from transformers.models.llama.modeling_llama import LlamaRMSNormdiff --git a/unsloth/kernels/rope_embedding.py b/unsloth/kernels/rope_embedding.pyindex 31c0260136e..45e31fa1e46 100644--- a/unsloth/kernels/rope_embedding.py+++ b/unsloth/kernels/rope_embedding.py@@ -14,7 +14,15 @@ import tritonimport triton.language as tlimport torchfrom ..device_type import DEVICE_COUNT+from typing import Tuplefrom .utils import calculate_settings, torch_gpu_device, torch_device_stream+from .rms_layernorm import (+ _BF16_TRACEABLE,+ _TRACEABLE,+ _eager_kernel,+ _tag_compile_cache,+ _traced_kernel,+)def _rope_embedding_QK(@@ -163,6 +171,33 @@ _rope_embedding = triton.heuristics()(_rope_embedding)+def _rope_rows(Q, cos, sin, seq_len, n_heads, head_dim, backward, wrap):+ # Rotates Q [batch * seq_len, n_heads * head_dim] in place; BLOCK_SIZE = head_dim//2 was racy.+ BLOCK_SIZE, num_warps = calculate_settings(head_dim // 2)+ n_groups = (+ n_heads + ROPE_GROUP_SIZE - 1+ ) // ROPE_GROUP_SIZE # also a SymInt under dynamic=True+ wrap(_rope_embedding)[+ (+ Q.shape[0],+ n_groups,+ )+ ](+ Q,+ Q.stride(0),+ cos,+ cos.stride(0),+ sin,+ sin.stride(0),+ seq_len,+ head_dim,+ n_heads,+ BACKWARD_PASS = backward,+ BLOCK_SIZE = BLOCK_SIZE,+ num_warps = num_warps,+ )++class Fast_RoPE_Embedding(torch.autograd.Function):@staticmethoddef forward(ctx, Q, cos, sin):@@ -173,44 +208,9 @@ class Fast_RoPE_Embedding(torch.autograd.Function):head_dim: intbatch, seq_len, n_heads, head_dim = Q.shapeQ = Q.reshape(batch * seq_len, n_heads * head_dim)- n_rows: int- n_cols: int- n_rows, n_cols = Q.shapeassert seq_len <= cos.shape[0]-- # Changing blocksize to head_dim//2 showed concurrency / non-deterministic issues; group_size too- # large also hurts performance.- BLOCK_SIZE, num_warps = calculate_settings(head_dim // 2)-- # group_size = 4 # 4 or 8, too large group_size can hurt performance.- div: int- mod: int- div, mod = divmod(n_heads, ROPE_GROUP_SIZE)- n_groups: int = div + (mod != 0)-with torch_gpu_device(Q.device):- _rope_embedding[- (- n_rows,- n_groups,- )- ](- Q,- Q.stride(0),- cos,- cos.stride(0),- sin,- sin.stride(0),- seq_len,- head_dim,- n_heads,- BACKWARD_PASS = False,- BLOCK_SIZE = BLOCK_SIZE,- num_warps = num_warps,- )- ctx.BLOCK_SIZE = BLOCK_SIZE- ctx.num_warps = num_warps- ctx.n_groups = n_groups+ _rope_rows(Q, cos, sin, seq_len, n_heads, head_dim, False, _eager_kernel)ctx.cos = cosctx.sin = sinreturn Q.reshape(batch, seq_len, n_heads, head_dim)@@ -222,35 +222,10 @@ class Fast_RoPE_Embedding(torch.autograd.Function):n_heads: inthead_dim: intbatch, seq_len, n_heads, head_dim = dY.shape- # The kernel writes in place: reshape of an expanded grad (e.g. from .sum()) stays a stride-0 view.- dY = dY.contiguous().view(batch * seq_len, n_heads * head_dim)- n_rows: int- n_cols: int- n_rows, n_cols = dY.shape-- cos = ctx.cos- sin = ctx.sin-+ # contiguous: an expanded gradient (e.g. from Q.sum()) would be rotated through stride 0.+ dY = dY.contiguous().reshape(batch * seq_len, n_heads * head_dim)with torch_gpu_device(dY.device):- _rope_embedding[- (- n_rows,- ctx.n_groups,- )- ](- dY,- dY.stride(0),- cos,- cos.stride(0),- sin,- sin.stride(0),- seq_len,- head_dim,- n_heads,- BACKWARD_PASS = True,- BLOCK_SIZE = ctx.BLOCK_SIZE,- num_warps = ctx.num_warps,- )+ _rope_rows(dY, ctx.cos, ctx.sin, seq_len, n_heads, head_dim, True, _eager_kernel)dY = dY.reshape(batch, seq_len, n_heads, head_dim)return (dY,@@ -259,23 +234,33 @@ class Fast_RoPE_Embedding(torch.autograd.Function):)-# RoPE embedding does not torch.compile properly; reason unknown.-def fast_rope_embedding(- Q,- K,- cos,- sin,- rope_embedding_indices = None,-):- if rope_embedding_indices is not None:- Q_out, K_out = Fast_RoPE_Embedding_QK.apply(Q, K, cos, sin, rope_embedding_indices)- else:- Q_out = Fast_RoPE_Embedding.apply(Q.transpose(1, 2).contiguous(), cos, sin).transpose(1, 2)- K_out = Fast_RoPE_Embedding.apply(K.transpose(1, 2).contiguous(), cos, sin).transpose(1, 2)- if DEVICE_COUNT > 1:- torch_device_stream(Q.device).synchronize()- return Q_out, K_out+def _rope_qk(Q, K, cos, sin, rope_ptr, has_indices, backward, wrap):+ # Rotates Q [batch, n_heads_Q, seq_len, head_dim] and K in place, at any strides.+ batch, n_heads_Q, seq_len, head_dim = Q.shape+ n_heads_K = K.shape[1]+ BLOCK_SIZE, num_warps = calculate_settings(head_dim)+ wrap(_rope_embedding_QK)[(batch * seq_len, n_heads_Q)](+ Q,+ Q.stride(0),+ Q.stride(1),+ Q.stride(2),+ K,+ K.stride(0),+ K.stride(1),+ K.stride(2),+ cos,+ cos.stride(0),+ sin,+ sin.stride(0),+ rope_ptr,+ seq_len,+ head_dim = head_dim,+ n_heads_K = n_heads_K,+ BACKWARD_PASS = backward,+ HAS_ROPE_INDICES = has_indices,+ BLOCK_SIZE = BLOCK_SIZE,+ num_warps = num_warps,+ )class Fast_RoPE_Embedding_QK(torch.autograd.Function):@@ -284,9 +269,6 @@ class Fast_RoPE_Embedding_QK(torch.autograd.Function):has_indices = rope_indices is not Nonecos, sin = cos.squeeze(), sin.squeeze()- batch, n_heads_Q, seq_len, head_dim = Q.shape- _, n_heads_K, _, _ = K.shape-# Inplace rotary embedding is generally fine.Q_out = Q.clone() if not Q.is_contiguous() else QK_out = K.clone() if not K.is_contiguous() else K@@ -297,52 +279,13 @@ class Fast_RoPE_Embedding_QK(torch.autograd.Function):else:rope_ptr = cos.new_empty(1, dtype = torch.int32)- BLOCK_SIZE, num_warps = calculate_settings(head_dim)-- Q_batch_stride, Q_head_stride, Q_seq_stride = (- Q_out.stride(0),- Q_out.stride(1),- Q_out.stride(2),- )- K_batch_stride, K_head_stride, K_seq_stride = (- K_out.stride(0),- K_out.stride(1),- K_out.stride(2),- )-with torch_gpu_device(Q.device):- _rope_embedding_QK[(batch * seq_len, n_heads_Q)](- Q_out,- Q_batch_stride,- Q_head_stride,- Q_seq_stride,- K_out,- K_batch_stride,- K_head_stride,- K_seq_stride,- cos,- cos.stride(0),- sin,- sin.stride(0),- rope_ptr,- seq_len,- head_dim = head_dim,- n_heads_K = n_heads_K,- BACKWARD_PASS = False,- HAS_ROPE_INDICES = has_indices,- BLOCK_SIZE = BLOCK_SIZE,- num_warps = num_warps,- )+ _rope_qk(Q_out, K_out, cos, sin, rope_ptr, has_indices, False, _eager_kernel)- ctx.block_size = BLOCK_SIZE- ctx.num_warps = num_warpsctx.has_indices = has_indicesctx.cos = cosctx.sin = sinctx.rope_indices = rope_ptr if has_indices else None- ctx.seq_len = seq_len- ctx.n_heads_Q = n_heads_Q- ctx.n_heads_K = n_heads_Kreturn (Q_out,@@ -351,52 +294,110 @@ class Fast_RoPE_Embedding_QK(torch.autograd.Function):@staticmethoddef backward(ctx, dQ, dK):- batch, _, _, head_dim = dQ.shape-rope_ptr = ctx.rope_indices if ctx.has_indices else ctx.cos.new_empty(1, dtype = torch.int32)# Inplace rotary embedding is generally fine.dQ_out = dQ.clone() if not dQ.is_contiguous() else dQdK_out = dK.clone() if not dK.is_contiguous() else dK- Q_batch_stride, Q_head_stride, Q_seq_stride = (- dQ_out.stride(0),- dQ_out.stride(1),- dQ_out.stride(2),- )- K_batch_stride, K_head_stride, K_seq_stride = (- dK_out.stride(0),- dK_out.stride(1),- dK_out.stride(2),- )-with torch_gpu_device(dQ.device):- _rope_embedding_QK[(batch * ctx.seq_len, ctx.n_heads_Q)](- dQ_out,- Q_batch_stride,- Q_head_stride,- Q_seq_stride,- dK_out,- K_batch_stride,- K_head_stride,- K_seq_stride,- ctx.cos,- ctx.cos.stride(0),- ctx.sin,- ctx.sin.stride(0),- rope_ptr,- ctx.seq_len,- head_dim = head_dim,- n_heads_K = ctx.n_heads_K,- BACKWARD_PASS = True,- HAS_ROPE_INDICES = ctx.has_indices,- BLOCK_SIZE = ctx.block_size,- num_warps = ctx.num_warps,+ _rope_qk(+ dQ_out, dK_out, ctx.cos, ctx.sin, rope_ptr, ctx.has_indices, True, _eager_kernel)return (dQ_out, dK_out, None, None, None)+# Compiled graphs take these ops: the same kernels on fresh copies (the Functions rotate in place).+if _TRACEABLE:++ @torch.library.triton_op("unsloth::rope_embedding", mutates_args = ())+ def _rope_embedding_op(+ Q: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor, backward: bool+ ) -> torch.Tensor:+ # Q: [batch, seq_len, n_heads, head_dim] at any strides.+ batch, seq_len, n_heads, head_dim = Q.shape+ out = Q.clone(memory_format = torch.contiguous_format)+ rows = out.view(batch * seq_len, n_heads * head_dim)+ _rope_rows(rows, cos, sin, seq_len, n_heads, head_dim, backward, _traced_kernel)+ return out++ def _rope_setup_context(ctx, inputs, output):+ ctx.save_for_backward(inputs[1], inputs[2])++ def _rope_embedding_op_backward(ctx, dY):+ cos, sin = ctx.saved_tensors+ return torch.ops.unsloth.rope_embedding(dY, cos, sin, True), None, None, None++ _rope_embedding_op.register_autograd(+ _rope_embedding_op_backward, setup_context = _rope_setup_context+ )++ @torch.library.triton_op("unsloth::rope_embedding_qk", mutates_args = ())+ def _rope_embedding_qk_op(+ Q: torch.Tensor,+ K: torch.Tensor,+ cos: torch.Tensor,+ sin: torch.Tensor,+ rope_indices: torch.Tensor,+ backward: bool,+ ) -> Tuple[torch.Tensor, torch.Tensor]:+ Q_out, K_out = Q.clone(), K.clone()+ _rope_qk(Q_out, K_out, cos, sin, rope_indices, True, backward, _traced_kernel)+ return Q_out, K_out++ def _rope_qk_setup_context(ctx, inputs, output):+ ctx.save_for_backward(inputs[2], inputs[3], inputs[4])++ def _rope_embedding_qk_op_backward(ctx, dQ, dK):+ cos, sin, rope_indices = ctx.saved_tensors+ dQ, dK = torch.ops.unsloth.rope_embedding_qk(dQ, dK, cos, sin, rope_indices, True)+ return dQ, dK, None, None, None, None++ _rope_embedding_qk_op.register_autograd(+ _rope_embedding_qk_op_backward, setup_context = _rope_qk_setup_context+ )+ _tag_compile_cache(__file__)+++def _fast_rope_embedding_eager(Q, K, cos, sin, rope_embedding_indices):+ if rope_embedding_indices is not None:+ Q_out, K_out = Fast_RoPE_Embedding_QK.apply(Q, K, cos, sin, rope_embedding_indices)+ else:+ Q_out = Fast_RoPE_Embedding.apply(Q.transpose(1, 2).contiguous(), cos, sin).transpose(1, 2)+ K_out = Fast_RoPE_Embedding.apply(K.transpose(1, 2).contiguous(), cos, sin).transpose(1, 2)+ if DEVICE_COUNT > 1:+ torch_device_stream(Q.device).synchronize()+ return Q_out, K_out+++_fast_rope_embedding_untraced = torch.compiler.disable(_fast_rope_embedding_eager)+++def fast_rope_embedding(+ Q,+ K,+ cos,+ sin,+ rope_embedding_indices = None,+):+ if not torch.compiler.is_compiling():+ return _fast_rope_embedding_eager(Q, K, cos, sin, rope_embedding_indices)+ if (+ not _TRACEABLE+ or Q.device.type != "cuda"+ or (Q.dtype == torch.bfloat16 and not _BF16_TRACEABLE)+ ):+ return _fast_rope_embedding_untraced(Q, K, cos, sin, rope_embedding_indices)+ cos, sin = cos.squeeze(), sin.squeeze()+ if rope_embedding_indices is not None:+ rope_ptr = rope_embedding_indices.reshape(-1).to(dtype = torch.int32, device = Q.device)+ return torch.ops.unsloth.rope_embedding_qk(Q, K, cos, sin, rope_ptr, False)+ Q_out = torch.ops.unsloth.rope_embedding(Q.transpose(1, 2), cos, sin, False).transpose(1, 2)+ K_out = torch.ops.unsloth.rope_embedding(K.transpose(1, 2), cos, sin, False).transpose(1, 2)+ return Q_out, K_out++class Slow_RoPE_Embedding(torch.autograd.Function):@staticmethoddef forward(ctx, Q, cos, sin, position_ids):diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex c3fea244978..26963b9e20f 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -13,6 +13,7 @@ from .._version import __version____all__ = ["SUPPORTS_BFLOAT16",+ "config_return_dict","is_bfloat16_supported","_requested_float32","_mark_requested_float32",@@ -111,6 +112,7 @@ __all__ = ["patch_flex_attention_kernel_options",]+import torchfrom typing import Union, Optional, List, Any, Callable, Tuple, Iteratorfrom platform import system as platform_system@@ -4297,6 +4299,11 @@ def offload_output_embeddings(model, temporary_location: str = "_unsloth_temporareturn+def config_return_dict(config):+ # use_return_dict without its transformers 5 deprecation warning, a torch.compile graph break.+ return getattr(config, "return_dict", True) and not getattr(config, "torchscript", False)++def is_bfloat16_supported():return SUPPORTS_BFLOAT16diff --git a/unsloth/models/falcon_h1.py b/unsloth/models/falcon_h1.pyindex ed295061d92..543ecf762f2 100644--- a/unsloth/models/falcon_h1.py+++ b/unsloth/models/falcon_h1.py@@ -646,7 +646,7 @@ def fix_prepare_inputs_for_generation(module):class Unsloth_FalconH1RMSNorm(FalconH1RMSNorm):- """fast_rms_layernorm (compiler-disabled, fp32 eps) avoids the float64 torch.compile RMSNorm kernel that fails on Intel Arc DG2 (issue #6555)."""+ """fast_rms_layernorm (compiler-disabled on XPU, fp32 eps) avoids the float64 torch.compile RMSNorm kernel that fails on Intel Arc DG2 (issue #6555)."""def forward(self, hidden_states):return fast_rms_layernorm(self, hidden_states, gemma = False)diff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex 0f46195381b..87ed091d1fa 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -969,7 +969,7 @@ def LlamaModel_fast_forward()use_cache = use_cache if use_cache is not None else self.config.use_cache- return_dict = return_dict if return_dict is not None else self.config.use_return_dict+ return_dict = return_dict if return_dict is not None else config_return_dict(self.config)if input_ids is not None and inputs_embeds is not None:raise ValueError(@@ -1562,7 +1562,9 @@ def CausalLM_fast_forward(fast_forward_inference):if output_hidden_states is not Noneelse self.config.output_hidden_states)- return_dict = return_dict if return_dict is not None else self.config.use_return_dict+ return_dict = (+ return_dict if return_dict is not None else config_return_dict(self.config)+ )self.model._has_no_labels = labels is Noneoutputs = self.model(input_ids = input_ids,diff --git a/unsloth/models/mistral.py b/unsloth/models/mistral.pyindex 156bbac7a8e..b4609619831 100644--- a/unsloth/models/mistral.py+++ b/unsloth/models/mistral.py@@ -12,7 +12,7 @@from .loader_utils import DEFAULT_DEVICE_MAPfrom .llama import *import os-from ._utils import __version__+from ._utils import __version__, config_return_dictfrom unsloth_zoo.utils import _get_dtypefrom unsloth_zoo.hf_utils import dtype_from_configfrom ..utils.packing import (@@ -262,7 +262,7 @@ def MistralForCausalLM_fast_forward(if output_hidden_states is not Noneelse self.config.output_hidden_states)- return_dict = return_dict if return_dict is not None else self.config.use_return_dict+ return_dict = return_dict if return_dict is not None else config_return_dict(self.config)self.model._has_no_labels = labels is Nonediff --git a/unsloth/models/sentence_transformer.py b/unsloth/models/sentence_transformer.pyindex fb4a0ccedc6..cd1ed0359ea 100644--- a/unsloth/models/sentence_transformer.py+++ b/unsloth/models/sentence_transformer.py@@ -19,6 +19,7 @@ from .loader_utils import (unmarked_device_map,)from ._utils import (+ config_return_dict,SUPPORTS_BFLOAT16,resolve_model_class,resolve_encoder_attention_implementation,@@ -835,7 +836,9 @@ class FastSentenceTransformer(FastModel):if output_hidden_states is not Noneelse self.config.output_hidden_states)- return_dict = return_dict if return_dict is not None else self.config.use_return_dict+ return_dict = (+ return_dict if return_dict is not None else config_return_dict(self.config)+ )if input_ids is not None and inputs_embeds is not None:raise ValueError(