Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Make RMSNorm, RoPE and the input-embedding hook traceable by torch.compile (#12171)

Daniel Han5d75cdeb8b8a
diff --git a/tests/test_rms_rope_compile.py b/tests/test_rms_rope_compile.py
new file mode 100644
index 00000000000..3dc049cd5f1
--- /dev/null
+++ b/tests/test_rms_rope_compile.py
@@ -0,0 +1,506 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""fast_rms_layernorm / fast_rope_embedding trace under torch.compile with eager's bytes."""
+
+import hashlib
+import math
+import os
+
+import pytest
+import torch
+
+if not torch.cuda.is_available():
+ pytest.skip("needs a CUDA device", allow_module_level = True)
+
+pytest.importorskip("triton")
+
+import unsloth # noqa: F401 (patches first)
+import torch._dynamo.utils as dynamo_utils
+from unsloth.kernels import fast_lora, rms_layernorm, rope_embedding
+from unsloth.kernels.rms_layernorm import Fast_RMS_Layernorm, fast_rms_layernorm
+from unsloth.kernels.rope_embedding import (
+ Fast_RoPE_Embedding,
+ Fast_RoPE_Embedding_QK,
+ fast_rope_embedding,
+)
+
+# Without triton_op the implementation may keep a graph break; results must still be right.
+TRACEABLE = hasattr(torch.library, "triton_op") and hasattr(torch.library, "wrap_triton")
+# Pre-Ampere GPUs (T4) keep bf16 on the untraced path, so bf16 byte / graph-break checks need bf16 there.
+BF16 = pytest.param(
+ torch.bfloat16,
+ marks = pytest.mark.skipif(
+ not rms_layernorm._BF16_TRACEABLE, reason = "no native bf16: bf16 stays untraced"
+ ),
+)
+DTYPES = [BF16, torch.float16, torch.float32]
+CDTYPE = torch.bfloat16 if rms_layernorm._BF16_TRACEABLE else torch.float16
+
+
[email protected](autouse = True)
+def _fresh_dynamo():
+ torch._dynamo.reset()
+ dynamo_utils.counters.clear()
+ yield
+ torch._dynamo.reset()
+
+
+def _compile(fn):
+ return torch.compile(fn, fullgraph = TRACEABLE)
+
+
+def _graph_breaks():
+ return sum(dynamo_utils.counters["graph_break"].values())
+
+
+def _bytes_equal(a, b):
+ if a is None or b is None:
+ return a is None and b is None
+ assert a.shape == b.shape and a.dtype == b.dtype
+ iv = {2: torch.int16, 4: torch.int32}[a.element_size()]
+ return torch.equal(a.contiguous().view(iv), b.contiguous().view(iv))
+
+
+class _Norm(torch.nn.Module):
+ def __init__(self, dim, dtype, gemma):
+ super().__init__()
+ g = torch.Generator(device = "cuda").manual_seed(1)
+ w = torch.randn(dim, device = "cuda", generator = g, dtype = torch.float32) * 0.1
+ self.weight = torch.nn.Parameter((w if gemma else w + 1).to(dtype))
+ self.variance_epsilon = 1e-6
+
+
+def _rms_case(fn, shape, dtype, gemma, non_contiguous):
+ g = torch.Generator(device = "cuda").manual_seed(0)
+ norm = _Norm(shape[-1], dtype, gemma)
+ if non_contiguous:
+ base = torch.randn(*shape[:-1], 2 * shape[-1], device = "cuda", generator = g, dtype = dtype)
+ X = base[..., ::2].detach().requires_grad_(True)
+ else:
+ X = torch.randn(shape, device = "cuda", generator = g, dtype = dtype).requires_grad_(True)
+ dY = torch.randn(shape, device = "cuda", generator = g, dtype = dtype)
+ before = X.detach().clone()
+ Y = fn(norm, X, gemma)
+ Y.backward(dY)
+ assert _bytes_equal(X.detach(), before), "the input was mutated"
+ return Y.detach(), X.grad, norm.weight.grad
+
+
[email protected]("dtype", DTYPES, ids = str)
[email protected]("gemma", [False, True], ids = ["llama", "gemma"])
+ "shape,non_contiguous",
+ [((64, 256), False), ((2, 33, 384), False), ((2, 33, 384), True)],
+ ids = ["2d", "3d", "3d_strided"],
+)
+def test_rms_layernorm_compiled_matches_eager(shape, non_contiguous, gemma, dtype):
+ """Compiled forward and backward are byte-identical to eager, with no graph break."""
+ eager = _rms_case(fast_rms_layernorm, shape, dtype, gemma, non_contiguous)
+ compiled = _rms_case(
+ _compile(lambda n, x, gm: fast_rms_layernorm(n, x, gm)), shape, dtype, gemma, non_contiguous
+ )
+ for e, c, name in zip(eager, compiled, ("Y", "dX", "dW")):
+ assert _bytes_equal(e, c), name
+ if TRACEABLE:
+ assert _graph_breaks() == 0
+
+
[email protected]("gemma", [False, True], ids = ["llama", "gemma"])
+def test_rms_layernorm_eager_is_the_autograd_function(gemma):
+ """Outside torch.compile the public function is still Fast_RMS_Layernorm, bytes for bytes."""
+ got = _rms_case(fast_rms_layernorm, (2, 17, 256), torch.bfloat16, gemma, False)
+ ref = _rms_case(
+ lambda n, x, gm: Fast_RMS_Layernorm.apply(x, n.weight, n.variance_epsilon, gm),
+ (2, 17, 256),
+ torch.bfloat16,
+ gemma,
+ False,
+ )
+ for e, c, name in zip(ref, got, ("Y", "dX", "dW")):
+ assert _bytes_equal(e, c), name
+
+
+def _cos_sin(rope_size, head_dim, dtype):
+ # As LlamaRotaryEmbedding builds it: fp32 outer product, cat, cast to Q's dtype.
+ inv_freq = 1.0 / (10000 ** (torch.arange(0, head_dim, 2, dtype = torch.int64).float() / head_dim))
+ freqs = torch.outer(torch.arange(rope_size).float(), inv_freq)
+ emb = torch.cat((freqs, freqs), dim = -1).cuda()
+ return emb.cos().to(dtype), emb.sin().to(dtype)
+
+
+def _rope_case(
+ fn,
+ bsz,
+ seq,
+ n_heads,
+ n_kv,
+ head_dim,
+ dtype,
+ with_indices,
+ pure = False,
+):
+ g = torch.Generator(device = "cuda").manual_seed(0)
+ q_lin = torch.randn(bsz, seq, n_heads * head_dim, device = "cuda", generator = g, dtype = dtype)
+ k_lin = torch.randn(bsz, seq, n_kv * head_dim, device = "cuda", generator = g, dtype = dtype)
+ q_lin.requires_grad_(True)
+ k_lin.requires_grad_(True)
+ Q = q_lin.view(bsz, seq, n_heads, head_dim).transpose(1, 2)
+ K = k_lin.view(bsz, seq, n_kv, head_dim).transpose(1, 2)
+ cos, sin = _cos_sin(max(seq, 64) + 16, head_dim, dtype)
+ indices = None
+ if with_indices:
+ # TRL-style packed position ids: every packed sequence restarts at 0, int32.
+ lens = [seq // 3, seq // 3, seq - 2 * (seq // 3)]
+ row = torch.cat([torch.arange(n, dtype = torch.int32) for n in lens])
+ indices = row.repeat(bsz, 1).cuda()
+ dQ = torch.randn(bsz, n_heads, seq, head_dim, device = "cuda", generator = g, dtype = dtype)
+ dK = torch.randn(bsz, n_kv, seq, head_dim, device = "cuda", generator = g, dtype = dtype)
+ q_before, k_before = q_lin.detach().clone(), k_lin.detach().clone()
+ Q_out, K_out = fn(Q, K, cos, sin, indices)
+ torch.autograd.backward((Q_out, K_out), (dQ, dK))
+ if pure:
+ assert _bytes_equal(q_lin.detach(), q_before), "the q projection was mutated"
+ assert _bytes_equal(k_lin.detach(), k_before), "the k projection was mutated"
+ elif with_indices:
+ if n_heads > 1:
+ assert _bytes_equal(q_lin.detach(), q_before), "the q projection was mutated"
+ if n_kv > 1:
+ assert _bytes_equal(k_lin.detach(), k_before), "the k projection was mutated"
+ return Q_out.detach(), K_out.detach(), q_lin.grad, k_lin.grad
+
+
+ROPE_SHAPES = [(2, 37, 8, 8, 64), (1, 64, 32, 8, 128), (3, 20, 4, 1, 64)]
+
+
[email protected]("dtype", [BF16, torch.float16], ids = str)
[email protected]("with_indices", [False, True], ids = ["positions", "rope_indices"])
+ "shape", ROPE_SHAPES, ids = lambda s: f"b{s[0]}_s{s[1]}_h{s[2]}_kv{s[3]}_d{s[4]}"
+)
+def test_rope_compiled_matches_eager(shape, with_indices, dtype):
+ """Compiled forward and backward (GQA included) are byte-identical to eager, no graph break."""
+ eager = _rope_case(fast_rope_embedding, *shape, dtype, with_indices)
+ compiled = _rope_case(
+ _compile(lambda q, k, c, s, i: fast_rope_embedding(q, k, c, s, i)),
+ *shape,
+ dtype,
+ with_indices,
+ pure = TRACEABLE,
+ )
+ for e, c, name in zip(eager, compiled, ("Q", "K", "dQ", "dK")):
+ assert _bytes_equal(e, c), name
+ if TRACEABLE:
+ assert _graph_breaks() == 0
+
+
[email protected]("with_indices", [False, True], ids = ["positions", "rope_indices"])
+def test_rope_eager_is_the_autograd_function(with_indices):
+ """Outside torch.compile the public function is still the Fast_RoPE autograd Functions."""
+
+ def direct(Q, K, cos, sin, indices):
+ if indices is not None:
+ return Fast_RoPE_Embedding_QK.apply(Q, K, cos, sin, indices)
+ Q_out = Fast_RoPE_Embedding.apply(Q.transpose(1, 2).contiguous(), cos, sin).transpose(1, 2)
+ K_out = Fast_RoPE_Embedding.apply(K.transpose(1, 2).contiguous(), cos, sin).transpose(1, 2)
+ return Q_out, K_out
+
+ ref = _rope_case(direct, 2, 37, 8, 4, 64, torch.bfloat16, with_indices)
+ got = _rope_case(fast_rope_embedding, 2, 37, 8, 4, 64, torch.bfloat16, with_indices)
+ for e, c, name in zip(ref, got, ("Q", "K", "dQ", "dK")):
+ assert _bytes_equal(e, c), name
+
+
+def test_rope_matches_the_rotation_formula():
+ """Control for the byte tests: the kernels compute Q * cos + rotate_half(Q) * sin."""
+ Q_out, K_out, _, _ = _rope_case(fast_rope_embedding, 2, 16, 4, 2, 64, torch.float32, False)
+ g = torch.Generator(device = "cuda").manual_seed(0)
+ q = torch.randn(2, 16, 4 * 64, device = "cuda", generator = g).view(2, 16, 4, 64).transpose(1, 2)
+ cos, sin = _cos_sin(80, 64, torch.float32)
+ rot = torch.cat((-q[..., 32:], q[..., :32]), dim = -1)
+ torch.testing.assert_close(Q_out, q * cos[:16] + rot * sin[:16], rtol = 1e-5, atol = 1e-5)
+
+
+def test_bf16_tracing_needs_native_bf16_on_every_gpu(monkeypatch):
+ """A pre-Ampere GPU anywhere in the process keeps bf16 untraced (a layer can land on it)."""
+ monkeypatch.setattr(torch.version, "hip", None)
+ monkeypatch.setattr(torch.cuda, "device_count", lambda: 2)
+ for caps, expected in (({0: (8, 0), 1: (8, 9)}, True), ({0: (8, 0), 1: (7, 5)}, False)):
+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda i, caps = caps: caps[i])
+ assert rms_layernorm._bf16_traceable() is expected, caps
+ monkeypatch.setattr(torch.version, "hip", "6.4")
+ assert rms_layernorm._bf16_traceable() is True
+
+
+def test_bf16_without_native_support_stays_untraced(monkeypatch):
+ """Where bf16 cannot be traced, bf16 inputs keep the old graph break and match eager."""
+ if not TRACEABLE:
+ pytest.skip("this torch has no torch.library.triton_op")
+ monkeypatch.setattr(rms_layernorm, "_BF16_TRACEABLE", False)
+ monkeypatch.setattr(rope_embedding, "_BF16_TRACEABLE", False)
+ eager = _rms_case(fast_rms_layernorm, (64, 256), torch.float16, False, False)
+ compiled = _rms_case(
+ torch.compile(lambda n, x, gm: fast_rms_layernorm(n, x, gm)),
+ (64, 256),
+ torch.float16,
+ False,
+ False,
+ )
+ assert all(_bytes_equal(e, c) for e, c in zip(eager, compiled)) and _graph_breaks() == 0
+ torch._dynamo.reset()
+ dynamo_utils.counters.clear()
+ eager = _rms_case(fast_rms_layernorm, (64, 256), torch.bfloat16, False, False)
+ compiled = _rms_case(
+ torch.compile(lambda n, x, gm: fast_rms_layernorm(n, x, gm)),
+ (64, 256),
+ torch.bfloat16,
+ False,
+ False,
+ )
+ assert all(_bytes_equal(e, c) for e, c in zip(eager, compiled)) and _graph_breaks() > 0
+
+
+def test_rope_dynamic_shapes():
+ """dynamic=True makes the launch grid SymInts (divmod rejected them)."""
+ if not TRACEABLE:
+ pytest.skip("this torch has no torch.library.triton_op")
+ cos, sin = _cos_sin(64, 64, CDTYPE)
+
+ def f(q_lin, k_lin, bsz, seq, n_heads, n_kv):
+ Q = q_lin.view(bsz, seq, n_heads, 64).transpose(1, 2)
+ K = k_lin.view(bsz, seq, n_kv, 64).transpose(1, 2)
+ return fast_rope_embedding(Q, K, cos, sin, None)
+
+ compiled = torch.compile(f, fullgraph = True, dynamic = True)
+ for seq in (16, 24, 40):
+ g = torch.Generator(device = "cuda").manual_seed(seq)
+ q_lin = torch.randn(2, seq, 8 * 64, device = "cuda", generator = g, dtype = CDTYPE)
+ k_lin = torch.randn(2, seq, 2 * 64, device = "cuda", generator = g, dtype = CDTYPE)
+ for e, c in zip(
+ f(q_lin.clone(), k_lin.clone(), 2, seq, 8, 2), compiled(q_lin, k_lin, 2, seq, 8, 2)
+ ):
+ assert _bytes_equal(e, c), seq
+
+
[email protected]("compiled", [False, True], ids = ["eager", "compiled"])
[email protected]("with_indices", [False, True], ids = ["positions", "rope_indices"])
+def test_rope_expanded_gradient(with_indices, compiled):
+ """Q.sum() gives a stride-0 gradient; rotating it in place gave wrong dQ / dK."""
+ g = torch.Generator(device = "cuda").manual_seed(0)
+ q_lin = torch.randn(2, 16, 4 * 64, device = "cuda", generator = g).requires_grad_(True)
+ k_lin = torch.randn(2, 16, 2 * 64, device = "cuda", generator = g).requires_grad_(True)
+ cos, sin = _cos_sin(80, 64, torch.float32)
+ indices = torch.arange(16, dtype = torch.int32).repeat(2, 1).cuda() if with_indices else None
+ fn = _compile(fast_rope_embedding) if compiled else fast_rope_embedding
+ Q_out, K_out = fn(
+ q_lin.view(2, 16, 4, 64).transpose(1, 2),
+ k_lin.view(2, 16, 2, 64).transpose(1, 2),
+ cos,
+ sin,
+ indices,
+ )
+ (Q_out.sum() + K_out.sum()).backward()
+ # d/dx of sum(x * cos + rotate_half(x) * sin): cos + rotate_half^T(ones) * sin, per position.
+ ones = torch.ones(16, 64, device = "cuda")
+ rot_t = torch.cat((ones[..., 32:], -ones[..., :32]), dim = -1)
+ per_pos = cos[:16] + rot_t * sin[:16]
+ torch.testing.assert_close(
+ q_lin.grad.view(2, 16, 4, 64),
+ per_pos[None, :, None].expand(2, 16, 4, 64),
+ rtol = 1e-5,
+ atol = 1e-5,
+ )
+ torch.testing.assert_close(
+ k_lin.grad.view(2, 16, 2, 64),
+ per_pos[None, :, None].expand(2, 16, 2, 64),
+ rtol = 1e-5,
+ atol = 1e-5,
+ )
+
+
+def test_tiny_llama_decoder_layers_compile_fullgraph():
+ """Decoder layers compile inside Unsloth's gradient checkpointing and match eager."""
+ if not TRACEABLE:
+ pytest.skip("this torch has no torch.library.triton_op")
+ from unsloth import FastLanguageModel
+
+ model, _ = FastLanguageModel.from_pretrained(
+ "hf-internal-testing/tiny-random-LlamaForCausalLM",
+ max_seq_length = 64,
+ load_in_4bit = False,
+ dtype = torch.bfloat16,
+ )
+ model = FastLanguageModel.get_peft_model(
+ model,
+ r = 8,
+ lora_alpha = 16,
+ lora_dropout = 0,
+ target_modules = [
+ "q_proj",
+ "k_proj",
+ "v_proj",
+ "o_proj",
+ "gate_proj",
+ "up_proj",
+ "down_proj",
+ ],
+ use_gradient_checkpointing = "unsloth",
+ random_state = 3407,
+ )
+ model.train()
+ vocab = model.config.vocab_size
+ g = torch.Generator().manual_seed(0)
+ # With several GPUs visible Unsloth may place the model on the emptiest one, not cuda:0.
+ ids = torch.randint(0, vocab, (2, 48), generator = g).to(
+ model.get_input_embeddings().weight.device
+ )
+
+ params = [p for p in model.parameters() if p.requires_grad]
+ eager_loss = model(input_ids = ids, labels = ids).loss
+ eager_loss.backward()
+ eager_grads = [p.grad.clone() for p in params]
+ model.zero_grad(set_to_none = True)
+
+ # Before torch 2.11 fast_lora's Functions stay opaque: the only graph breaks allowed.
+ trace_lora = fast_lora.TRACE_LORA_FUNCTIONS
+ layers = model.base_model.model.model.layers
+ for layer in layers:
+ layer.forward = torch.compile(layer.forward, fullgraph = trace_lora)
+ loss = model(input_ids = ids, labels = ids).loss
+ loss.backward()
+ reasons = [" ".join(str(k).split()) for k in dynamo_utils.counters["graph_break"]]
+ assert all("_apply" in r for r in reasons), reasons
+ assert math.isfinite(loss.item())
+ assert abs(loss.item() - eager_loss.item()) <= 1e-3 * abs(eager_loss.item())
+ grads = [p.grad for p in params]
+ assert grads and all(gr is not None and torch.isfinite(gr).all() for gr in grads)
+ scale = max(g.abs().max().item() for g in eager_grads)
+ worst = max((a.float() - b.float()).abs().max().item() for a, b in zip(grads, eager_grads))
+ # Two bf16 rounding steps of the largest gradient: compiled kernels sum in another order (one
+ # step over 1% on an RTX PRO 6000).
+ assert worst <= 2 * torch.finfo(torch.bfloat16).eps * scale, (worst, scale)
+
+
+def test_tiny_llama_causal_lm_compiles_fullgraph():
+ """The whole causal LM compiles with no requires-grad hook or use_return_dict graph break,
+ and its LoRA gradients match eager (torch 2.10 still breaks in the fused loss)."""
+ if not TRACEABLE:
+ pytest.skip("this torch has no torch.library.triton_op")
+ from unsloth import FastLanguageModel
+ from unsloth_zoo.utils import Version
+ import unsloth_zoo.fused_losses.cross_entropy_loss as fused_ce
+
+ model, _ = FastLanguageModel.from_pretrained(
+ "hf-internal-testing/tiny-random-LlamaForCausalLM",
+ max_seq_length = 64,
+ load_in_4bit = False,
+ dtype = torch.bfloat16,
+ )
+ model = FastLanguageModel.get_peft_model(
+ model,
+ r = 8,
+ lora_alpha = 16,
+ lora_dropout = 0,
+ target_modules = [
+ "q_proj",
+ "k_proj",
+ "v_proj",
+ "o_proj",
+ "gate_proj",
+ "up_proj",
+ "down_proj",
+ ],
+ use_gradient_checkpointing = False,
+ random_state = 3407,
+ )
+ model.train()
+ embeddings = model.get_input_embeddings()
+ assert embeddings._forward_hooks, "enable_input_require_grads registered no hook"
+ g = torch.Generator().manual_seed(0)
+ ids = torch.randint(0, model.config.vocab_size, (2, 48), generator = g).to(
+ embeddings.weight.device
+ )
+ params = [p for p in model.parameters() if p.requires_grad]
+
+ eager_loss = model(input_ids = ids, labels = ids).loss
+ eager_loss.backward()
+ eager_grads = [p.grad.detach().float().clone() for p in params]
+ model.zero_grad(set_to_none = True)
+
+ loss_opaque = getattr(fused_ce, "_FUSED_LOSS_OPAQUE", False)
+ fullgraph = (
+ fast_lora.TRACE_LORA_FUNCTIONS
+ and Version(torch.__version__) >= Version("2.11.0")
+ and not loss_opaque
+ )
+ causal_lm = model.base_model.model
+ causal_lm.forward = torch.compile(causal_lm.forward, fullgraph = fullgraph)
+ loss = model(input_ids = ids, labels = ids).loss
+ loss.backward()
+ reasons = [" ".join(str(k).split()) for k in dynamo_utils.counters["graph_break"]]
+ assert not any("requires_grad_()" in r or "logging.Logger" in r for r in reasons), reasons
+ if loss_opaque and fast_lora.TRACE_LORA_FUNCTIONS:
+ assert all("_fused_loss_opaque" in r for r in reasons), reasons
+ assert abs(loss.item() - eager_loss.item()) <= 1e-3 * abs(eager_loss.item())
+ total = sum(float(e.norm()) for e in eager_grads)
+ assert total > 0
+ for p, e in zip(params, eager_grads):
+ assert p.grad is not None
+ err = float((p.grad.float() - e).norm())
+ assert err <= 0.05 * float(e.norm()) + 1e-6, (err, float(e.norm()))
+
+
+def test_input_require_grads_hook_is_exact():
+ """Compiled, the hook returns x + -0.0 (exact) needing grad; eager flips requires_grad."""
+ from transformers import AutoModelForCausalLM
+
+ model = AutoModelForCausalLM.from_pretrained(
+ "hf-internal-testing/tiny-random-LlamaForCausalLM"
+ ).cuda()
+ model.requires_grad_(False)
+ model.enable_input_require_grads()
+ emb = model.get_input_embeddings()
+ with torch.no_grad():
+ emb.weight[0].fill_(-0.0)
+ ids = torch.tensor([[0, 1, 2]], device = "cuda")
+ eager = emb(ids)
+ assert eager.requires_grad
+ compiled = torch.compile(lambda i: emb(i), fullgraph = TRACEABLE)(ids)
+ assert compiled.requires_grad
+ assert _bytes_equal(compiled.detach(), eager.detach())
+ assert torch.signbit(compiled[0, 0]).all()
+
+
+def test_input_require_grads_hook_is_a_no_op_in_compiled_no_grad_decode():
+ """A compiled decode step runs without grad: the hook leaves the embedding output alone."""
+ from transformers import AutoModelForCausalLM
+
+ model = AutoModelForCausalLM.from_pretrained(
+ "hf-internal-testing/tiny-random-LlamaForCausalLM"
+ ).cuda()
+ model.requires_grad_(False)
+ model.enable_input_require_grads()
+ emb = model.get_input_embeddings()
+ ids = torch.tensor([[0, 1, 2]], device = "cuda")
+ dynamo_utils.counters.clear()
+ with torch.no_grad():
+ compiled = torch.compile(lambda i: emb(i), fullgraph = TRACEABLE)(ids)
+ eager = emb(ids)
+ assert not compiled.requires_grad
+ assert _bytes_equal(compiled, eager)
+ assert not dynamo_utils.counters["graph_break"]
+
+
+def test_compile_cache_key_covers_the_kernel_source():
+ """A warm FX graph cache served an edited kernel as the old one; key it on the files."""
+ from unsloth.kernels import rms_layernorm, rope_embedding
+
+ if not hasattr(getattr(torch.compiler, "config", None), "cache_key_tag"):
+ pytest.skip("torch has no compile cache key tag")
+ if not rms_layernorm._TRACEABLE:
+ pytest.skip("no triton_op, nothing is cached by op name")
+ tags = torch.compiler.config.cache_key_tag.split(",")
+ for mod in (rms_layernorm, rope_embedding):
+ with open(mod.__file__, "rb") as file:
+ digest = hashlib.sha256(file.read()).hexdigest()[:16]
+ assert f"unsloth/{os.path.basename(mod.__file__)}:{digest}" in tags
diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py
index aca464a67dd..ed1e6b04b2e 100644
--- a/unsloth/import_fixes.py
+++ b/unsloth/import_fixes.py
@@ -4568,28 +4568,54 @@ def check_fbgemm_gpu_version():
def patch_enable_input_require_grads():
"""Patch PreTrainedModel.enable_input_require_grads to tolerate vision models
- that raise NotImplementedError from get_input_embeddings()."""
+ that raise NotImplementedError from get_input_embeddings(), and so that its hook traces
+ under torch.compile."""
import inspect
+ import torch
from transformers import PreTrainedModel
- # Only patch the new variant that iterates over self.modules(); see huggingface/transformers#41993.
try:
original_source = inspect.getsource(PreTrainedModel.enable_input_require_grads)
except:
return
+ class _RequireGrad(torch.autograd.Function):
+ # x + -0.0 is x exactly (signed zeros too); the anchor only makes the output need grad.
+ @staticmethod
+ def forward(ctx, x, anchor):
+ return x + anchor
+
+ @staticmethod
+ def backward(ctx, grad):
+ return grad, None
+
+ # Created here: torch.compile cannot create a tensor that requires grad inside a graph.
+ anchor = torch.tensor(-0.0, requires_grad = True)
+
+ def make_inputs_require_grads(module, input, output):
+ # requires_grad_() on an intermediate is a graph break under torch.compile.
+ if torch.compiler.is_compiling():
+ if not torch.is_grad_enabled():
+ return # a compiled decode step records no grad
+ return _RequireGrad.apply(output, anchor)
+ output.requires_grad_(True)
+
+ # Older transformers hooks a single embedding (huggingface/transformers#41993 added the loop).
+ # wraps keeps inspect.getsource on transformers' source for later source checks.
+ original = PreTrainedModel.enable_input_require_grads
if "for module in self.modules()" not in original_source:
- return
- def _patched_enable_input_require_grads(self):
- import torch
+ @functools.wraps(original)
+ def _patched_single_enable_input_require_grads(self):
+ self._require_grads_hook = self.get_input_embeddings().register_forward_hook(
+ make_inputs_require_grads
+ )
- def make_inputs_require_grads(module, input, output):
- # Dynamo graph-breaks on requires_grad_(); a compiled decode step records no grad anyway.
- if torch.compiler.is_compiling() and not torch.is_grad_enabled():
- return
- output.requires_grad_(True)
+ PreTrainedModel.enable_input_require_grads = _patched_single_enable_input_require_grads
+ return
+ @functools.wraps(original)
+ def _patched_enable_input_require_grads(self):
hooks = []
seen_modules = set()
diff --git a/unsloth/kernels/rms_layernorm.py b/unsloth/kernels/rms_layernorm.py
index c9976b24361..fa4940c5f49 100644
--- a/unsloth/kernels/rms_layernorm.py
+++ b/unsloth/kernels/rms_layernorm.py
@@ -9,9 +9,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import hashlib
+import os
import triton
import triton.language as tl
import torch
+from typing import Tuple
from .utils import calculate_settings, torch_gpu_device
@@ -154,6 +157,55 @@ def _gemma_rms_layernorm_forward(
tl.store(Y + col_offsets, output, mask = mask)
+def _rms_forward(X, W, eps, gemma, wrap):
+ n_rows, n_cols = X.shape
+ BLOCK_SIZE, num_warps = calculate_settings(n_cols)
+ Y = torch.empty((n_rows, n_cols), dtype = X.dtype, device = X.device)
+ r = torch.empty(n_rows, dtype = torch.float32, device = X.device)
+ fx = _gemma_rms_layernorm_forward if gemma else _rms_layernorm_forward
+ wrap(fx)[(n_rows,)](
+ Y,
+ Y.stride(0),
+ X,
+ X.stride(0),
+ W,
+ W.stride(0),
+ r,
+ r.stride(0),
+ n_cols,
+ eps,
+ BLOCK_SIZE = BLOCK_SIZE,
+ num_warps = num_warps,
+ )
+ return Y, r
+
+
+def _rms_backward(dY, dX, X, W, r, eps, gemma, wrap):
+ # Non-Gemma writes dX over dY (the kernel ignores dX); Gemma writes into dX.
+ n_rows, n_cols = dY.shape
+ BLOCK_SIZE, num_warps = calculate_settings(n_cols)
+ wrap(_rms_layernorm_backward)[(n_rows,)](
+ dY,
+ dY.stride(0),
+ dX,
+ dX.stride(0),
+ X,
+ X.stride(0),
+ W,
+ W.stride(0),
+ r,
+ r.stride(0),
+ n_cols,
+ eps,
+ GEMMA = gemma,
+ BLOCK_SIZE = BLOCK_SIZE,
+ num_warps = num_warps,
+ )
+
+
+_eager_kernel = lambda kernel: kernel
+
+
class Fast_RMS_Layernorm(torch.autograd.Function):
@staticmethod
def forward(
@@ -168,36 +220,9 @@ class Fast_RMS_Layernorm(torch.autograd.Function):
X = X.reshape(-1, dim).contiguous()
# kernels read W at unit stride, and this W is the one saved for backward.
W = W.contiguous()
- n_rows: int
- n_cols: int
- n_rows, n_cols = X.shape
- BLOCK_SIZE: int
- num_warps: int
- BLOCK_SIZE, num_warps = calculate_settings(n_cols)
- device = X.device
-
- Y = torch.empty((n_rows, n_cols), dtype = X.dtype, device = device)
- r = torch.empty(n_rows, dtype = torch.float32, device = device)
-
- fx = _gemma_rms_layernorm_forward if gemma else _rms_layernorm_forward
- with torch_gpu_device(device):
- fx[(n_rows,)](
- Y,
- Y.stride(0),
- X,
- X.stride(0),
- W,
- W.stride(0),
- r,
- r.stride(0),
- n_cols,
- eps,
- BLOCK_SIZE = BLOCK_SIZE,
- num_warps = num_warps,
- )
+ with torch_gpu_device(X.device):
+ Y, r = _rms_forward(X, W, eps, gemma, _eager_kernel)
ctx.eps = eps
- ctx.BLOCK_SIZE = BLOCK_SIZE
- ctx.num_warps = num_warps
ctx.GEMMA = gemma
ctx.save_for_backward(X, W, r)
return Y.view(*shape)
@@ -208,35 +233,88 @@ class Fast_RMS_Layernorm(torch.autograd.Function):
dim: int = shape[-1]
dY = dY.reshape(-1, dim).contiguous()
X, W, r = ctx.saved_tensors
- n_rows: int
- n_cols: int
- n_rows, n_cols = dY.shape
dX = torch.empty_like(dY) if ctx.GEMMA else dY
-
with torch_gpu_device(dY.device):
- _rms_layernorm_backward[(n_rows,)](
- dY,
- dY.stride(0),
- dX,
- dX.stride(0),
- X,
- X.stride(0),
- W,
- W.stride(0),
- r,
- r.stride(0),
- n_cols,
- ctx.eps,
- GEMMA = ctx.GEMMA,
- BLOCK_SIZE = ctx.BLOCK_SIZE,
- num_warps = ctx.num_warps,
- )
+ _rms_backward(dY, dX, X, W, r, ctx.eps, ctx.GEMMA, _eager_kernel)
dX = dX.view(*shape)
return dX, None, None, None
-# RMS Layernorm does not torch.compile properly; reason unknown.
+# Compiled graphs take these ops: same kernels, fresh outputs (the Function writes dX over dY).
+_TRACEABLE = hasattr(torch.library, "triton_op") and hasattr(torch.library, "wrap_triton")
+
+
+def _bf16_traceable():
+ # Dynamo cannot trace bf16 Triton without native bf16 (T4); any visible GPU may hold a layer.
+ try:
+ if torch.version.hip:
+ return True
+ n = torch.cuda.device_count()
+ return n > 0 and all(torch.cuda.get_device_capability(i)[0] >= 8 for i in range(n))
+ except Exception:
+ return False
+
+
+_BF16_TRACEABLE = _TRACEABLE and _bf16_traceable()
+
+
+def _tag_compile_cache(path):
+ # Inductor's FX cache keys a triton_op without its source; key on the file so upgrades miss.
+ config = getattr(torch.compiler, "config", None)
+ if config is None or not hasattr(config, "cache_key_tag"):
+ return
+ with open(path, "rb") as file:
+ tag = f"unsloth/{os.path.basename(path)}:{hashlib.sha256(file.read()).hexdigest()[:16]}"
+ tags = [t for t in config.cache_key_tag.split(",") if t]
+ if tag not in tags:
+ config.cache_key_tag = ",".join(tags + [tag])
+
+
+def _traced_kernel(kernel):
+ # wrap_triton needs the JITFunction under triton.heuristics; callers pass its constexprs.
+ if isinstance(kernel, triton.runtime.autotuner.Heuristics):
+ kernel = kernel.fn
+ return torch.library.wrap_triton(kernel)
+
+
+if _TRACEABLE:
+
+ @torch.library.triton_op("unsloth::rms_layernorm", mutates_args = ())
+ def _rms_layernorm_op(
+ X: torch.Tensor, W: torch.Tensor, eps: float, gemma: bool
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
+ return _rms_forward(X, W, eps, gemma, _traced_kernel)
+
+ @torch.library.triton_op("unsloth::rms_layernorm_backward", mutates_args = ())
+ def _rms_layernorm_backward_op(
+ dY: torch.Tensor, X: torch.Tensor, W: torch.Tensor, r: torch.Tensor, eps: float, gemma: bool
+ ) -> torch.Tensor:
+ dX = torch.empty_like(dY)
+ # Non-Gemma rewrites its dY argument, so it gets a copy.
+ _rms_backward(dY if gemma else dX.copy_(dY), dX, X, W, r, eps, gemma, _traced_kernel)
+ return dX
+
+ def _rms_setup_context(ctx, inputs, output):
+ X, W, eps, gemma = inputs
+ ctx.eps, ctx.gemma = eps, gemma
+ ctx.save_for_backward(X, W, output[1])
+
+ def _rms_layernorm_op_backward(ctx, dY, dr):
+ X, W, r = ctx.saved_tensors
+ dX = torch.ops.unsloth.rms_layernorm_backward(dY.contiguous(), X, W, r, ctx.eps, ctx.gemma)
+ return dX, None, None, None
+
+ _rms_layernorm_op.register_autograd(
+ _rms_layernorm_op_backward, setup_context = _rms_setup_context
+ )
+ _tag_compile_cache(__file__)
+
+
@torch.compiler.disable
+def _fast_rms_layernorm_untraced(X, W, eps, gemma):
+ return Fast_RMS_Layernorm.apply(X, W, eps, gemma)
+
+
def fast_rms_layernorm(
layernorm,
X: torch.Tensor,
@@ -246,8 +324,19 @@ def fast_rms_layernorm(
eps: float = (
layernorm.variance_epsilon if hasattr(layernorm, "variance_epsilon") else layernorm.eps
)
- out = Fast_RMS_Layernorm.apply(X, W, eps, gemma)
- return out
+ if not torch.compiler.is_compiling():
+ return Fast_RMS_Layernorm.apply(X, W, eps, gemma)
+ if (
+ not _TRACEABLE
+ or X.device.type != "cuda"
+ or (X.dtype == torch.bfloat16 and not _BF16_TRACEABLE)
+ ):
+ return _fast_rms_layernorm_untraced(X, W, eps, gemma)
+ shape = X.shape
+ Y, _ = torch.ops.unsloth.rms_layernorm(
+ X.reshape(-1, shape[-1]).contiguous(), W.contiguous(), eps, gemma
+ )
+ return Y.view(shape)
from transformers.models.llama.modeling_llama import LlamaRMSNorm
diff --git a/unsloth/kernels/rope_embedding.py b/unsloth/kernels/rope_embedding.py
index 31c0260136e..45e31fa1e46 100644
--- a/unsloth/kernels/rope_embedding.py
+++ b/unsloth/kernels/rope_embedding.py
@@ -14,7 +14,15 @@ import triton
import triton.language as tl
import torch
from ..device_type import DEVICE_COUNT
+from typing import Tuple
from .utils import calculate_settings, torch_gpu_device, torch_device_stream
+from .rms_layernorm import (
+ _BF16_TRACEABLE,
+ _TRACEABLE,
+ _eager_kernel,
+ _tag_compile_cache,
+ _traced_kernel,
+)
def _rope_embedding_QK(
@@ -163,6 +171,33 @@ _rope_embedding = triton.heuristics(
)(_rope_embedding)
+def _rope_rows(Q, cos, sin, seq_len, n_heads, head_dim, backward, wrap):
+ # Rotates Q [batch * seq_len, n_heads * head_dim] in place; BLOCK_SIZE = head_dim//2 was racy.
+ BLOCK_SIZE, num_warps = calculate_settings(head_dim // 2)
+ n_groups = (
+ n_heads + ROPE_GROUP_SIZE - 1
+ ) // ROPE_GROUP_SIZE # also a SymInt under dynamic=True
+ wrap(_rope_embedding)[
+ (
+ Q.shape[0],
+ n_groups,
+ )
+ ](
+ Q,
+ Q.stride(0),
+ cos,
+ cos.stride(0),
+ sin,
+ sin.stride(0),
+ seq_len,
+ head_dim,
+ n_heads,
+ BACKWARD_PASS = backward,
+ BLOCK_SIZE = BLOCK_SIZE,
+ num_warps = num_warps,
+ )
+
+
class Fast_RoPE_Embedding(torch.autograd.Function):
@staticmethod
def forward(ctx, Q, cos, sin):
@@ -173,44 +208,9 @@ class Fast_RoPE_Embedding(torch.autograd.Function):
head_dim: int
batch, seq_len, n_heads, head_dim = Q.shape
Q = Q.reshape(batch * seq_len, n_heads * head_dim)
- n_rows: int
- n_cols: int
- n_rows, n_cols = Q.shape
assert seq_len <= cos.shape[0]
-
- # Changing blocksize to head_dim//2 showed concurrency / non-deterministic issues; group_size too
- # large also hurts performance.
- BLOCK_SIZE, num_warps = calculate_settings(head_dim // 2)
-
- # group_size = 4 # 4 or 8, too large group_size can hurt performance.
- div: int
- mod: int
- div, mod = divmod(n_heads, ROPE_GROUP_SIZE)
- n_groups: int = div + (mod != 0)
-
with torch_gpu_device(Q.device):
- _rope_embedding[
- (
- n_rows,
- n_groups,
- )
- ](
- Q,
- Q.stride(0),
- cos,
- cos.stride(0),
- sin,
- sin.stride(0),
- seq_len,
- head_dim,
- n_heads,
- BACKWARD_PASS = False,
- BLOCK_SIZE = BLOCK_SIZE,
- num_warps = num_warps,
- )
- ctx.BLOCK_SIZE = BLOCK_SIZE
- ctx.num_warps = num_warps
- ctx.n_groups = n_groups
+ _rope_rows(Q, cos, sin, seq_len, n_heads, head_dim, False, _eager_kernel)
ctx.cos = cos
ctx.sin = sin
return Q.reshape(batch, seq_len, n_heads, head_dim)
@@ -222,35 +222,10 @@ class Fast_RoPE_Embedding(torch.autograd.Function):
n_heads: int
head_dim: int
batch, seq_len, n_heads, head_dim = dY.shape
- # The kernel writes in place: reshape of an expanded grad (e.g. from .sum()) stays a stride-0 view.
- dY = dY.contiguous().view(batch * seq_len, n_heads * head_dim)
- n_rows: int
- n_cols: int
- n_rows, n_cols = dY.shape
-
- cos = ctx.cos
- sin = ctx.sin
-
+ # contiguous: an expanded gradient (e.g. from Q.sum()) would be rotated through stride 0.
+ dY = dY.contiguous().reshape(batch * seq_len, n_heads * head_dim)
with torch_gpu_device(dY.device):
- _rope_embedding[
- (
- n_rows,
- ctx.n_groups,
- )
- ](
- dY,
- dY.stride(0),
- cos,
- cos.stride(0),
- sin,
- sin.stride(0),
- seq_len,
- head_dim,
- n_heads,
- BACKWARD_PASS = True,
- BLOCK_SIZE = ctx.BLOCK_SIZE,
- num_warps = ctx.num_warps,
- )
+ _rope_rows(dY, ctx.cos, ctx.sin, seq_len, n_heads, head_dim, True, _eager_kernel)
dY = dY.reshape(batch, seq_len, n_heads, head_dim)
return (
dY,
@@ -259,23 +234,33 @@ class Fast_RoPE_Embedding(torch.autograd.Function):
)
-# RoPE embedding does not torch.compile properly; reason unknown.
-def fast_rope_embedding(
- Q,
- K,
- cos,
- sin,
- rope_embedding_indices = None,
-):
- if rope_embedding_indices is not None:
- Q_out, K_out = Fast_RoPE_Embedding_QK.apply(Q, K, cos, sin, rope_embedding_indices)
- else:
- Q_out = Fast_RoPE_Embedding.apply(Q.transpose(1, 2).contiguous(), cos, sin).transpose(1, 2)
- K_out = Fast_RoPE_Embedding.apply(K.transpose(1, 2).contiguous(), cos, sin).transpose(1, 2)
- if DEVICE_COUNT > 1:
- torch_device_stream(Q.device).synchronize()
- return Q_out, K_out
+def _rope_qk(Q, K, cos, sin, rope_ptr, has_indices, backward, wrap):
+ # Rotates Q [batch, n_heads_Q, seq_len, head_dim] and K in place, at any strides.
+ batch, n_heads_Q, seq_len, head_dim = Q.shape
+ n_heads_K = K.shape[1]
+ BLOCK_SIZE, num_warps = calculate_settings(head_dim)
+ wrap(_rope_embedding_QK)[(batch * seq_len, n_heads_Q)](
+ Q,
+ Q.stride(0),
+ Q.stride(1),
+ Q.stride(2),
+ K,
+ K.stride(0),
+ K.stride(1),
+ K.stride(2),
+ cos,
+ cos.stride(0),
+ sin,
+ sin.stride(0),
+ rope_ptr,
+ seq_len,
+ head_dim = head_dim,
+ n_heads_K = n_heads_K,
+ BACKWARD_PASS = backward,
+ HAS_ROPE_INDICES = has_indices,
+ BLOCK_SIZE = BLOCK_SIZE,
+ num_warps = num_warps,
+ )
class Fast_RoPE_Embedding_QK(torch.autograd.Function):
@@ -284,9 +269,6 @@ class Fast_RoPE_Embedding_QK(torch.autograd.Function):
has_indices = rope_indices is not None
cos, sin = cos.squeeze(), sin.squeeze()
- batch, n_heads_Q, seq_len, head_dim = Q.shape
- _, n_heads_K, _, _ = K.shape
-
# Inplace rotary embedding is generally fine.
Q_out = Q.clone() if not Q.is_contiguous() else Q
K_out = K.clone() if not K.is_contiguous() else K
@@ -297,52 +279,13 @@ class Fast_RoPE_Embedding_QK(torch.autograd.Function):
else:
rope_ptr = cos.new_empty(1, dtype = torch.int32)
- BLOCK_SIZE, num_warps = calculate_settings(head_dim)
-
- Q_batch_stride, Q_head_stride, Q_seq_stride = (
- Q_out.stride(0),
- Q_out.stride(1),
- Q_out.stride(2),
- )
- K_batch_stride, K_head_stride, K_seq_stride = (
- K_out.stride(0),
- K_out.stride(1),
- K_out.stride(2),
- )
-
with torch_gpu_device(Q.device):
- _rope_embedding_QK[(batch * seq_len, n_heads_Q)](
- Q_out,
- Q_batch_stride,
- Q_head_stride,
- Q_seq_stride,
- K_out,
- K_batch_stride,
- K_head_stride,
- K_seq_stride,
- cos,
- cos.stride(0),
- sin,
- sin.stride(0),
- rope_ptr,
- seq_len,
- head_dim = head_dim,
- n_heads_K = n_heads_K,
- BACKWARD_PASS = False,
- HAS_ROPE_INDICES = has_indices,
- BLOCK_SIZE = BLOCK_SIZE,
- num_warps = num_warps,
- )
+ _rope_qk(Q_out, K_out, cos, sin, rope_ptr, has_indices, False, _eager_kernel)
- ctx.block_size = BLOCK_SIZE
- ctx.num_warps = num_warps
ctx.has_indices = has_indices
ctx.cos = cos
ctx.sin = sin
ctx.rope_indices = rope_ptr if has_indices else None
- ctx.seq_len = seq_len
- ctx.n_heads_Q = n_heads_Q
- ctx.n_heads_K = n_heads_K
return (
Q_out,
@@ -351,52 +294,110 @@ class Fast_RoPE_Embedding_QK(torch.autograd.Function):
@staticmethod
def backward(ctx, dQ, dK):
- batch, _, _, head_dim = dQ.shape
-
rope_ptr = ctx.rope_indices if ctx.has_indices else ctx.cos.new_empty(1, dtype = torch.int32)
# Inplace rotary embedding is generally fine.
dQ_out = dQ.clone() if not dQ.is_contiguous() else dQ
dK_out = dK.clone() if not dK.is_contiguous() else dK
- Q_batch_stride, Q_head_stride, Q_seq_stride = (
- dQ_out.stride(0),
- dQ_out.stride(1),
- dQ_out.stride(2),
- )
- K_batch_stride, K_head_stride, K_seq_stride = (
- dK_out.stride(0),
- dK_out.stride(1),
- dK_out.stride(2),
- )
-
with torch_gpu_device(dQ.device):
- _rope_embedding_QK[(batch * ctx.seq_len, ctx.n_heads_Q)](
- dQ_out,
- Q_batch_stride,
- Q_head_stride,
- Q_seq_stride,
- dK_out,
- K_batch_stride,
- K_head_stride,
- K_seq_stride,
- ctx.cos,
- ctx.cos.stride(0),
- ctx.sin,
- ctx.sin.stride(0),
- rope_ptr,
- ctx.seq_len,
- head_dim = head_dim,
- n_heads_K = ctx.n_heads_K,
- BACKWARD_PASS = True,
- HAS_ROPE_INDICES = ctx.has_indices,
- BLOCK_SIZE = ctx.block_size,
- num_warps = ctx.num_warps,
+ _rope_qk(
+ dQ_out, dK_out, ctx.cos, ctx.sin, rope_ptr, ctx.has_indices, True, _eager_kernel
)
return (dQ_out, dK_out, None, None, None)
+# Compiled graphs take these ops: the same kernels on fresh copies (the Functions rotate in place).
+if _TRACEABLE:
+
+ @torch.library.triton_op("unsloth::rope_embedding", mutates_args = ())
+ def _rope_embedding_op(
+ Q: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor, backward: bool
+ ) -> torch.Tensor:
+ # Q: [batch, seq_len, n_heads, head_dim] at any strides.
+ batch, seq_len, n_heads, head_dim = Q.shape
+ out = Q.clone(memory_format = torch.contiguous_format)
+ rows = out.view(batch * seq_len, n_heads * head_dim)
+ _rope_rows(rows, cos, sin, seq_len, n_heads, head_dim, backward, _traced_kernel)
+ return out
+
+ def _rope_setup_context(ctx, inputs, output):
+ ctx.save_for_backward(inputs[1], inputs[2])
+
+ def _rope_embedding_op_backward(ctx, dY):
+ cos, sin = ctx.saved_tensors
+ return torch.ops.unsloth.rope_embedding(dY, cos, sin, True), None, None, None
+
+ _rope_embedding_op.register_autograd(
+ _rope_embedding_op_backward, setup_context = _rope_setup_context
+ )
+
+ @torch.library.triton_op("unsloth::rope_embedding_qk", mutates_args = ())
+ def _rope_embedding_qk_op(
+ Q: torch.Tensor,
+ K: torch.Tensor,
+ cos: torch.Tensor,
+ sin: torch.Tensor,
+ rope_indices: torch.Tensor,
+ backward: bool,
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
+ Q_out, K_out = Q.clone(), K.clone()
+ _rope_qk(Q_out, K_out, cos, sin, rope_indices, True, backward, _traced_kernel)
+ return Q_out, K_out
+
+ def _rope_qk_setup_context(ctx, inputs, output):
+ ctx.save_for_backward(inputs[2], inputs[3], inputs[4])
+
+ def _rope_embedding_qk_op_backward(ctx, dQ, dK):
+ cos, sin, rope_indices = ctx.saved_tensors
+ dQ, dK = torch.ops.unsloth.rope_embedding_qk(dQ, dK, cos, sin, rope_indices, True)
+ return dQ, dK, None, None, None, None
+
+ _rope_embedding_qk_op.register_autograd(
+ _rope_embedding_qk_op_backward, setup_context = _rope_qk_setup_context
+ )
+ _tag_compile_cache(__file__)
+
+
+def _fast_rope_embedding_eager(Q, K, cos, sin, rope_embedding_indices):
+ if rope_embedding_indices is not None:
+ Q_out, K_out = Fast_RoPE_Embedding_QK.apply(Q, K, cos, sin, rope_embedding_indices)
+ else:
+ Q_out = Fast_RoPE_Embedding.apply(Q.transpose(1, 2).contiguous(), cos, sin).transpose(1, 2)
+ K_out = Fast_RoPE_Embedding.apply(K.transpose(1, 2).contiguous(), cos, sin).transpose(1, 2)
+ if DEVICE_COUNT > 1:
+ torch_device_stream(Q.device).synchronize()
+ return Q_out, K_out
+
+
+_fast_rope_embedding_untraced = torch.compiler.disable(_fast_rope_embedding_eager)
+
+
+def fast_rope_embedding(
+ Q,
+ K,
+ cos,
+ sin,
+ rope_embedding_indices = None,
+):
+ if not torch.compiler.is_compiling():
+ return _fast_rope_embedding_eager(Q, K, cos, sin, rope_embedding_indices)
+ if (
+ not _TRACEABLE
+ or Q.device.type != "cuda"
+ or (Q.dtype == torch.bfloat16 and not _BF16_TRACEABLE)
+ ):
+ return _fast_rope_embedding_untraced(Q, K, cos, sin, rope_embedding_indices)
+ cos, sin = cos.squeeze(), sin.squeeze()
+ if rope_embedding_indices is not None:
+ rope_ptr = rope_embedding_indices.reshape(-1).to(dtype = torch.int32, device = Q.device)
+ return torch.ops.unsloth.rope_embedding_qk(Q, K, cos, sin, rope_ptr, False)
+ Q_out = torch.ops.unsloth.rope_embedding(Q.transpose(1, 2), cos, sin, False).transpose(1, 2)
+ K_out = torch.ops.unsloth.rope_embedding(K.transpose(1, 2), cos, sin, False).transpose(1, 2)
+ return Q_out, K_out
+
+
class Slow_RoPE_Embedding(torch.autograd.Function):
@staticmethod
def forward(ctx, Q, cos, sin, position_ids):
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index c3fea244978..26963b9e20f 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -13,6 +13,7 @@ from .._version import __version__
__all__ = [
"SUPPORTS_BFLOAT16",
+ "config_return_dict",
"is_bfloat16_supported",
"_requested_float32",
"_mark_requested_float32",
@@ -111,6 +112,7 @@ __all__ = [
"patch_flex_attention_kernel_options",
]
+
import torch
from typing import Union, Optional, List, Any, Callable, Tuple, Iterator
from platform import system as platform_system
@@ -4297,6 +4299,11 @@ def offload_output_embeddings(model, temporary_location: str = "_unsloth_tempora
return
+def config_return_dict(config):
+ # use_return_dict without its transformers 5 deprecation warning, a torch.compile graph break.
+ return getattr(config, "return_dict", True) and not getattr(config, "torchscript", False)
+
+
def is_bfloat16_supported():
return SUPPORTS_BFLOAT16
diff --git a/unsloth/models/falcon_h1.py b/unsloth/models/falcon_h1.py
index ed295061d92..543ecf762f2 100644
--- a/unsloth/models/falcon_h1.py
+++ b/unsloth/models/falcon_h1.py
@@ -646,7 +646,7 @@ def fix_prepare_inputs_for_generation(module):
class Unsloth_FalconH1RMSNorm(FalconH1RMSNorm):
- """fast_rms_layernorm (compiler-disabled, fp32 eps) avoids the float64 torch.compile RMSNorm kernel that fails on Intel Arc DG2 (issue #6555)."""
+ """fast_rms_layernorm (compiler-disabled on XPU, fp32 eps) avoids the float64 torch.compile RMSNorm kernel that fails on Intel Arc DG2 (issue #6555)."""
def forward(self, hidden_states):
return fast_rms_layernorm(self, hidden_states, gemma = False)
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index 0f46195381b..87ed091d1fa 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -969,7 +969,7 @@ def LlamaModel_fast_forward(
)
use_cache = use_cache if use_cache is not None else self.config.use_cache
- return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+ return_dict = return_dict if return_dict is not None else config_return_dict(self.config)
if input_ids is not None and inputs_embeds is not None:
raise ValueError(
@@ -1562,7 +1562,9 @@ def CausalLM_fast_forward(fast_forward_inference):
if output_hidden_states is not None
else self.config.output_hidden_states
)
- return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+ return_dict = (
+ return_dict if return_dict is not None else config_return_dict(self.config)
+ )
self.model._has_no_labels = labels is None
outputs = self.model(
input_ids = input_ids,
diff --git a/unsloth/models/mistral.py b/unsloth/models/mistral.py
index 156bbac7a8e..b4609619831 100644
--- a/unsloth/models/mistral.py
+++ b/unsloth/models/mistral.py
@@ -12,7 +12,7 @@
from .loader_utils import DEFAULT_DEVICE_MAP
from .llama import *
import os
-from ._utils import __version__
+from ._utils import __version__, config_return_dict
from unsloth_zoo.utils import _get_dtype
from unsloth_zoo.hf_utils import dtype_from_config
from ..utils.packing import (
@@ -262,7 +262,7 @@ def MistralForCausalLM_fast_forward(
if output_hidden_states is not None
else self.config.output_hidden_states
)
- return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+ return_dict = return_dict if return_dict is not None else config_return_dict(self.config)
self.model._has_no_labels = labels is None
diff --git a/unsloth/models/sentence_transformer.py b/unsloth/models/sentence_transformer.py
index fb4a0ccedc6..cd1ed0359ea 100644
--- a/unsloth/models/sentence_transformer.py
+++ b/unsloth/models/sentence_transformer.py
@@ -19,6 +19,7 @@ from .loader_utils import (
unmarked_device_map,
)
from ._utils import (
+ config_return_dict,
SUPPORTS_BFLOAT16,
resolve_model_class,
resolve_encoder_attention_implementation,
@@ -835,7 +836,9 @@ class FastSentenceTransformer(FastModel):
if output_hidden_states is not None
else self.config.output_hidden_states
)
- return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+ return_dict = (
+ return_dict if return_dict is not None else config_return_dict(self.config)
+ )
if input_ids is not None and inputs_embeds is not None:
raise ValueError(