Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

FastLanguageModel LoRA / QLoRA: compile each decoder block as one region (#14398)

Daniel Hane24affffda3f
diff --git a/tests/test_attention_dispatch_compiled_block.py b/tests/test_attention_dispatch_compiled_block.py
new file mode 100644
index 00000000000..40d0bfe6414
--- /dev/null
+++ b/tests/test_attention_dispatch_compiled_block.py
@@ -0,0 +1,83 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""Attention dispatch traced fullgraph with a dynamic sequence length (CPU)."""
+
+import pytest
+import torch
+
+ad = pytest.importorskip("unsloth.utils.attention_dispatch")
+
+
+def _run(backend, causal_mask = None):
+ def attend(Q, K, V):
+ bsz, n_heads, q_len, head_dim = Q.shape
+ config = ad.AttentionConfig(
+ backend = backend, n_kv_heads = K.shape[1], n_groups = n_heads // K.shape[1]
+ )
+ context = ad.AttentionContext(
+ bsz = bsz,
+ q_len = q_len,
+ kv_seq_len = K.shape[-2],
+ n_heads = n_heads,
+ head_dim = head_dim,
+ requires_grad = True,
+ seq_info = None,
+ attention_mask = None,
+ causal_mask = causal_mask,
+ )
+ return ad.run_attention(config = config, context = context, Q = Q, K = K, V = V)
+
+ return attend
+
+
+def _qkv(q_len):
+ g = torch.Generator().manual_seed(0)
+ Q = torch.randn(1, 4, q_len, 16, generator = g)
+ K = torch.randn(1, 2, q_len, 16, generator = g)
+ V = torch.randn(1, 2, q_len, 16, generator = g)
+ return Q, K, V
+
+
+def _reference(Q, K, V):
+ K = K.repeat_interleave(2, dim = 1)
+ V = V.repeat_interleave(2, dim = 1)
+ return torch.nn.functional.scaled_dot_product_attention(Q, K, V, is_causal = True).transpose(1, 2)
+
+
[email protected]("backend", ["sdpa", "xformers"])
+def test_plain_causal_attention_traces_with_a_dynamic_length(backend):
+ backend = {"sdpa": ad.SDPA, "xformers": ad.XFORMERS}[backend]
+ torch._dynamo.reset()
+ compiled = torch.compile(_run(backend), fullgraph = True, backend = "eager", dynamic = True)
+ for q_len in (8, 12):
+ Q, K, V = _qkv(q_len)
+ out = compiled(Q, K, V)
+ torch.testing.assert_close(out.reshape(1, q_len, 4, 16), _reference(Q, K, V))
+
+
+def test_xformers_eager_choice_is_unchanged(monkeypatch):
+ # Outside a trace the backend is the caller's: the SDPA route is only taken while compiling.
+ seen = []
+ monkeypatch.setattr(
+ ad, "build_xformers_block_causal_mask", lambda *a, **k: seen.append(1) or None
+ )
+ monkeypatch.setattr(ad, "move_xformers_attention_bias", lambda bias, device: bias)
+ monkeypatch.setattr(ad, "xformers_attention", lambda Q, K, V, attn_bias = None, **kw: Q)
+ Q, K, V = _qkv(8)
+ _run(ad.XFORMERS)(Q, K, V)
+ assert seen
+
+
+def test_rerouted_xformers_keeps_the_fp32_downcast(monkeypatch):
+ # DoRA fp32 Q/K/V on a GPU whose xformers op rejects fp32: the SDPA reroute must downcast like xformers.
+ monkeypatch.setattr(ad, "_XFORMERS_FP32_UNSUPPORTED", True)
+ monkeypatch.setattr(ad, "SUPPORTS_BFLOAT16", True)
+ torch._dynamo.reset()
+ compiled = torch.compile(_run(ad.XFORMERS), fullgraph = True, backend = "eager", dynamic = True)
+ Q, K, V = _qkv(8)
+ out = compiled(Q, K, V)
+ assert out.dtype == torch.bfloat16
+ torch.testing.assert_close(
+ out.float().reshape(1, 8, 4, 16), _reference(Q, K, V), atol = 3e-2, rtol = 3e-2
+ )
diff --git a/tests/test_legacy_block_compile_install.py b/tests/test_legacy_block_compile_install.py
new file mode 100644
index 00000000000..d9c812ac168
--- /dev/null
+++ b/tests/test_legacy_block_compile_install.py
@@ -0,0 +1,105 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""FastLanguageModel LoRA installs the per-block compile, with its own kill switch."""
+
+import ast
+import pathlib
+
+import pytest
+
+_LLAMA = pathlib.Path(__file__).resolve().parents[1] / "unsloth" / "models" / "llama.py"
+
+
+def test_patch_peft_model_installs_block_compile():
+ tree = ast.parse(_LLAMA.read_text())
+ cls = next(n for n in tree.body if isinstance(n, ast.ClassDef) and n.name == "FastLlamaModel")
+ fn = next(
+ n for n in cls.body if isinstance(n, ast.FunctionDef) and n.name == "patch_peft_model"
+ )
+ final = [n for n in fn.body if isinstance(n, ast.Return)][-1]
+ before = fn.body[fn.body.index(final) - 1]
+ assert isinstance(before, ast.Expr) and ast.unparse(before) == "_legacy_block_compile(model)"
+
+
+def llama(monkeypatch):
+ mod = pytest.importorskip("unsloth.models.llama")
+ from unsloth.models import _block_compile
+
+ calls = []
+ monkeypatch.setattr(
+ _block_compile, "patch_block_compile", lambda model: calls.append(model) or ["X"]
+ )
+ monkeypatch.delenv("UNSLOTH_BLOCK_COMPILE_LEGACY", raising = False)
+ return mod, calls
+
+
+ "env, traceable, installed",
+ [
+ (None, True, True),
+ (None, False, False),
+ ("1", False, True),
+ ("0", True, False),
+ ],
+)
+def test_legacy_gate(llama, monkeypatch, env, traceable, installed):
+ mod, calls = llama
+ from unsloth.kernels import fast_lora
+
+ monkeypatch.setattr(fast_lora, "TRACE_LORA_FUNCTIONS", traceable)
+ if env is not None:
+ monkeypatch.setenv("UNSLOTH_BLOCK_COMPILE_LEGACY", env)
+ model = object()
+ assert mod._legacy_block_compile(model) == (["X"] if installed else [])
+ assert calls == ([model] if installed else [])
+
+
+def test_packed_calls_keep_the_unsloth_layer_forward(monkeypatch):
+ # FastLanguageModel's layer forwards read packed lengths on the host: packed calls stay eager.
+ import importlib.util
+ import sys
+
+ import torch
+
+ path = _LLAMA.parent / "_block_compile.py"
+ spec = importlib.util.spec_from_file_location("_ublock_legacy", path)
+ M = importlib.util.module_from_spec(spec)
+ sys.modules[spec.name] = M
+ spec.loader.exec_module(M)
+ calls = []
+
+ def fake_compile(fn, **kwargs):
+ def compiled(*args, **kw):
+ calls.append(1)
+ return fn(*args, **kw)
+
+ return compiled
+
+ monkeypatch.setattr(torch, "compile", fake_compile)
+
+ def forward(
+ self,
+ hidden_states,
+ packed_seq_lengths = None,
+ **kwargs,
+ ):
+ return self.proj(hidden_states)
+
+ forward.__module__ = "unsloth.models.llama"
+
+ def __init__(self):
+ torch.nn.Module.__init__(self)
+ self.proj = torch.nn.Linear(8, 8)
+
+ Legacy = type(
+ "LegacyToyDecoderLayer", (torch.nn.Module,), {"__init__": __init__, "forward": forward}
+ )
+ M._wrap(Legacy, {})
+ layer = Legacy()
+ x = torch.randn(1, 4, 8, requires_grad = True)
+ layer(x, packed_seq_lengths = torch.tensor([2, 2])).sum().backward()
+ assert calls == []
+ layer(x).sum().backward()
+ assert calls
diff --git a/unsloth/models/_block_compile.py b/unsloth/models/_block_compile.py
index 41a5770e76b..6c8c60e208e 100644
--- a/unsloth/models/_block_compile.py
+++ b/unsloth/models/_block_compile.py
@@ -660,6 +660,8 @@ def _layer_kind(layer):
def _wrap(cls, compile_options):
original = cls.forward
+ # Unsloth layer forwards read packed lengths on the host (packing.py): packed calls stay stock.
+ unsloth_forward = getattr(original, "__module__", "").startswith("unsloth.models.")
compiled = torch.compile(original, fullgraph = False, dynamic = None, options = compile_options)
try:
from unsloth_zoo.temporary_patches.utils import UNSLOTH_DECODE_COMPILE as inline_flag
@@ -694,6 +696,7 @@ def _wrap(cls, compile_options):
or not _enabled()
or _forced_float32(self)
or _zero3_partitioned(self)
+ or (unsloth_forward and kwargs.get("packed_seq_lengths") is not None)
):
return original(self, hidden_states, *args, **kwargs)
if not self.__dict__.get("_unsloth_block_moe_done", False):
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index 87422c0e37c..e63fa142401 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -3131,6 +3131,24 @@ def _decline_fused_lora_for_fsdp(model, reason = None) -> int:
return n
+def _legacy_block_compile(model):
+ """Per-block compile for FastLanguageModel LoRA. Off before torch 2.11 unless =1: untraced
+ fused LoRA (fast_lora.TRACE_LORA_FUNCTIONS) splits each block."""
+ mode = os.environ.get("UNSLOTH_BLOCK_COMPILE_LEGACY", "auto")
+ if mode == "0":
+ return []
+ if mode != "1":
+ from ..kernels.fast_lora import TRACE_LORA_FUNCTIONS
+ if not TRACE_LORA_FUNCTIONS:
+ return []
+ try:
+ from ._block_compile import patch_block_compile
+ return patch_block_compile(model)
+ except Exception as e:
+ logger.warning(f"Unsloth: per-block compile not installed: {e}")
+ return []
+
+
class FastLlamaModel:
@staticmethod
def _prepare_for_qat(model, qat_scheme):
@@ -5169,6 +5187,7 @@ class FastLlamaModel:
m.for_inference = functools.partial(FastBaseModel.for_inference, m)
m = m.model
_unsloth_install_pretrain_detector(model)
+ _legacy_block_compile(model)
return model
@staticmethod
diff --git a/unsloth/utils/attention_dispatch.py b/unsloth/utils/attention_dispatch.py
index 52c808d56a7..753e8c324dd 100644
--- a/unsloth/utils/attention_dispatch.py
+++ b/unsloth/utils/attention_dispatch.py
@@ -398,6 +398,11 @@ def _sdpa_takes_gqa(
needs_grad = torch.is_grad_enabled() and any(
t is not None and t.requires_grad for t in (Q, K, V)
)
+ # Branch, not bool(): assume_constant_result cannot take a dynamic SymBool.
+ single_query = True if Q.shape[-2] == 1 else False
+ single_key = True if K.shape[-2] == 1 else False
+ is_causal = True if is_causal else False
+ has_mask = True if has_mask else False
return _sdpa_gqa_state(
str(Q.device),
Q.dtype,
@@ -408,8 +413,8 @@ def _sdpa_takes_gqa(
bool(has_mask),
bool(needs_grad),
float(dropout_p or 0.0),
- Q.shape[-2] == 1,
- K.shape[-2] == 1,
+ single_query,
+ single_key,
)
@@ -906,6 +911,24 @@ def _run_attention(
)
backend = SDPA
+ # Traced xformers bias objects break the graph per layer: plain causal calls take SDPA.
+ xformers_rerouted = False
+ if (
+ backend == XFORMERS
+ and torch.compiler.is_compiling()
+ and context.seq_info is None
+ and context.attention_mask is None
+ and context.is_causal
+ and context.q_len == context.kv_seq_len
+ and (
+ context.causal_mask is None
+ or type(context.causal_mask).__name__ == "LowerTriangularMask"
+ )
+ and not config.xformers_kwargs
+ and _configured_softcap(config) is None
+ ):
+ backend = SDPA
+ xformers_rerouted = True
flash_dense_kwargs = config.flash_dense_kwargs or {}
flash_varlen_kwargs = config.flash_varlen_kwargs or {}
sdpa_kwargs = config.sdpa_kwargs or {}
@@ -939,7 +962,7 @@ def _run_attention(
cp_out_dtype = Q.dtype if ring else None
if (
backend in (FLASH_DENSE, FLASH_VARLEN)
- or (backend == XFORMERS and xformers_fp32_unsupported)
+ or ((backend == XFORMERS or xformers_rerouted) and xformers_fp32_unsupported)
or ring
) and torch.float32 in (
Q.dtype,
@@ -1166,7 +1189,9 @@ def _run_attention(
)
is_causal_local = (
- context.is_causal and local_mask is None and q_len_local == k_len_local
+ True
+ if context.is_causal and local_mask is None and q_len_local == k_len_local
+ else False
)
kwargs = dict(sdpa_kwargs)