Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
FastLanguageModel LoRA / QLoRA: compile each decoder block as one region (#14398)
Daniel Hane24affffda3f
diff --git a/tests/test_attention_dispatch_compiled_block.py b/tests/test_attention_dispatch_compiled_block.pynew file mode 100644index 00000000000..40d0bfe6414--- /dev/null+++ b/tests/test_attention_dispatch_compiled_block.py@@ -0,0 +1,83 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""Attention dispatch traced fullgraph with a dynamic sequence length (CPU)."""++import pytest+import torch++ad = pytest.importorskip("unsloth.utils.attention_dispatch")+++def _run(backend, causal_mask = None):+ def attend(Q, K, V):+ bsz, n_heads, q_len, head_dim = Q.shape+ config = ad.AttentionConfig(+ backend = backend, n_kv_heads = K.shape[1], n_groups = n_heads // K.shape[1]+ )+ context = ad.AttentionContext(+ bsz = bsz,+ q_len = q_len,+ kv_seq_len = K.shape[-2],+ n_heads = n_heads,+ head_dim = head_dim,+ requires_grad = True,+ seq_info = None,+ attention_mask = None,+ causal_mask = causal_mask,+ )+ return ad.run_attention(config = config, context = context, Q = Q, K = K, V = V)++ return attend+++def _qkv(q_len):+ g = torch.Generator().manual_seed(0)+ Q = torch.randn(1, 4, q_len, 16, generator = g)+ K = torch.randn(1, 2, q_len, 16, generator = g)+ V = torch.randn(1, 2, q_len, 16, generator = g)+ return Q, K, V+++def _reference(Q, K, V):+ K = K.repeat_interleave(2, dim = 1)+ V = V.repeat_interleave(2, dim = 1)+ return torch.nn.functional.scaled_dot_product_attention(Q, K, V, is_causal = True).transpose(1, 2)++[email protected]("backend", ["sdpa", "xformers"])+def test_plain_causal_attention_traces_with_a_dynamic_length(backend):+ backend = {"sdpa": ad.SDPA, "xformers": ad.XFORMERS}[backend]+ torch._dynamo.reset()+ compiled = torch.compile(_run(backend), fullgraph = True, backend = "eager", dynamic = True)+ for q_len in (8, 12):+ Q, K, V = _qkv(q_len)+ out = compiled(Q, K, V)+ torch.testing.assert_close(out.reshape(1, q_len, 4, 16), _reference(Q, K, V))+++def test_xformers_eager_choice_is_unchanged(monkeypatch):+ # Outside a trace the backend is the caller's: the SDPA route is only taken while compiling.+ seen = []+ monkeypatch.setattr(+ ad, "build_xformers_block_causal_mask", lambda *a, **k: seen.append(1) or None+ )+ monkeypatch.setattr(ad, "move_xformers_attention_bias", lambda bias, device: bias)+ monkeypatch.setattr(ad, "xformers_attention", lambda Q, K, V, attn_bias = None, **kw: Q)+ Q, K, V = _qkv(8)+ _run(ad.XFORMERS)(Q, K, V)+ assert seen+++def test_rerouted_xformers_keeps_the_fp32_downcast(monkeypatch):+ # DoRA fp32 Q/K/V on a GPU whose xformers op rejects fp32: the SDPA reroute must downcast like xformers.+ monkeypatch.setattr(ad, "_XFORMERS_FP32_UNSUPPORTED", True)+ monkeypatch.setattr(ad, "SUPPORTS_BFLOAT16", True)+ torch._dynamo.reset()+ compiled = torch.compile(_run(ad.XFORMERS), fullgraph = True, backend = "eager", dynamic = True)+ Q, K, V = _qkv(8)+ out = compiled(Q, K, V)+ assert out.dtype == torch.bfloat16+ torch.testing.assert_close(+ out.float().reshape(1, 8, 4, 16), _reference(Q, K, V), atol = 3e-2, rtol = 3e-2+ )diff --git a/tests/test_legacy_block_compile_install.py b/tests/test_legacy_block_compile_install.pynew file mode 100644index 00000000000..d9c812ac168--- /dev/null+++ b/tests/test_legacy_block_compile_install.py@@ -0,0 +1,105 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""FastLanguageModel LoRA installs the per-block compile, with its own kill switch."""++import ast+import pathlib++import pytest++_LLAMA = pathlib.Path(__file__).resolve().parents[1] / "unsloth" / "models" / "llama.py"+++def test_patch_peft_model_installs_block_compile():+ tree = ast.parse(_LLAMA.read_text())+ cls = next(n for n in tree.body if isinstance(n, ast.ClassDef) and n.name == "FastLlamaModel")+ fn = next(+ n for n in cls.body if isinstance(n, ast.FunctionDef) and n.name == "patch_peft_model"+ )+ final = [n for n in fn.body if isinstance(n, ast.Return)][-1]+ before = fn.body[fn.body.index(final) - 1]+ assert isinstance(before, ast.Expr) and ast.unparse(before) == "_legacy_block_compile(model)"+++def llama(monkeypatch):+ mod = pytest.importorskip("unsloth.models.llama")+ from unsloth.models import _block_compile++ calls = []+ monkeypatch.setattr(+ _block_compile, "patch_block_compile", lambda model: calls.append(model) or ["X"]+ )+ monkeypatch.delenv("UNSLOTH_BLOCK_COMPILE_LEGACY", raising = False)+ return mod, calls+++ "env, traceable, installed",+ [+ (None, True, True),+ (None, False, False),+ ("1", False, True),+ ("0", True, False),+ ],+)+def test_legacy_gate(llama, monkeypatch, env, traceable, installed):+ mod, calls = llama+ from unsloth.kernels import fast_lora++ monkeypatch.setattr(fast_lora, "TRACE_LORA_FUNCTIONS", traceable)+ if env is not None:+ monkeypatch.setenv("UNSLOTH_BLOCK_COMPILE_LEGACY", env)+ model = object()+ assert mod._legacy_block_compile(model) == (["X"] if installed else [])+ assert calls == ([model] if installed else [])+++def test_packed_calls_keep_the_unsloth_layer_forward(monkeypatch):+ # FastLanguageModel's layer forwards read packed lengths on the host: packed calls stay eager.+ import importlib.util+ import sys++ import torch++ path = _LLAMA.parent / "_block_compile.py"+ spec = importlib.util.spec_from_file_location("_ublock_legacy", path)+ M = importlib.util.module_from_spec(spec)+ sys.modules[spec.name] = M+ spec.loader.exec_module(M)+ calls = []++ def fake_compile(fn, **kwargs):+ def compiled(*args, **kw):+ calls.append(1)+ return fn(*args, **kw)++ return compiled++ monkeypatch.setattr(torch, "compile", fake_compile)++ def forward(+ self,+ hidden_states,+ packed_seq_lengths = None,+ **kwargs,+ ):+ return self.proj(hidden_states)++ forward.__module__ = "unsloth.models.llama"++ def __init__(self):+ torch.nn.Module.__init__(self)+ self.proj = torch.nn.Linear(8, 8)++ Legacy = type(+ "LegacyToyDecoderLayer", (torch.nn.Module,), {"__init__": __init__, "forward": forward}+ )+ M._wrap(Legacy, {})+ layer = Legacy()+ x = torch.randn(1, 4, 8, requires_grad = True)+ layer(x, packed_seq_lengths = torch.tensor([2, 2])).sum().backward()+ assert calls == []+ layer(x).sum().backward()+ assert callsdiff --git a/unsloth/models/_block_compile.py b/unsloth/models/_block_compile.pyindex 41a5770e76b..6c8c60e208e 100644--- a/unsloth/models/_block_compile.py+++ b/unsloth/models/_block_compile.py@@ -660,6 +660,8 @@ def _layer_kind(layer):def _wrap(cls, compile_options):original = cls.forward+ # Unsloth layer forwards read packed lengths on the host (packing.py): packed calls stay stock.+ unsloth_forward = getattr(original, "__module__", "").startswith("unsloth.models.")compiled = torch.compile(original, fullgraph = False, dynamic = None, options = compile_options)try:from unsloth_zoo.temporary_patches.utils import UNSLOTH_DECODE_COMPILE as inline_flag@@ -694,6 +696,7 @@ def _wrap(cls, compile_options):or not _enabled()or _forced_float32(self)or _zero3_partitioned(self)+ or (unsloth_forward and kwargs.get("packed_seq_lengths") is not None)):return original(self, hidden_states, *args, **kwargs)if not self.__dict__.get("_unsloth_block_moe_done", False):diff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex 87422c0e37c..e63fa142401 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -3131,6 +3131,24 @@ def _decline_fused_lora_for_fsdp(model, reason = None) -> int:return n+def _legacy_block_compile(model):+ """Per-block compile for FastLanguageModel LoRA. Off before torch 2.11 unless =1: untraced+ fused LoRA (fast_lora.TRACE_LORA_FUNCTIONS) splits each block."""+ mode = os.environ.get("UNSLOTH_BLOCK_COMPILE_LEGACY", "auto")+ if mode == "0":+ return []+ if mode != "1":+ from ..kernels.fast_lora import TRACE_LORA_FUNCTIONS+ if not TRACE_LORA_FUNCTIONS:+ return []+ try:+ from ._block_compile import patch_block_compile+ return patch_block_compile(model)+ except Exception as e:+ logger.warning(f"Unsloth: per-block compile not installed: {e}")+ return []++class FastLlamaModel:@staticmethoddef _prepare_for_qat(model, qat_scheme):@@ -5169,6 +5187,7 @@ class FastLlamaModel:m.for_inference = functools.partial(FastBaseModel.for_inference, m)m = m.model_unsloth_install_pretrain_detector(model)+ _legacy_block_compile(model)return model@staticmethoddiff --git a/unsloth/utils/attention_dispatch.py b/unsloth/utils/attention_dispatch.pyindex 52c808d56a7..753e8c324dd 100644--- a/unsloth/utils/attention_dispatch.py+++ b/unsloth/utils/attention_dispatch.py@@ -398,6 +398,11 @@ def _sdpa_takes_gqa(needs_grad = torch.is_grad_enabled() and any(t is not None and t.requires_grad for t in (Q, K, V))+ # Branch, not bool(): assume_constant_result cannot take a dynamic SymBool.+ single_query = True if Q.shape[-2] == 1 else False+ single_key = True if K.shape[-2] == 1 else False+ is_causal = True if is_causal else False+ has_mask = True if has_mask else Falsereturn _sdpa_gqa_state(str(Q.device),Q.dtype,@@ -408,8 +413,8 @@ def _sdpa_takes_gqa(bool(has_mask),bool(needs_grad),float(dropout_p or 0.0),- Q.shape[-2] == 1,- K.shape[-2] == 1,+ single_query,+ single_key,)@@ -906,6 +911,24 @@ def _run_attention()backend = SDPA+ # Traced xformers bias objects break the graph per layer: plain causal calls take SDPA.+ xformers_rerouted = False+ if (+ backend == XFORMERS+ and torch.compiler.is_compiling()+ and context.seq_info is None+ and context.attention_mask is None+ and context.is_causal+ and context.q_len == context.kv_seq_len+ and (+ context.causal_mask is None+ or type(context.causal_mask).__name__ == "LowerTriangularMask"+ )+ and not config.xformers_kwargs+ and _configured_softcap(config) is None+ ):+ backend = SDPA+ xformers_rerouted = Trueflash_dense_kwargs = config.flash_dense_kwargs or {}flash_varlen_kwargs = config.flash_varlen_kwargs or {}sdpa_kwargs = config.sdpa_kwargs or {}@@ -939,7 +962,7 @@ def _run_attention(cp_out_dtype = Q.dtype if ring else Noneif (backend in (FLASH_DENSE, FLASH_VARLEN)- or (backend == XFORMERS and xformers_fp32_unsupported)+ or ((backend == XFORMERS or xformers_rerouted) and xformers_fp32_unsupported)or ring) and torch.float32 in (Q.dtype,@@ -1166,7 +1189,9 @@ def _run_attention()is_causal_local = (- context.is_causal and local_mask is None and q_len_local == k_len_local+ True+ if context.is_causal and local_mask is None and q_len_local == k_len_local+ else False)kwargs = dict(sdpa_kwargs)