Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Send maskless causal flex calls to SDPA is_causal (#12773)
Daniel Hanb6a6dcdbf001
diff --git a/tests/test_flex_maskless_sdpa.py b/tests/test_flex_maskless_sdpa.pynew file mode 100644index 00000000000..b359d3f1bbc--- /dev/null+++ b/tests/test_flex_maskless_sdpa.py@@ -0,0 +1,183 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++import copy+import types++import pytest+import torch++from real_accelerator import has_real_cuda++if not has_real_cuda():+ pytest.skip("needs a CUDA device", allow_module_level = True)++import unsloth # noqa: F401+import unsloth.models._utils as U+from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS++B, H, KV, T, D = 2, 8, 2, 256, 64+++class Causal(torch.nn.Module):+ is_causal = True+ num_key_value_groups = H // KV+ training = True+++def qkv():+ torch.manual_seed(0)+ q = torch.randn(B, H, T, D, device = "cuda", dtype = torch.bfloat16)+ k = torch.randn(B, KV, T, D, device = "cuda", dtype = torch.bfloat16)+ v = torch.randn(B, KV, T, D, device = "cuda", dtype = torch.bfloat16)+ return q, k, v+++def test_the_wrapper_advertises_the_reroute():+ U.patch_flex_attention_kernel_options()+ function = ALL_ATTENTION_FUNCTIONS["flex_attention"]+ assert (+ getattr(function, "_unsloth_maskless_causal_sdpa", False) is U._FLEX_MASKLESS_SDPA_ENABLED+ )+++def _flex():+ U.patch_flex_attention_kernel_options()+ return ALL_ATTENTION_FUNCTIONS["flex_attention"]+++def _causal_reference(q, k, v, scale):+ return torch.nn.functional.scaled_dot_product_attention(+ q.float(),+ k.float(),+ v.float(),+ is_causal = True,+ scale = scale,+ enable_gqa = True,+ ).transpose(1, 2)+++def test_a_dropped_mask_runs_sdpa_is_causal(qkv):+ q, k, v = qkv+ before = dict(U.FLEX_MASKLESS_SDPA_STATS)+ out, _ = _flex()(Causal(), q, k, v, None, scaling = D**-0.5)+ assert U.FLEX_MASKLESS_SDPA_STATS["sdpa"] == before["sdpa"] + 1+ reference = torch.nn.functional.scaled_dot_product_attention(+ q,+ k,+ v,+ is_causal = True,+ scale = D**-0.5,+ enable_gqa = True,+ ).transpose(1, 2)+ assert torch.equal(out, reference)++[email protected]("case", ["softcap", "head_dim_512"])+def test_a_dropped_mask_that_stays_on_flex_is_still_causal(case):+ # Never hand flex a None mask for a causal call: it would attend to future tokens.+ torch.manual_seed(0)+ head_dim = 512 if case == "head_dim_512" else D+ q = torch.randn(1, H, 128, head_dim, device = "cuda", dtype = torch.bfloat16)+ k = torch.randn(1, KV, 128, head_dim, device = "cuda", dtype = torch.bfloat16)+ v = torch.randn(1, KV, 128, head_dim, device = "cuda", dtype = torch.bfloat16)+ kwargs = {"softcap": 1e4} if case == "softcap" else {} # a softcap this large is a no-op+ before = dict(U.FLEX_MASKLESS_SDPA_STATS)+ out, _ = _flex()(Causal(), q, k, v, None, scaling = head_dim**-0.5, **kwargs)+ assert U.FLEX_MASKLESS_SDPA_STATS["sdpa"] == before["sdpa"]+ reference = _causal_reference(q, k, v, head_dim**-0.5)+ assert (out.float() - reference).abs().max() < 2e-2+++def test_counters_do_not_mutate_while_compiling(qkv, monkeypatch):+ q, k, v = qkv+ monkeypatch.setattr(torch.compiler, "is_compiling", lambda: True)+ before = dict(U.FLEX_MASKLESS_SDPA_STATS)+ _flex()(Causal(), q, k, v, None, scaling = D**-0.5)+ assert U.FLEX_MASKLESS_SDPA_STATS == before+++ "case",+ [+ "mask",+ "softcap",+ "s_aux",+ "not_causal",+ "no_is_causal",+ "is_causal_false",+ "head_dim_512",+ "decode",+ "cache",+ "sdpa_excluded",+ ],+)+def test_everything_else_stays_on_flex(qkv, case):+ q, k, v = qkv+ module, args, kwargs = Causal(), (None,), {}+ if case == "mask":+ args = (torch.ones(B, 1, T, T, device = "cuda", dtype = torch.bool),)+ if case == "softcap":+ kwargs = {"softcap": 30.0}+ if case == "s_aux":+ kwargs = {"s_aux": torch.zeros(H, device = "cuda")}+ if case == "not_causal":+ module.is_causal = False+ if case == "no_is_causal":+ module = torch.nn.Module() # vision callers: None means bidirectional+ if case == "is_causal_false":+ kwargs = {"is_causal": False} # per-call bidirectional override+ if case == "head_dim_512":+ q, k, v = (torch.randn(*t.shape[:3], 512, device = "cuda", dtype = t.dtype) for t in (q, k, v))+ if case == "decode":+ q = q[:, :, :1]+ if case == "cache":+ q = q[:, :, : T // 2]+ if case == "sdpa_excluded":+ module.config = types.SimpleNamespace(model_type = "gemma3_text")+ assert U._maskless_causal_sdpa_forward(module, q, k, v, args, kwargs) is None+++def _softcap_call(T, requires_grad = False):+ q, k, v = (torch.randn(1, n, T, D, device = "cuda", dtype = torch.bfloat16) for n in (H, KV, KV))+ q.requires_grad_(requires_grad)+ return _flex()(Causal(), q, k, v, None, scaling = D**-0.5, softcap = 1e4), q+++def test_a_fallback_mask_first_built_under_inference_mode_still_trains():+ U._CAUSAL_BLOCK_MASKS.clear()+ with torch.inference_mode():+ _softcap_call(128)+ (out, _), q = _softcap_call(128, requires_grad = True)+ out.float().sum().backward()+ assert q.grad is not None and torch.isfinite(q.grad).all()+++def test_the_fallback_mask_cache_is_bounded():+ U._CAUSAL_BLOCK_MASKS.clear()+ for T in range(128, 128 * 13, 128):+ _softcap_call(T)+ assert 0 < len(U._CAUSAL_BLOCK_MASKS) <= 8+++def test_the_wrapper_vetoes_configs_whose_layers_cannot_take_sdpa():+ from transformers import Gemma2Config, Gemma3TextConfig, LlamaConfig, Qwen3Config++ accepts = _flex()._unsloth_maskless_causal_sdpa_accepts+ assert accepts(LlamaConfig()) and accepts(Qwen3Config(head_dim = 256))+ assert not accepts(Qwen3Config(head_dim = 512))+ assert not accepts(Gemma3TextConfig()) # SDPA disabled for Gemma 3+ assert not accepts(Gemma2Config()) # softcap+++def test_a_class_level_sdpa_opt_out_keeps_the_mask():+ from transformers import LlamaConfig, LlamaForCausalLM++ no_sdpa = type("NoSdpaLlama", (LlamaForCausalLM,), {"_supports_sdpa": False})+ config = LlamaConfig()+ U.resolve_attention_implementation(no_sdpa, config)+ accepts = _flex()._unsloth_maskless_causal_sdpa_accepts+ assert not accepts(copy.deepcopy(config)) # from_pretrained copies the config it is given+ U._NO_SDPA_MODEL_TYPES.discard("llama")+ assert accepts(LlamaConfig())diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex 74a0949fce6..220c7c27317 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -654,6 +654,118 @@ def _flex_call_needs_backward(query, key, value):return False+# unsloth_zoo drops the causal mask of unpadded, cache-free batches; route that None to SDPA is_causal.+# Kill switch: UNSLOTH_FLEX_MASKLESS_SDPA=0.+_FLEX_MASKLESS_SDPA_ENABLED = os.environ.get("UNSLOTH_FLEX_MASKLESS_SDPA", "1") != "0"+FLEX_MASKLESS_SDPA_STATS = {"sdpa": 0, "flex": 0}+++def _dropped_causal_mask(module, query, key, args, kwargs):+ """A flex call whose causal mask unsloth_zoo dropped (stock flex reads None as bidirectional)."""+ attention_mask = args[0] if len(args) > 0 else kwargs.get("attention_mask", None)+ if attention_mask is not None:+ return False+ # Vision callers pass None meaning bidirectional, and is_causal=False is a per-call override.+ if getattr(module, "is_causal", None) is not True or kwargs.get("is_causal", None) is False:+ return False+ if not (hasattr(query, "dim") and query.dim() == 4 and key.dim() == 4):+ return False+ return query.shape[2] >= 2 and query.shape[2] == key.shape[2]+++# Model types the resolver loaded without SDPA support (class opt-out or exclusion list). Keyed by+# model_type, not config identity: from_pretrained deep-copies the config it was given.+_NO_SDPA_MODEL_TYPES = set()+++def _remember_no_sdpa_config(config):+ for attention_config in _iter_attention_configs(config):+ model_type = _config_get(attention_config, "model_type", None)+ if isinstance(model_type, str) and model_type:+ _NO_SDPA_MODEL_TYPES.add(model_type.lower())+++def _maskless_causal_sdpa_accepts(config):+ """False when some decoder layer could not take the SDPA reroute, so unsloth_zoo keeps the mask."""+ for attention_config in _text_attention_configs(config) + [config]:+ model_type = (_config_get(attention_config, "model_type", None) or "").lower()+ if model_type in _NO_SDPA_MODEL_TYPES:+ return False+ if _is_sdpa_excluded(_config_get(attention_config, "model_type", None) or ""):+ return False+ if _config_get(attention_config, "attn_logit_softcapping", None) is not None:+ return False+ head_dim = _text_attention_head_dim(config)+ return head_dim is None or head_dim <= 256+++def _maskless_causal_sdpa_forward(module, query, key, value, args, kwargs):+ """SDPA is_causal output for a dropped causal mask, else None (the call stays on flex)."""+ if not _dropped_causal_mask(module, query, key, args, kwargs):+ return None+ if _is_sdpa_excluded(_config_get(getattr(module, "config", None), "model_type", None) or ""):+ return None+ # flex positional order: attention_mask, scaling, softcap, s_aux.+ if any(arg is not None for arg in args[2:]):+ return None+ if kwargs.get("softcap", None) is not None or kwargs.get("s_aux", None) is not None:+ return None+ if kwargs.get("position_bias", None) is not None:+ return None+ # SDPA has flash / cuDNN kernels only up to head_dim 256; above that flex is faster.+ if query.shape[-1] > 256:+ return None+ try:+ from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS+ sdpa_forward = ALL_ATTENTION_FUNCTIONS["sdpa"]+ except Exception:+ return None+ scaling = args[1] if len(args) > 1 else kwargs.get("scaling", None)+ return sdpa_forward(+ module,+ query,+ key,+ value,+ None,+ dropout = kwargs.get("dropout", 0.0),+ scaling = scaling,+ is_causal = True,+ )+++_CAUSAL_BLOCK_MASKS = {}+++def _causal_block_mask(query, key):+ """The causal BlockMask unsloth_zoo dropped, for calls that cannot go to SDPA."""+ cache_key = (query.shape[2], key.shape[2], query.device)+ block_mask = _CAUSAL_BLOCK_MASKS.get(cache_key)+ if block_mask is None:+ from torch.nn.attention.flex_attention import create_block_mask++ # Uncompiled builds materialize Q x KV; inference tensors break a later backward.+ with torch.inference_mode(False):+ block_mask = create_block_mask(+ lambda b, h, q_idx, kv_idx: q_idx >= kv_idx,+ None,+ None,+ query.shape[2],+ key.shape[2],+ device = query.device,+ _compile = True,+ )+ if len(_CAUSAL_BLOCK_MASKS) >= 8:+ _CAUSAL_BLOCK_MASKS.pop(next(iter(_CAUSAL_BLOCK_MASKS)))+ _CAUSAL_BLOCK_MASKS[cache_key] = block_mask+ return block_mask+++def _count_flex_reroute(path):+ # A Python counter mutated inside a compiled region makes Dynamo recompile every call.+ if not torch.compiler.is_compiling():+ FLEX_MASKLESS_SDPA_STATS[path] += 1++def _wrap_flex_attention_forward(flex_attention_forward):"""Add kernel_options to a registered `flex_attention` function: block sizes above head_dim256, and the main flex kernel for calls that need a backward."""@@ -662,6 +774,19 @@ def _wrap_flex_attention_forward(flex_attention_forward):@functools.wraps(flex_attention_forward)def unsloth_flex_attention_forward(module, query, key, value, *args, **kwargs):+ if _FLEX_MASKLESS_SDPA_ENABLED and _dropped_causal_mask(module, query, key, args, kwargs):+ output = _maskless_causal_sdpa_forward(module, query, key, value, args, kwargs)+ if output is not None:+ _count_flex_reroute("sdpa")+ return output+ # Softcap, sinks or head_dim > 256: stay on flex, but never with a None mask.+ _count_flex_reroute("flex")+ if len(args) > 0:+ args = (_causal_block_mask(query, key),) + tuple(args[1:])+ else:+ kwargs["attention_mask"] = _causal_block_mask(query, key)+ elif _FLEX_MASKLESS_SDPA_ENABLED:+ _count_flex_reroute("flex")try:# Some vision callers reuse the interface with a non-4D query.kernel_options = (@@ -687,6 +812,10 @@ def _wrap_flex_attention_forward(flex_attention_forward):return flex_attention_forward(module, query, key, value, *args, **kwargs)unsloth_flex_attention_forward._unsloth_flex_kernel_options = True+ unsloth_flex_attention_forward._unsloth_maskless_causal_sdpa = _FLEX_MASKLESS_SDPA_ENABLED+ unsloth_flex_attention_forward._unsloth_maskless_causal_sdpa_accepts = (+ _maskless_causal_sdpa_accepts+ )unsloth_flex_attention_forward._unsloth_original_forward = flex_attention_forwardreturn unsloth_flex_attention_forward@@ -2264,6 +2393,8 @@ def resolve_attention_implementation(supports_sdpa = model_class is not None and getattr(model_class, "_supports_sdpa", False)if _is_sdpa_excluded(model_type) or _declares_no_sdpa(model_class):supports_sdpa = False+ if not supports_sdpa:+ _remember_no_sdpa_config(config)supports_flash_attention = _model_class_supports_flash_attention(model_class) and not _is_flash_excluded(model_type)