Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Send maskless causal flex calls to SDPA is_causal (#12773)

Daniel Hanb6a6dcdbf001
diff --git a/tests/test_flex_maskless_sdpa.py b/tests/test_flex_maskless_sdpa.py
new file mode 100644
index 00000000000..b359d3f1bbc
--- /dev/null
+++ b/tests/test_flex_maskless_sdpa.py
@@ -0,0 +1,183 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+import copy
+import types
+
+import pytest
+import torch
+
+from real_accelerator import has_real_cuda
+
+if not has_real_cuda():
+ pytest.skip("needs a CUDA device", allow_module_level = True)
+
+import unsloth # noqa: F401
+import unsloth.models._utils as U
+from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
+
+B, H, KV, T, D = 2, 8, 2, 256, 64
+
+
+class Causal(torch.nn.Module):
+ is_causal = True
+ num_key_value_groups = H // KV
+ training = True
+
+
+def qkv():
+ torch.manual_seed(0)
+ q = torch.randn(B, H, T, D, device = "cuda", dtype = torch.bfloat16)
+ k = torch.randn(B, KV, T, D, device = "cuda", dtype = torch.bfloat16)
+ v = torch.randn(B, KV, T, D, device = "cuda", dtype = torch.bfloat16)
+ return q, k, v
+
+
+def test_the_wrapper_advertises_the_reroute():
+ U.patch_flex_attention_kernel_options()
+ function = ALL_ATTENTION_FUNCTIONS["flex_attention"]
+ assert (
+ getattr(function, "_unsloth_maskless_causal_sdpa", False) is U._FLEX_MASKLESS_SDPA_ENABLED
+ )
+
+
+def _flex():
+ U.patch_flex_attention_kernel_options()
+ return ALL_ATTENTION_FUNCTIONS["flex_attention"]
+
+
+def _causal_reference(q, k, v, scale):
+ return torch.nn.functional.scaled_dot_product_attention(
+ q.float(),
+ k.float(),
+ v.float(),
+ is_causal = True,
+ scale = scale,
+ enable_gqa = True,
+ ).transpose(1, 2)
+
+
+def test_a_dropped_mask_runs_sdpa_is_causal(qkv):
+ q, k, v = qkv
+ before = dict(U.FLEX_MASKLESS_SDPA_STATS)
+ out, _ = _flex()(Causal(), q, k, v, None, scaling = D**-0.5)
+ assert U.FLEX_MASKLESS_SDPA_STATS["sdpa"] == before["sdpa"] + 1
+ reference = torch.nn.functional.scaled_dot_product_attention(
+ q,
+ k,
+ v,
+ is_causal = True,
+ scale = D**-0.5,
+ enable_gqa = True,
+ ).transpose(1, 2)
+ assert torch.equal(out, reference)
+
+
[email protected]("case", ["softcap", "head_dim_512"])
+def test_a_dropped_mask_that_stays_on_flex_is_still_causal(case):
+ # Never hand flex a None mask for a causal call: it would attend to future tokens.
+ torch.manual_seed(0)
+ head_dim = 512 if case == "head_dim_512" else D
+ q = torch.randn(1, H, 128, head_dim, device = "cuda", dtype = torch.bfloat16)
+ k = torch.randn(1, KV, 128, head_dim, device = "cuda", dtype = torch.bfloat16)
+ v = torch.randn(1, KV, 128, head_dim, device = "cuda", dtype = torch.bfloat16)
+ kwargs = {"softcap": 1e4} if case == "softcap" else {} # a softcap this large is a no-op
+ before = dict(U.FLEX_MASKLESS_SDPA_STATS)
+ out, _ = _flex()(Causal(), q, k, v, None, scaling = head_dim**-0.5, **kwargs)
+ assert U.FLEX_MASKLESS_SDPA_STATS["sdpa"] == before["sdpa"]
+ reference = _causal_reference(q, k, v, head_dim**-0.5)
+ assert (out.float() - reference).abs().max() < 2e-2
+
+
+def test_counters_do_not_mutate_while_compiling(qkv, monkeypatch):
+ q, k, v = qkv
+ monkeypatch.setattr(torch.compiler, "is_compiling", lambda: True)
+ before = dict(U.FLEX_MASKLESS_SDPA_STATS)
+ _flex()(Causal(), q, k, v, None, scaling = D**-0.5)
+ assert U.FLEX_MASKLESS_SDPA_STATS == before
+
+
+ "case",
+ [
+ "mask",
+ "softcap",
+ "s_aux",
+ "not_causal",
+ "no_is_causal",
+ "is_causal_false",
+ "head_dim_512",
+ "decode",
+ "cache",
+ "sdpa_excluded",
+ ],
+)
+def test_everything_else_stays_on_flex(qkv, case):
+ q, k, v = qkv
+ module, args, kwargs = Causal(), (None,), {}
+ if case == "mask":
+ args = (torch.ones(B, 1, T, T, device = "cuda", dtype = torch.bool),)
+ if case == "softcap":
+ kwargs = {"softcap": 30.0}
+ if case == "s_aux":
+ kwargs = {"s_aux": torch.zeros(H, device = "cuda")}
+ if case == "not_causal":
+ module.is_causal = False
+ if case == "no_is_causal":
+ module = torch.nn.Module() # vision callers: None means bidirectional
+ if case == "is_causal_false":
+ kwargs = {"is_causal": False} # per-call bidirectional override
+ if case == "head_dim_512":
+ q, k, v = (torch.randn(*t.shape[:3], 512, device = "cuda", dtype = t.dtype) for t in (q, k, v))
+ if case == "decode":
+ q = q[:, :, :1]
+ if case == "cache":
+ q = q[:, :, : T // 2]
+ if case == "sdpa_excluded":
+ module.config = types.SimpleNamespace(model_type = "gemma3_text")
+ assert U._maskless_causal_sdpa_forward(module, q, k, v, args, kwargs) is None
+
+
+def _softcap_call(T, requires_grad = False):
+ q, k, v = (torch.randn(1, n, T, D, device = "cuda", dtype = torch.bfloat16) for n in (H, KV, KV))
+ q.requires_grad_(requires_grad)
+ return _flex()(Causal(), q, k, v, None, scaling = D**-0.5, softcap = 1e4), q
+
+
+def test_a_fallback_mask_first_built_under_inference_mode_still_trains():
+ U._CAUSAL_BLOCK_MASKS.clear()
+ with torch.inference_mode():
+ _softcap_call(128)
+ (out, _), q = _softcap_call(128, requires_grad = True)
+ out.float().sum().backward()
+ assert q.grad is not None and torch.isfinite(q.grad).all()
+
+
+def test_the_fallback_mask_cache_is_bounded():
+ U._CAUSAL_BLOCK_MASKS.clear()
+ for T in range(128, 128 * 13, 128):
+ _softcap_call(T)
+ assert 0 < len(U._CAUSAL_BLOCK_MASKS) <= 8
+
+
+def test_the_wrapper_vetoes_configs_whose_layers_cannot_take_sdpa():
+ from transformers import Gemma2Config, Gemma3TextConfig, LlamaConfig, Qwen3Config
+
+ accepts = _flex()._unsloth_maskless_causal_sdpa_accepts
+ assert accepts(LlamaConfig()) and accepts(Qwen3Config(head_dim = 256))
+ assert not accepts(Qwen3Config(head_dim = 512))
+ assert not accepts(Gemma3TextConfig()) # SDPA disabled for Gemma 3
+ assert not accepts(Gemma2Config()) # softcap
+
+
+def test_a_class_level_sdpa_opt_out_keeps_the_mask():
+ from transformers import LlamaConfig, LlamaForCausalLM
+
+ no_sdpa = type("NoSdpaLlama", (LlamaForCausalLM,), {"_supports_sdpa": False})
+ config = LlamaConfig()
+ U.resolve_attention_implementation(no_sdpa, config)
+ accepts = _flex()._unsloth_maskless_causal_sdpa_accepts
+ assert not accepts(copy.deepcopy(config)) # from_pretrained copies the config it is given
+ U._NO_SDPA_MODEL_TYPES.discard("llama")
+ assert accepts(LlamaConfig())
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index 74a0949fce6..220c7c27317 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -654,6 +654,118 @@ def _flex_call_needs_backward(query, key, value):
return False
+# unsloth_zoo drops the causal mask of unpadded, cache-free batches; route that None to SDPA is_causal.
+# Kill switch: UNSLOTH_FLEX_MASKLESS_SDPA=0.
+_FLEX_MASKLESS_SDPA_ENABLED = os.environ.get("UNSLOTH_FLEX_MASKLESS_SDPA", "1") != "0"
+FLEX_MASKLESS_SDPA_STATS = {"sdpa": 0, "flex": 0}
+
+
+def _dropped_causal_mask(module, query, key, args, kwargs):
+ """A flex call whose causal mask unsloth_zoo dropped (stock flex reads None as bidirectional)."""
+ attention_mask = args[0] if len(args) > 0 else kwargs.get("attention_mask", None)
+ if attention_mask is not None:
+ return False
+ # Vision callers pass None meaning bidirectional, and is_causal=False is a per-call override.
+ if getattr(module, "is_causal", None) is not True or kwargs.get("is_causal", None) is False:
+ return False
+ if not (hasattr(query, "dim") and query.dim() == 4 and key.dim() == 4):
+ return False
+ return query.shape[2] >= 2 and query.shape[2] == key.shape[2]
+
+
+# Model types the resolver loaded without SDPA support (class opt-out or exclusion list). Keyed by
+# model_type, not config identity: from_pretrained deep-copies the config it was given.
+_NO_SDPA_MODEL_TYPES = set()
+
+
+def _remember_no_sdpa_config(config):
+ for attention_config in _iter_attention_configs(config):
+ model_type = _config_get(attention_config, "model_type", None)
+ if isinstance(model_type, str) and model_type:
+ _NO_SDPA_MODEL_TYPES.add(model_type.lower())
+
+
+def _maskless_causal_sdpa_accepts(config):
+ """False when some decoder layer could not take the SDPA reroute, so unsloth_zoo keeps the mask."""
+ for attention_config in _text_attention_configs(config) + [config]:
+ model_type = (_config_get(attention_config, "model_type", None) or "").lower()
+ if model_type in _NO_SDPA_MODEL_TYPES:
+ return False
+ if _is_sdpa_excluded(_config_get(attention_config, "model_type", None) or ""):
+ return False
+ if _config_get(attention_config, "attn_logit_softcapping", None) is not None:
+ return False
+ head_dim = _text_attention_head_dim(config)
+ return head_dim is None or head_dim <= 256
+
+
+def _maskless_causal_sdpa_forward(module, query, key, value, args, kwargs):
+ """SDPA is_causal output for a dropped causal mask, else None (the call stays on flex)."""
+ if not _dropped_causal_mask(module, query, key, args, kwargs):
+ return None
+ if _is_sdpa_excluded(_config_get(getattr(module, "config", None), "model_type", None) or ""):
+ return None
+ # flex positional order: attention_mask, scaling, softcap, s_aux.
+ if any(arg is not None for arg in args[2:]):
+ return None
+ if kwargs.get("softcap", None) is not None or kwargs.get("s_aux", None) is not None:
+ return None
+ if kwargs.get("position_bias", None) is not None:
+ return None
+ # SDPA has flash / cuDNN kernels only up to head_dim 256; above that flex is faster.
+ if query.shape[-1] > 256:
+ return None
+ try:
+ from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
+ sdpa_forward = ALL_ATTENTION_FUNCTIONS["sdpa"]
+ except Exception:
+ return None
+ scaling = args[1] if len(args) > 1 else kwargs.get("scaling", None)
+ return sdpa_forward(
+ module,
+ query,
+ key,
+ value,
+ None,
+ dropout = kwargs.get("dropout", 0.0),
+ scaling = scaling,
+ is_causal = True,
+ )
+
+
+_CAUSAL_BLOCK_MASKS = {}
+
+
+def _causal_block_mask(query, key):
+ """The causal BlockMask unsloth_zoo dropped, for calls that cannot go to SDPA."""
+ cache_key = (query.shape[2], key.shape[2], query.device)
+ block_mask = _CAUSAL_BLOCK_MASKS.get(cache_key)
+ if block_mask is None:
+ from torch.nn.attention.flex_attention import create_block_mask
+
+ # Uncompiled builds materialize Q x KV; inference tensors break a later backward.
+ with torch.inference_mode(False):
+ block_mask = create_block_mask(
+ lambda b, h, q_idx, kv_idx: q_idx >= kv_idx,
+ None,
+ None,
+ query.shape[2],
+ key.shape[2],
+ device = query.device,
+ _compile = True,
+ )
+ if len(_CAUSAL_BLOCK_MASKS) >= 8:
+ _CAUSAL_BLOCK_MASKS.pop(next(iter(_CAUSAL_BLOCK_MASKS)))
+ _CAUSAL_BLOCK_MASKS[cache_key] = block_mask
+ return block_mask
+
+
+def _count_flex_reroute(path):
+ # A Python counter mutated inside a compiled region makes Dynamo recompile every call.
+ if not torch.compiler.is_compiling():
+ FLEX_MASKLESS_SDPA_STATS[path] += 1
+
+
def _wrap_flex_attention_forward(flex_attention_forward):
"""Add kernel_options to a registered `flex_attention` function: block sizes above head_dim
256, and the main flex kernel for calls that need a backward."""
@@ -662,6 +774,19 @@ def _wrap_flex_attention_forward(flex_attention_forward):
@functools.wraps(flex_attention_forward)
def unsloth_flex_attention_forward(module, query, key, value, *args, **kwargs):
+ if _FLEX_MASKLESS_SDPA_ENABLED and _dropped_causal_mask(module, query, key, args, kwargs):
+ output = _maskless_causal_sdpa_forward(module, query, key, value, args, kwargs)
+ if output is not None:
+ _count_flex_reroute("sdpa")
+ return output
+ # Softcap, sinks or head_dim > 256: stay on flex, but never with a None mask.
+ _count_flex_reroute("flex")
+ if len(args) > 0:
+ args = (_causal_block_mask(query, key),) + tuple(args[1:])
+ else:
+ kwargs["attention_mask"] = _causal_block_mask(query, key)
+ elif _FLEX_MASKLESS_SDPA_ENABLED:
+ _count_flex_reroute("flex")
try:
# Some vision callers reuse the interface with a non-4D query.
kernel_options = (
@@ -687,6 +812,10 @@ def _wrap_flex_attention_forward(flex_attention_forward):
return flex_attention_forward(module, query, key, value, *args, **kwargs)
unsloth_flex_attention_forward._unsloth_flex_kernel_options = True
+ unsloth_flex_attention_forward._unsloth_maskless_causal_sdpa = _FLEX_MASKLESS_SDPA_ENABLED
+ unsloth_flex_attention_forward._unsloth_maskless_causal_sdpa_accepts = (
+ _maskless_causal_sdpa_accepts
+ )
unsloth_flex_attention_forward._unsloth_original_forward = flex_attention_forward
return unsloth_flex_attention_forward
@@ -2264,6 +2393,8 @@ def resolve_attention_implementation(
supports_sdpa = model_class is not None and getattr(model_class, "_supports_sdpa", False)
if _is_sdpa_excluded(model_type) or _declares_no_sdpa(model_class):
supports_sdpa = False
+ if not supports_sdpa:
+ _remember_no_sdpa_config(config)
supports_flash_attention = _model_class_supports_flash_attention(
model_class
) and not _is_flash_excluded(model_type)