Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Flex wrapper: keep the causal mask inside compiled blocks on torch < 2.10 (#16234)

Yufan Zhuang2bf0053696a4
diff --git a/tests/test_flex_large_head_dim_kernel_options.py b/tests/test_flex_large_head_dim_kernel_options.py
index 4fcaf81a77f..2ddc0082450 100644
--- a/tests/test_flex_large_head_dim_kernel_options.py
+++ b/tests/test_flex_large_head_dim_kernel_options.py
@@ -77,6 +77,18 @@ def test_wrapper_injects_kernel_options_above_256():
assert _call(512)["kernel_options"]["BLOCK_M"] == 32
+def test_wrapper_injects_kernel_options_inside_a_compiled_block():
+ import torch
+
+ original, seen = _record()
+ wrapped = u._wrap_flex_attention_forward(original)
+ torch._dynamo.reset()
+ torch.compile(lambda q: wrapped(None, q, q, q, None), backend = "eager")(
+ torch.zeros(1, 2, 8, 512)
+ )
+ assert seen["kernel_options"]["BLOCK_M"] == 32
+
+
def test_wrapper_leaves_256_alone():
# Must be absent, not an empty dict, so torch keeps its own default.
assert _call(256).get("kernel_options") is None
diff --git a/tests/test_flex_maskless_sdpa.py b/tests/test_flex_maskless_sdpa.py
index b359d3f1bbc..7f7d2c8b213 100644
--- a/tests/test_flex_maskless_sdpa.py
+++ b/tests/test_flex_maskless_sdpa.py
@@ -74,6 +74,13 @@ def test_a_dropped_mask_runs_sdpa_is_causal(qkv):
assert torch.equal(out, reference)
+def test_a_dropped_mask_is_still_causal_in_a_compiled_block(qkv):
+ q, k, v = qkv
+ torch._dynamo.reset()
+ out, _ = torch.compile(_flex(), backend = "eager")(Causal(), q, k, v, None, scaling = D**-0.5)
+ assert (out.float() - _causal_reference(q, k, v, D**-0.5)).abs().max() < 2e-2
+
+
@pytest.mark.parametrize("case", ["softcap", "head_dim_512"])
def test_a_dropped_mask_that_stays_on_flex_is_still_causal(case):
# Never hand flex a None mask for a causal call: it would attend to future tokens.
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index ff1161bc73e..1d3d159c240 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -1107,6 +1107,13 @@ _FLEX_MASKLESS_SDPA_ENABLED = os.environ.get("UNSLOTH_FLEX_MASKLESS_SDPA", "1")
FLEX_MASKLESS_SDPA_STATS = {"sdpa": 0, "flex": 0}
+def _is_4d(tensor):
+ # Dynamo < 2.10 reads hasattr / getattr on a traced tensor as missing (pytorch/pytorch#167651).
+ if isinstance(tensor, torch.Tensor):
+ return tensor.dim() == 4
+ return callable(getattr(tensor, "dim", None)) and tensor.dim() == 4
+
+
def _dropped_causal_mask(module, query, key, args, kwargs):
"""A flex call whose causal mask unsloth_zoo dropped (stock flex reads None as bidirectional)."""
attention_mask = args[0] if len(args) > 0 else kwargs.get("attention_mask", None)
@@ -1115,7 +1122,7 @@ def _dropped_causal_mask(module, query, key, args, kwargs):
# Vision callers pass None meaning bidirectional, and is_causal=False is a per-call override.
if getattr(module, "is_causal", None) is not True or kwargs.get("is_causal", None) is False:
return False
- if not (hasattr(query, "dim") and query.dim() == 4 and key.dim() == 4):
+ if not (_is_4d(query) and key.dim() == 4):
return False
return query.shape[2] >= 2 and query.shape[2] == key.shape[2]
@@ -1256,9 +1263,7 @@ def _wrap_flex_attention_forward(flex_attention_forward):
try:
# Some vision callers reuse the interface with a non-4D query.
kernel_options = (
- _flex_kernel_options_for_head_dim(query.shape[-1])
- if hasattr(query, "dim") and query.dim() == 4
- else None
+ _flex_kernel_options_for_head_dim(query.shape[-1]) if _is_4d(query) else None
)
except Exception:
kernel_options = None
@@ -1268,8 +1273,7 @@ def _wrap_flex_attention_forward(flex_attention_forward):
if (
kernel_options is None
and small_smem_device_ok
- and hasattr(query, "dim")
- and query.dim() == 4
+ and _is_4d(query)
and (
getattr(query, "dtype", None) in (torch.float16, torch.bfloat16)
or (
@@ -1312,8 +1316,7 @@ def _wrap_flex_attention_forward(flex_attention_forward):
if (
kernel_options is None
and tuned_bwd_device_ok
- and hasattr(query, "dim")
- and query.dim() == 4
+ and _is_4d(query)
and _flex_call_needs_backward(query, key, value)
and _flex_tuned_bwd_fits(args, kwargs)
):