Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Flex wrapper: keep the causal mask inside compiled blocks on torch < 2.10 (#16234)
Yufan Zhuang2bf0053696a4
diff --git a/tests/test_flex_large_head_dim_kernel_options.py b/tests/test_flex_large_head_dim_kernel_options.pyindex 4fcaf81a77f..2ddc0082450 100644--- a/tests/test_flex_large_head_dim_kernel_options.py+++ b/tests/test_flex_large_head_dim_kernel_options.py@@ -77,6 +77,18 @@ def test_wrapper_injects_kernel_options_above_256():assert _call(512)["kernel_options"]["BLOCK_M"] == 32+def test_wrapper_injects_kernel_options_inside_a_compiled_block():+ import torch++ original, seen = _record()+ wrapped = u._wrap_flex_attention_forward(original)+ torch._dynamo.reset()+ torch.compile(lambda q: wrapped(None, q, q, q, None), backend = "eager")(+ torch.zeros(1, 2, 8, 512)+ )+ assert seen["kernel_options"]["BLOCK_M"] == 32++def test_wrapper_leaves_256_alone():# Must be absent, not an empty dict, so torch keeps its own default.assert _call(256).get("kernel_options") is Nonediff --git a/tests/test_flex_maskless_sdpa.py b/tests/test_flex_maskless_sdpa.pyindex b359d3f1bbc..7f7d2c8b213 100644--- a/tests/test_flex_maskless_sdpa.py+++ b/tests/test_flex_maskless_sdpa.py@@ -74,6 +74,13 @@ def test_a_dropped_mask_runs_sdpa_is_causal(qkv):assert torch.equal(out, reference)+def test_a_dropped_mask_is_still_causal_in_a_compiled_block(qkv):+ q, k, v = qkv+ torch._dynamo.reset()+ out, _ = torch.compile(_flex(), backend = "eager")(Causal(), q, k, v, None, scaling = D**-0.5)+ assert (out.float() - _causal_reference(q, k, v, D**-0.5)).abs().max() < 2e-2++@pytest.mark.parametrize("case", ["softcap", "head_dim_512"])def test_a_dropped_mask_that_stays_on_flex_is_still_causal(case):# Never hand flex a None mask for a causal call: it would attend to future tokens.diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex ff1161bc73e..1d3d159c240 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -1107,6 +1107,13 @@ _FLEX_MASKLESS_SDPA_ENABLED = os.environ.get("UNSLOTH_FLEX_MASKLESS_SDPA", "1")FLEX_MASKLESS_SDPA_STATS = {"sdpa": 0, "flex": 0}+def _is_4d(tensor):+ # Dynamo < 2.10 reads hasattr / getattr on a traced tensor as missing (pytorch/pytorch#167651).+ if isinstance(tensor, torch.Tensor):+ return tensor.dim() == 4+ return callable(getattr(tensor, "dim", None)) and tensor.dim() == 4++def _dropped_causal_mask(module, query, key, args, kwargs):"""A flex call whose causal mask unsloth_zoo dropped (stock flex reads None as bidirectional)."""attention_mask = args[0] if len(args) > 0 else kwargs.get("attention_mask", None)@@ -1115,7 +1122,7 @@ def _dropped_causal_mask(module, query, key, args, kwargs):# Vision callers pass None meaning bidirectional, and is_causal=False is a per-call override.if getattr(module, "is_causal", None) is not True or kwargs.get("is_causal", None) is False:return False- if not (hasattr(query, "dim") and query.dim() == 4 and key.dim() == 4):+ if not (_is_4d(query) and key.dim() == 4):return Falsereturn query.shape[2] >= 2 and query.shape[2] == key.shape[2]@@ -1256,9 +1263,7 @@ def _wrap_flex_attention_forward(flex_attention_forward):try:# Some vision callers reuse the interface with a non-4D query.kernel_options = (- _flex_kernel_options_for_head_dim(query.shape[-1])- if hasattr(query, "dim") and query.dim() == 4- else None+ _flex_kernel_options_for_head_dim(query.shape[-1]) if _is_4d(query) else None)except Exception:kernel_options = None@@ -1268,8 +1273,7 @@ def _wrap_flex_attention_forward(flex_attention_forward):if (kernel_options is Noneand small_smem_device_ok- and hasattr(query, "dim")- and query.dim() == 4+ and _is_4d(query)and (getattr(query, "dtype", None) in (torch.float16, torch.bfloat16)or (@@ -1312,8 +1316,7 @@ def _wrap_flex_attention_forward(flex_attention_forward):if (kernel_options is Noneand tuned_bwd_device_ok- and hasattr(query, "dim")- and query.dim() == 4+ and _is_4d(query)and _flex_call_needs_backward(query, key, value)and _flex_tuned_bwd_fits(args, kwargs)):