Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Update the flex large head dim mask tests to the unpadded causal-mask skip (#13018)

Daniel Han637fd54ce348
diff --git a/tests/test_flex_large_head_dim_fallback.py b/tests/test_flex_large_head_dim_fallback.py
index 2793f8c0370..9550180688d 100644
--- a/tests/test_flex_large_head_dim_fallback.py
+++ b/tests/test_flex_large_head_dim_fallback.py
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""The large-head-dim flex routing: decoder-only scoping, opt-outs, config-driven detection,
-and why the mask is always present under Unsloth's compiled mask wrapper."""
+and when Unsloth's mask wrapper skips the causal mask."""
import pytest
@@ -255,9 +255,8 @@ def test_forcing_the_env_var_cannot_override_an_architecture_opt_out(monkeypatch
assert u._enable_flex_attention_support(cls, "t5gemma2") is False
-# Upstream skips the mask for an unpadded batch, but `_ignore_causal_mask_sdpa` returns False
-# while tracing, so Unsloth's compiled create_causal_mask always builds one. Pins both halves,
-# and that with UNSLOTH_COMPILE_DISABLE=1 the uncompiled wrapper keeps upstream's skip.
+# `_ignore_causal_mask_sdpa` returns False while tracing, so a compiled create_causal_mask always
+# builds a mask; unsloth-zoo#1575 restores upstream's unpadded skip by running eagerly.
def _mask_for(
@@ -266,9 +265,13 @@ def _mask_for(
q_len = 64,
head_dim = 256,
bsz = 2,
+ position_ids = "arange",
):
+ import inspect
+
import torch
import transformers as T
+ from transformers import masking_utils
cfg = T.LlamaConfig(
hidden_size = 2048,
@@ -279,13 +282,39 @@ def _mask_for(
vocab_size = 128,
)
cfg._attn_implementation = "sdpa"
- return create(
- config = cfg,
- inputs_embeds = torch.zeros(bsz, q_len, cfg.hidden_size, dtype = torch.bfloat16),
- attention_mask = attention_mask,
- past_key_values = None,
- position_ids = torch.arange(q_len).unsqueeze(0).expand(bsz, -1),
+ # Upstream signature, not the wrapper's: <= 5.1 `input_embeds` + `cache_position`, 5.9 no cache.
+ original = getattr(
+ masking_utils, "_unsloth_original_create_causal_mask", masking_utils.create_causal_mask
)
+ parameters = inspect.signature(original).parameters
+ embeds_name = "input_embeds" if "input_embeds" in parameters else "inputs_embeds"
+ if isinstance(position_ids, str):
+ position_ids = torch.arange(q_len).unsqueeze(0).expand(bsz, -1)
+ kwargs = {
+ "config": cfg,
+ embeds_name: torch.zeros(bsz, q_len, cfg.hidden_size, dtype = torch.bfloat16),
+ "attention_mask": attention_mask,
+ "past_key_values": None,
+ "position_ids": position_ids,
+ }
+ if "cache_position" in parameters:
+ kwargs["cache_position"] = torch.arange(q_len)
+ return create(**kwargs)
+
+
+def _padded_masks():
+ import torch
+
+ right = torch.ones(2, 64, dtype = torch.long)
+ right[0, -8:] = 0
+ left = torch.ones(2, 64, dtype = torch.long)
+ left[0, :8] = 0
+ return right, left
+
+
+def _packed_position_ids():
+ import torch
+ return torch.arange(32).repeat(2).unsqueeze(0).expand(2, -1)
def _uncompiled_create_causal_mask():
@@ -297,22 +326,37 @@ def _uncompiled_create_causal_mask():
return original
+def _wrapper_skips_unpadded_masks():
+ """Whether the installed unsloth_zoo has the unpadded causal-mask skip (unsloth-zoo#1575)."""
+ try:
+ from unsloth_zoo.temporary_patches import misc
+ except Exception:
+ return False
+ return hasattr(misc, "CAUSAL_MASK_SKIP_STATS") and hasattr(misc, "_maskless_causal_arguments")
+
+
+def _skip_count():
+ from unsloth_zoo.temporary_patches.misc import CAUSAL_MASK_SKIP_STATS
+ return CAUSAL_MASK_SKIP_STATS["skipped"]
+
+
+def _skip_on(monkeypatch):
+ monkeypatch.delenv("UNSLOTH_SKIP_CAUSAL_MASK", raising = False)
+
+
def test_upstream_skips_the_mask_for_an_unpadded_batch():
import torch
create = _uncompiled_create_causal_mask()
- assert _mask_for(create, None) is None
+ # transformers 4.x treats any position_ids as possibly packed and keeps the mask.
+ assert _mask_for(create, None, position_ids = None) is None
assert _mask_for(create, torch.ones(2, 64, dtype = torch.long)) is None
def test_upstream_still_materialises_a_mask_when_padded():
- import torch
-
create = _uncompiled_create_causal_mask()
- right = torch.ones(2, 64, dtype = torch.long)
- right[0, -8:] = 0
- left = torch.ones(2, 64, dtype = torch.long)
- left[0, :8] = 0
+ right, left = _padded_masks()
assert _mask_for(create, right) is not None
assert _mask_for(create, left) is not None
@@ -339,25 +383,68 @@ def _mask_wrapper_is_compiled():
return inner is not original
-def test_our_compiled_wrapper_is_what_defeats_the_skip():
- """Pins the cause, so this is a deliberate trade and not an accident nobody noticed."""
+def test_compiled_wrapper_skips_the_mask_for_an_unpadded_batch(_skip_on):
+ """An unpadded, cache-free SDPA batch gets no mask, so SDPA can take its is_causal kernels."""
+ import torch
from transformers import masking_utils
- # Skips when the pair does not stash the original or the wrapper cannot be read.
if not _mask_wrapper_is_compiled():
pytest.skip("the mask wrapper calls the uncompiled original in this run")
- assert _mask_for(masking_utils.create_causal_mask, None) is not None
+ create = masking_utils.create_causal_mask
+ if not _wrapper_skips_unpadded_masks():
+ # Older unsloth_zoo: tracing makes `_ignore_causal_mask_sdpa` refuse, so the mask is built.
+ assert _mask_for(create, None) is not None
+ return
+ before = _skip_count()
+ assert _mask_for(create, None) is None
+ assert _mask_for(create, torch.ones(2, 64, dtype = torch.long)) is None
+ assert _skip_count() == before + 2
+
+
+def test_compiled_wrapper_keeps_the_mask_when_padded_or_packed(_skip_on):
+ from transformers import masking_utils
+ create = masking_utils.create_causal_mask
+ right, left = _padded_masks()
+ before = _skip_count() if _wrapper_skips_unpadded_masks() else None
+ assert _mask_for(create, right) is not None
+ assert _mask_for(create, left) is not None
+ assert _mask_for(create, None, position_ids = _packed_position_ids()) is not None
+ if before is not None:
+ assert _skip_count() == before
-def test_an_uncompiled_wrapper_keeps_the_upstream_skip():
- """The other half of the cause: without compilation the wrapper changes nothing here.
+
+def test_kill_switch_keeps_the_mask_for_an_unpadded_batch(monkeypatch):
+ import torch
+ from transformers import masking_utils
+
+ if not _wrapper_skips_unpadded_masks():
+ pytest.skip("this unsloth_zoo has no unpadded causal-mask skip")
+ if not _mask_wrapper_is_compiled():
+ pytest.skip("uncompiled, the original skips the mask itself")
+ monkeypatch.setenv("UNSLOTH_SKIP_CAUSAL_MASK", "0")
+ create = masking_utils.create_causal_mask
+ before = _skip_count()
+ assert _mask_for(create, None) is not None
+ assert _mask_for(create, torch.ones(2, 64, dtype = torch.long)) is not None
+ assert _skip_count() == before
+
+
+def test_an_uncompiled_wrapper_keeps_the_upstream_skip(_skip_on):
+ """Without compilation the wrapper agrees with upstream on unpadded and padded batches.
unsloth-zoo#1335 made the wrapper install under UNSLOTH_COMPILE_DISABLE=1 too, around the
- uncompiled original. The mask is then skipped exactly as upstream skips it, so a
- materialised mask on an unpadded batch comes from compiling, not from wrapping.
+ uncompiled original. The mask is then skipped exactly as upstream skips it, and kept exactly
+ where upstream keeps it.
"""
+ import torch
from transformers import masking_utils
if _mask_wrapper_is_compiled():
pytest.skip("the mask wrapper is compiled in this run")
- assert _mask_for(masking_utils.create_causal_mask, None) is None
+ create = masking_utils.create_causal_mask
+ assert _mask_for(create, None, position_ids = None) is None
+ assert _mask_for(create, torch.ones(2, 64, dtype = torch.long)) is None
+ right, left = _padded_masks()
+ assert _mask_for(create, right) is not None
+ assert _mask_for(create, left) is not None