Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Build flex attention masks without graph breaks under torch.compile (#12837)

Daniel Han4eef0117dcf1
diff --git a/tests/test_flex_mask_graph_breaks.py b/tests/test_flex_mask_graph_breaks.py
new file mode 100644
index 00000000000..355112ca4a7
--- /dev/null
+++ b/tests/test_flex_mask_graph_breaks.py
@@ -0,0 +1,136 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""Compiled flex mask: no graph breaks, same mask as eager."""
+
+import pytest
+
+torch = pytest.importorskip("torch")
+masking_utils = pytest.importorskip("transformers.masking_utils")
+qwen3 = pytest.importorskip("transformers.models.qwen3.configuration_qwen3")
+
+from unsloth.import_fixes import ( # noqa: E402
+ _FLEX_MASK_PATCH_FLAG,
+ _flex_mask_reads_padding_values,
+ fix_transformers_flex_mask_graph_breaks,
+)
+
+import inspect # noqa: E402
+
+# transformers before 5.0 builds flex masks from cache_position and never reads the padding values.
+NEW_SIGNATURE = "q_length" in inspect.signature(masking_utils.flex_attention_mask).parameters
+needs_new_signature = pytest.mark.skipif(not NEW_SIGNATURE, reason = "old flex mask signature")
+
+create_causal_mask = getattr(
+ masking_utils, "_unsloth_original_create_causal_mask", masking_utils.create_causal_mask
+)
+
+
+def _config():
+ config = qwen3.Qwen3Config(
+ hidden_size = 16,
+ num_attention_heads = 2,
+ num_key_value_heads = 1,
+ head_dim = 8,
+ num_hidden_layers = 2,
+ )
+ config._attn_implementation = "flex_attention"
+ return config
+
+
+def _build(
+ config,
+ embeds,
+ attention_mask,
+ position_ids = None,
+):
+ return create_causal_mask(
+ config = config,
+ inputs_embeds = embeds,
+ attention_mask = attention_mask,
+ past_key_values = None,
+ position_ids = position_ids,
+ )
+
+
+def _elements(block_mask, length):
+ q, kv = torch.arange(length)[:, None], torch.arange(length)[None]
+ zero = torch.zeros((), dtype = torch.long)
+ return torch.stack(
+ [block_mask.mask_mod(torch.tensor(b), zero, q, kv) for b in range(block_mask.shape[0])]
+ )
+
+
+@needs_new_signature
+def test_compiled_flex_mask_has_no_graph_breaks_and_matches_eager():
+ fix_transformers_flex_mask_graph_breaks()
+ fix_transformers_flex_mask_graph_breaks()
+ assert getattr(masking_utils.create_block_mask, _FLEX_MASK_PATCH_FLAG, False)
+ config = _config()
+ compiled = torch.compile(_build, backend = "eager", fullgraph = True, dynamic = True)
+ cases = [
+ (torch.ones(2, 200, dtype = torch.long), None),
+ (torch.tensor([[1] * 300, [1] * 250 + [0] * 50]), None),
+ (None, torch.cat([torch.arange(100), torch.arange(60), torch.arange(40)])[None]),
+ ]
+ for attention_mask, position_ids in cases:
+ length = (attention_mask if attention_mask is not None else position_ids).shape[-1]
+ embeds = torch.zeros(1 if attention_mask is None else 2, length, 16)
+ eager = _build(config, embeds, attention_mask, position_ids)
+ traced = compiled(config, embeds, attention_mask, position_ids)
+ assert torch.equal(eager.to_dense(), traced.to_dense())
+ assert torch.equal(_elements(eager, length), _elements(traced, length))
+ assert not _elements(traced, 200)[0, 100:160, :100].any()
+
+
+def test_probe_reads_only_value_branching_builders():
+ def branches(
+ batch_size,
+ q_length,
+ kv_length,
+ attention_mask = None,
+ device = "cpu",
+ **kwargs,
+ ):
+ if attention_mask is not None and not attention_mask.all():
+ pass
+ return masking_utils.create_block_mask(None)
+
+ def applies(
+ batch_size,
+ q_length,
+ kv_length,
+ attention_mask = None,
+ device = "cpu",
+ **kwargs,
+ ):
+ return masking_utils.create_block_mask(None)
+
+ builder = masking_utils.create_block_mask
+ assert _flex_mask_reads_padding_values(masking_utils, branches)
+ assert not _flex_mask_reads_padding_values(masking_utils, applies)
+ assert masking_utils.create_block_mask is builder
+
+
+@needs_new_signature
+def test_eager_calls_reach_the_original():
+ fix_transformers_flex_mask_graph_breaks()
+ patched = masking_utils.ALL_MASK_ATTENTION_FUNCTIONS["flex_attention"]
+ original = getattr(patched, "__wrapped__", patched)
+ calls = []
+ builder = masking_utils.create_block_mask
+ masking_utils.create_block_mask = lambda *args, **kwargs: calls.append(kwargs) or "mask"
+ try:
+ mask = torch.ones(1, 4, dtype = torch.bool)
+ kwargs = dict(batch_size = 1, q_length = 4, kv_length = 4, attention_mask = mask)
+ assert patched(**kwargs) == original(**kwargs) == "mask"
+ assert calls[0].keys() == calls[1].keys()
+ finally:
+ masking_utils.create_block_mask = builder
+
+
[email protected](NEW_SIGNATURE, reason = "new flex mask signature")
+def test_old_flex_mask_builder_is_left_alone():
+ original = masking_utils.ALL_MASK_ATTENTION_FUNCTIONS["flex_attention"]
+ fix_transformers_flex_mask_graph_breaks()
+ assert masking_utils.ALL_MASK_ATTENTION_FUNCTIONS["flex_attention"] is original
diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.py
index 9a48f080c8a..195507ab15e 100644
--- a/unsloth/_gpu_init.py
+++ b/unsloth/_gpu_init.py
@@ -312,6 +312,7 @@ from .import_fixes import (
fix_transformers_untrusted_config_fields,
fix_transformers_chat_template_path_traversal,
fix_transformers_chunked_mask_block_sequence_ids,
+ fix_transformers_flex_mask_graph_breaks,
fix_transformers_longcat_lsa_config,
fix_transformers_rope_scaling_drops_theta,
fix_transformers_fp8_modulelist_experts,
@@ -371,6 +372,7 @@ fix_transformers5_bare_annotation_configs()
# same process is covered too (#9708).
fix_transformers_fully_masked_rows()
fix_transformers_chunked_mask_block_sequence_ids()
+fix_transformers_flex_mask_graph_breaks()
# CVE-2026-4372 / 5241 / 9856, no-ops once transformers carries the fix; before any config loads.
fix_transformers_untrusted_config_fields()
fix_transformers_chat_template_path_traversal()
diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py
index 1fac14719df..317f4f7eb38 100644
--- a/unsloth/import_fixes.py
+++ b/unsloth/import_fixes.py
@@ -2194,6 +2194,113 @@ def fix_transformers_chunked_mask_block_sequence_ids():
logger.info(f"Unsloth: Failed patching create_chunked_causal_mask ({e})")
+_FLEX_MASK_PATCH_FLAG = "_unsloth_patched_flex_mask_graph_breaks"
+
+
+def _is_dynamo_compiling():
+ import torch
+ is_compiling = getattr(getattr(torch, "compiler", None), "is_compiling", None)
+ return bool(is_compiling()) if callable(is_compiling) else False
+
+
+def _flex_mask_reads_padding_values(masking_utils, flex_attention_mask):
+ """True if building from a meta mask fails (values read) while no mask succeeds."""
+ import torch
+
+ try:
+ if "q_length" not in inspect.signature(flex_attention_mask).parameters:
+ return False
+ except (TypeError, ValueError):
+ return False
+ builder = masking_utils.create_block_mask
+ masking_utils.create_block_mask = lambda *args, **kwargs: None
+ kwargs = dict(batch_size = 1, q_length = 2, kv_length = 2, device = "meta")
+ try:
+ try:
+ flex_attention_mask(attention_mask = None, **kwargs)
+ except Exception:
+ return False
+ try:
+ mask = torch.ones((1, 2), dtype = torch.bool, device = "meta")
+ flex_attention_mask(attention_mask = mask, **kwargs)
+ except Exception:
+ return True
+ return False
+ finally:
+ masking_utils.create_block_mask = builder
+
+
+def fix_transformers_flex_mask_graph_breaks():
+ """While dynamo traces, always apply the padding mask (all-ones gives an equal BlockMask) instead
+ of branching on `fast_all`, and drop `_compile`, whose deprecation warnings.warn breaks the graph."""
+ try:
+ from transformers import masking_utils
+ except Exception as e:
+ logger.info(f"Unsloth: Skipping the flex mask graph break fix ({e})")
+ return
+ interface = getattr(masking_utils, "ALL_MASK_ATTENTION_FUNCTIONS", None)
+ builder = getattr(masking_utils, "create_block_mask", None)
+ if interface is None or builder is None:
+ return
+ try:
+ if not getattr(builder, _FLEX_MASK_PATCH_FLAG, False):
+
+ @functools.wraps(builder)
+ def create_block_mask(
+ *args,
+ _original = builder,
+ **kwargs,
+ ):
+ if _is_dynamo_compiling():
+ kwargs.pop("_compile", None)
+ return _original(*args, **kwargs)
+
+ setattr(create_block_mask, _FLEX_MASK_PATCH_FLAG, True)
+ masking_utils.create_block_mask = create_block_mask
+
+ current = interface["flex_attention"]
+ if getattr(current, _FLEX_MASK_PATCH_FLAG, False):
+ return
+ # Before torch 2.6 the padded branch also pads the mask to the 128 block size.
+ if not getattr(masking_utils, "_is_torch_greater_or_equal_than_2_6", False):
+ return
+ if not _flex_mask_reads_padding_values(masking_utils, current):
+ return
+ original = current
+
+ @functools.wraps(original)
+ def flex_attention_mask(*args, **kwargs):
+ attention_mask = kwargs.get("attention_mask")
+ if (
+ args
+ or attention_mask is None
+ or "kv_length" not in kwargs
+ or not _is_dynamo_compiling()
+ ):
+ return original(*args, **kwargs)
+ padding_mask = masking_utils.prepare_padding_mask(
+ attention_mask, kwargs["kv_length"], kwargs.get("kv_offset", 0)
+ )
+ kwargs["mask_function"] = masking_utils.and_masks(
+ kwargs.get("mask_function", masking_utils.causal_mask_function),
+ masking_utils.padding_mask_function(padding_mask),
+ )
+ kwargs["attention_mask"] = None
+ return original(**kwargs)
+
+ flex_attention_mask.__wrapped__ = original
+ setattr(flex_attention_mask, _FLEX_MASK_PATCH_FLAG, True)
+ interface.register("flex_attention", flex_attention_mask)
+ if getattr(masking_utils, "flex_attention_mask", None) is original:
+ masking_utils.flex_attention_mask = flex_attention_mask
+ logger.info(
+ "Unsloth: Patching transformers `flex_attention_mask` so a compiled mask "
+ "builds without graph breaks"
+ )
+ except Exception as e:
+ logger.info(f"Unsloth: Failed patching flex_attention_mask ({e})")
+
+
_COMPOSITE_PREFIX_RENAMING_FLAG = "_unsloth_patched_composite_prefix_renaming"
# unsloth_zoo marks its own copy of this repair with this. Spelled as a literal rather than