Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Build flex attention masks without graph breaks under torch.compile (#12837)
Daniel Han4eef0117dcf1
diff --git a/tests/test_flex_mask_graph_breaks.py b/tests/test_flex_mask_graph_breaks.pynew file mode 100644index 00000000000..355112ca4a7--- /dev/null+++ b/tests/test_flex_mask_graph_breaks.py@@ -0,0 +1,136 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""Compiled flex mask: no graph breaks, same mask as eager."""++import pytest++torch = pytest.importorskip("torch")+masking_utils = pytest.importorskip("transformers.masking_utils")+qwen3 = pytest.importorskip("transformers.models.qwen3.configuration_qwen3")++from unsloth.import_fixes import ( # noqa: E402+ _FLEX_MASK_PATCH_FLAG,+ _flex_mask_reads_padding_values,+ fix_transformers_flex_mask_graph_breaks,+)++import inspect # noqa: E402++# transformers before 5.0 builds flex masks from cache_position and never reads the padding values.+NEW_SIGNATURE = "q_length" in inspect.signature(masking_utils.flex_attention_mask).parameters+needs_new_signature = pytest.mark.skipif(not NEW_SIGNATURE, reason = "old flex mask signature")++create_causal_mask = getattr(+ masking_utils, "_unsloth_original_create_causal_mask", masking_utils.create_causal_mask+)+++def _config():+ config = qwen3.Qwen3Config(+ hidden_size = 16,+ num_attention_heads = 2,+ num_key_value_heads = 1,+ head_dim = 8,+ num_hidden_layers = 2,+ )+ config._attn_implementation = "flex_attention"+ return config+++def _build(+ config,+ embeds,+ attention_mask,+ position_ids = None,+):+ return create_causal_mask(+ config = config,+ inputs_embeds = embeds,+ attention_mask = attention_mask,+ past_key_values = None,+ position_ids = position_ids,+ )+++def _elements(block_mask, length):+ q, kv = torch.arange(length)[:, None], torch.arange(length)[None]+ zero = torch.zeros((), dtype = torch.long)+ return torch.stack(+ [block_mask.mask_mod(torch.tensor(b), zero, q, kv) for b in range(block_mask.shape[0])]+ )+++@needs_new_signature+def test_compiled_flex_mask_has_no_graph_breaks_and_matches_eager():+ fix_transformers_flex_mask_graph_breaks()+ fix_transformers_flex_mask_graph_breaks()+ assert getattr(masking_utils.create_block_mask, _FLEX_MASK_PATCH_FLAG, False)+ config = _config()+ compiled = torch.compile(_build, backend = "eager", fullgraph = True, dynamic = True)+ cases = [+ (torch.ones(2, 200, dtype = torch.long), None),+ (torch.tensor([[1] * 300, [1] * 250 + [0] * 50]), None),+ (None, torch.cat([torch.arange(100), torch.arange(60), torch.arange(40)])[None]),+ ]+ for attention_mask, position_ids in cases:+ length = (attention_mask if attention_mask is not None else position_ids).shape[-1]+ embeds = torch.zeros(1 if attention_mask is None else 2, length, 16)+ eager = _build(config, embeds, attention_mask, position_ids)+ traced = compiled(config, embeds, attention_mask, position_ids)+ assert torch.equal(eager.to_dense(), traced.to_dense())+ assert torch.equal(_elements(eager, length), _elements(traced, length))+ assert not _elements(traced, 200)[0, 100:160, :100].any()+++def test_probe_reads_only_value_branching_builders():+ def branches(+ batch_size,+ q_length,+ kv_length,+ attention_mask = None,+ device = "cpu",+ **kwargs,+ ):+ if attention_mask is not None and not attention_mask.all():+ pass+ return masking_utils.create_block_mask(None)++ def applies(+ batch_size,+ q_length,+ kv_length,+ attention_mask = None,+ device = "cpu",+ **kwargs,+ ):+ return masking_utils.create_block_mask(None)++ builder = masking_utils.create_block_mask+ assert _flex_mask_reads_padding_values(masking_utils, branches)+ assert not _flex_mask_reads_padding_values(masking_utils, applies)+ assert masking_utils.create_block_mask is builder+++@needs_new_signature+def test_eager_calls_reach_the_original():+ fix_transformers_flex_mask_graph_breaks()+ patched = masking_utils.ALL_MASK_ATTENTION_FUNCTIONS["flex_attention"]+ original = getattr(patched, "__wrapped__", patched)+ calls = []+ builder = masking_utils.create_block_mask+ masking_utils.create_block_mask = lambda *args, **kwargs: calls.append(kwargs) or "mask"+ try:+ mask = torch.ones(1, 4, dtype = torch.bool)+ kwargs = dict(batch_size = 1, q_length = 4, kv_length = 4, attention_mask = mask)+ assert patched(**kwargs) == original(**kwargs) == "mask"+ assert calls[0].keys() == calls[1].keys()+ finally:+ masking_utils.create_block_mask = builder++[email protected](NEW_SIGNATURE, reason = "new flex mask signature")+def test_old_flex_mask_builder_is_left_alone():+ original = masking_utils.ALL_MASK_ATTENTION_FUNCTIONS["flex_attention"]+ fix_transformers_flex_mask_graph_breaks()+ assert masking_utils.ALL_MASK_ATTENTION_FUNCTIONS["flex_attention"] is originaldiff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.pyindex 9a48f080c8a..195507ab15e 100644--- a/unsloth/_gpu_init.py+++ b/unsloth/_gpu_init.py@@ -312,6 +312,7 @@ from .import_fixes import (fix_transformers_untrusted_config_fields,fix_transformers_chat_template_path_traversal,fix_transformers_chunked_mask_block_sequence_ids,+ fix_transformers_flex_mask_graph_breaks,fix_transformers_longcat_lsa_config,fix_transformers_rope_scaling_drops_theta,fix_transformers_fp8_modulelist_experts,@@ -371,6 +372,7 @@ fix_transformers5_bare_annotation_configs()# same process is covered too (#9708).fix_transformers_fully_masked_rows()fix_transformers_chunked_mask_block_sequence_ids()+fix_transformers_flex_mask_graph_breaks()# CVE-2026-4372 / 5241 / 9856, no-ops once transformers carries the fix; before any config loads.fix_transformers_untrusted_config_fields()fix_transformers_chat_template_path_traversal()diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.pyindex 1fac14719df..317f4f7eb38 100644--- a/unsloth/import_fixes.py+++ b/unsloth/import_fixes.py@@ -2194,6 +2194,113 @@ def fix_transformers_chunked_mask_block_sequence_ids():logger.info(f"Unsloth: Failed patching create_chunked_causal_mask ({e})")+_FLEX_MASK_PATCH_FLAG = "_unsloth_patched_flex_mask_graph_breaks"+++def _is_dynamo_compiling():+ import torch+ is_compiling = getattr(getattr(torch, "compiler", None), "is_compiling", None)+ return bool(is_compiling()) if callable(is_compiling) else False+++def _flex_mask_reads_padding_values(masking_utils, flex_attention_mask):+ """True if building from a meta mask fails (values read) while no mask succeeds."""+ import torch++ try:+ if "q_length" not in inspect.signature(flex_attention_mask).parameters:+ return False+ except (TypeError, ValueError):+ return False+ builder = masking_utils.create_block_mask+ masking_utils.create_block_mask = lambda *args, **kwargs: None+ kwargs = dict(batch_size = 1, q_length = 2, kv_length = 2, device = "meta")+ try:+ try:+ flex_attention_mask(attention_mask = None, **kwargs)+ except Exception:+ return False+ try:+ mask = torch.ones((1, 2), dtype = torch.bool, device = "meta")+ flex_attention_mask(attention_mask = mask, **kwargs)+ except Exception:+ return True+ return False+ finally:+ masking_utils.create_block_mask = builder+++def fix_transformers_flex_mask_graph_breaks():+ """While dynamo traces, always apply the padding mask (all-ones gives an equal BlockMask) instead+ of branching on `fast_all`, and drop `_compile`, whose deprecation warnings.warn breaks the graph."""+ try:+ from transformers import masking_utils+ except Exception as e:+ logger.info(f"Unsloth: Skipping the flex mask graph break fix ({e})")+ return+ interface = getattr(masking_utils, "ALL_MASK_ATTENTION_FUNCTIONS", None)+ builder = getattr(masking_utils, "create_block_mask", None)+ if interface is None or builder is None:+ return+ try:+ if not getattr(builder, _FLEX_MASK_PATCH_FLAG, False):++ @functools.wraps(builder)+ def create_block_mask(+ *args,+ _original = builder,+ **kwargs,+ ):+ if _is_dynamo_compiling():+ kwargs.pop("_compile", None)+ return _original(*args, **kwargs)++ setattr(create_block_mask, _FLEX_MASK_PATCH_FLAG, True)+ masking_utils.create_block_mask = create_block_mask++ current = interface["flex_attention"]+ if getattr(current, _FLEX_MASK_PATCH_FLAG, False):+ return+ # Before torch 2.6 the padded branch also pads the mask to the 128 block size.+ if not getattr(masking_utils, "_is_torch_greater_or_equal_than_2_6", False):+ return+ if not _flex_mask_reads_padding_values(masking_utils, current):+ return+ original = current++ @functools.wraps(original)+ def flex_attention_mask(*args, **kwargs):+ attention_mask = kwargs.get("attention_mask")+ if (+ args+ or attention_mask is None+ or "kv_length" not in kwargs+ or not _is_dynamo_compiling()+ ):+ return original(*args, **kwargs)+ padding_mask = masking_utils.prepare_padding_mask(+ attention_mask, kwargs["kv_length"], kwargs.get("kv_offset", 0)+ )+ kwargs["mask_function"] = masking_utils.and_masks(+ kwargs.get("mask_function", masking_utils.causal_mask_function),+ masking_utils.padding_mask_function(padding_mask),+ )+ kwargs["attention_mask"] = None+ return original(**kwargs)++ flex_attention_mask.__wrapped__ = original+ setattr(flex_attention_mask, _FLEX_MASK_PATCH_FLAG, True)+ interface.register("flex_attention", flex_attention_mask)+ if getattr(masking_utils, "flex_attention_mask", None) is original:+ masking_utils.flex_attention_mask = flex_attention_mask+ logger.info(+ "Unsloth: Patching transformers `flex_attention_mask` so a compiled mask "+ "builds without graph breaks"+ )+ except Exception as e:+ logger.info(f"Unsloth: Failed patching flex_attention_mask ({e})")++_COMPOSITE_PREFIX_RENAMING_FLAG = "_unsloth_patched_composite_prefix_renaming"# unsloth_zoo marks its own copy of this repair with this. Spelled as a literal rather than