Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Keep flash attention off sub-models that do not support it (LFM2.5-VL SigLIP2 tower) (#11959)
Daniel Han6a4af104bef3
diff --git a/tests/test_flash_attention_sub_config_fallback.py b/tests/test_flash_attention_sub_config_fallback.pynew file mode 100644index 00000000000..9797512e922--- /dev/null+++ b/tests/test_flash_attention_sub_config_fallback.py@@ -0,0 +1,119 @@+# SPDX-License-Identifier: AGPL-3.0-only+# A plain "flash_attention_2" reaches every sub-config; LFM2-VL's SigLIP2 tower raises at init on it.+import pytest+import unsloth # noqa: F401+import transformers++from unsloth.models import _utils++pytestmark = pytest.mark.skipif(+ not hasattr(transformers, "Lfm2VlConfig"), reason = "needs transformers with LFM2-VL"+)+++def _flash_available(monkeypatch):+ monkeypatch.setattr(_utils, "HAS_FLASH_ATTENTION", True)+ monkeypatch.setattr(_utils, "_get_flash_attention_disable_reason", lambda config: None)+++def _lfm2_vl():+ from transformers.models.lfm2_vl.modeling_lfm2_vl import Lfm2VlForConditionalGeneration+ return Lfm2VlForConditionalGeneration, transformers.Lfm2VlConfig()+++def test_lfm2_vl_keeps_flash_off_the_siglip2_tower(monkeypatch):+ _flash_available(monkeypatch)+ model_class, config = _lfm2_vl()+ assert model_class._supports_flash_attn+ impl = _utils.resolve_attention_implementation(model_class, config, supports_sdpa = True)+ assert impl == {"": "flash_attention_2", "vision_config": "sdpa"}+ assert config.vision_config._attn_implementation == "sdpa"+ assert config.text_config._attn_implementation == "flash_attention_2"+++def test_explicit_flash_request_is_scoped_too(monkeypatch):+ _flash_available(monkeypatch)+ model_class, config = _lfm2_vl()+ impl = _utils.resolve_attention_implementation(+ model_class,+ config,+ requested_attn_implementation = "flash_attention_2",+ supports_sdpa = True,+ )+ assert impl == {"": "flash_attention_2", "vision_config": "sdpa"}+++def test_scoped_mapping_constructs_the_model(monkeypatch):+ _flash_available(monkeypatch)+ import torch+ from transformers import modeling_utils++ if not hasattr(modeling_utils, "lazy_import_flash_attention") or not hasattr(+ modeling_utils.PreTrainedModel, "_flash_attn_import_error"+ ):+ pytest.skip(reason = "flash_attn package checks not stubbable on this Transformers")+ # Stub only the package / kernel import so the per-class support check runs without flash_attn.+ monkeypatch.setattr(+ modeling_utils.PreTrainedModel, "_flash_attn_import_error", lambda self, **kwargs: None+ )+ monkeypatch.setattr(modeling_utils, "lazy_import_flash_attention", lambda *a, **k: None)+ model_class, config = _lfm2_vl()+ config.text_config.num_hidden_layers = 2+ config.text_config.layer_types = ["full_attention", "conv"]+ config.vision_config.num_hidden_layers = 1+ impl = _utils.resolve_attention_implementation(model_class, config, supports_sdpa = True)+ with torch.device("meta"):+ model_class._from_config(config, attn_implementation = impl, dtype = torch.bfloat16)+ with torch.device("meta"), pytest.raises(ValueError, match = "Flash Attention 2"):+ model_class._from_config(+ transformers.Lfm2VlConfig(),+ attn_implementation = "flash_attention_2",+ dtype = torch.bfloat16,+ )+++def test_all_flash_capable_sub_models_keep_the_plain_string(monkeypatch):+ _flash_available(monkeypatch)+ from transformers.models.gemma3.modeling_gemma3 import Gemma3ForConditionalGeneration++ config = transformers.Gemma3Config()+ assert _utils._flash_unsupported_sub_configs(config) == {}+ assert _utils._scoped_flash_attention(config, True) == "flash_attention_2"+ assert Gemma3ForConditionalGeneration is not None+++def test_text_only_config_is_unchanged(monkeypatch):+ _flash_available(monkeypatch)+ from transformers.models.lfm2.modeling_lfm2 import Lfm2ForCausalLM++ config = transformers.Lfm2Config()+ impl = _utils.resolve_attention_implementation(Lfm2ForCausalLM, config, supports_sdpa = True)+ assert impl == "flash_attention_2"+++ "unsupported, expected",+ [({"vision_config": "sdpa"}, "sdpa"), ({"vision_config": "eager"}, "eager")],+)+def test_without_the_mapping_form_the_global_fallback_suits_every_tower(+ monkeypatch, unsupported, expected+):+ monkeypatch.setattr(_utils, "_transformers_supports_attn_impl_mapping", lambda: False)+ monkeypatch.setattr(_utils, "_flash_unsupported_sub_configs", lambda config: unsupported)+ assert _utils._scoped_flash_attention(object(), True) == expected+ assert _utils._scoped_flash_attention(object(), False) == "eager"+++def test_encoder_decoder_towers_are_scoped_by_their_declared_names(monkeypatch):+ # Donut: flash-capable VisionEncoderDecoderModel over a DonutSwin encoder with neither flash nor SDPA.+ _flash_available(monkeypatch)+ from transformers import DonutSwinConfig, GPT2Config, VisionEncoderDecoderConfig+ from transformers import VisionEncoderDecoderModel++ config = VisionEncoderDecoderConfig.from_encoder_decoder_configs(+ DonutSwinConfig(depths = [1, 1], num_heads = [2, 4]), GPT2Config(n_layer = 1)+ )+ impl = _utils.resolve_attention_implementation(+ VisionEncoderDecoderModel, config, supports_sdpa = True+ )+ assert impl == {"": "flash_attention_2", "encoder": "eager"}diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex 87870e1c7cc..6d0582f83aa 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -728,6 +728,54 @@ def _flex_attn_impl_for(config, other_attn_implementation):return None+def _flash_unsupported_sub_configs(config):+ """{sub-config: fallback} for towers lacking flash, which Transformers rejects (LFM2-VL SigLIP2)."""+ try:+ from transformers import AutoModel, AutoModelForCausalLM+ mappings = (AutoModel._model_mapping, AutoModelForCausalLM._model_mapping)+ except Exception:+ return {}+ # Encoder-decoder composites (Donut, MusicGen) name their towers "encoder" / "decoder".+ declared = getattr(config, "sub_configs", None) or ()+ out = {}+ for field_name, child_config in _config_items(config):+ if not isinstance(field_name, str):+ continue+ if not (field_name.endswith("_config") or field_name in declared):+ continue+ if not hasattr(child_config, "model_type"):+ continue+ child_class = None+ for mapping in mappings:+ try:+ child_class = mapping[type(child_config)]+ break+ except Exception:+ continue+ if isinstance(child_class, (list, tuple)):+ child_class = child_class[0] if child_class else None+ if child_class is None:+ continue+ if getattr(child_class, "_supports_flash_attn", False) or getattr(+ child_class, "_supports_flash_attn_2", False+ ):+ continue+ out[field_name] = "sdpa" if getattr(child_class, "_supports_sdpa", False) else "eager"+ return out+++def _scoped_flash_attention(config, supports_sdpa):+ unsupported = _flash_unsupported_sub_configs(config)+ if not unsupported:+ return "flash_attention_2"+ if not _transformers_supports_attn_impl_mapping():+ # One plain value reaches every tower, so it must suit the weakest (Pixtral on 4.51).+ if not supports_sdpa or "eager" in unsupported.values():+ return "eager"+ return "sdpa"+ return {"": "flash_attention_2", **unsupported}++def _flex_support_anchor_class(model_class):"""The architecture's own PreTrainedModel base, so `_supports_flex_attn` also covers the innertext model Transformers validates separately under a mapping."""@@ -2008,7 +2056,7 @@ def resolve_attention_implementation(and supports_flash_attentionand not flex_forced_for_head_dim):- attn_impl = _set_attn_impl(config, "flash_attention_2")+ attn_impl = _set_attn_impl(config, _scoped_flash_attention(config, supports_sdpa))elif flash_attention_disabled:attn_impl = _disable_flash_attention_if_needed(config,@@ -2056,6 +2104,8 @@ def resolve_attention_implementation()else:final_attn_impl = requested_attn_implementation+ if final_attn_impl == "flash_attention_2":+ final_attn_impl = _scoped_flash_attention(config, supports_sdpa)_set_attn_impl(config, final_attn_impl)# An explicit "sdpa" is kept even on a conservatively unsupported model, except where SDPA is known-broken, which still downgrades to eager just as flex falls back for _FLEX_EXCLUDED_MODELS. A synthesized default sdpa (requested is None) also downgrades.