Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Keep Llama 3.2 Vision off flash attention (vision and cross attention have no is_causal) (#12033)
Daniel Han7b33e85ca0a1
diff --git a/tests/test_mllama_flash_attention_excluded.py b/tests/test_mllama_flash_attention_excluded.pynew file mode 100644index 00000000000..ce8f9e613df--- /dev/null+++ b/tests/test_mllama_flash_attention_excluded.py@@ -0,0 +1,137 @@+# SPDX-License-Identifier: AGPL-3.0-only+import pytest+import torch+import unsloth # noqa: F401+import transformers++from unsloth.models import _utils++pytestmark = pytest.mark.skipif(+ not hasattr(transformers, "MllamaConfig"), reason = "needs transformers with Mllama"+)+++def _tiny_mllama():+ from transformers.models.mllama.modeling_mllama import MllamaForConditionalGeneration++ config = transformers.MllamaConfig()+ v = config.vision_config+ v.hidden_size, v.attention_heads, v.intermediate_size = 64, 4, 128+ v.num_hidden_layers, v.num_global_layers, v.vision_output_dim = 2, 1, 128+ v.image_size, v.patch_size, v.max_num_tiles = 56, 14, 1+ v.intermediate_layers_indices, v.supported_aspect_ratios = [0], [[1, 1]]+ t = config.text_config+ t.hidden_size, t.num_attention_heads, t.num_key_value_heads, t.intermediate_size = 64, 4, 2, 128+ t.num_hidden_layers, t.cross_attention_layers, t.vocab_size, t.pad_token_id = 3, [1], 256, 0+ # Transformers 4.x defaults rope_scaling to None, which MllamaRotaryEmbedding subscripts.+ if getattr(t, "rope_scaling", None) is None:+ t.rope_scaling = {"rope_type": "default"}+ config.image_token_index = 255+ return MllamaForConditionalGeneration, config+++def test_mllama_resolves_off_flash_even_when_flash_is_available(monkeypatch):+ monkeypatch.setattr(_utils, "HAS_FLASH_ATTENTION", True)+ model_class, config = _tiny_mllama()+ assert _utils._model_class_supports_flash_attention(model_class)+ impl = _utils.resolve_attention_implementation(model_class, config, supports_sdpa = True)+ assert impl in ("sdpa", "eager")+ assert _utils._get_flash_attention_disable_reason(config) is not None+++ "requested",+ [{"": "eager"}, {"": "sdpa", "vision_config": "eager"}],+)+def test_explicit_non_flash_mappings_are_kept(monkeypatch, requested):+ monkeypatch.setattr(_utils, "HAS_FLASH_ATTENTION", True)+ model_class, config = _tiny_mllama()+ impl = _utils.resolve_attention_implementation(+ model_class, config, requested_attn_implementation = dict(requested), supports_sdpa = True+ )+ assert impl == requested+++def test_flash_entries_of_a_mapping_fall_back(monkeypatch):+ monkeypatch.setattr(_utils, "HAS_FLASH_ATTENTION", True)+ model_class, config = _tiny_mllama()+ impl = _utils.resolve_attention_implementation(+ model_class,+ config,+ requested_attn_implementation = {"": "flash_attention_2", "vision_config": "eager"},+ supports_sdpa = True,+ )+ assert impl == {"": "sdpa", "vision_config": "eager"}++[email protected](not torch.cuda.is_available(), reason = "needs CUDA")+def test_mllama_image_forward_runs_on_the_resolved_implementation(monkeypatch):+ monkeypatch.setattr(_utils, "HAS_FLASH_ATTENTION", True)+ model_class, config = _tiny_mllama()+ impl = _utils.resolve_attention_implementation(model_class, config, supports_sdpa = True)+ dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16+ # No dtype keyword: Transformers 4.x names it torch_dtype, 5.x dtype.+ model = model_class._from_config(config, attn_implementation = impl).to("cuda", dtype).eval()+ ids = torch.tensor([[255, 5, 6, 7, 8, 9]], device = "cuda")+ pixels = torch.randn(1, 1, 1, 3, 56, 56, device = "cuda", dtype = dtype)+ with torch.no_grad():+ out = model(+ input_ids = ids,+ pixel_values = pixels,+ aspect_ratio_ids = torch.tensor([[1]], device = "cuda"),+ aspect_ratio_mask = torch.ones(1, 1, 1, device = "cuda", dtype = torch.long),+ cross_attention_mask = torch.ones(1, 6, 1, 1, device = "cuda", dtype = torch.long),+ )+ assert torch.isfinite(out.logits.float()).all()+++def test_mapping_entries_keep_the_backend_exclusions(monkeypatch):+ # A mapping must not re-enable what a scalar request cannot: flex on mllama, sdpa on gpt_oss.+ monkeypatch.setattr(_utils, "HAS_FLASH_ATTENTION", True)+ model_class, config = _tiny_mllama()+ impl = _utils.resolve_attention_implementation(+ model_class,+ config,+ requested_attn_implementation = {"": "flex_attention", "vision_config": "eager"},+ supports_sdpa = True,+ )+ assert impl == {"": "sdpa", "vision_config": "eager"}+ if not hasattr(transformers, "GptOssConfig"):+ return+ from transformers.models.gpt_oss.modeling_gpt_oss import GptOssForCausalLM++ impl = _utils.resolve_attention_implementation(+ GptOssForCausalLM,+ transformers.GptOssConfig(),+ requested_attn_implementation = {"": "sdpa"},+ )+ values = impl.values() if isinstance(impl, dict) else [impl]+ assert "sdpa" not in values+++def test_mapping_fallback_is_applied_per_key(monkeypatch):+ # A large-head VLM falls back to {"": sdpa, text_config: flex}; a dict request must not collapse it to sdpa.+ if not hasattr(transformers, "Qwen2VLConfig"):+ pytest.skip("needs transformers with Qwen2-VL")+ from transformers.models.qwen2_vl.modeling_qwen2_vl import Qwen2VLForConditionalGeneration++ monkeypatch.setattr(_utils, "HAS_FLASH_ATTENTION", True)++ def config():+ c = transformers.Qwen2VLConfig()+ t = c.text_config+ t.hidden_size, t.num_attention_heads, t.num_key_value_heads, t.head_dim = 2048, 4, 2, 512+ return c++ scalar = _utils.resolve_attention_implementation(+ Qwen2VLForConditionalGeneration, config(), "flash_attention_2", supports_sdpa = True+ )+ if not isinstance(scalar, dict):+ pytest.skip("flex attention cannot be scoped to the decoder here")+ impl = _utils.resolve_attention_implementation(+ Qwen2VLForConditionalGeneration,+ config(),+ {"": "flash_attention_2", "text_config": "flash_attention_2"},+ supports_sdpa = True,+ )+ assert impl == {"": scalar[""], "text_config": scalar["text_config"]}diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex 6d0582f83aa..bdfccc5bbe8 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -384,8 +384,9 @@ DISABLE_SDPA_MODEL_NAMES = ["gemma3_text", # Gemma3TextModel (EmbeddingGemma) - substring match, keep underscore"gpt_oss",]-_FLASH_EXCLUDED_MODELS = ("gpt_oss", "deepseek_v4")+_FLASH_EXCLUDED_MODELS = ("gpt_oss", "deepseek_v4", "mllama")# deepseek_v4's custom attention is sdpa/flash-incompatible: force eager, and it is excluded above so an explicit request cannot re-enable the crash.+# mllama declares flash support, but its vision and cross attention modules have no is_causal, which the flash path reads._EAGER_ONLY_PREFIXES = ("gemma3n", "deepseek_v4")_FLASH_ATTENTION_MAX_HEAD_DIM = 256_FLASH_ATTENTION_DISABLED_WARNED = set()@@ -1046,6 +1047,8 @@ def _get_max_attention_head_dim(config):def _get_flash_attention_disable_reason(config):model_type = _config_get(config, "model_type", "").lower()if _is_flash_excluded(model_type):+ if model_type == "mllama":+ return "mllama vision and cross attention do not run under Flash Attention 2"return f"{model_type} uses custom sink attention kernels"max_head_dim = _get_max_attention_head_dim(config)if max_head_dim is not None and max_head_dim > _FLASH_ATTENTION_MAX_HEAD_DIM:@@ -1083,6 +1086,43 @@ def _disable_flash_attention_if_needed(# Only an implementation passed by the caller is an explicit request: config values are synthesized by the loaders or come from Transformers defaults.explicit_request = attn_implementation+ # A per-sub-config mapping keeps the entries a scalar request would keep; flash and excluded backends take the fallback.+ if isinstance(explicit_request, dict):+ excluded_model_type = _config_get(config, "model_type", "").lower()++ def _needs_fallback(impl):+ if _is_flash_attention_requested(impl):+ return True+ if impl == "sdpa":+ return _is_sdpa_excluded(excluded_model_type)+ if impl == "flex_attention":+ return not supports_flex_attention+ return False++ fallback = _disable_flash_attention_if_needed(+ config,+ supports_sdpa = supports_sdpa,+ supports_flex_attention = supports_flex_attention,+ would_use_flash_attention = any(+ _is_flash_attention_requested(v) for v in explicit_request.values()+ ),+ disable_reason = disable_reason,+ honor_config_attn_implementation = False,+ )++ def _fallback_for(key):+ if isinstance(fallback, dict):+ return fallback.get(key, fallback.get("", "eager"))+ return fallback++ return _set_attn_impl(+ config,+ {+ k: (_fallback_for(k) if _needs_fallback(v) else v)+ for k, v in explicit_request.items()+ },+ )+# Off for a float32 load: with no flash-specific reason the config never steered the choice, so a config-seeded "eager" must not drag an fp32 load from sdpa down to eager.requested_attn_implementation = attn_implementationif honor_config_attn_implementation: