Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Keep Llama 3.2 Vision off flash attention (vision and cross attention have no is_causal) (#12033)

Daniel Han7b33e85ca0a1
diff --git a/tests/test_mllama_flash_attention_excluded.py b/tests/test_mllama_flash_attention_excluded.py
new file mode 100644
index 00000000000..ce8f9e613df
--- /dev/null
+++ b/tests/test_mllama_flash_attention_excluded.py
@@ -0,0 +1,137 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+import pytest
+import torch
+import unsloth # noqa: F401
+import transformers
+
+from unsloth.models import _utils
+
+pytestmark = pytest.mark.skipif(
+ not hasattr(transformers, "MllamaConfig"), reason = "needs transformers with Mllama"
+)
+
+
+def _tiny_mllama():
+ from transformers.models.mllama.modeling_mllama import MllamaForConditionalGeneration
+
+ config = transformers.MllamaConfig()
+ v = config.vision_config
+ v.hidden_size, v.attention_heads, v.intermediate_size = 64, 4, 128
+ v.num_hidden_layers, v.num_global_layers, v.vision_output_dim = 2, 1, 128
+ v.image_size, v.patch_size, v.max_num_tiles = 56, 14, 1
+ v.intermediate_layers_indices, v.supported_aspect_ratios = [0], [[1, 1]]
+ t = config.text_config
+ t.hidden_size, t.num_attention_heads, t.num_key_value_heads, t.intermediate_size = 64, 4, 2, 128
+ t.num_hidden_layers, t.cross_attention_layers, t.vocab_size, t.pad_token_id = 3, [1], 256, 0
+ # Transformers 4.x defaults rope_scaling to None, which MllamaRotaryEmbedding subscripts.
+ if getattr(t, "rope_scaling", None) is None:
+ t.rope_scaling = {"rope_type": "default"}
+ config.image_token_index = 255
+ return MllamaForConditionalGeneration, config
+
+
+def test_mllama_resolves_off_flash_even_when_flash_is_available(monkeypatch):
+ monkeypatch.setattr(_utils, "HAS_FLASH_ATTENTION", True)
+ model_class, config = _tiny_mllama()
+ assert _utils._model_class_supports_flash_attention(model_class)
+ impl = _utils.resolve_attention_implementation(model_class, config, supports_sdpa = True)
+ assert impl in ("sdpa", "eager")
+ assert _utils._get_flash_attention_disable_reason(config) is not None
+
+
+ "requested",
+ [{"": "eager"}, {"": "sdpa", "vision_config": "eager"}],
+)
+def test_explicit_non_flash_mappings_are_kept(monkeypatch, requested):
+ monkeypatch.setattr(_utils, "HAS_FLASH_ATTENTION", True)
+ model_class, config = _tiny_mllama()
+ impl = _utils.resolve_attention_implementation(
+ model_class, config, requested_attn_implementation = dict(requested), supports_sdpa = True
+ )
+ assert impl == requested
+
+
+def test_flash_entries_of_a_mapping_fall_back(monkeypatch):
+ monkeypatch.setattr(_utils, "HAS_FLASH_ATTENTION", True)
+ model_class, config = _tiny_mllama()
+ impl = _utils.resolve_attention_implementation(
+ model_class,
+ config,
+ requested_attn_implementation = {"": "flash_attention_2", "vision_config": "eager"},
+ supports_sdpa = True,
+ )
+ assert impl == {"": "sdpa", "vision_config": "eager"}
+
+
[email protected](not torch.cuda.is_available(), reason = "needs CUDA")
+def test_mllama_image_forward_runs_on_the_resolved_implementation(monkeypatch):
+ monkeypatch.setattr(_utils, "HAS_FLASH_ATTENTION", True)
+ model_class, config = _tiny_mllama()
+ impl = _utils.resolve_attention_implementation(model_class, config, supports_sdpa = True)
+ dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16
+ # No dtype keyword: Transformers 4.x names it torch_dtype, 5.x dtype.
+ model = model_class._from_config(config, attn_implementation = impl).to("cuda", dtype).eval()
+ ids = torch.tensor([[255, 5, 6, 7, 8, 9]], device = "cuda")
+ pixels = torch.randn(1, 1, 1, 3, 56, 56, device = "cuda", dtype = dtype)
+ with torch.no_grad():
+ out = model(
+ input_ids = ids,
+ pixel_values = pixels,
+ aspect_ratio_ids = torch.tensor([[1]], device = "cuda"),
+ aspect_ratio_mask = torch.ones(1, 1, 1, device = "cuda", dtype = torch.long),
+ cross_attention_mask = torch.ones(1, 6, 1, 1, device = "cuda", dtype = torch.long),
+ )
+ assert torch.isfinite(out.logits.float()).all()
+
+
+def test_mapping_entries_keep_the_backend_exclusions(monkeypatch):
+ # A mapping must not re-enable what a scalar request cannot: flex on mllama, sdpa on gpt_oss.
+ monkeypatch.setattr(_utils, "HAS_FLASH_ATTENTION", True)
+ model_class, config = _tiny_mllama()
+ impl = _utils.resolve_attention_implementation(
+ model_class,
+ config,
+ requested_attn_implementation = {"": "flex_attention", "vision_config": "eager"},
+ supports_sdpa = True,
+ )
+ assert impl == {"": "sdpa", "vision_config": "eager"}
+ if not hasattr(transformers, "GptOssConfig"):
+ return
+ from transformers.models.gpt_oss.modeling_gpt_oss import GptOssForCausalLM
+
+ impl = _utils.resolve_attention_implementation(
+ GptOssForCausalLM,
+ transformers.GptOssConfig(),
+ requested_attn_implementation = {"": "sdpa"},
+ )
+ values = impl.values() if isinstance(impl, dict) else [impl]
+ assert "sdpa" not in values
+
+
+def test_mapping_fallback_is_applied_per_key(monkeypatch):
+ # A large-head VLM falls back to {"": sdpa, text_config: flex}; a dict request must not collapse it to sdpa.
+ if not hasattr(transformers, "Qwen2VLConfig"):
+ pytest.skip("needs transformers with Qwen2-VL")
+ from transformers.models.qwen2_vl.modeling_qwen2_vl import Qwen2VLForConditionalGeneration
+
+ monkeypatch.setattr(_utils, "HAS_FLASH_ATTENTION", True)
+
+ def config():
+ c = transformers.Qwen2VLConfig()
+ t = c.text_config
+ t.hidden_size, t.num_attention_heads, t.num_key_value_heads, t.head_dim = 2048, 4, 2, 512
+ return c
+
+ scalar = _utils.resolve_attention_implementation(
+ Qwen2VLForConditionalGeneration, config(), "flash_attention_2", supports_sdpa = True
+ )
+ if not isinstance(scalar, dict):
+ pytest.skip("flex attention cannot be scoped to the decoder here")
+ impl = _utils.resolve_attention_implementation(
+ Qwen2VLForConditionalGeneration,
+ config(),
+ {"": "flash_attention_2", "text_config": "flash_attention_2"},
+ supports_sdpa = True,
+ )
+ assert impl == {"": scalar[""], "text_config": scalar["text_config"]}
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index 6d0582f83aa..bdfccc5bbe8 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -384,8 +384,9 @@ DISABLE_SDPA_MODEL_NAMES = [
"gemma3_text", # Gemma3TextModel (EmbeddingGemma) - substring match, keep underscore
"gpt_oss",
]
-_FLASH_EXCLUDED_MODELS = ("gpt_oss", "deepseek_v4")
+_FLASH_EXCLUDED_MODELS = ("gpt_oss", "deepseek_v4", "mllama")
# deepseek_v4's custom attention is sdpa/flash-incompatible: force eager, and it is excluded above so an explicit request cannot re-enable the crash.
+# mllama declares flash support, but its vision and cross attention modules have no is_causal, which the flash path reads.
_EAGER_ONLY_PREFIXES = ("gemma3n", "deepseek_v4")
_FLASH_ATTENTION_MAX_HEAD_DIM = 256
_FLASH_ATTENTION_DISABLED_WARNED = set()
@@ -1046,6 +1047,8 @@ def _get_max_attention_head_dim(config):
def _get_flash_attention_disable_reason(config):
model_type = _config_get(config, "model_type", "").lower()
if _is_flash_excluded(model_type):
+ if model_type == "mllama":
+ return "mllama vision and cross attention do not run under Flash Attention 2"
return f"{model_type} uses custom sink attention kernels"
max_head_dim = _get_max_attention_head_dim(config)
if max_head_dim is not None and max_head_dim > _FLASH_ATTENTION_MAX_HEAD_DIM:
@@ -1083,6 +1086,43 @@ def _disable_flash_attention_if_needed(
# Only an implementation passed by the caller is an explicit request: config values are synthesized by the loaders or come from Transformers defaults.
explicit_request = attn_implementation
+ # A per-sub-config mapping keeps the entries a scalar request would keep; flash and excluded backends take the fallback.
+ if isinstance(explicit_request, dict):
+ excluded_model_type = _config_get(config, "model_type", "").lower()
+
+ def _needs_fallback(impl):
+ if _is_flash_attention_requested(impl):
+ return True
+ if impl == "sdpa":
+ return _is_sdpa_excluded(excluded_model_type)
+ if impl == "flex_attention":
+ return not supports_flex_attention
+ return False
+
+ fallback = _disable_flash_attention_if_needed(
+ config,
+ supports_sdpa = supports_sdpa,
+ supports_flex_attention = supports_flex_attention,
+ would_use_flash_attention = any(
+ _is_flash_attention_requested(v) for v in explicit_request.values()
+ ),
+ disable_reason = disable_reason,
+ honor_config_attn_implementation = False,
+ )
+
+ def _fallback_for(key):
+ if isinstance(fallback, dict):
+ return fallback.get(key, fallback.get("", "eager"))
+ return fallback
+
+ return _set_attn_impl(
+ config,
+ {
+ k: (_fallback_for(k) if _needs_fallback(v) else v)
+ for k, v in explicit_request.items()
+ },
+ )
+
# Off for a float32 load: with no flash-specific reason the config never steered the choice, so a config-seeded "eager" must not drag an fp32 load from sdpa down to eager.
requested_attn_implementation = attn_implementation
if honor_config_attn_implementation: