Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Attention resolver: respect a declared _supports_sdpa = False, and skip flash_attention_2 when a class's compatible flash kernels exclude it (MiMo-V2-Flash) (#12147)
Daniel Hanc86708ffa086
diff --git a/tests/test_declared_no_sdpa.py b/tests/test_declared_no_sdpa.pynew file mode 100644index 00000000000..85e77c4077f--- /dev/null+++ b/tests/test_declared_no_sdpa.py@@ -0,0 +1,73 @@+# SPDX-License-Identifier: AGPL-3.0-only+"""A class declaring `_supports_sdpa = False` (MiMo-V2-Flash) resolves to eager, not sdpa."""++from types import SimpleNamespace++import pytest+import unsloth # noqa: F401++from transformers import PretrainedConfig+from transformers.modeling_utils import PreTrainedModel++from unsloth.models import _utils+++class _Pre(PreTrainedModel):+ config_class = PretrainedConfig+ _supports_sdpa = True+++class DeclaresNoSdpa(_Pre):+ _supports_sdpa = False+++class Inherits(_Pre):+ pass+++class BarePreTrained(PreTrainedModel):+ config_class = PretrainedConfig+++def _config():+ return SimpleNamespace(model_type = "declared_no_sdpa_test", attention_dropout = 0)++[email protected](autouse = True)+def _no_flash(monkeypatch):+ monkeypatch.setattr(_utils, "HAS_FLASH_ATTENTION", False, raising = False)+++def test_declared_false_wins_over_a_source_level_sdpa_guess():+ assert (+ _utils.resolve_attention_implementation(DeclaresNoSdpa, _config(), supports_sdpa = True)+ == "eager"+ )+++def test_classes_that_do_not_opt_out_keep_sdpa():+ assert (+ _utils.resolve_attention_implementation(Inherits, _config(), supports_sdpa = True) == "sdpa"+ )+ assert (+ _utils.resolve_attention_implementation(BarePreTrained, _config(), supports_sdpa = True)+ == "sdpa"+ )+++def test_explicit_sdpa_request_is_still_honoured():+ impl = _utils.resolve_attention_implementation(+ DeclaresNoSdpa, _config(), requested_attn_implementation = "sdpa", supports_sdpa = True+ )+ assert impl == "sdpa"+++def test_real_mimo_v2_flash_resolves_to_eager():+ modeling = pytest.importorskip("transformers.models.mimo_v2_flash.modeling_mimo_v2_flash")+ cls = modeling.MiMoV2FlashForCausalLM+ if cls._supports_sdpa:+ pytest.skip(+ reason = "this transformers gives mimo_v2_flash an sdpa path, nothing to resolve away"+ )+ config = SimpleNamespace(model_type = "mimo_v2_flash", attention_dropout = 0)+ assert _utils.resolve_attention_implementation(cls, config, supports_sdpa = True) == "eager"diff --git a/tests/test_flash_compatible_implementations.py b/tests/test_flash_compatible_implementations.pynew file mode 100644index 00000000000..da814fd8efe--- /dev/null+++ b/tests/test_flash_compatible_implementations.py@@ -0,0 +1,79 @@+# SPDX-License-Identifier: AGPL-3.0-only+"""A class whose `_compatible_flash_implementations` excludes FA2 (MiMo-V2-Flash: FA4 only) is not given FA2."""++from types import SimpleNamespace++import pytest+import unsloth # noqa: F401++from unsloth.models import _utils+++class _Base:+ _supports_flash_attn_2 = True+ _supports_flash_attn = True+ _supports_flex_attn = False+ _supports_sdpa = True+++class Fa4Only(_Base):+ _compatible_flash_implementations = ["flash_attention_4"]+++class Fa4OnlyNoSdpa(Fa4Only):+ _supports_sdpa = False+++class ListsFa2(_Base):+ _compatible_flash_implementations = ["flash_attention_4", "flash_attention_2"]++[email protected](autouse = True)+def _no_fa3_fa4_or_hub_kernels(monkeypatch):+ monkeypatch.setattr(_utils, "_flash_implementation_available", lambda name: False)+++def _config(model_type):+ return SimpleNamespace(model_type = model_type, attention_dropout = 0)+++def test_class_without_fa2_in_its_compatible_list_is_not_flash():+ assert _utils._model_class_supports_flash_attention(Fa4Only) is False+ assert _utils._model_class_supports_flash_attention(ListsFa2) is True+ assert _utils._model_class_supports_flash_attention(_Base) is True+++def test_resolver_picks_a_non_flash_backend(monkeypatch):+ monkeypatch.setattr(_utils, "HAS_FLASH_ATTENTION", True, raising = False)+ impl = _utils.resolve_attention_implementation(+ Fa4Only, _config("fa4_only_test"), supports_sdpa = True+ )+ assert impl == "sdpa"+ impl = _utils.resolve_attention_implementation(+ Fa4OnlyNoSdpa, _config("fa4_only_test"), supports_sdpa = False+ )+ assert impl == "eager"+++def test_real_mimo_v2_flash_class_is_not_given_flash_attention_2():+ modeling = pytest.importorskip("transformers.models.mimo_v2_flash.modeling_mimo_v2_flash")+ cls = modeling.MiMoV2FlashForCausalLM+ if "flash_attention_2" in (+ getattr(cls, "_compatible_flash_implementations", None) or ["flash_attention_2"]+ ):+ pytest.skip(+ reason = "this transformers lists flash_attention_2 as compatible for mimo_v2_flash"+ )+ assert _utils._model_class_supports_flash_attention(cls) is False+++def test_flash_is_kept_when_the_switched_to_kernel_is_installed(monkeypatch):+ monkeypatch.setattr(+ _utils, "_flash_implementation_available", lambda name: name == "flash_attention_4"+ )+ assert _utils._model_class_supports_flash_attention(Fa4Only) is True++ class HubOnly(_Base):+ _compatible_flash_implementations = ["kernels-community/some-flash"]++ assert _utils._model_class_supports_flash_attention(HubOnly) is Falsediff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex ff0c37d0bc1..2ca50e825e9 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -826,6 +826,22 @@ def _declares_flex_support(model_class):return None+def _declares_no_sdpa(model_class):+ # An explicit `_supports_sdpa = False` below PreTrainedModel (MiMo-V2-Flash sinks) beats the zoo's source-level guess.+ try:+ from transformers.modeling_utils import PreTrainedModel+ except Exception:+ return False+ for klass in getattr(model_class, "__mro__", ()):+ if klass is PreTrainedModel:+ break+ if not isinstance(klass, type):+ continue+ if "_supports_sdpa" in vars(klass):+ return vars(klass)["_supports_sdpa"] is False+ return False++def _model_class_supports_flash_attention(model_class):"""Whether installed transformers lets this class dispatch flash attention."""if model_class is None:@@ -837,6 +853,8 @@ def _model_class_supports_flash_attention(model_class):new_flag_dispatched = PreTrainedModel is not None and hasattr(PreTrainedModel, "_supports_flash_attn")+ if not _flash_attention_2_is_compatible(model_class):+ return Falseif new_flag_dispatched and not _flash_dispatch_reads_legacy_flag(PreTrainedModel):return bool(getattr(model_class, "_supports_flash_attn", False))return bool(@@ -845,6 +863,40 @@ def _model_class_supports_flash_attention(model_class):)+def _flash_attention_2_is_compatible(model_class):+ # transformers 5 silently rewrites flash_attention_2 to compatible[0] (_check_and_adjust_attn_implementation).+ compatible = getattr(model_class, "_compatible_flash_implementations", None)+ if not isinstance(compatible, (list, tuple)) or not compatible:+ return True+ if any(str(name).split("|")[-1] == "flash_attention_2" for name in compatible):+ return True+ return _flash_implementation_available(str(compatible[0]).split("|")[-1])+++def _flash_implementation_available(name):+ try:+ from transformers.utils import import_utils+ except Exception:+ return False+ checks = {+ "flash_attention_3": "is_flash_attn_3_available",+ "flash_attention_4": "is_flash_attn_4_available",+ }+ if name in checks:+ check = getattr(import_utils, checks[name], None)+ try:+ return bool(check()) if check is not None else False+ except Exception:+ return False+ if "/" in name:+ check = getattr(import_utils, "is_kernels_available", None)+ try:+ return bool(check()) if check is not None else False+ except Exception:+ return False+ return False++def _flash_dispatch_reads_legacy_flag(PreTrainedModel) -> bool:# Read from source: later versions keep the legacy name only in the error message.import inspect@@ -2098,7 +2150,7 @@ def resolve_attention_implementation(model_type = model_type_name.lower()if supports_sdpa is None:supports_sdpa = model_class is not None and getattr(model_class, "_supports_sdpa", False)- if _is_sdpa_excluded(model_type):+ if _is_sdpa_excluded(model_type) or _declares_no_sdpa(model_class):supports_sdpa = Falsesupports_flash_attention = _model_class_supports_flash_attention(model_class