Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Load Mamba / FalconMamba with eager instead of a refused sdpa (#15076)

Daniel Hand4dab132e04c
diff --git a/tests/test_native_no_sdpa_class.py b/tests/test_native_no_sdpa_class.py
new file mode 100644
index 00000000000..226ca7ca9da
--- /dev/null
+++ b/tests/test_native_no_sdpa_class.py
@@ -0,0 +1,62 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+"""A transformers class whose _supports_sdpa resolves False (Mamba, FalconMamba) resolves to eager even when
+the zoo's source probe reports sdpa: transformers raises "does not support ... scaled_dot_product_attention"."""
+
+from types import SimpleNamespace
+
+import pytest
+import unsloth # noqa: F401
+
+from unsloth.models import _utils
+
+
[email protected](autouse = True)
+def _no_flash(monkeypatch):
+ monkeypatch.setattr(_utils, "HAS_FLASH_ATTENTION", False, raising = False)
+
+
+ "module_name, class_name, model_type",
+ [
+ ("transformers.models.mamba.modeling_mamba", "MambaForCausalLM", "mamba"),
+ (
+ "transformers.models.falcon_mamba.modeling_falcon_mamba",
+ "FalconMambaForCausalLM",
+ "falcon_mamba",
+ ),
+ ],
+)
+def test_real_mamba_classes_resolve_to_eager(module_name, class_name, model_type):
+ cls = getattr(pytest.importorskip(module_name), class_name)
+ if cls._supports_sdpa:
+ pytest.skip(reason = f"this transformers gives {class_name} an sdpa path")
+ config = SimpleNamespace(model_type = model_type)
+ # supports_sdpa = True is what unsloth_compile_transformers reports for these classes.
+ impl = _utils.resolve_attention_implementation(cls, config, supports_sdpa = True)
+ assert impl == "eager"
+ # The resolved value is what the loader stamps on the config and passes to from_pretrained.
+ assert config._attn_implementation == "eager"
+
+
+def test_native_class_with_sdpa_keeps_sdpa():
+ from transformers.models.llama.modeling_llama import LlamaForCausalLM
+
+ assert LlamaForCausalLM._supports_sdpa
+ config = SimpleNamespace(model_type = "llama", attention_dropout = 0)
+ assert (
+ _utils.resolve_attention_implementation(LlamaForCausalLM, config, supports_sdpa = True)
+ == "sdpa"
+ )
+
+
+def test_only_transformers_classes_are_checked():
+ # Remote code that does not declare _supports_sdpa keeps the source-probe answer.
+ remote = type("RemoteForCausalLM", (), {"__module__": "transformers_modules.repo.modeling"})
+ remote._supports_sdpa = False
+ assert not _utils._native_class_refuses_sdpa(remote)
+ native = type("NativeForCausalLM", (), {"__module__": "transformers.models.x.modeling_x"})
+ native._supports_sdpa = False
+ assert _utils._native_class_refuses_sdpa(native)
+ native._supports_sdpa = True
+ assert not _utils._native_class_refuses_sdpa(native)
+ assert not _utils._native_class_refuses_sdpa(None)
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index 764e3508653..855dfc61688 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -1171,6 +1171,15 @@ def _declares_no_sdpa(model_class):
return False
+def _native_class_refuses_sdpa(model_class):
+ # A transformers-shipped class whose _supports_sdpa resolves False (Mamba, FalconMamba inherit PreTrainedModel's
+ # default): transformers raises on sdpa for it, so the zoo's registry-based "sdpa" guess must not win. Remote code keeps the guess.
+ module = getattr(model_class, "__module__", None) or ""
+ if not (module == "transformers" or module.startswith("transformers.")):
+ return False
+ return getattr(model_class, "_supports_sdpa", None) is False
+
+
def _model_class_supports_flash_attention(model_class):
"""Whether installed transformers lets this class dispatch flash attention."""
if model_class is None:
@@ -2649,7 +2658,11 @@ def resolve_attention_implementation(
model_type = model_type_name.lower()
if supports_sdpa is None:
supports_sdpa = model_class is not None and getattr(model_class, "_supports_sdpa", False)
- if _is_sdpa_excluded(model_type) or _declares_no_sdpa(model_class):
+ if (
+ _is_sdpa_excluded(model_type)
+ or _declares_no_sdpa(model_class)
+ or _native_class_refuses_sdpa(model_class)
+ ):
supports_sdpa = False
if not supports_sdpa:
_remember_no_sdpa_config(config)