Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Carry the resolved attention implementation to nested configs a remote config baked flash attention into (Nemotron 3 Nano Omni) (#12099)

Daniel Hanf53b1031f64f
diff --git a/tests/test_attn_impl_baked_nested_config.py b/tests/test_attn_impl_baked_nested_config.py
new file mode 100644
index 00000000000..a72b76ad294
--- /dev/null
+++ b/tests/test_attn_impl_baked_nested_config.py
@@ -0,0 +1,83 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+import pytest
+import unsloth # noqa: F401
+from transformers import PretrainedConfig
+
+from unsloth.models import _utils
+
+
+class _Llm(PretrainedConfig):
+ model_type = "baked_llm"
+
+
+class _Vision(PretrainedConfig):
+ model_type = "baked_vision"
+
+
+class _OmniLike(PretrainedConfig):
+ model_type = "baked_omni"
+
+ def __init__(
+ self,
+ attn_implementation = "flash_attention_2",
+ **kwargs,
+ ):
+ super().__init__(**kwargs)
+ self.llm_config = _Llm()
+ self.vision_config = _Vision()
+ self._attn_implementation = attn_implementation
+ self.vision_config.use_flash_attn = "flash_attention" in attn_implementation
+ self.llm_config._attn_implementation = attn_implementation
+
+
+class _FlashOnlyModel:
+ _supports_flash_attn = True
+ _supports_flash_attn_2 = True
+ _supports_sdpa = False
+
+
[email protected]("requested", [None, "sdpa", "eager"])
+def test_non_flash_resolution_reaches_the_baked_decoder(monkeypatch, requested):
+ monkeypatch.setattr(_utils, "HAS_FLASH_ATTENTION", False)
+ config = _OmniLike()
+ impl = _utils.resolve_attention_implementation(
+ _FlashOnlyModel, config, requested_attn_implementation = requested, supports_sdpa = False
+ )
+ assert not str(impl).startswith("flash")
+ assert config.llm_config._attn_implementation == impl
+ assert config.vision_config.use_flash_attn is False
+
+
+def test_flash_resolution_keeps_the_decoder_on_flash(monkeypatch):
+ monkeypatch.setattr(_utils, "HAS_FLASH_ATTENTION", True)
+ monkeypatch.setattr(_utils, "_get_flash_attention_disable_reason", lambda config: None)
+ config = _OmniLike()
+ impl = _utils.resolve_attention_implementation(_FlashOnlyModel, config, supports_sdpa = False)
+ assert impl == "flash_attention_2"
+ assert config.llm_config._attn_implementation == "flash_attention_2"
+ assert config.vision_config.use_flash_attn is True
+
+
+def test_nested_config_with_its_own_choice_is_left_alone():
+ config = _OmniLike(attn_implementation = "sdpa")
+ config.llm_config._attn_implementation = "eager"
+ _utils._set_attn_impl(config, "flex_attention")
+ assert config.llm_config._attn_implementation == "eager"
+
+
+class _PropertySubConfigs(PretrainedConfig):
+ # transformers 4.57 DPT / DETR / VitMatte expose sub_configs as a property.
+ model_type = "baked_property_sub_configs"
+
+ @property
+ def sub_configs(self):
+ return {"backbone_config": _Vision}
+
+
+def test_sub_configs_property_is_read_from_the_instance():
+ config = _PropertySubConfigs()
+ config.backbone_config = _Vision()
+ config.llm_config = _Llm()
+ config.llm_config._attn_implementation = "flash_attention_2"
+ _utils._set_attn_impl(config, "eager")
+ assert config.llm_config._attn_implementation == "eager"
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index bdfccc5bbe8..17f48bd6d28 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -1191,10 +1191,52 @@ def _attn_impl_label(impl):
return impl
+def _undeclared_nested_configs(config):
+ """Nested configs outside `sub_configs`, which transformers' attn setter never reaches (Nemotron-Omni `llm_config`)."""
+ try:
+ from transformers import PretrainedConfig
+ except Exception:
+ return []
+ if not isinstance(config, PretrainedConfig):
+ return []
+ # Read from the instance: DPT / DETR / VitMatte on 4.57 define `sub_configs` as a property.
+ declared = getattr(config, "sub_configs", None)
+ declared = set(declared) if isinstance(declared, dict) else set()
+ return [
+ value
+ for name, value in vars(config).items()
+ if name not in declared and value is not config and isinstance(value, PretrainedConfig)
+ ]
+
+
+def _sync_baked_attn_impl(config, previous, impl):
+ # Remote __init__ bakes its flash default into nested configs; follow the top value over stale or flash copies.
+ if not isinstance(impl, str):
+ return
+ for nested in _undeclared_nested_configs(config):
+ current = getattr(nested, "_attn_implementation", None)
+ if (
+ current is not None
+ and current != impl
+ and (
+ current == previous
+ or (
+ _is_flash_attention_requested(current)
+ and not _is_flash_attention_requested(impl)
+ )
+ )
+ ):
+ _write_attn_impl(nested, impl)
+ if isinstance(getattr(nested, "use_flash_attn", None), bool):
+ nested.use_flash_attn = _is_flash_attention_requested(impl)
+
+
def _write_attn_impl(config, impl):
+ previous = _config_get(config, "_attn_implementation", None)
_config_set(config, "_attn_implementation", impl)
if isinstance(config, dict) or hasattr(config, "attn_implementation"):
_config_set(config, "attn_implementation", impl)
+ _sync_baked_attn_impl(config, previous, impl)
def _set_attn_impl(config, impl):