Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Carry the resolved attention implementation to nested configs a remote config baked flash attention into (Nemotron 3 Nano Omni) (#12099)
Daniel Hanf53b1031f64f
diff --git a/tests/test_attn_impl_baked_nested_config.py b/tests/test_attn_impl_baked_nested_config.pynew file mode 100644index 00000000000..a72b76ad294--- /dev/null+++ b/tests/test_attn_impl_baked_nested_config.py@@ -0,0 +1,83 @@+# SPDX-License-Identifier: AGPL-3.0-only+import pytest+import unsloth # noqa: F401+from transformers import PretrainedConfig++from unsloth.models import _utils+++class _Llm(PretrainedConfig):+ model_type = "baked_llm"+++class _Vision(PretrainedConfig):+ model_type = "baked_vision"+++class _OmniLike(PretrainedConfig):+ model_type = "baked_omni"++ def __init__(+ self,+ attn_implementation = "flash_attention_2",+ **kwargs,+ ):+ super().__init__(**kwargs)+ self.llm_config = _Llm()+ self.vision_config = _Vision()+ self._attn_implementation = attn_implementation+ self.vision_config.use_flash_attn = "flash_attention" in attn_implementation+ self.llm_config._attn_implementation = attn_implementation+++class _FlashOnlyModel:+ _supports_flash_attn = True+ _supports_flash_attn_2 = True+ _supports_sdpa = False++[email protected]("requested", [None, "sdpa", "eager"])+def test_non_flash_resolution_reaches_the_baked_decoder(monkeypatch, requested):+ monkeypatch.setattr(_utils, "HAS_FLASH_ATTENTION", False)+ config = _OmniLike()+ impl = _utils.resolve_attention_implementation(+ _FlashOnlyModel, config, requested_attn_implementation = requested, supports_sdpa = False+ )+ assert not str(impl).startswith("flash")+ assert config.llm_config._attn_implementation == impl+ assert config.vision_config.use_flash_attn is False+++def test_flash_resolution_keeps_the_decoder_on_flash(monkeypatch):+ monkeypatch.setattr(_utils, "HAS_FLASH_ATTENTION", True)+ monkeypatch.setattr(_utils, "_get_flash_attention_disable_reason", lambda config: None)+ config = _OmniLike()+ impl = _utils.resolve_attention_implementation(_FlashOnlyModel, config, supports_sdpa = False)+ assert impl == "flash_attention_2"+ assert config.llm_config._attn_implementation == "flash_attention_2"+ assert config.vision_config.use_flash_attn is True+++def test_nested_config_with_its_own_choice_is_left_alone():+ config = _OmniLike(attn_implementation = "sdpa")+ config.llm_config._attn_implementation = "eager"+ _utils._set_attn_impl(config, "flex_attention")+ assert config.llm_config._attn_implementation == "eager"+++class _PropertySubConfigs(PretrainedConfig):+ # transformers 4.57 DPT / DETR / VitMatte expose sub_configs as a property.+ model_type = "baked_property_sub_configs"++ @property+ def sub_configs(self):+ return {"backbone_config": _Vision}+++def test_sub_configs_property_is_read_from_the_instance():+ config = _PropertySubConfigs()+ config.backbone_config = _Vision()+ config.llm_config = _Llm()+ config.llm_config._attn_implementation = "flash_attention_2"+ _utils._set_attn_impl(config, "eager")+ assert config.llm_config._attn_implementation == "eager"diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex bdfccc5bbe8..17f48bd6d28 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -1191,10 +1191,52 @@ def _attn_impl_label(impl):return impl+def _undeclared_nested_configs(config):+ """Nested configs outside `sub_configs`, which transformers' attn setter never reaches (Nemotron-Omni `llm_config`)."""+ try:+ from transformers import PretrainedConfig+ except Exception:+ return []+ if not isinstance(config, PretrainedConfig):+ return []+ # Read from the instance: DPT / DETR / VitMatte on 4.57 define `sub_configs` as a property.+ declared = getattr(config, "sub_configs", None)+ declared = set(declared) if isinstance(declared, dict) else set()+ return [+ value+ for name, value in vars(config).items()+ if name not in declared and value is not config and isinstance(value, PretrainedConfig)+ ]+++def _sync_baked_attn_impl(config, previous, impl):+ # Remote __init__ bakes its flash default into nested configs; follow the top value over stale or flash copies.+ if not isinstance(impl, str):+ return+ for nested in _undeclared_nested_configs(config):+ current = getattr(nested, "_attn_implementation", None)+ if (+ current is not None+ and current != impl+ and (+ current == previous+ or (+ _is_flash_attention_requested(current)+ and not _is_flash_attention_requested(impl)+ )+ )+ ):+ _write_attn_impl(nested, impl)+ if isinstance(getattr(nested, "use_flash_attn", None), bool):+ nested.use_flash_attn = _is_flash_attention_requested(impl)++def _write_attn_impl(config, impl):+ previous = _config_get(config, "_attn_implementation", None)_config_set(config, "_attn_implementation", impl)if isinstance(config, dict) or hasattr(config, "attn_implementation"):_config_set(config, "attn_implementation", impl)+ _sync_baked_attn_impl(config, previous, impl)def _set_attn_impl(config, impl):