Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Resolve remote-code model classes, reach per-expert submodules with LoRA, run the root dispatch hook on the embedding's device (Nemotron-Labs-Teacher) (#11543)
Daniel Han98fc7fee6e33
diff --git a/tests/python/test_fast_language_model_text_only.py b/tests/python/test_fast_language_model_text_only.pyindex 1f842792acb..c6bbf795d0e 100644--- a/tests/python/test_fast_language_model_text_only.py+++ b/tests/python/test_fast_language_model_text_only.py@@ -79,6 +79,7 @@ def _load_text_only_namespace():if isinstance(node, ast.FunctionDef)}for name in (+ "_resolve_remote_model_class","resolve_model_class","_is_family_text_decoder","_remap_text_only_skip_modules",diff --git a/tests/test_attention_implementation.py b/tests/test_attention_implementation.pyindex 4fff5f0eeea..ebdb312a813 100644--- a/tests/test_attention_implementation.py+++ b/tests/test_attention_implementation.py@@ -1,3 +1,4 @@+# SPDX-License-Identifier: AGPL-3.0-onlyfrom types import SimpleNamespaceimport torch@@ -15,6 +16,7 @@ class SupportsFlexAndSdpa:class SupportsFlashAndSdpa:_supports_flash_attn_2 = True+ _supports_flash_attn = True # the flag transformers >= 4.53 dispatches on_supports_flex_attn = False_supports_sdpa = Truediff --git a/tests/test_attn_dtype_custom_datatype.py b/tests/test_attn_dtype_custom_datatype.pyindex d98cf2dc0e0..59a4d8ad27b 100644--- a/tests/test_attn_dtype_custom_datatype.py+++ b/tests/test_attn_dtype_custom_datatype.py@@ -122,6 +122,7 @@ def test_custom_datatype_load_does_not_disable_flash_attention():class SupportsFlashAndSdpa:_supports_flash_attn_2 = True+ _supports_flash_attn = True # the flag transformers >= 4.53 dispatches on_supports_flex_attn = False_supports_sdpa = Truediff --git a/tests/test_remote_code_class_and_expert_submodules.py b/tests/test_remote_code_class_and_expert_submodules.pynew file mode 100644index 00000000000..64679d95e5a--- /dev/null+++ b/tests/test_remote_code_class_and_expert_submodules.py@@ -0,0 +1,586 @@+# SPDX-License-Identifier: AGPL-3.0-only+"""Remote code whose config shares a native class name (Nemotron-H hub checkpoints), and+LoRA targets on per-expert submodules (`mixer.experts.<i>.up_proj`)."""++import re+import sys+import types+from types import SimpleNamespace++import pytest+import torch+++def _utils():+ from unsloth.models import _utils+ return _utils+++def test_old_flag_alone_is_not_flash_support_on_new_transformers():+ U = _utils()+ from transformers.modeling_utils import PreTrainedModel++ class OldRemote:+ _supports_flash_attn_2 = True++ class NewNative:+ _supports_flash_attn = True++ class Neither:+ pass++ # 5.0 to 5.3 define the new flag but their dispatch check still accepts the old one.+ legacy_ok = not hasattr(PreTrainedModel, "_supports_flash_attn") or (+ U._flash_dispatch_reads_legacy_flag(PreTrainedModel)+ )+ assert U._model_class_supports_flash_attention(OldRemote) is legacy_ok+ assert U._model_class_supports_flash_attention(NewNative) is True+ assert U._model_class_supports_flash_attention(Neither) is False+ assert U._model_class_supports_flash_attention(None) is False+++def test_resolver_does_not_request_flash_for_old_flag_remote_class(monkeypatch):+ U = _utils()+ # Without flash-attn installed the ladder never reaches flash, and this would pass either way.+ monkeypatch.setattr(U, "HAS_FLASH_ATTENTION", True)+ from transformers.modeling_utils import PreTrainedModel++ if not hasattr(PreTrainedModel, "_supports_flash_attn") or (+ U._flash_dispatch_reads_legacy_flag(PreTrainedModel)+ ):+ pytest.skip("transformers still dispatches on _supports_flash_attn_2")++ class OldRemote:+ _supports_flash_attn_2 = True+ _supports_sdpa = True++ class NewRemote:+ _supports_flash_attn = True+ _supports_sdpa = True++ config = SimpleNamespace(model_type = "nemotron_h", _attn_implementation = None)+ impl = U.resolve_attention_implementation(OldRemote, config, dtype = torch.bfloat16)+ assert "flash" not in str(impl)+ # Control: the same config does reach flash for a class carrying the dispatched flag.+ config = SimpleNamespace(model_type = "nemotron_h", _attn_implementation = None)+ impl = U.resolve_attention_implementation(NewRemote, config, dtype = torch.bfloat16)+ assert impl == "flash_attention_2"+++def _install_fake_remote_modules(monkeypatch, package = "transformers_modules.fake_repo.abc123"):+ """A remote config/model pair whose config class name collides with a native one."""+ from transformers import PretrainedConfig+ from transformers.models.llama.configuration_llama import LlamaConfig # noqa: F401 (the collision target)++ pkg = types.ModuleType(package)+ cfg_mod = types.ModuleType(package + ".configuration_llama")+ model_mod = types.ModuleType(package + ".modeling_llama")++ class LlamaConfig(PretrainedConfig): # same name as the native class on purpose+ model_type = "llama"++ class LlamaForCausalLM:+ _supports_flash_attn_2 = True++ LlamaConfig.__module__ = cfg_mod.__name__+ LlamaForCausalLM.__module__ = model_mod.__name__+ cfg_mod.LlamaConfig = LlamaConfig+ model_mod.LlamaForCausalLM = LlamaForCausalLM+ for m in (pkg, cfg_mod, model_mod):+ monkeypatch.setitem(sys.modules, m.__name__, m)+ config = LlamaConfig()+ config.auto_map = {+ "AutoConfig": "configuration_llama.LlamaConfig",+ "AutoModelForCausalLM": "modeling_llama.LlamaForCausalLM",+ }+ config._name_or_path = "fake/repo"+ return config, LlamaForCausalLM+++def test_remote_config_resolves_to_remote_model_class(monkeypatch):+ U = _utils()+ from transformers import AutoModelForCausalLM++ config, remote_cls = _install_fake_remote_modules(monkeypatch)+ assert U.resolve_model_class(AutoModelForCausalLM, config) is remote_cls+++def test_fetch_of_a_missing_modeling_module_uses_the_load_options(monkeypatch):+ """A cold-cache fetch uses the load's revision, token and offline flag."""+ U = _utils()+ from transformers import AutoModelForCausalLM+ import transformers.dynamic_module_utils as dmu++ config, _ = _install_fake_remote_modules(monkeypatch)+ monkeypatch.delitem(sys.modules, "transformers_modules.fake_repo.abc123.modeling_llama")+ seen = {}++ class Fetched:+ pass++ def fake_get(class_ref, repo_id, **kw):+ seen.update(class_ref = class_ref, repo_id = repo_id, **kw)+ return Fetched++ monkeypatch.setattr(dmu, "get_class_from_dynamic_module", fake_get)+ got = U.resolve_model_class(+ AutoModelForCausalLM,+ config,+ trust_remote_code = True,+ revision = "deadbeef",+ code_revision = "cafe",+ token = "tok",+ cache_dir = "/c",+ local_files_only = True,+ proxies = {"https": "http://proxy"},+ )+ assert got is Fetched+ assert seen == dict(+ class_ref = "modeling_llama.LlamaForCausalLM",+ repo_id = "fake/repo",+ revision = "deadbeef",+ code_revision = "cafe",+ token = "tok",+ cache_dir = "/c",+ local_files_only = True,+ proxies = {"https": "http://proxy"},+ )+++def test_cross_repository_auto_map_skips_the_local_sibling(monkeypatch):+ """An `other/repo--module.Class` reference must not resolve to a same-named sibling module."""+ U = _utils()+ from transformers import AutoModelForCausalLM+ import transformers.dynamic_module_utils as dmu++ config, local_cls = _install_fake_remote_modules(monkeypatch)+ config.auto_map["AutoModelForCausalLM"] = "other/repo--modeling_llama.LlamaForCausalLM"+ seen = {}++ class Remote:+ pass++ def fake_get(class_ref, repo_id, **kw):+ seen.update(class_ref = class_ref, repo_id = repo_id)+ return Remote++ monkeypatch.setattr(dmu, "get_class_from_dynamic_module", fake_get)+ got = U.resolve_model_class(AutoModelForCausalLM, config, trust_remote_code = True)+ assert got is Remote and got is not local_cls+ # Unsplit, with the model path, as from_pretrained calls it.+ assert seen == dict(+ class_ref = "other/repo--modeling_llama.LlamaForCausalLM", repo_id = "fake/repo"+ )+++def test_native_config_keeps_native_resolution():+ U = _utils()+ from transformers import AutoModelForCausalLM, LlamaConfig+ from transformers.models.llama.modeling_llama import LlamaForCausalLM++ config = LlamaConfig()+ config.auto_map = {+ "AutoModelForCausalLM": "modeling_llama.LlamaForCausalLM"+ } # ignored: not remote+ assert U.resolve_model_class(AutoModelForCausalLM, config) is LlamaForCausalLM+++def test_remote_config_without_auto_class_entry_stays_native(monkeypatch):+ U = _utils()+ from transformers import AutoModelForCausalLM+ from transformers.models.llama.modeling_llama import LlamaForCausalLM++ config, _ = _install_fake_remote_modules(monkeypatch)+ config.auto_map = {"AutoConfig": "configuration_llama.LlamaConfig"}+ assert U.resolve_model_class(AutoModelForCausalLM, config) is LlamaForCausalLM+++class _Expert(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.up_proj = torch.nn.Linear(8, 16, bias = False)+ self.down_proj = torch.nn.Linear(16, 8, bias = False)+++class _MoE(torch.nn.Module):+ def __init__(self, n = 4):+ super().__init__()+ self.experts = torch.nn.ModuleList([_Expert() for _ in range(n)])+ self.shared_experts = _Expert()+ self.fc1_latent_proj = torch.nn.Identity()+ self.gate = _Router(n)+++# Nemotron-Labs-Teacher's expert classes come from the checkpoint's own modeling file.+_Expert.__module__ = "transformers_modules.fake_teacher.modeling_nemotron_h"+sys.modules.setdefault(_Expert.__module__, sys.modules[__name__])+++class _Router(torch.nn.Module): # a Parameter-backed router, as in Nemotron-H+ def __init__(self, n):+ super().__init__()+ self.weight = torch.nn.Parameter(torch.zeros(n, 8))+++class _Mamba(torch.nn.Module): # a mixer with a Linear directly under it, like the Mamba layers+ def __init__(self):+ super().__init__()+ self.in_proj = torch.nn.Linear(8, 32, bias = False)+++class _Layer(torch.nn.Module):+ def __init__(self, mixer):+ super().__init__()+ self.mixer = mixer+++class _Inner(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.layers = torch.nn.ModuleList(+ [_Layer(_Mamba()), _Layer(_MoE()), _Layer(_Mamba()), _Layer(_MoE())]+ )+++class _Model(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.config = SimpleNamespace(n_routed_experts = 4, model_type = "nemotron_h")+ self.model = _Inner()+++def _text_only_regex():+ """The default FastModel.get_peft_model regex misses nested experts in a text-only model."""+ import importlib++ stub = sys.modules.get("unsloth_zoo.peft_utils")+ if stub is not None and getattr(stub, "__file__", None) is None:+ # Another test file leaves a stub in sys.modules; the real module is wanted here.+ del sys.modules["unsloth_zoo.peft_utils"]+ peft_utils = importlib.import_module("unsloth_zoo.peft_utils")+ return peft_utils.get_peft_regex(_Model())+++def test_text_only_regex_misses_nested_experts_on_its_own():+ regex = _text_only_regex()+ names = [n for n, m in _Model().named_modules() if isinstance(m, torch.nn.Linear)]+ assert not any(re.fullmatch(regex, n) for n in names if ".experts." in n)+++def test_expert_submodule_leaves_and_regex_reach_every_expert():+ U = _utils()+ model = _Model()+ regex = _text_only_regex()+ leaves = U.get_moe_expert_submodule_leaves(model, regex)+ assert leaves == ["down_proj", "up_proj"]+ extended = f"(?:{regex})|(?:{U.moe_expert_submodule_regex(leaves)})"+ matched = {n for n, m in model.named_modules() if re.fullmatch(extended, n)}+ linears = {n for n, m in model.named_modules() if isinstance(m, torch.nn.Linear)}+ experts = {n for n in linears if ".experts." in n or ".shared_experts." in n}+ assert experts <= matched+ assert "model.layers.1.mixer.gate" not in matched # the router is not a Linear+ assert "model.layers.1.mixer.fc1_latent_proj" not in matched # Identity, not a Linear+ assert "model.layers.0.mixer.in_proj" in matched # the block-level leaves stay+ assert all(isinstance(dict(model.named_modules())[n], torch.nn.Linear) for n in matched)+++def test_expert_submodule_leaves_follow_the_request():+ U = _utils()+ model = _Model()+ assert U.get_moe_expert_submodule_leaves(model, ["down_proj"]) == ["down_proj"]+ assert U.get_moe_expert_submodule_leaves(model, ["q_proj", "k_proj"]) == []+ assert U.get_moe_expert_submodule_leaves(model, None) == []+++def test_non_moe_model_adds_nothing():+ U = _utils()+ model = _Model()+ model.config = SimpleNamespace(model_type = "llama")+ assert U.get_moe_expert_submodule_leaves(model, ["up_proj", "down_proj"]) == []+++def test_only_a_generated_regex_is_widened_to_the_routed_experts():+ """A caller-written regex is never widened; the generated text-only regex is."""+ U = _utils()+ model = _Model()+ caller_regex = r".*\.shared_experts\.down_proj"+ kept, detect, leaves = U.widen_target_regex_to_expert_submodules(+ model, caller_regex, caller_regex, auto_regex = False+ )+ assert kept == caller_regex and detect == caller_regex and leaves == []+ matched = {n for n, _ in model.named_modules() if re.fullmatch(kept, n)}+ assert matched and all(".shared_experts." in n for n in matched)++ generated = _text_only_regex()+ widened, detect, leaves = U.widen_target_regex_to_expert_submodules(+ model, generated, generated, auto_regex = True+ )+ assert leaves == ["down_proj", "up_proj"]+ assert detect == widened != generated+ matched = {n for n, _ in model.named_modules() if re.fullmatch(widened, n)}+ assert any(".experts." in n for n in matched)++ # A leaf list as the detection target keeps its own identity through the widening.+ widened, detect, leaves = U.widen_target_regex_to_expert_submodules(+ model, generated, ["down_proj"], auto_regex = True+ )+ assert leaves == ["down_proj"] and detect == ["down_proj"] and widened != generated+++def test_gate_and_up_expert_leaves_stay_separate():+ U = _utils()+ model = _Model()+ assert U.get_moe_expert_submodule_leaves(model, ["up_proj"]) == ["up_proj"]+ assert (+ U.get_moe_expert_submodule_leaves(model, ["gate_proj"]) == []+ ) # the fixture has no gate leaf+ assert U.get_moe_expert_submodule_leaves(model, ["gate_up_proj"]) == ["up_proj"]+++def test_legacy_flash_flag_follows_the_installed_dispatch_check():+ """The helper reads the installed dispatch check for the legacy flag."""+ U = _utils()++ class OldDispatch:+ _supports_flash_attn = False++ def _flash_attn_can_dispatch(self):+ if not (self._supports_flash_attn or getattr(self, "_supports_flash_attn_2", False)):+ raise ValueError("no")++ class NewDispatch:+ _supports_flash_attn = False++ def _flash_attn_can_dispatch(self):+ if not self._supports_flash_attn:+ message = "x"+ if self._supports_flash_attn or getattr(self, "_supports_flash_attn_2", False):+ message += ", "+ raise ValueError(message)++ assert U._flash_dispatch_reads_legacy_flag(OldDispatch) is True+ assert U._flash_dispatch_reads_legacy_flag(NewDispatch) is False+++def test_force_download_bypasses_the_imported_sibling(monkeypatch):+ U = _utils()+ calls = []++ def fake_get_class(class_ref, repo_id, **kw):+ calls.append((class_ref, repo_id, kw))+ return type("Built", (), {})++ import transformers.dynamic_module_utils as dmu++ monkeypatch.setattr(dmu, "get_class_from_dynamic_module", fake_get_class)+ import sys, types++ module = types.ModuleType("transformers_modules.fake.configuration_fake")+ sys.modules["transformers_modules.fake.configuration_fake"] = module+ sibling = types.ModuleType("transformers_modules.fake.modeling_fake")+ sibling.FakeForCausalLM = type("FakeForCausalLM", (), {})+ sys.modules["transformers_modules.fake.modeling_fake"] = sibling+ try:+ cfg_cls = type("FakeConfig", (), {})+ cfg_cls.__module__ = "transformers_modules.fake.configuration_fake"+ cfg = cfg_cls()+ cfg.auto_map = {"AutoModelForCausalLM": "modeling_fake.FakeForCausalLM"}+ cfg._name_or_path = "fake/repo"+ auto = type("AutoModelForCausalLM", (), {})+ assert U._resolve_remote_model_class(auto, cfg) is sibling.FakeForCausalLM+ assert calls == []+ built = U._resolve_remote_model_class(+ auto, cfg, trust_remote_code = True, force_download = True+ )+ assert (+ built is not sibling.FakeForCausalLM+ and calls+ and calls[0][2].get("force_download") is True+ )+ assert U._resolve_remote_model_class(auto, cfg, trust_remote_code = False) is None+ finally:+ sys.modules.pop("transformers_modules.fake.configuration_fake", None)+ sys.modules.pop("transformers_modules.fake.modeling_fake", None)+++def test_every_resolver_probe_forwards_the_trust_decision():+ """Class probes pass trust_remote_code and the load's hub kwargs."""+ import ast, inspect+ from unsloth.models import llama, loader, loader_utils, vision++ probes = ("resolve_model_class", "_resolve_omni_auto_model")+ planner = next(+ node+ for node in ast.walk(ast.parse(inspect.getsource(loader_utils)))+ if isinstance(node, ast.FunctionDef) and node.name == "planner_model_class"+ )+ for module, tree in (+ (loader, ast.parse(inspect.getsource(loader))),+ (vision, ast.parse(inspect.getsource(vision))),+ (llama, ast.parse(inspect.getsource(llama))),+ (loader_utils, planner),+ ):+ for node in ast.walk(tree):+ if not (isinstance(node, ast.Call) and getattr(node.func, "id", None) in probes):+ continue+ names = {k.arg for k in node.keywords}+ splats = [getattr(k.value, "id", "") for k in node.keywords if k.arg is None]+ assert "trust_remote_code" in names or "_probe_hub_kwargs" in splats, (+ module.__name__,+ node.lineno,+ )+ if module in (loader, vision) and not splats:+ assert {"revision", "token", "local_files_only", "proxies"} <= names, (+ module.__name__,+ node.lineno,+ )+++def test_a_code_revision_skips_the_materialised_sibling(monkeypatch):+ """With a code_revision the resolver must ask transformers, not the sibling module."""+ import importlib+ import types++ from unsloth.models import _utils++ sibling = types.ModuleType("transformers_modules.tiny_rev.modeling_tiny")++ class FromConfigRevision:+ pass++ class FromCodeRevision:+ pass++ sibling.TinyForCausalLM = FromConfigRevision+ config_cls = type(+ "TinyConfig", (), {"__module__": "transformers_modules.tiny_rev.configuration_tiny"}+ )+ config = config_cls()+ config.auto_map = {"AutoModelForCausalLM": "modeling_tiny.TinyForCausalLM"}+ config._name_or_path = "someone/tiny"+ monkeypatch.setitem(__import__("sys").modules, sibling.__name__, sibling)+ import transformers.dynamic_module_utils as dmu++ monkeypatch.setattr(dmu, "get_class_from_dynamic_module", lambda *a, **k: FromCodeRevision)+ auto = type("AutoModelForCausalLM", (), {})+ assert _utils._resolve_remote_model_class(auto, config) is FromConfigRevision+ assert (+ _utils._resolve_remote_model_class(+ auto, config, trust_remote_code = True, code_revision = "abc123"+ )+ is FromCodeRevision+ )+ # A load revision is the code revision for same-repo code, so it skips the sibling too.+ assert (+ _utils._resolve_remote_model_class(auto, config, trust_remote_code = True, revision = "b")+ is FromCodeRevision+ )+++def test_native_per_expert_layouts_are_not_widened():+ """Qwen3-MoE on transformers 4.x nests native per-expert Linears the same way; those+ keep main's targets instead of gaining LoRA on every routed expert."""+ U = _utils()++ class _NativeExpert(_Expert):+ pass++ _NativeExpert.__module__ = "transformers.models.qwen3_moe.modeling_qwen3_moe"+ model = _Model()+ for layer in model.model.layers:+ mixer = layer.mixer+ if isinstance(mixer, _MoE):+ mixer.experts = torch.nn.ModuleList([_NativeExpert() for _ in mixer.experts])+ mixer.shared_experts = _NativeExpert()+ generated = _text_only_regex()+ kept, detect, leaves = U.widen_target_regex_to_expert_submodules(+ model, generated, generated, auto_regex = True+ )+ assert kept == generated and detect == generated and leaves == []+++def test_a_native_expert_tower_next_to_remote_code_is_not_widened():+ """Only the remote expert blocks' own parents gain the nested alternative."""+ U = _utils()++ class _NativeExpert(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.up_proj = torch.nn.Linear(8, 16, bias = False)+ self.down_proj = torch.nn.Linear(16, 8, bias = False)++ class _NativeTower(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.experts = torch.nn.ModuleList([_NativeExpert() for _ in range(4)])++ _NativeExpert.__module__ = "transformers.models.some_vlm.modeling_some_vlm"+ model = _Model()+ model.visual = torch.nn.ModuleList([_NativeTower(), _NativeTower()])+ generated = _text_only_regex()+ widened, _, leaves = U.widen_target_regex_to_expert_submodules(+ model, generated, generated, auto_regex = True+ )+ assert leaves == ["down_proj", "up_proj"]+ matched = {n for n, _ in model.named_modules() if re.fullmatch(widened, n)}+ assert any(n.startswith("model.layers.") and ".experts." in n for n in matched)+ assert not any(n.startswith("visual.") for n in matched)+++def test_a_load_that_did_not_ask_for_remote_code_never_fetches(monkeypatch):+ """Without trust_remote_code the resolver may reuse an imported sibling, never the Hub."""+ import transformers.dynamic_module_utils as dmu+ from unsloth.models import _utils++ fetched = []+ monkeypatch.setattr(+ dmu, "get_class_from_dynamic_module", lambda *a, **k: fetched.append(k) or object+ )+ config_cls = type(+ "TinyConfig", (), {"__module__": "transformers_modules.not_imported.configuration_tiny"}+ )+ config = config_cls()+ config.auto_map = {"AutoModel": "modeling_tiny.TinyModel"}+ config._name_or_path = "someone/tiny"+ auto = type("AutoModel", (), {})+ for trust in (None, False):+ assert _utils._resolve_remote_model_class(auto, config, trust_remote_code = trust) is None+ assert fetched == []+++def test_sentence_transformer_probes_use_the_loads_options(monkeypatch):+ """Both FastSentenceTransformer class probes see the load's trust, revision, token,+ cache and offline mode."""+ import ast+ import inspect++ from unsloth.models import _utils, sentence_transformer++ seen = []+ monkeypatch.setattr(+ _utils, "resolve_model_class", lambda auto, config, **kw: seen.append(kw) or None+ )+ options = dict(+ trust_remote_code = True,+ revision = "abc",+ token = "t",+ local_files_only = True,+ proxies = {"https": "http://proxy"},+ )+ _utils.resolve_encoder_attention_implementation(object, object(), **options)+ monkeypatch.setattr(+ sentence_transformer, "resolve_model_class", lambda auto, config, **kw: seen.append(kw)+ )+ sentence_transformer.FastSentenceTransformer._has_add_pooling_layer(object(), object, **options)+ assert seen == [options, options]+ probes = {"resolve_encoder_attention_implementation", "_has_add_pooling_layer"}+ calls = [+ node+ for node in ast.walk(ast.parse(inspect.getsource(sentence_transformer)))+ if isinstance(node, ast.Call)+ and (getattr(node.func, "id", None) in probes or getattr(node.func, "attr", None) in probes)+ ]+ assert len(calls) == 2+ for call in calls:+ splats = [getattr(k.value, "id", "") for k in call.keywords if k.arg is None]+ assert "_remote_class_probe_kwargs" in splats, call.linenodiff --git a/tests/test_root_hook_follows_input_embedding.py b/tests/test_root_hook_follows_input_embedding.pynew file mode 100644index 00000000000..ccb7bc0530f--- /dev/null+++ b/tests/test_root_hook_follows_input_embedding.py@@ -0,0 +1,122 @@+# SPDX-License-Identifier: AGPL-3.0-only+"""The root dispatch hook runs where the input embedding lives, not on the first device in+the map, so attention_mask follows input_ids."""++from types import SimpleNamespace++import pytest+import torch+++def _helper():+ from unsloth.models.vision import _align_root_hook_with_input_embeddings+ return _align_root_hook_with_input_embeddings+++class _RemoteModel(SimpleNamespace):+ pass+++# The Teacher's model class comes from the checkpoint's own modeling file.+_RemoteModel.__module__ = "transformers_modules.teacher.modeling_nemotron_h"+++def _model(+ device_map,+ root_device,+ embedding_device,+ cls = _RemoteModel,+ config = None,+):+ embedding = SimpleNamespace(weight = SimpleNamespace(device = embedding_device))+ return cls(+ hf_device_map = device_map,+ _hf_hook = SimpleNamespace(execution_device = root_device),+ get_input_embeddings = lambda: embedding,+ config = config or SimpleNamespace(),+ )+++def test_root_moves_to_the_embedding_device():+ align = _helper()+ model = _model(+ {"model.layers.1": 2, "model.embeddings": 1, "lm_head": 2, "model.layers.0": 0},+ 0,+ torch.device("cuda", 1),+ )+ assert align(model) == torch.device("cuda", 1)+ assert model._hf_hook.execution_device == torch.device("cuda", 1)+++def test_root_already_on_the_embedding_device_is_untouched():+ align = _helper()+ for root in (0, torch.device("cuda", 0), "cuda:0"):+ model = _model({"model.embeddings": 0, "lm_head": 1}, root, torch.device("cuda", 0))+ assert align(model) is None+ assert model._hf_hook.execution_device == root+++def test_single_device_and_unhooked_models_are_untouched():+ align = _helper()+ model = _model({"": 0}, 0, torch.device("cuda", 1))+ assert align(model) is None+ model = _model({"model.embeddings": 0, "lm_head": 1}, 0, torch.device("cuda", 1))+ model._hf_hook = None+ assert align(model) is None+ model = _model({"model.embeddings": 0, "lm_head": 1}, 0, torch.device("cuda", 1))+ del model._hf_hook+ assert align(model) is None+++def test_offloaded_or_meta_embedding_is_left_to_its_hook():+ align = _helper()+ for device in (torch.device("cpu"), torch.device("meta")):+ model = _model({"model.embeddings": "cpu", "lm_head": 0}, 0, device)+ assert align(model) is None+ assert model._hf_hook.execution_device == 0+++def test_model_without_embedding_accessor_is_untouched():+ align = _helper()+ model = _model({"model.embeddings": 1, "lm_head": 0}, 0, torch.device("cuda", 1))++ def broken():+ raise NotImplementedError++ model.get_input_embeddings = broken+ assert align(model) is None+++def test_native_and_multimodal_models_are_untouched():+ """Native code moves the mask itself; the root hook moves every input, so a vision or+ audio tower would get its tensors on the text card."""+ align = _helper()+ split = {"model.layers.1": 2, "model.embeddings": 1, "lm_head": 2, "model.layers.0": 0}+ native = _model(split, 0, torch.device("cuda", 1), cls = SimpleNamespace)+ assert align(native) is None+ assert native._hf_hook.execution_device == 0+ for name in ("vision_config", "audio_config"):+ config = SimpleNamespace(**{name: SimpleNamespace()})+ multimodal = _model(split, 0, torch.device("cuda", 1), config = config)+ assert align(multimodal) is None+ assert multimodal._hf_hook.execution_device == 0+++def test_omni_config_with_nested_towers_is_untouched():+ """Omni checkpoints keep vision / audio under thinker_config; they are still multimodal."""+ align = _helper()+ split = {"model.layers.1": 2, "model.embeddings": 1, "lm_head": 2, "model.layers.0": 0}+ config = SimpleNamespace(thinker_config = SimpleNamespace(vision_config = SimpleNamespace()))+ omni = _model(split, 0, torch.device("cuda", 1), config = config)+ assert align(omni) is None+ assert omni._hf_hook.execution_device == 0++[email protected]("tower", ["vision_encoder_config", "audio_encoder_config", "encoder_config"])+def test_encoder_spelled_tower_configs_are_untouched(tower):+ """The same spellings _uses_flash_attention_for_generation already treats as non-language."""+ align = _helper()+ split = {"model.layers.1": 2, "model.embeddings": 1, "lm_head": 2, "model.layers.0": 0}+ multimodal = _model(split, 0, torch.device("cuda", 1), config = SimpleNamespace(**{tower: SimpleNamespace()}))+ assert align(multimodal) is None+ assert multimodal._hf_hook.execution_device == 0diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex 272e6c19321..af78c2cb240 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -86,6 +86,9 @@ __all__ = ["is_moe_model","get_moe_target_parameters","get_moe_target_modules",+ "get_moe_expert_submodule_leaves",+ "moe_expert_submodule_regex",+ "widen_target_regex_to_expert_submodules","warn_if_zoo_cannot_merge_moe_experts","_select_moe_detection_targets","EMBEDDING_MODULES",@@ -773,6 +776,43 @@ def _declares_flex_support(model_class):return None+def _model_class_supports_flash_attention(model_class):+ """Whether installed transformers lets this class dispatch flash attention."""+ if model_class is None:+ return False+ try:+ from transformers.modeling_utils import PreTrainedModel+ except Exception:+ PreTrainedModel = None+ new_flag_dispatched = PreTrainedModel is not None and hasattr(+ PreTrainedModel, "_supports_flash_attn"+ )+ if new_flag_dispatched and not _flash_dispatch_reads_legacy_flag(PreTrainedModel):+ return bool(getattr(model_class, "_supports_flash_attn", False))+ return bool(+ getattr(model_class, "_supports_flash_attn_2", False)+ or getattr(model_class, "_supports_flash_attn", False)+ )+++def _flash_dispatch_reads_legacy_flag(PreTrainedModel) -> bool:+ # Read from source: later versions keep the legacy name only in the error message.+ import inspect+ for name in ("_flash_attn_can_dispatch", "_flash_attn_2_can_dispatch"):+ check = getattr(PreTrainedModel, name, None)+ if check is None:+ continue+ try:+ source = inspect.getsource(check)+ except (OSError, TypeError):+ return False+ return any(+ "_supports_flash_attn_2" in line and line.lstrip().startswith("if not")+ for line in source.splitlines()+ )+ return False++def _enable_flex_attention_support(model_class, model_type = ""):"""Set `_supports_flex_attn` on an interface-based architecture that leaves it unset (qwen3_5,qwen3_5_moe). Returns True when flex is now permitted."""@@ -1087,7 +1127,73 @@ def _set_attn_impl(config, impl):return impl-def resolve_model_class(auto_model, config):+_REMOTE_CODE_HUB_KWARGS = (+ "revision",+ "code_revision",+ "token",+ "cache_dir",+ "local_files_only",+ "force_download",+ "proxies",+)+++def _resolve_remote_model_class(auto_model, config, **hub_kwargs):+ """Remote-code class `auto_model` would build for `config`, or None when native.+ Auto mapping keys on config class name, so a remote `NemotronHConfig` would resolve native."""+ if hub_kwargs.get("trust_remote_code", None) is False:+ return None+ auto_name = getattr(auto_model, "__name__", None)+ auto_map = getattr(config, "auto_map", None)+ if not auto_name or not isinstance(auto_map, dict) or auto_name not in auto_map:+ return None+ if not str(getattr(type(config), "__module__", "")).startswith("transformers_modules"):+ return None+ class_ref = auto_map[auto_name]+ if not isinstance(class_ref, str) or "." not in class_ref:+ return None+ repo_id = getattr(config, "_name_or_path", None) or getattr(config, "name_or_path", None)+ full_ref = class_ref+ cross_repo = "--" in class_ref+ if cross_repo:+ _, class_ref = class_ref.split("--", 1)+ module_name, class_name = class_ref.rsplit(".", 1)+ # These may replace an already imported sibling module (revision = code revision same-repo).+ if (+ not cross_repo+ and not hub_kwargs.get("force_download", False)+ and not hub_kwargs.get("code_revision", None)+ and not hub_kwargs.get("revision", None)+ ):+ config_module = str(type(config).__module__)+ try:+ import importlib++ sibling = importlib.import_module(f"{config_module.rsplit('.', 1)[0]}.{module_name}")+ klass = getattr(sibling, class_name, None)+ if isinstance(klass, type):+ return klass+ except Exception:+ pass+ if not repo_id or not hub_kwargs.get("trust_remote_code", None):+ return None+ try:+ from transformers.dynamic_module_utils import get_class_from_dynamic_module++ passed = {+ k: v for k, v in hub_kwargs.items() if k in _REMOTE_CODE_HUB_KWARGS and v is not None+ }+ # Same call as from_pretrained, so `revision` applies only to same-repo code.+ klass = get_class_from_dynamic_module(full_ref, repo_id, **passed)+ return klass if isinstance(klass, type) else None+ except Exception:+ return None+++def resolve_model_class(auto_model, config, **hub_kwargs):+ remote_class = _resolve_remote_model_class(auto_model, config, **hub_kwargs)+ if remote_class is not None:+ return remote_classmapping = getattr(auto_model, "_model_mapping", {})try:result = mapping[config.__class__]@@ -1415,14 +1521,9 @@ def resolve_attention_implementation(supports_sdpa = model_class is not None and getattr(model_class, "_supports_sdpa", False)if _is_sdpa_excluded(model_type):supports_sdpa = False- supports_flash_attention = (- model_class is not None- and (- getattr(model_class, "_supports_flash_attn_2", False)- or getattr(model_class, "_supports_flash_attn", False)- )- and not _is_flash_excluded(model_type)- )+ supports_flash_attention = _model_class_supports_flash_attention(+ model_class+ ) and not _is_flash_excluded(model_type)supports_flex_attention = _supports_flex_attention(model_class, config, model_type)# Before the ladder, since every branch below reads supports_flex_attention.prefers_flex_for_head_dim = _prefers_flex_for_head_dim(config)@@ -1526,8 +1627,9 @@ def resolve_encoder_attention_implementation(config,model_type = "",disable_sdpa_model_names = (),+ **hub_kwargs,):- model_class = resolve_model_class(auto_model, config)+ model_class = resolve_model_class(auto_model, config, **hub_kwargs)supports_sdpa = model_class is not None and getattr(model_class, "_supports_sdpa", False)if any(name in model_type.lower() for name in disable_sdpa_model_names):return "eager"@@ -5098,7 +5200,9 @@ def get_moe_target_modules(model, target_modules = None) -> List[str]:return []# Scope the suffixes to the requested leaves, matching get_moe_target_parameters: gate/up/gate_up map to the fused gate_up ModuleList and down_proj to the down one, so a down-only request must not pull in the other projection.- want_gate_up = bool(target_set & {"gate_proj", "up_proj", "gate_up_proj"})+ # gate_proj alone must not collect up_proj leaves; only gate_up_proj asks for both.+ want_gate = bool(target_set & {"gate_proj", "gate_up_proj"})+ want_up = bool(target_set & {"up_proj", "gate_up_proj"})want_down = "down_proj" in target_settargets = set()@@ -5117,10 +5221,17 @@ def get_moe_target_modules(model, target_modules = None) -> List[str]:continueleaf_lower = leaf.lower()is_down = "down" in leaf_lower- is_gate_up = (not is_down) and ("gate" in leaf_lower or "up" in leaf_lower)+ is_gate = (not is_down) and "gate" in leaf_lower+ is_up = (not is_down) and "up" in leaf_lower+ is_gate_up = is_gate or is_upif is_down and not want_down:continue- if is_gate_up and not want_gate_up:+ if is_gate and is_up: # a fused gate_up leaf: either request reaches it+ if not (want_gate or want_up):+ continue+ elif is_gate and not want_gate:+ continue+ elif is_up and not want_up:continue# One entry per expert index; leaf.<i> matches expert i in every layer.for expert_index in range(len(module)):@@ -5129,6 +5240,113 @@ def get_moe_target_modules(model, target_modules = None) -> List[str]:return sorted(targets)+_EXPERT_SUBMODULE_PATTERN = re.compile(r"(?:^|\.)(?:experts\.\d+|shared_experts?)\.([A-Za-z_]\w*)$")+_EXPERT_BLOCK_PATTERN = re.compile(r"(?:^|\.)(?:experts\.\d+|shared_experts?)$")+++def get_moe_expert_submodule_leaves(model, target_modules = None) -> List[str]:+ """Requested MLP leaves inside expert submodules, unreached by the text-only regex."""+ if not is_moe_model(model):+ return []+ if target_modules is None or not hasattr(model, "named_modules"):+ return []+ if isinstance(target_modules, str):+ target_set = _moe_target_set_from_string(target_modules)+ else:+ target_set = {+ target+ for target in target_modules or ()+ if (isinstance(target, str) and "." not in target and target in _MOE_BROAD_MLP_TARGETS)+ }+ if not (target_set & _MOE_BROAD_MLP_TARGETS):+ return []+ # gate_proj alone must not collect up_proj leaves; only gate_up_proj asks for both.+ want_gate = bool(target_set & {"gate_proj", "gate_up_proj"})+ want_up = bool(target_set & {"up_proj", "gate_up_proj"})+ want_down = "down_proj" in target_set++ leaves = set()+ for name, module in model.named_modules():+ matched = _EXPERT_SUBMODULE_PATTERN.search(name)+ if matched is None:+ continue+ if not (+ isinstance(module, torch.nn.Linear)+ or isinstance(getattr(module, "base_layer", None), torch.nn.Linear)+ ):+ continue+ leaf = matched.group(1)+ leaf_lower = leaf.lower()+ is_down = "down" in leaf_lower+ is_gate = (not is_down) and "gate" in leaf_lower+ is_up = (not is_down) and "up" in leaf_lower+ is_gate_up = is_gate or is_up+ if is_down and not want_down:+ continue+ if is_gate and is_up: # a fused gate_up leaf: either request reaches it+ if not (want_gate or want_up):+ continue+ elif is_gate and not want_gate:+ continue+ elif is_up and not want_up:+ continue+ if not (is_down or is_gate_up):+ continue+ leaves.add(leaf)+ return sorted(leaves)+++def moe_expert_submodule_regex(leaves, prefixes = None) -> str:+ parent = r".*" if not prefixes else "(?:" + "|".join(sorted(prefixes)) + ")"+ return (+ parent+ + r"\.(?:experts\.\d+|shared_experts?)\.(?:"+ + "|".join(re.escape(leaf) for leaf in leaves)+ + r")"+ )+++def _remote_expert_parents(model):+ parents = set()+ for name, module in model.named_modules():+ matched = _EXPERT_BLOCK_PATTERN.search(name)+ if matched is None:+ continue+ if "transformers_modules" not in (getattr(type(module), "__module__", "") or ""):+ continue+ parent = name[: matched.start()]+ parents.add(+ r"\.".join(r"\d+" if part.isdigit() else re.escape(part) for part in parent.split("."))+ )+ return parents+++def widen_target_regex_to_expert_submodules(+ model, target_modules, detect_targets, auto_regex: bool+):+ """Widen a generated regex to ``experts.<i>.<leaf>``; a user-written regex is never widened."""+ if not auto_regex or not isinstance(target_modules, str):+ return target_modules, detect_targets, []+ # Remote expert blocks only: widening native ones (Qwen3-MoE on 4.x) LoRAs every routed expert.+ parents = _remote_expert_parents(model)+ if not parents:+ return target_modules, detect_targets, []+ leaves = get_moe_expert_submodule_leaves(model, detect_targets)+ if not leaves:+ return target_modules, detect_targets, []+ if any(+ re.fullmatch(target_modules, name)+ for name, _ in model.named_modules()+ if ".experts." in name+ ):+ return target_modules, detect_targets, []+ detect_was_the_regex = detect_targets is target_modules+ target_modules = f"(?:{target_modules})|(?:{moe_expert_submodule_regex(leaves, parents)})"+ if detect_was_the_regex:+ detect_targets = target_modules+ return target_modules, detect_targets, leaves++def warn_if_zoo_cannot_merge_moe_experts():"""Warn once when the installed unsloth_zoo cannot fold per-expert Linear MoE LoRA into a merged_16bit checkpoint. Older zoo releases keep the fused gate_up_proj / down_proj tensors and drop the per-expert deltas, so save_pretrained_merged("merged_16bit") would silently lose the expert training (the LoRA adapter itself still saves and reloads correctly)."""try:diff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex 618cc19bc2c..6779bba4d87 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -2618,7 +2618,11 @@ class FastLlamaModel:if _planner_skip_reason is None and num_labels is not None:_planner_skip_reason = (planner_class_mismatch_reason(- resolve_model_class(AutoModelForSequenceClassification, model_config),+ resolve_model_class(+ AutoModelForSequenceClassification,+ model_config,+ trust_remote_code = trust_remote_code,+ ),planner_model_class(model_config, trust_remote_code = trust_remote_code),)or "num_labels loads a task head the repo config does not describe"diff --git a/unsloth/models/loader.py b/unsloth/models/loader.pyindex ba4dfd7f51b..0fce2d6152e 100644--- a/unsloth/models/loader.py+++ b/unsloth/models/loader.py@@ -319,7 +319,11 @@ def _config_has_native_class(auto_class, config):return False-def _resolve_omni_auto_model(model_config):+def _resolve_omni_auto_model(+ model_config,+ trust_remote_code = None,+ **hub_kwargs,+):"""A multimodal auto class that really maps this config, or None.Qwen3-Omni names Qwen3OmniMoeForConditionalGeneration so it reads as a VLM,@@ -334,7 +338,12 @@ def _resolve_omni_auto_model(model_config):if auto_class is None:continuetry:- if resolve_model_class(auto_class, model_config) is not None:+ if (+ resolve_model_class(+ auto_class, model_config, trust_remote_code = trust_remote_code, **hub_kwargs+ )+ is not None+ ):return auto_classexcept Exception:continue@@ -1962,12 +1971,25 @@ class FastModel(FastBaseModel):is_vlm = any(x.endswith("ForConditionalGeneration") for x in architectures)is_vlm = is_vlm or hasattr(model_config, "vision_config")load_text_only = text_only and auto_model is None+ # Class probes below fetch remote modeling code exactly as the load will.+ _probe_hub_kwargs = dict(+ trust_remote_code = trust_remote_code,+ revision = base_revision if not is_peft else None,+ code_revision = kwargs.get("code_revision", None),+ token = token,+ cache_dir = kwargs.get("cache_dir", None),+ local_files_only = local_files_only,+ force_download = kwargs.get("force_download", None),+ proxies = kwargs.get("proxies", None),+ )text_only_decoder = Falseif load_text_only:if hasattr(model_config, "vision_config"):text_config = _get_text_only_config(model_config, old_model_name)# Skip the vision tower only for families with their own text decoder (Gemma 3); others would load random weights, so keep the full model.- text_class = resolve_model_class(AutoModelForCausalLM, text_config)+ text_class = resolve_model_class(+ AutoModelForCausalLM, text_config, **_probe_hub_kwargs+ )if text_class is None or not _is_family_text_decoder(getattr(model_config, "model_type", ""),getattr(text_config, "model_type", ""),@@ -2024,8 +2046,11 @@ class FastModel(FastBaseModel):auto_model = AutoModelForVision2Seq# Only when the image-text class has no mapping, so anything that# resolves today keeps the class it resolves to now.- if resolve_model_class(auto_model, model_config) is None:- auto_model = _resolve_omni_auto_model(model_config) or auto_model+ if resolve_model_class(auto_model, model_config, **_probe_hub_kwargs) is None:+ auto_model = (+ _resolve_omni_auto_model(model_config, **_probe_hub_kwargs)+ or auto_model+ )else:auto_model = AutoModelForCausalLMdiff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.pyindex addce0e59d6..5088dc11766 100644--- a/unsloth/models/loader_utils.py+++ b/unsloth/models/loader_utils.py@@ -258,7 +258,7 @@ def planner_model_class(config, trust_remote_code = False):from ._utils import resolve_model_classauto_class = _auto_class_for(config, trust_remote_code = trust_remote_code)- return resolve_model_class(auto_class, config)+ return resolve_model_class(auto_class, config, trust_remote_code = trust_remote_code)except Exception:# Unknown, not mismatched: an unsloth_zoo without this has no planner to feed.return Nonediff --git a/unsloth/models/sentence_transformer.py b/unsloth/models/sentence_transformer.pyindex ea45ada465a..7d5189c14d5 100644--- a/unsloth/models/sentence_transformer.py+++ b/unsloth/models/sentence_transformer.py@@ -885,12 +885,16 @@ class FastSentenceTransformer(FastModel):modeling_distilbert.DistilBertModel.forward = forward@staticmethod- def _has_add_pooling_layer(config, auto_model_class = None):+ def _has_add_pooling_layer(+ config,+ auto_model_class = None,+ **hub_kwargs,+ ):"""Check if the model class accepts the `add_pooling_layer` argument."""try:if auto_model_class is None:auto_model_class = AutoModel- model_class = resolve_model_class(auto_model_class, config)+ model_class = resolve_model_class(auto_model_class, config, **hub_kwargs)if model_class:sig = inspect.signature(model_class.__init__)@@ -1471,6 +1475,16 @@ class FastSentenceTransformer(FastModel):"Run `pip install sentence-transformers` to install it.")+ # Remote-class probes must use this load's trust, revision and hub options.+ _remote_class_probe_kwargs = dict(+ trust_remote_code = trust_remote_code,+ revision = revision,+ token = token,+ cache_dir = kwargs.get("cache_folder", kwargs.get("cache_dir", None)),+ local_files_only = kwargs.get("local_files_only", None),+ proxies = kwargs.get("proxies", None),+ )+# The other leaf loaders resolve the "unsloth" sentinel by planning; this one declines. st_device below hands device_map to SentenceTransformer(device=), which ends in self.to(device): the sentinel raises there, and that same .to() would pull a split model back onto one card. The env-var opt-in is resolved too, or UNSLOTH_AUTO_DEVICE_MAP=1 asks for a plan without naming the sentinel.device_map = requested_device_map(device_map)# Always "sequential", never the asked-for name's own declined value: the st_device blocks normalise only dicts, "auto" and "sequential", so "balanced" would reach .to("balanced"). isinstance first, since a caller's explicit dict is unhashable and `in` alone raises.@@ -1602,6 +1616,7 @@ class FastSentenceTransformer(FastModel):config,model_type = model_type,disable_sdpa_model_names = DISABLE_SDPA_MODEL_NAMES,+ **_remote_class_probe_kwargs,)supports_sdpa = encoder_attn_impl == "sdpa"if encoder_attn_impl is not None:@@ -1742,7 +1757,7 @@ class FastSentenceTransformer(FastModel):if "add_pooling_layer" not in kwargs:supported = FastSentenceTransformer._has_add_pooling_layer(- config, kwargs.get("auto_model", AutoModel)+ config, kwargs.get("auto_model", AutoModel), **_remote_class_probe_kwargs)if supported:kwargs["add_pooling_layer"] = Falsediff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex 84371b0b6eb..dae92111181 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -348,6 +348,61 @@ def _lift_endpoint_hooks_onto_adapters(model):return lifted+_MODALITY_SUB_CONFIGS = (+ "vision_config",+ "audio_config",+ "speech_config",+ "sound_config",+ "vision_encoder_config",+ "audio_encoder_config",+ "encoder_config",+)+++def _align_root_hook_with_input_embeddings(model):+ """Point the root dispatch hook at the input embedding's device (remote text models only).+ accelerate picks the first mapped device, so remote code building its mask there mismatches;+ vision/audio towers are skipped since the hook moves every input to the text card."""+ if "transformers_modules" not in (getattr(type(model), "__module__", "") or ""):+ return None+ config = getattr(model, "config", None)+ # Omni configs nest the towers one level down (thinker_config.vision_config).+ nested = [config] + [getattr(config, name, None) for name in ("thinker_config",)]+ if any(+ getattr(sub, name, None) is not None+ for sub in nested+ if sub is not None+ for name in _MODALITY_SUB_CONFIGS+ ):+ return None+ device_map = getattr(model, "hf_device_map", None)+ if not device_map or len(set(device_map.values())) < 2:+ return None+ hook = getattr(model, "_hf_hook", None)+ current = getattr(hook, "execution_device", None)+ if current is None:+ return None+ try:+ embedding = model.get_input_embeddings()+ except Exception:+ return None+ weight = getattr(embedding, "weight", None)+ target = getattr(weight, "device", None)+ if target is None or target.type in ("cpu", "meta") or target.index is None:+ return None+ if isinstance(current, int):+ current = torch.device(target.type, current)+ else:+ try:+ current = torch.device(current)+ except (TypeError, RuntimeError):+ return None+ if current == target:+ return None+ hook.execution_device = target+ return target++def _attach_bnb_multidevice_hooks(model, load_in_4bit, load_in_8bit, offload_embedding, fast_inference):@@ -1557,7 +1612,18 @@ class FastBaseModel:if text_only and hasattr(auto_config, "vision_config"):parent_config = auto_configtext_config = _get_text_only_config(parent_config, model_name)- text_class = resolve_model_class(AutoModelForCausalLM, text_config)+ text_class = resolve_model_class(+ AutoModelForCausalLM,+ text_config,+ revision = _revision,+ code_revision = kwargs.get("code_revision"),+ token = token,+ cache_dir = kwargs.get("cache_dir"),+ proxies = kwargs.get("proxies"),+ local_files_only = local_files_only,+ force_download = kwargs.get("force_download", None),+ trust_remote_code = trust_remote_code,+ )if text_class is not None and _is_family_text_decoder(getattr(parent_config, "model_type", ""),getattr(text_config, "model_type", ""),@@ -1733,7 +1799,18 @@ class FastBaseModel:local_files_only = local_files_only,**{k: kwargs.get(k, None) for k in _REMOTE_CLASS_HUB_OPTIONS},)- model_class = resolve_model_class(auto_model, auto_config)+ model_class = _remote_class or resolve_model_class(+ auto_model,+ auto_config,+ revision = _revision,+ code_revision = kwargs.get("code_revision"),+ token = token,+ cache_dir = kwargs.get("cache_dir"),+ proxies = kwargs.get("proxies"),+ local_files_only = local_files_only,+ force_download = kwargs.get("force_download", None),+ trust_remote_code = trust_remote_code,+ )# Forced float32 loads in bfloat16 then casts to float16. Resolved here, not at the load, because attention resolution and the device-map planner both size the same dtype.torch_dtype = dtypeif do_forced_float32:@@ -2064,6 +2141,12 @@ class FastBaseModel:offload_embedding = offload_embedding,fast_inference = fast_inference,)+ _aligned_root_device = _align_root_hook_with_input_embeddings(model)+ if _aligned_root_device is not None:+ logger.info(+ f"Unsloth: inputs now go straight to {_aligned_root_device}, where the input "+ "embedding lives, instead of through the first device in the map."+ )# Re-apply block-fp8 weight_scale_inv tensors transformers dropped on load (#6200)._restore_dropped_fp8_scales(model,@@ -2624,6 +2707,8 @@ class FastBaseModel:)_raise_if_fast_inference_modules_to_save(model, modules_to_save)+ # Only a regex generated here may be widened to expert submodules below.+ _target_modules_auto_regex = Falseif target_modules is None or target_modules == "all-linear":target_modules = get_peft_regex(model,@@ -2633,6 +2718,7 @@ class FastBaseModel:finetune_mlp_modules = finetune_mlp_modules,**_audio_kwargs,)+ _target_modules_auto_regex = Trueelse:assert type(target_modules) in (list, tuple, str)# Route an explicit list through get_peft_regex when the caller scoped a layer family or opted into audio, so the new audio/embedder branches are considered. finetune_audio_layers is a POSITIVE term: negating it would force every explicit list through the filter.@@ -2658,6 +2744,7 @@ class FastBaseModel:target_modules = list(target_modules),**_audio_kwargs,)+ _target_modules_auto_regex = Trueif hasattr(model, "vllm_engine"):if (@@ -2695,6 +2782,21 @@ class FastBaseModel:finetune_language_layers = finetune_language_layers,)+ # Widen to per-expert submodules before expert detection; never for vision-only.+ target_modules, _moe_module_detect, _expert_submodule_leaves = (+ widen_target_regex_to_expert_submodules(+ model,+ target_modules,+ _moe_module_detect,+ auto_regex = _target_modules_auto_regex and bool(finetune_language_layers),+ )+ )+ if _expert_submodule_leaves:+ print(+ f"Unsloth: Detected MoE model with per-expert submodules. "+ f"Enabling LoRA on expert projections {_expert_submodule_leaves}."+ )+# Per-expert Linear layouts (gpt-oss bnb-4bit) target experts via target_modules, not fused Parameters. Extend either form PEFT accepts: a leaf list, or a regex string._moe_module_targets = get_moe_target_modules(model, _moe_module_detect)