Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Resolve remote-code model classes, reach per-expert submodules with LoRA, run the root dispatch hook on the embedding's device (Nemotron-Labs-Teacher) (#11543)

Daniel Han98fc7fee6e33
diff --git a/tests/python/test_fast_language_model_text_only.py b/tests/python/test_fast_language_model_text_only.py
index 1f842792acb..c6bbf795d0e 100644
--- a/tests/python/test_fast_language_model_text_only.py
+++ b/tests/python/test_fast_language_model_text_only.py
@@ -79,6 +79,7 @@ def _load_text_only_namespace():
if isinstance(node, ast.FunctionDef)
}
for name in (
+ "_resolve_remote_model_class",
"resolve_model_class",
"_is_family_text_decoder",
"_remap_text_only_skip_modules",
diff --git a/tests/test_attention_implementation.py b/tests/test_attention_implementation.py
index 4fff5f0eeea..ebdb312a813 100644
--- a/tests/test_attention_implementation.py
+++ b/tests/test_attention_implementation.py
@@ -1,3 +1,4 @@
+# SPDX-License-Identifier: AGPL-3.0-only
from types import SimpleNamespace
import torch
@@ -15,6 +16,7 @@ class SupportsFlexAndSdpa:
class SupportsFlashAndSdpa:
_supports_flash_attn_2 = True
+ _supports_flash_attn = True # the flag transformers >= 4.53 dispatches on
_supports_flex_attn = False
_supports_sdpa = True
diff --git a/tests/test_attn_dtype_custom_datatype.py b/tests/test_attn_dtype_custom_datatype.py
index d98cf2dc0e0..59a4d8ad27b 100644
--- a/tests/test_attn_dtype_custom_datatype.py
+++ b/tests/test_attn_dtype_custom_datatype.py
@@ -122,6 +122,7 @@ def test_custom_datatype_load_does_not_disable_flash_attention():
class SupportsFlashAndSdpa:
_supports_flash_attn_2 = True
+ _supports_flash_attn = True # the flag transformers >= 4.53 dispatches on
_supports_flex_attn = False
_supports_sdpa = True
diff --git a/tests/test_remote_code_class_and_expert_submodules.py b/tests/test_remote_code_class_and_expert_submodules.py
new file mode 100644
index 00000000000..64679d95e5a
--- /dev/null
+++ b/tests/test_remote_code_class_and_expert_submodules.py
@@ -0,0 +1,586 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+"""Remote code whose config shares a native class name (Nemotron-H hub checkpoints), and
+LoRA targets on per-expert submodules (`mixer.experts.<i>.up_proj`)."""
+
+import re
+import sys
+import types
+from types import SimpleNamespace
+
+import pytest
+import torch
+
+
+def _utils():
+ from unsloth.models import _utils
+ return _utils
+
+
+def test_old_flag_alone_is_not_flash_support_on_new_transformers():
+ U = _utils()
+ from transformers.modeling_utils import PreTrainedModel
+
+ class OldRemote:
+ _supports_flash_attn_2 = True
+
+ class NewNative:
+ _supports_flash_attn = True
+
+ class Neither:
+ pass
+
+ # 5.0 to 5.3 define the new flag but their dispatch check still accepts the old one.
+ legacy_ok = not hasattr(PreTrainedModel, "_supports_flash_attn") or (
+ U._flash_dispatch_reads_legacy_flag(PreTrainedModel)
+ )
+ assert U._model_class_supports_flash_attention(OldRemote) is legacy_ok
+ assert U._model_class_supports_flash_attention(NewNative) is True
+ assert U._model_class_supports_flash_attention(Neither) is False
+ assert U._model_class_supports_flash_attention(None) is False
+
+
+def test_resolver_does_not_request_flash_for_old_flag_remote_class(monkeypatch):
+ U = _utils()
+ # Without flash-attn installed the ladder never reaches flash, and this would pass either way.
+ monkeypatch.setattr(U, "HAS_FLASH_ATTENTION", True)
+ from transformers.modeling_utils import PreTrainedModel
+
+ if not hasattr(PreTrainedModel, "_supports_flash_attn") or (
+ U._flash_dispatch_reads_legacy_flag(PreTrainedModel)
+ ):
+ pytest.skip("transformers still dispatches on _supports_flash_attn_2")
+
+ class OldRemote:
+ _supports_flash_attn_2 = True
+ _supports_sdpa = True
+
+ class NewRemote:
+ _supports_flash_attn = True
+ _supports_sdpa = True
+
+ config = SimpleNamespace(model_type = "nemotron_h", _attn_implementation = None)
+ impl = U.resolve_attention_implementation(OldRemote, config, dtype = torch.bfloat16)
+ assert "flash" not in str(impl)
+ # Control: the same config does reach flash for a class carrying the dispatched flag.
+ config = SimpleNamespace(model_type = "nemotron_h", _attn_implementation = None)
+ impl = U.resolve_attention_implementation(NewRemote, config, dtype = torch.bfloat16)
+ assert impl == "flash_attention_2"
+
+
+def _install_fake_remote_modules(monkeypatch, package = "transformers_modules.fake_repo.abc123"):
+ """A remote config/model pair whose config class name collides with a native one."""
+ from transformers import PretrainedConfig
+ from transformers.models.llama.configuration_llama import LlamaConfig # noqa: F401 (the collision target)
+
+ pkg = types.ModuleType(package)
+ cfg_mod = types.ModuleType(package + ".configuration_llama")
+ model_mod = types.ModuleType(package + ".modeling_llama")
+
+ class LlamaConfig(PretrainedConfig): # same name as the native class on purpose
+ model_type = "llama"
+
+ class LlamaForCausalLM:
+ _supports_flash_attn_2 = True
+
+ LlamaConfig.__module__ = cfg_mod.__name__
+ LlamaForCausalLM.__module__ = model_mod.__name__
+ cfg_mod.LlamaConfig = LlamaConfig
+ model_mod.LlamaForCausalLM = LlamaForCausalLM
+ for m in (pkg, cfg_mod, model_mod):
+ monkeypatch.setitem(sys.modules, m.__name__, m)
+ config = LlamaConfig()
+ config.auto_map = {
+ "AutoConfig": "configuration_llama.LlamaConfig",
+ "AutoModelForCausalLM": "modeling_llama.LlamaForCausalLM",
+ }
+ config._name_or_path = "fake/repo"
+ return config, LlamaForCausalLM
+
+
+def test_remote_config_resolves_to_remote_model_class(monkeypatch):
+ U = _utils()
+ from transformers import AutoModelForCausalLM
+
+ config, remote_cls = _install_fake_remote_modules(monkeypatch)
+ assert U.resolve_model_class(AutoModelForCausalLM, config) is remote_cls
+
+
+def test_fetch_of_a_missing_modeling_module_uses_the_load_options(monkeypatch):
+ """A cold-cache fetch uses the load's revision, token and offline flag."""
+ U = _utils()
+ from transformers import AutoModelForCausalLM
+ import transformers.dynamic_module_utils as dmu
+
+ config, _ = _install_fake_remote_modules(monkeypatch)
+ monkeypatch.delitem(sys.modules, "transformers_modules.fake_repo.abc123.modeling_llama")
+ seen = {}
+
+ class Fetched:
+ pass
+
+ def fake_get(class_ref, repo_id, **kw):
+ seen.update(class_ref = class_ref, repo_id = repo_id, **kw)
+ return Fetched
+
+ monkeypatch.setattr(dmu, "get_class_from_dynamic_module", fake_get)
+ got = U.resolve_model_class(
+ AutoModelForCausalLM,
+ config,
+ trust_remote_code = True,
+ revision = "deadbeef",
+ code_revision = "cafe",
+ token = "tok",
+ cache_dir = "/c",
+ local_files_only = True,
+ proxies = {"https": "http://proxy"},
+ )
+ assert got is Fetched
+ assert seen == dict(
+ class_ref = "modeling_llama.LlamaForCausalLM",
+ repo_id = "fake/repo",
+ revision = "deadbeef",
+ code_revision = "cafe",
+ token = "tok",
+ cache_dir = "/c",
+ local_files_only = True,
+ proxies = {"https": "http://proxy"},
+ )
+
+
+def test_cross_repository_auto_map_skips_the_local_sibling(monkeypatch):
+ """An `other/repo--module.Class` reference must not resolve to a same-named sibling module."""
+ U = _utils()
+ from transformers import AutoModelForCausalLM
+ import transformers.dynamic_module_utils as dmu
+
+ config, local_cls = _install_fake_remote_modules(monkeypatch)
+ config.auto_map["AutoModelForCausalLM"] = "other/repo--modeling_llama.LlamaForCausalLM"
+ seen = {}
+
+ class Remote:
+ pass
+
+ def fake_get(class_ref, repo_id, **kw):
+ seen.update(class_ref = class_ref, repo_id = repo_id)
+ return Remote
+
+ monkeypatch.setattr(dmu, "get_class_from_dynamic_module", fake_get)
+ got = U.resolve_model_class(AutoModelForCausalLM, config, trust_remote_code = True)
+ assert got is Remote and got is not local_cls
+ # Unsplit, with the model path, as from_pretrained calls it.
+ assert seen == dict(
+ class_ref = "other/repo--modeling_llama.LlamaForCausalLM", repo_id = "fake/repo"
+ )
+
+
+def test_native_config_keeps_native_resolution():
+ U = _utils()
+ from transformers import AutoModelForCausalLM, LlamaConfig
+ from transformers.models.llama.modeling_llama import LlamaForCausalLM
+
+ config = LlamaConfig()
+ config.auto_map = {
+ "AutoModelForCausalLM": "modeling_llama.LlamaForCausalLM"
+ } # ignored: not remote
+ assert U.resolve_model_class(AutoModelForCausalLM, config) is LlamaForCausalLM
+
+
+def test_remote_config_without_auto_class_entry_stays_native(monkeypatch):
+ U = _utils()
+ from transformers import AutoModelForCausalLM
+ from transformers.models.llama.modeling_llama import LlamaForCausalLM
+
+ config, _ = _install_fake_remote_modules(monkeypatch)
+ config.auto_map = {"AutoConfig": "configuration_llama.LlamaConfig"}
+ assert U.resolve_model_class(AutoModelForCausalLM, config) is LlamaForCausalLM
+
+
+class _Expert(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.up_proj = torch.nn.Linear(8, 16, bias = False)
+ self.down_proj = torch.nn.Linear(16, 8, bias = False)
+
+
+class _MoE(torch.nn.Module):
+ def __init__(self, n = 4):
+ super().__init__()
+ self.experts = torch.nn.ModuleList([_Expert() for _ in range(n)])
+ self.shared_experts = _Expert()
+ self.fc1_latent_proj = torch.nn.Identity()
+ self.gate = _Router(n)
+
+
+# Nemotron-Labs-Teacher's expert classes come from the checkpoint's own modeling file.
+_Expert.__module__ = "transformers_modules.fake_teacher.modeling_nemotron_h"
+sys.modules.setdefault(_Expert.__module__, sys.modules[__name__])
+
+
+class _Router(torch.nn.Module): # a Parameter-backed router, as in Nemotron-H
+ def __init__(self, n):
+ super().__init__()
+ self.weight = torch.nn.Parameter(torch.zeros(n, 8))
+
+
+class _Mamba(torch.nn.Module): # a mixer with a Linear directly under it, like the Mamba layers
+ def __init__(self):
+ super().__init__()
+ self.in_proj = torch.nn.Linear(8, 32, bias = False)
+
+
+class _Layer(torch.nn.Module):
+ def __init__(self, mixer):
+ super().__init__()
+ self.mixer = mixer
+
+
+class _Inner(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.layers = torch.nn.ModuleList(
+ [_Layer(_Mamba()), _Layer(_MoE()), _Layer(_Mamba()), _Layer(_MoE())]
+ )
+
+
+class _Model(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.config = SimpleNamespace(n_routed_experts = 4, model_type = "nemotron_h")
+ self.model = _Inner()
+
+
+def _text_only_regex():
+ """The default FastModel.get_peft_model regex misses nested experts in a text-only model."""
+ import importlib
+
+ stub = sys.modules.get("unsloth_zoo.peft_utils")
+ if stub is not None and getattr(stub, "__file__", None) is None:
+ # Another test file leaves a stub in sys.modules; the real module is wanted here.
+ del sys.modules["unsloth_zoo.peft_utils"]
+ peft_utils = importlib.import_module("unsloth_zoo.peft_utils")
+ return peft_utils.get_peft_regex(_Model())
+
+
+def test_text_only_regex_misses_nested_experts_on_its_own():
+ regex = _text_only_regex()
+ names = [n for n, m in _Model().named_modules() if isinstance(m, torch.nn.Linear)]
+ assert not any(re.fullmatch(regex, n) for n in names if ".experts." in n)
+
+
+def test_expert_submodule_leaves_and_regex_reach_every_expert():
+ U = _utils()
+ model = _Model()
+ regex = _text_only_regex()
+ leaves = U.get_moe_expert_submodule_leaves(model, regex)
+ assert leaves == ["down_proj", "up_proj"]
+ extended = f"(?:{regex})|(?:{U.moe_expert_submodule_regex(leaves)})"
+ matched = {n for n, m in model.named_modules() if re.fullmatch(extended, n)}
+ linears = {n for n, m in model.named_modules() if isinstance(m, torch.nn.Linear)}
+ experts = {n for n in linears if ".experts." in n or ".shared_experts." in n}
+ assert experts <= matched
+ assert "model.layers.1.mixer.gate" not in matched # the router is not a Linear
+ assert "model.layers.1.mixer.fc1_latent_proj" not in matched # Identity, not a Linear
+ assert "model.layers.0.mixer.in_proj" in matched # the block-level leaves stay
+ assert all(isinstance(dict(model.named_modules())[n], torch.nn.Linear) for n in matched)
+
+
+def test_expert_submodule_leaves_follow_the_request():
+ U = _utils()
+ model = _Model()
+ assert U.get_moe_expert_submodule_leaves(model, ["down_proj"]) == ["down_proj"]
+ assert U.get_moe_expert_submodule_leaves(model, ["q_proj", "k_proj"]) == []
+ assert U.get_moe_expert_submodule_leaves(model, None) == []
+
+
+def test_non_moe_model_adds_nothing():
+ U = _utils()
+ model = _Model()
+ model.config = SimpleNamespace(model_type = "llama")
+ assert U.get_moe_expert_submodule_leaves(model, ["up_proj", "down_proj"]) == []
+
+
+def test_only_a_generated_regex_is_widened_to_the_routed_experts():
+ """A caller-written regex is never widened; the generated text-only regex is."""
+ U = _utils()
+ model = _Model()
+ caller_regex = r".*\.shared_experts\.down_proj"
+ kept, detect, leaves = U.widen_target_regex_to_expert_submodules(
+ model, caller_regex, caller_regex, auto_regex = False
+ )
+ assert kept == caller_regex and detect == caller_regex and leaves == []
+ matched = {n for n, _ in model.named_modules() if re.fullmatch(kept, n)}
+ assert matched and all(".shared_experts." in n for n in matched)
+
+ generated = _text_only_regex()
+ widened, detect, leaves = U.widen_target_regex_to_expert_submodules(
+ model, generated, generated, auto_regex = True
+ )
+ assert leaves == ["down_proj", "up_proj"]
+ assert detect == widened != generated
+ matched = {n for n, _ in model.named_modules() if re.fullmatch(widened, n)}
+ assert any(".experts." in n for n in matched)
+
+ # A leaf list as the detection target keeps its own identity through the widening.
+ widened, detect, leaves = U.widen_target_regex_to_expert_submodules(
+ model, generated, ["down_proj"], auto_regex = True
+ )
+ assert leaves == ["down_proj"] and detect == ["down_proj"] and widened != generated
+
+
+def test_gate_and_up_expert_leaves_stay_separate():
+ U = _utils()
+ model = _Model()
+ assert U.get_moe_expert_submodule_leaves(model, ["up_proj"]) == ["up_proj"]
+ assert (
+ U.get_moe_expert_submodule_leaves(model, ["gate_proj"]) == []
+ ) # the fixture has no gate leaf
+ assert U.get_moe_expert_submodule_leaves(model, ["gate_up_proj"]) == ["up_proj"]
+
+
+def test_legacy_flash_flag_follows_the_installed_dispatch_check():
+ """The helper reads the installed dispatch check for the legacy flag."""
+ U = _utils()
+
+ class OldDispatch:
+ _supports_flash_attn = False
+
+ def _flash_attn_can_dispatch(self):
+ if not (self._supports_flash_attn or getattr(self, "_supports_flash_attn_2", False)):
+ raise ValueError("no")
+
+ class NewDispatch:
+ _supports_flash_attn = False
+
+ def _flash_attn_can_dispatch(self):
+ if not self._supports_flash_attn:
+ message = "x"
+ if self._supports_flash_attn or getattr(self, "_supports_flash_attn_2", False):
+ message += ", "
+ raise ValueError(message)
+
+ assert U._flash_dispatch_reads_legacy_flag(OldDispatch) is True
+ assert U._flash_dispatch_reads_legacy_flag(NewDispatch) is False
+
+
+def test_force_download_bypasses_the_imported_sibling(monkeypatch):
+ U = _utils()
+ calls = []
+
+ def fake_get_class(class_ref, repo_id, **kw):
+ calls.append((class_ref, repo_id, kw))
+ return type("Built", (), {})
+
+ import transformers.dynamic_module_utils as dmu
+
+ monkeypatch.setattr(dmu, "get_class_from_dynamic_module", fake_get_class)
+ import sys, types
+
+ module = types.ModuleType("transformers_modules.fake.configuration_fake")
+ sys.modules["transformers_modules.fake.configuration_fake"] = module
+ sibling = types.ModuleType("transformers_modules.fake.modeling_fake")
+ sibling.FakeForCausalLM = type("FakeForCausalLM", (), {})
+ sys.modules["transformers_modules.fake.modeling_fake"] = sibling
+ try:
+ cfg_cls = type("FakeConfig", (), {})
+ cfg_cls.__module__ = "transformers_modules.fake.configuration_fake"
+ cfg = cfg_cls()
+ cfg.auto_map = {"AutoModelForCausalLM": "modeling_fake.FakeForCausalLM"}
+ cfg._name_or_path = "fake/repo"
+ auto = type("AutoModelForCausalLM", (), {})
+ assert U._resolve_remote_model_class(auto, cfg) is sibling.FakeForCausalLM
+ assert calls == []
+ built = U._resolve_remote_model_class(
+ auto, cfg, trust_remote_code = True, force_download = True
+ )
+ assert (
+ built is not sibling.FakeForCausalLM
+ and calls
+ and calls[0][2].get("force_download") is True
+ )
+ assert U._resolve_remote_model_class(auto, cfg, trust_remote_code = False) is None
+ finally:
+ sys.modules.pop("transformers_modules.fake.configuration_fake", None)
+ sys.modules.pop("transformers_modules.fake.modeling_fake", None)
+
+
+def test_every_resolver_probe_forwards_the_trust_decision():
+ """Class probes pass trust_remote_code and the load's hub kwargs."""
+ import ast, inspect
+ from unsloth.models import llama, loader, loader_utils, vision
+
+ probes = ("resolve_model_class", "_resolve_omni_auto_model")
+ planner = next(
+ node
+ for node in ast.walk(ast.parse(inspect.getsource(loader_utils)))
+ if isinstance(node, ast.FunctionDef) and node.name == "planner_model_class"
+ )
+ for module, tree in (
+ (loader, ast.parse(inspect.getsource(loader))),
+ (vision, ast.parse(inspect.getsource(vision))),
+ (llama, ast.parse(inspect.getsource(llama))),
+ (loader_utils, planner),
+ ):
+ for node in ast.walk(tree):
+ if not (isinstance(node, ast.Call) and getattr(node.func, "id", None) in probes):
+ continue
+ names = {k.arg for k in node.keywords}
+ splats = [getattr(k.value, "id", "") for k in node.keywords if k.arg is None]
+ assert "trust_remote_code" in names or "_probe_hub_kwargs" in splats, (
+ module.__name__,
+ node.lineno,
+ )
+ if module in (loader, vision) and not splats:
+ assert {"revision", "token", "local_files_only", "proxies"} <= names, (
+ module.__name__,
+ node.lineno,
+ )
+
+
+def test_a_code_revision_skips_the_materialised_sibling(monkeypatch):
+ """With a code_revision the resolver must ask transformers, not the sibling module."""
+ import importlib
+ import types
+
+ from unsloth.models import _utils
+
+ sibling = types.ModuleType("transformers_modules.tiny_rev.modeling_tiny")
+
+ class FromConfigRevision:
+ pass
+
+ class FromCodeRevision:
+ pass
+
+ sibling.TinyForCausalLM = FromConfigRevision
+ config_cls = type(
+ "TinyConfig", (), {"__module__": "transformers_modules.tiny_rev.configuration_tiny"}
+ )
+ config = config_cls()
+ config.auto_map = {"AutoModelForCausalLM": "modeling_tiny.TinyForCausalLM"}
+ config._name_or_path = "someone/tiny"
+ monkeypatch.setitem(__import__("sys").modules, sibling.__name__, sibling)
+ import transformers.dynamic_module_utils as dmu
+
+ monkeypatch.setattr(dmu, "get_class_from_dynamic_module", lambda *a, **k: FromCodeRevision)
+ auto = type("AutoModelForCausalLM", (), {})
+ assert _utils._resolve_remote_model_class(auto, config) is FromConfigRevision
+ assert (
+ _utils._resolve_remote_model_class(
+ auto, config, trust_remote_code = True, code_revision = "abc123"
+ )
+ is FromCodeRevision
+ )
+ # A load revision is the code revision for same-repo code, so it skips the sibling too.
+ assert (
+ _utils._resolve_remote_model_class(auto, config, trust_remote_code = True, revision = "b")
+ is FromCodeRevision
+ )
+
+
+def test_native_per_expert_layouts_are_not_widened():
+ """Qwen3-MoE on transformers 4.x nests native per-expert Linears the same way; those
+ keep main's targets instead of gaining LoRA on every routed expert."""
+ U = _utils()
+
+ class _NativeExpert(_Expert):
+ pass
+
+ _NativeExpert.__module__ = "transformers.models.qwen3_moe.modeling_qwen3_moe"
+ model = _Model()
+ for layer in model.model.layers:
+ mixer = layer.mixer
+ if isinstance(mixer, _MoE):
+ mixer.experts = torch.nn.ModuleList([_NativeExpert() for _ in mixer.experts])
+ mixer.shared_experts = _NativeExpert()
+ generated = _text_only_regex()
+ kept, detect, leaves = U.widen_target_regex_to_expert_submodules(
+ model, generated, generated, auto_regex = True
+ )
+ assert kept == generated and detect == generated and leaves == []
+
+
+def test_a_native_expert_tower_next_to_remote_code_is_not_widened():
+ """Only the remote expert blocks' own parents gain the nested alternative."""
+ U = _utils()
+
+ class _NativeExpert(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.up_proj = torch.nn.Linear(8, 16, bias = False)
+ self.down_proj = torch.nn.Linear(16, 8, bias = False)
+
+ class _NativeTower(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.experts = torch.nn.ModuleList([_NativeExpert() for _ in range(4)])
+
+ _NativeExpert.__module__ = "transformers.models.some_vlm.modeling_some_vlm"
+ model = _Model()
+ model.visual = torch.nn.ModuleList([_NativeTower(), _NativeTower()])
+ generated = _text_only_regex()
+ widened, _, leaves = U.widen_target_regex_to_expert_submodules(
+ model, generated, generated, auto_regex = True
+ )
+ assert leaves == ["down_proj", "up_proj"]
+ matched = {n for n, _ in model.named_modules() if re.fullmatch(widened, n)}
+ assert any(n.startswith("model.layers.") and ".experts." in n for n in matched)
+ assert not any(n.startswith("visual.") for n in matched)
+
+
+def test_a_load_that_did_not_ask_for_remote_code_never_fetches(monkeypatch):
+ """Without trust_remote_code the resolver may reuse an imported sibling, never the Hub."""
+ import transformers.dynamic_module_utils as dmu
+ from unsloth.models import _utils
+
+ fetched = []
+ monkeypatch.setattr(
+ dmu, "get_class_from_dynamic_module", lambda *a, **k: fetched.append(k) or object
+ )
+ config_cls = type(
+ "TinyConfig", (), {"__module__": "transformers_modules.not_imported.configuration_tiny"}
+ )
+ config = config_cls()
+ config.auto_map = {"AutoModel": "modeling_tiny.TinyModel"}
+ config._name_or_path = "someone/tiny"
+ auto = type("AutoModel", (), {})
+ for trust in (None, False):
+ assert _utils._resolve_remote_model_class(auto, config, trust_remote_code = trust) is None
+ assert fetched == []
+
+
+def test_sentence_transformer_probes_use_the_loads_options(monkeypatch):
+ """Both FastSentenceTransformer class probes see the load's trust, revision, token,
+ cache and offline mode."""
+ import ast
+ import inspect
+
+ from unsloth.models import _utils, sentence_transformer
+
+ seen = []
+ monkeypatch.setattr(
+ _utils, "resolve_model_class", lambda auto, config, **kw: seen.append(kw) or None
+ )
+ options = dict(
+ trust_remote_code = True,
+ revision = "abc",
+ token = "t",
+ local_files_only = True,
+ proxies = {"https": "http://proxy"},
+ )
+ _utils.resolve_encoder_attention_implementation(object, object(), **options)
+ monkeypatch.setattr(
+ sentence_transformer, "resolve_model_class", lambda auto, config, **kw: seen.append(kw)
+ )
+ sentence_transformer.FastSentenceTransformer._has_add_pooling_layer(object(), object, **options)
+ assert seen == [options, options]
+ probes = {"resolve_encoder_attention_implementation", "_has_add_pooling_layer"}
+ calls = [
+ node
+ for node in ast.walk(ast.parse(inspect.getsource(sentence_transformer)))
+ if isinstance(node, ast.Call)
+ and (getattr(node.func, "id", None) in probes or getattr(node.func, "attr", None) in probes)
+ ]
+ assert len(calls) == 2
+ for call in calls:
+ splats = [getattr(k.value, "id", "") for k in call.keywords if k.arg is None]
+ assert "_remote_class_probe_kwargs" in splats, call.lineno
diff --git a/tests/test_root_hook_follows_input_embedding.py b/tests/test_root_hook_follows_input_embedding.py
new file mode 100644
index 00000000000..ccb7bc0530f
--- /dev/null
+++ b/tests/test_root_hook_follows_input_embedding.py
@@ -0,0 +1,122 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+"""The root dispatch hook runs where the input embedding lives, not on the first device in
+the map, so attention_mask follows input_ids."""
+
+from types import SimpleNamespace
+
+import pytest
+import torch
+
+
+def _helper():
+ from unsloth.models.vision import _align_root_hook_with_input_embeddings
+ return _align_root_hook_with_input_embeddings
+
+
+class _RemoteModel(SimpleNamespace):
+ pass
+
+
+# The Teacher's model class comes from the checkpoint's own modeling file.
+_RemoteModel.__module__ = "transformers_modules.teacher.modeling_nemotron_h"
+
+
+def _model(
+ device_map,
+ root_device,
+ embedding_device,
+ cls = _RemoteModel,
+ config = None,
+):
+ embedding = SimpleNamespace(weight = SimpleNamespace(device = embedding_device))
+ return cls(
+ hf_device_map = device_map,
+ _hf_hook = SimpleNamespace(execution_device = root_device),
+ get_input_embeddings = lambda: embedding,
+ config = config or SimpleNamespace(),
+ )
+
+
+def test_root_moves_to_the_embedding_device():
+ align = _helper()
+ model = _model(
+ {"model.layers.1": 2, "model.embeddings": 1, "lm_head": 2, "model.layers.0": 0},
+ 0,
+ torch.device("cuda", 1),
+ )
+ assert align(model) == torch.device("cuda", 1)
+ assert model._hf_hook.execution_device == torch.device("cuda", 1)
+
+
+def test_root_already_on_the_embedding_device_is_untouched():
+ align = _helper()
+ for root in (0, torch.device("cuda", 0), "cuda:0"):
+ model = _model({"model.embeddings": 0, "lm_head": 1}, root, torch.device("cuda", 0))
+ assert align(model) is None
+ assert model._hf_hook.execution_device == root
+
+
+def test_single_device_and_unhooked_models_are_untouched():
+ align = _helper()
+ model = _model({"": 0}, 0, torch.device("cuda", 1))
+ assert align(model) is None
+ model = _model({"model.embeddings": 0, "lm_head": 1}, 0, torch.device("cuda", 1))
+ model._hf_hook = None
+ assert align(model) is None
+ model = _model({"model.embeddings": 0, "lm_head": 1}, 0, torch.device("cuda", 1))
+ del model._hf_hook
+ assert align(model) is None
+
+
+def test_offloaded_or_meta_embedding_is_left_to_its_hook():
+ align = _helper()
+ for device in (torch.device("cpu"), torch.device("meta")):
+ model = _model({"model.embeddings": "cpu", "lm_head": 0}, 0, device)
+ assert align(model) is None
+ assert model._hf_hook.execution_device == 0
+
+
+def test_model_without_embedding_accessor_is_untouched():
+ align = _helper()
+ model = _model({"model.embeddings": 1, "lm_head": 0}, 0, torch.device("cuda", 1))
+
+ def broken():
+ raise NotImplementedError
+
+ model.get_input_embeddings = broken
+ assert align(model) is None
+
+
+def test_native_and_multimodal_models_are_untouched():
+ """Native code moves the mask itself; the root hook moves every input, so a vision or
+ audio tower would get its tensors on the text card."""
+ align = _helper()
+ split = {"model.layers.1": 2, "model.embeddings": 1, "lm_head": 2, "model.layers.0": 0}
+ native = _model(split, 0, torch.device("cuda", 1), cls = SimpleNamespace)
+ assert align(native) is None
+ assert native._hf_hook.execution_device == 0
+ for name in ("vision_config", "audio_config"):
+ config = SimpleNamespace(**{name: SimpleNamespace()})
+ multimodal = _model(split, 0, torch.device("cuda", 1), config = config)
+ assert align(multimodal) is None
+ assert multimodal._hf_hook.execution_device == 0
+
+
+def test_omni_config_with_nested_towers_is_untouched():
+ """Omni checkpoints keep vision / audio under thinker_config; they are still multimodal."""
+ align = _helper()
+ split = {"model.layers.1": 2, "model.embeddings": 1, "lm_head": 2, "model.layers.0": 0}
+ config = SimpleNamespace(thinker_config = SimpleNamespace(vision_config = SimpleNamespace()))
+ omni = _model(split, 0, torch.device("cuda", 1), config = config)
+ assert align(omni) is None
+ assert omni._hf_hook.execution_device == 0
+
+
[email protected]("tower", ["vision_encoder_config", "audio_encoder_config", "encoder_config"])
+def test_encoder_spelled_tower_configs_are_untouched(tower):
+ """The same spellings _uses_flash_attention_for_generation already treats as non-language."""
+ align = _helper()
+ split = {"model.layers.1": 2, "model.embeddings": 1, "lm_head": 2, "model.layers.0": 0}
+ multimodal = _model(split, 0, torch.device("cuda", 1), config = SimpleNamespace(**{tower: SimpleNamespace()}))
+ assert align(multimodal) is None
+ assert multimodal._hf_hook.execution_device == 0
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index 272e6c19321..af78c2cb240 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -86,6 +86,9 @@ __all__ = [
"is_moe_model",
"get_moe_target_parameters",
"get_moe_target_modules",
+ "get_moe_expert_submodule_leaves",
+ "moe_expert_submodule_regex",
+ "widen_target_regex_to_expert_submodules",
"warn_if_zoo_cannot_merge_moe_experts",
"_select_moe_detection_targets",
"EMBEDDING_MODULES",
@@ -773,6 +776,43 @@ def _declares_flex_support(model_class):
return None
+def _model_class_supports_flash_attention(model_class):
+ """Whether installed transformers lets this class dispatch flash attention."""
+ if model_class is None:
+ return False
+ try:
+ from transformers.modeling_utils import PreTrainedModel
+ except Exception:
+ PreTrainedModel = None
+ new_flag_dispatched = PreTrainedModel is not None and hasattr(
+ PreTrainedModel, "_supports_flash_attn"
+ )
+ if new_flag_dispatched and not _flash_dispatch_reads_legacy_flag(PreTrainedModel):
+ return bool(getattr(model_class, "_supports_flash_attn", False))
+ return bool(
+ getattr(model_class, "_supports_flash_attn_2", False)
+ or getattr(model_class, "_supports_flash_attn", False)
+ )
+
+
+def _flash_dispatch_reads_legacy_flag(PreTrainedModel) -> bool:
+ # Read from source: later versions keep the legacy name only in the error message.
+ import inspect
+ for name in ("_flash_attn_can_dispatch", "_flash_attn_2_can_dispatch"):
+ check = getattr(PreTrainedModel, name, None)
+ if check is None:
+ continue
+ try:
+ source = inspect.getsource(check)
+ except (OSError, TypeError):
+ return False
+ return any(
+ "_supports_flash_attn_2" in line and line.lstrip().startswith("if not")
+ for line in source.splitlines()
+ )
+ return False
+
+
def _enable_flex_attention_support(model_class, model_type = ""):
"""Set `_supports_flex_attn` on an interface-based architecture that leaves it unset (qwen3_5,
qwen3_5_moe). Returns True when flex is now permitted."""
@@ -1087,7 +1127,73 @@ def _set_attn_impl(config, impl):
return impl
-def resolve_model_class(auto_model, config):
+_REMOTE_CODE_HUB_KWARGS = (
+ "revision",
+ "code_revision",
+ "token",
+ "cache_dir",
+ "local_files_only",
+ "force_download",
+ "proxies",
+)
+
+
+def _resolve_remote_model_class(auto_model, config, **hub_kwargs):
+ """Remote-code class `auto_model` would build for `config`, or None when native.
+ Auto mapping keys on config class name, so a remote `NemotronHConfig` would resolve native."""
+ if hub_kwargs.get("trust_remote_code", None) is False:
+ return None
+ auto_name = getattr(auto_model, "__name__", None)
+ auto_map = getattr(config, "auto_map", None)
+ if not auto_name or not isinstance(auto_map, dict) or auto_name not in auto_map:
+ return None
+ if not str(getattr(type(config), "__module__", "")).startswith("transformers_modules"):
+ return None
+ class_ref = auto_map[auto_name]
+ if not isinstance(class_ref, str) or "." not in class_ref:
+ return None
+ repo_id = getattr(config, "_name_or_path", None) or getattr(config, "name_or_path", None)
+ full_ref = class_ref
+ cross_repo = "--" in class_ref
+ if cross_repo:
+ _, class_ref = class_ref.split("--", 1)
+ module_name, class_name = class_ref.rsplit(".", 1)
+ # These may replace an already imported sibling module (revision = code revision same-repo).
+ if (
+ not cross_repo
+ and not hub_kwargs.get("force_download", False)
+ and not hub_kwargs.get("code_revision", None)
+ and not hub_kwargs.get("revision", None)
+ ):
+ config_module = str(type(config).__module__)
+ try:
+ import importlib
+
+ sibling = importlib.import_module(f"{config_module.rsplit('.', 1)[0]}.{module_name}")
+ klass = getattr(sibling, class_name, None)
+ if isinstance(klass, type):
+ return klass
+ except Exception:
+ pass
+ if not repo_id or not hub_kwargs.get("trust_remote_code", None):
+ return None
+ try:
+ from transformers.dynamic_module_utils import get_class_from_dynamic_module
+
+ passed = {
+ k: v for k, v in hub_kwargs.items() if k in _REMOTE_CODE_HUB_KWARGS and v is not None
+ }
+ # Same call as from_pretrained, so `revision` applies only to same-repo code.
+ klass = get_class_from_dynamic_module(full_ref, repo_id, **passed)
+ return klass if isinstance(klass, type) else None
+ except Exception:
+ return None
+
+
+def resolve_model_class(auto_model, config, **hub_kwargs):
+ remote_class = _resolve_remote_model_class(auto_model, config, **hub_kwargs)
+ if remote_class is not None:
+ return remote_class
mapping = getattr(auto_model, "_model_mapping", {})
try:
result = mapping[config.__class__]
@@ -1415,14 +1521,9 @@ def resolve_attention_implementation(
supports_sdpa = model_class is not None and getattr(model_class, "_supports_sdpa", False)
if _is_sdpa_excluded(model_type):
supports_sdpa = False
- supports_flash_attention = (
- model_class is not None
- and (
- getattr(model_class, "_supports_flash_attn_2", False)
- or getattr(model_class, "_supports_flash_attn", False)
- )
- and not _is_flash_excluded(model_type)
- )
+ supports_flash_attention = _model_class_supports_flash_attention(
+ model_class
+ ) and not _is_flash_excluded(model_type)
supports_flex_attention = _supports_flex_attention(model_class, config, model_type)
# Before the ladder, since every branch below reads supports_flex_attention.
prefers_flex_for_head_dim = _prefers_flex_for_head_dim(config)
@@ -1526,8 +1627,9 @@ def resolve_encoder_attention_implementation(
config,
model_type = "",
disable_sdpa_model_names = (),
+ **hub_kwargs,
):
- model_class = resolve_model_class(auto_model, config)
+ model_class = resolve_model_class(auto_model, config, **hub_kwargs)
supports_sdpa = model_class is not None and getattr(model_class, "_supports_sdpa", False)
if any(name in model_type.lower() for name in disable_sdpa_model_names):
return "eager"
@@ -5098,7 +5200,9 @@ def get_moe_target_modules(model, target_modules = None) -> List[str]:
return []
# Scope the suffixes to the requested leaves, matching get_moe_target_parameters: gate/up/gate_up map to the fused gate_up ModuleList and down_proj to the down one, so a down-only request must not pull in the other projection.
- want_gate_up = bool(target_set & {"gate_proj", "up_proj", "gate_up_proj"})
+ # gate_proj alone must not collect up_proj leaves; only gate_up_proj asks for both.
+ want_gate = bool(target_set & {"gate_proj", "gate_up_proj"})
+ want_up = bool(target_set & {"up_proj", "gate_up_proj"})
want_down = "down_proj" in target_set
targets = set()
@@ -5117,10 +5221,17 @@ def get_moe_target_modules(model, target_modules = None) -> List[str]:
continue
leaf_lower = leaf.lower()
is_down = "down" in leaf_lower
- is_gate_up = (not is_down) and ("gate" in leaf_lower or "up" in leaf_lower)
+ is_gate = (not is_down) and "gate" in leaf_lower
+ is_up = (not is_down) and "up" in leaf_lower
+ is_gate_up = is_gate or is_up
if is_down and not want_down:
continue
- if is_gate_up and not want_gate_up:
+ if is_gate and is_up: # a fused gate_up leaf: either request reaches it
+ if not (want_gate or want_up):
+ continue
+ elif is_gate and not want_gate:
+ continue
+ elif is_up and not want_up:
continue
# One entry per expert index; leaf.<i> matches expert i in every layer.
for expert_index in range(len(module)):
@@ -5129,6 +5240,113 @@ def get_moe_target_modules(model, target_modules = None) -> List[str]:
return sorted(targets)
+_EXPERT_SUBMODULE_PATTERN = re.compile(r"(?:^|\.)(?:experts\.\d+|shared_experts?)\.([A-Za-z_]\w*)$")
+_EXPERT_BLOCK_PATTERN = re.compile(r"(?:^|\.)(?:experts\.\d+|shared_experts?)$")
+
+
+def get_moe_expert_submodule_leaves(model, target_modules = None) -> List[str]:
+ """Requested MLP leaves inside expert submodules, unreached by the text-only regex."""
+ if not is_moe_model(model):
+ return []
+ if target_modules is None or not hasattr(model, "named_modules"):
+ return []
+ if isinstance(target_modules, str):
+ target_set = _moe_target_set_from_string(target_modules)
+ else:
+ target_set = {
+ target
+ for target in target_modules or ()
+ if (isinstance(target, str) and "." not in target and target in _MOE_BROAD_MLP_TARGETS)
+ }
+ if not (target_set & _MOE_BROAD_MLP_TARGETS):
+ return []
+ # gate_proj alone must not collect up_proj leaves; only gate_up_proj asks for both.
+ want_gate = bool(target_set & {"gate_proj", "gate_up_proj"})
+ want_up = bool(target_set & {"up_proj", "gate_up_proj"})
+ want_down = "down_proj" in target_set
+
+ leaves = set()
+ for name, module in model.named_modules():
+ matched = _EXPERT_SUBMODULE_PATTERN.search(name)
+ if matched is None:
+ continue
+ if not (
+ isinstance(module, torch.nn.Linear)
+ or isinstance(getattr(module, "base_layer", None), torch.nn.Linear)
+ ):
+ continue
+ leaf = matched.group(1)
+ leaf_lower = leaf.lower()
+ is_down = "down" in leaf_lower
+ is_gate = (not is_down) and "gate" in leaf_lower
+ is_up = (not is_down) and "up" in leaf_lower
+ is_gate_up = is_gate or is_up
+ if is_down and not want_down:
+ continue
+ if is_gate and is_up: # a fused gate_up leaf: either request reaches it
+ if not (want_gate or want_up):
+ continue
+ elif is_gate and not want_gate:
+ continue
+ elif is_up and not want_up:
+ continue
+ if not (is_down or is_gate_up):
+ continue
+ leaves.add(leaf)
+ return sorted(leaves)
+
+
+def moe_expert_submodule_regex(leaves, prefixes = None) -> str:
+ parent = r".*" if not prefixes else "(?:" + "|".join(sorted(prefixes)) + ")"
+ return (
+ parent
+ + r"\.(?:experts\.\d+|shared_experts?)\.(?:"
+ + "|".join(re.escape(leaf) for leaf in leaves)
+ + r")"
+ )
+
+
+def _remote_expert_parents(model):
+ parents = set()
+ for name, module in model.named_modules():
+ matched = _EXPERT_BLOCK_PATTERN.search(name)
+ if matched is None:
+ continue
+ if "transformers_modules" not in (getattr(type(module), "__module__", "") or ""):
+ continue
+ parent = name[: matched.start()]
+ parents.add(
+ r"\.".join(r"\d+" if part.isdigit() else re.escape(part) for part in parent.split("."))
+ )
+ return parents
+
+
+def widen_target_regex_to_expert_submodules(
+ model, target_modules, detect_targets, auto_regex: bool
+):
+ """Widen a generated regex to ``experts.<i>.<leaf>``; a user-written regex is never widened."""
+ if not auto_regex or not isinstance(target_modules, str):
+ return target_modules, detect_targets, []
+ # Remote expert blocks only: widening native ones (Qwen3-MoE on 4.x) LoRAs every routed expert.
+ parents = _remote_expert_parents(model)
+ if not parents:
+ return target_modules, detect_targets, []
+ leaves = get_moe_expert_submodule_leaves(model, detect_targets)
+ if not leaves:
+ return target_modules, detect_targets, []
+ if any(
+ re.fullmatch(target_modules, name)
+ for name, _ in model.named_modules()
+ if ".experts." in name
+ ):
+ return target_modules, detect_targets, []
+ detect_was_the_regex = detect_targets is target_modules
+ target_modules = f"(?:{target_modules})|(?:{moe_expert_submodule_regex(leaves, parents)})"
+ if detect_was_the_regex:
+ detect_targets = target_modules
+ return target_modules, detect_targets, leaves
+
+
def warn_if_zoo_cannot_merge_moe_experts():
"""Warn once when the installed unsloth_zoo cannot fold per-expert Linear MoE LoRA into a merged_16bit checkpoint. Older zoo releases keep the fused gate_up_proj / down_proj tensors and drop the per-expert deltas, so save_pretrained_merged("merged_16bit") would silently lose the expert training (the LoRA adapter itself still saves and reloads correctly)."""
try:
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index 618cc19bc2c..6779bba4d87 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -2618,7 +2618,11 @@ class FastLlamaModel:
if _planner_skip_reason is None and num_labels is not None:
_planner_skip_reason = (
planner_class_mismatch_reason(
- resolve_model_class(AutoModelForSequenceClassification, model_config),
+ resolve_model_class(
+ AutoModelForSequenceClassification,
+ model_config,
+ trust_remote_code = trust_remote_code,
+ ),
planner_model_class(model_config, trust_remote_code = trust_remote_code),
)
or "num_labels loads a task head the repo config does not describe"
diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py
index ba4dfd7f51b..0fce2d6152e 100644
--- a/unsloth/models/loader.py
+++ b/unsloth/models/loader.py
@@ -319,7 +319,11 @@ def _config_has_native_class(auto_class, config):
return False
-def _resolve_omni_auto_model(model_config):
+def _resolve_omni_auto_model(
+ model_config,
+ trust_remote_code = None,
+ **hub_kwargs,
+):
"""A multimodal auto class that really maps this config, or None.
Qwen3-Omni names Qwen3OmniMoeForConditionalGeneration so it reads as a VLM,
@@ -334,7 +338,12 @@ def _resolve_omni_auto_model(model_config):
if auto_class is None:
continue
try:
- if resolve_model_class(auto_class, model_config) is not None:
+ if (
+ resolve_model_class(
+ auto_class, model_config, trust_remote_code = trust_remote_code, **hub_kwargs
+ )
+ is not None
+ ):
return auto_class
except Exception:
continue
@@ -1962,12 +1971,25 @@ class FastModel(FastBaseModel):
is_vlm = any(x.endswith("ForConditionalGeneration") for x in architectures)
is_vlm = is_vlm or hasattr(model_config, "vision_config")
load_text_only = text_only and auto_model is None
+ # Class probes below fetch remote modeling code exactly as the load will.
+ _probe_hub_kwargs = dict(
+ trust_remote_code = trust_remote_code,
+ revision = base_revision if not is_peft else None,
+ code_revision = kwargs.get("code_revision", None),
+ token = token,
+ cache_dir = kwargs.get("cache_dir", None),
+ local_files_only = local_files_only,
+ force_download = kwargs.get("force_download", None),
+ proxies = kwargs.get("proxies", None),
+ )
text_only_decoder = False
if load_text_only:
if hasattr(model_config, "vision_config"):
text_config = _get_text_only_config(model_config, old_model_name)
# Skip the vision tower only for families with their own text decoder (Gemma 3); others would load random weights, so keep the full model.
- text_class = resolve_model_class(AutoModelForCausalLM, text_config)
+ text_class = resolve_model_class(
+ AutoModelForCausalLM, text_config, **_probe_hub_kwargs
+ )
if text_class is None or not _is_family_text_decoder(
getattr(model_config, "model_type", ""),
getattr(text_config, "model_type", ""),
@@ -2024,8 +2046,11 @@ class FastModel(FastBaseModel):
auto_model = AutoModelForVision2Seq
# Only when the image-text class has no mapping, so anything that
# resolves today keeps the class it resolves to now.
- if resolve_model_class(auto_model, model_config) is None:
- auto_model = _resolve_omni_auto_model(model_config) or auto_model
+ if resolve_model_class(auto_model, model_config, **_probe_hub_kwargs) is None:
+ auto_model = (
+ _resolve_omni_auto_model(model_config, **_probe_hub_kwargs)
+ or auto_model
+ )
else:
auto_model = AutoModelForCausalLM
diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py
index addce0e59d6..5088dc11766 100644
--- a/unsloth/models/loader_utils.py
+++ b/unsloth/models/loader_utils.py
@@ -258,7 +258,7 @@ def planner_model_class(config, trust_remote_code = False):
from ._utils import resolve_model_class
auto_class = _auto_class_for(config, trust_remote_code = trust_remote_code)
- return resolve_model_class(auto_class, config)
+ return resolve_model_class(auto_class, config, trust_remote_code = trust_remote_code)
except Exception:
# Unknown, not mismatched: an unsloth_zoo without this has no planner to feed.
return None
diff --git a/unsloth/models/sentence_transformer.py b/unsloth/models/sentence_transformer.py
index ea45ada465a..7d5189c14d5 100644
--- a/unsloth/models/sentence_transformer.py
+++ b/unsloth/models/sentence_transformer.py
@@ -885,12 +885,16 @@ class FastSentenceTransformer(FastModel):
modeling_distilbert.DistilBertModel.forward = forward
@staticmethod
- def _has_add_pooling_layer(config, auto_model_class = None):
+ def _has_add_pooling_layer(
+ config,
+ auto_model_class = None,
+ **hub_kwargs,
+ ):
"""Check if the model class accepts the `add_pooling_layer` argument."""
try:
if auto_model_class is None:
auto_model_class = AutoModel
- model_class = resolve_model_class(auto_model_class, config)
+ model_class = resolve_model_class(auto_model_class, config, **hub_kwargs)
if model_class:
sig = inspect.signature(model_class.__init__)
@@ -1471,6 +1475,16 @@ class FastSentenceTransformer(FastModel):
"Run `pip install sentence-transformers` to install it."
)
+ # Remote-class probes must use this load's trust, revision and hub options.
+ _remote_class_probe_kwargs = dict(
+ trust_remote_code = trust_remote_code,
+ revision = revision,
+ token = token,
+ cache_dir = kwargs.get("cache_folder", kwargs.get("cache_dir", None)),
+ local_files_only = kwargs.get("local_files_only", None),
+ proxies = kwargs.get("proxies", None),
+ )
+
# The other leaf loaders resolve the "unsloth" sentinel by planning; this one declines. st_device below hands device_map to SentenceTransformer(device=), which ends in self.to(device): the sentinel raises there, and that same .to() would pull a split model back onto one card. The env-var opt-in is resolved too, or UNSLOTH_AUTO_DEVICE_MAP=1 asks for a plan without naming the sentinel.
device_map = requested_device_map(device_map)
# Always "sequential", never the asked-for name's own declined value: the st_device blocks normalise only dicts, "auto" and "sequential", so "balanced" would reach .to("balanced"). isinstance first, since a caller's explicit dict is unhashable and `in` alone raises.
@@ -1602,6 +1616,7 @@ class FastSentenceTransformer(FastModel):
config,
model_type = model_type,
disable_sdpa_model_names = DISABLE_SDPA_MODEL_NAMES,
+ **_remote_class_probe_kwargs,
)
supports_sdpa = encoder_attn_impl == "sdpa"
if encoder_attn_impl is not None:
@@ -1742,7 +1757,7 @@ class FastSentenceTransformer(FastModel):
if "add_pooling_layer" not in kwargs:
supported = FastSentenceTransformer._has_add_pooling_layer(
- config, kwargs.get("auto_model", AutoModel)
+ config, kwargs.get("auto_model", AutoModel), **_remote_class_probe_kwargs
)
if supported:
kwargs["add_pooling_layer"] = False
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index 84371b0b6eb..dae92111181 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -348,6 +348,61 @@ def _lift_endpoint_hooks_onto_adapters(model):
return lifted
+_MODALITY_SUB_CONFIGS = (
+ "vision_config",
+ "audio_config",
+ "speech_config",
+ "sound_config",
+ "vision_encoder_config",
+ "audio_encoder_config",
+ "encoder_config",
+)
+
+
+def _align_root_hook_with_input_embeddings(model):
+ """Point the root dispatch hook at the input embedding's device (remote text models only).
+ accelerate picks the first mapped device, so remote code building its mask there mismatches;
+ vision/audio towers are skipped since the hook moves every input to the text card."""
+ if "transformers_modules" not in (getattr(type(model), "__module__", "") or ""):
+ return None
+ config = getattr(model, "config", None)
+ # Omni configs nest the towers one level down (thinker_config.vision_config).
+ nested = [config] + [getattr(config, name, None) for name in ("thinker_config",)]
+ if any(
+ getattr(sub, name, None) is not None
+ for sub in nested
+ if sub is not None
+ for name in _MODALITY_SUB_CONFIGS
+ ):
+ return None
+ device_map = getattr(model, "hf_device_map", None)
+ if not device_map or len(set(device_map.values())) < 2:
+ return None
+ hook = getattr(model, "_hf_hook", None)
+ current = getattr(hook, "execution_device", None)
+ if current is None:
+ return None
+ try:
+ embedding = model.get_input_embeddings()
+ except Exception:
+ return None
+ weight = getattr(embedding, "weight", None)
+ target = getattr(weight, "device", None)
+ if target is None or target.type in ("cpu", "meta") or target.index is None:
+ return None
+ if isinstance(current, int):
+ current = torch.device(target.type, current)
+ else:
+ try:
+ current = torch.device(current)
+ except (TypeError, RuntimeError):
+ return None
+ if current == target:
+ return None
+ hook.execution_device = target
+ return target
+
+
def _attach_bnb_multidevice_hooks(
model, load_in_4bit, load_in_8bit, offload_embedding, fast_inference
):
@@ -1557,7 +1612,18 @@ class FastBaseModel:
if text_only and hasattr(auto_config, "vision_config"):
parent_config = auto_config
text_config = _get_text_only_config(parent_config, model_name)
- text_class = resolve_model_class(AutoModelForCausalLM, text_config)
+ text_class = resolve_model_class(
+ AutoModelForCausalLM,
+ text_config,
+ revision = _revision,
+ code_revision = kwargs.get("code_revision"),
+ token = token,
+ cache_dir = kwargs.get("cache_dir"),
+ proxies = kwargs.get("proxies"),
+ local_files_only = local_files_only,
+ force_download = kwargs.get("force_download", None),
+ trust_remote_code = trust_remote_code,
+ )
if text_class is not None and _is_family_text_decoder(
getattr(parent_config, "model_type", ""),
getattr(text_config, "model_type", ""),
@@ -1733,7 +1799,18 @@ class FastBaseModel:
local_files_only = local_files_only,
**{k: kwargs.get(k, None) for k in _REMOTE_CLASS_HUB_OPTIONS},
)
- model_class = resolve_model_class(auto_model, auto_config)
+ model_class = _remote_class or resolve_model_class(
+ auto_model,
+ auto_config,
+ revision = _revision,
+ code_revision = kwargs.get("code_revision"),
+ token = token,
+ cache_dir = kwargs.get("cache_dir"),
+ proxies = kwargs.get("proxies"),
+ local_files_only = local_files_only,
+ force_download = kwargs.get("force_download", None),
+ trust_remote_code = trust_remote_code,
+ )
# Forced float32 loads in bfloat16 then casts to float16. Resolved here, not at the load, because attention resolution and the device-map planner both size the same dtype.
torch_dtype = dtype
if do_forced_float32:
@@ -2064,6 +2141,12 @@ class FastBaseModel:
offload_embedding = offload_embedding,
fast_inference = fast_inference,
)
+ _aligned_root_device = _align_root_hook_with_input_embeddings(model)
+ if _aligned_root_device is not None:
+ logger.info(
+ f"Unsloth: inputs now go straight to {_aligned_root_device}, where the input "
+ "embedding lives, instead of through the first device in the map."
+ )
# Re-apply block-fp8 weight_scale_inv tensors transformers dropped on load (#6200).
_restore_dropped_fp8_scales(
model,
@@ -2624,6 +2707,8 @@ class FastBaseModel:
)
_raise_if_fast_inference_modules_to_save(model, modules_to_save)
+ # Only a regex generated here may be widened to expert submodules below.
+ _target_modules_auto_regex = False
if target_modules is None or target_modules == "all-linear":
target_modules = get_peft_regex(
model,
@@ -2633,6 +2718,7 @@ class FastBaseModel:
finetune_mlp_modules = finetune_mlp_modules,
**_audio_kwargs,
)
+ _target_modules_auto_regex = True
else:
assert type(target_modules) in (list, tuple, str)
# Route an explicit list through get_peft_regex when the caller scoped a layer family or opted into audio, so the new audio/embedder branches are considered. finetune_audio_layers is a POSITIVE term: negating it would force every explicit list through the filter.
@@ -2658,6 +2744,7 @@ class FastBaseModel:
target_modules = list(target_modules),
**_audio_kwargs,
)
+ _target_modules_auto_regex = True
if hasattr(model, "vllm_engine"):
if (
@@ -2695,6 +2782,21 @@ class FastBaseModel:
finetune_language_layers = finetune_language_layers,
)
+ # Widen to per-expert submodules before expert detection; never for vision-only.
+ target_modules, _moe_module_detect, _expert_submodule_leaves = (
+ widen_target_regex_to_expert_submodules(
+ model,
+ target_modules,
+ _moe_module_detect,
+ auto_regex = _target_modules_auto_regex and bool(finetune_language_layers),
+ )
+ )
+ if _expert_submodule_leaves:
+ print(
+ f"Unsloth: Detected MoE model with per-expert submodules. "
+ f"Enabling LoRA on expert projections {_expert_submodule_leaves}."
+ )
+
# Per-expert Linear layouts (gpt-oss bnb-4bit) target experts via target_modules, not fused Parameters. Extend either form PEFT accepts: a leaf list, or a regex string.
_moe_module_targets = get_moe_target_modules(model, _moe_module_detect)