Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Resolve the auto class for omni checkpoints, and guard embedding lookups (#11467)

Daniel Han3945bde1a168
diff --git a/tests/test_for_training_generation_flag.py b/tests/test_for_training_generation_flag.py
index 03ec864fa5a..490c0e205ea 100644
--- a/tests/test_for_training_generation_flag.py
+++ b/tests/test_for_training_generation_flag.py
@@ -19,6 +19,27 @@ class _Namespace(dict):
return getattr(builtins, name, None)
+def _helpers_used_by(method, tree):
+ """Module-level functions the lifted method calls, so they run for real.
+
+ Without this they fall to `_Namespace.__missing__` and become None, and the
+ first call raises `TypeError: 'NoneType' object is not callable`: the test
+ then fails for a reason that has nothing to do with what it asserts.
+ """
+ defined = {
+ node.name: node
+ for node in tree.body
+ if isinstance(node, ast.FunctionDef) and not node.decorator_list
+ }
+ wanted, pending = {}, [method]
+ while pending:
+ for node in ast.walk(pending.pop()):
+ if isinstance(node, ast.Name) and node.id in defined and node.id not in wanted:
+ wanted[node.id] = defined[node.id]
+ pending.append(defined[node.id])
+ return list(wanted.values())
+
+
def _for_training(module, class_name):
path = Path(__file__).parents[1] / "unsloth" / "models" / module
tree = ast.parse(path.read_text(encoding = "utf-8"))
@@ -31,7 +52,7 @@ def _for_training(module, class_name):
if isinstance(node, ast.FunctionDef) and node.name == "for_training"
)
method.decorator_list = []
- compiled = ast.Module(body = [method], type_ignores = [])
+ compiled = ast.Module(body = _helpers_used_by(method, tree) + [method], type_ignores = [])
namespace = _Namespace(os = os)
exec(compile(ast.fix_missing_locations(compiled), str(path), "exec"), namespace)
return namespace["for_training"]
diff --git a/tests/test_offload_tied_autodisable.py b/tests/test_offload_tied_autodisable.py
index c1b2a3fd569..3b3720b4a47 100644
--- a/tests/test_offload_tied_autodisable.py
+++ b/tests/test_offload_tied_autodisable.py
@@ -143,10 +143,11 @@ def test_tied_model_disables_offload_instead_of_raising():
assert resolve(_tied_model(), True) is False
-def test_opaque_model_leaves_request_alone():
- # Cannot inspect it, so do not guess, and do not crash.
+def test_opaque_model_disables_offload():
+ # Used to return the request unchanged, which is the one answer that crashes: the caller
+ # acts on True by calling get_input_embeddings() unguarded, the same call that raised here.
with _as_platform("posix"):
- assert resolve(_Opaque(), True) is True
+ assert resolve(_Opaque(), True) is False
def test_wsl_and_windows_disable_offload():
diff --git a/tests/test_omni_auto_class_resolution.py b/tests/test_omni_auto_class_resolution.py
new file mode 100644
index 00000000000..e154b7f36f0
--- /dev/null
+++ b/tests/test_omni_auto_class_resolution.py
@@ -0,0 +1,233 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Auto-class resolution for omni checkpoints, and what it must NOT move.
+
+`Qwen/Qwen3-Omni-30B-A3B-Instruct` names `Qwen3OmniMoeForConditionalGeneration`
+and so reads as a VLM, but transformers registers `qwen3_omni_moe` only under
+`AutoModelForTextToWaveform`. Every other auto class raises "Unrecognized
+configuration class" on it, which is a hard load failure before the weights are
+touched.
+
+The resolver is consulted ONLY when the already-chosen class has no mapping, so
+these tests spend as much effort on the models that must keep taking exactly
+the branch they take today as on the one that changes.
+"""
+
+import pytest
+
+transformers = pytest.importorskip("transformers")
+
+from unsloth.models.loader import ( # noqa: E402
+ _resolve_omni_auto_model,
+ resolve_model_class,
+)
+from unsloth.models.vision import ( # noqa: E402
+ _embeddings_or_none,
+ _multimodal_auto_classes,
+)
+
+try:
+ from transformers import AutoModelForImageTextToText as IMAGE_TEXT_CLASS
+except ImportError: # transformers 4.x
+ from transformers import AutoModelForVision2Seq as IMAGE_TEXT_CLASS
+
+
+def _omni_config():
+ config_class = getattr(transformers, "Qwen3OmniMoeConfig", None)
+ if config_class is None:
+ pytest.skip("this transformers has no Qwen3-Omni")
+ return config_class()
+
+
+def test_the_defect_is_real_on_this_transformers():
+ """Without this, the test below would prove nothing on a build that maps it."""
+ config = _omni_config()
+ assert resolve_model_class(IMAGE_TEXT_CLASS, config) is None
+
+
+def test_omni_resolves_to_the_class_its_family_registered():
+ config = _omni_config()
+ resolved = _resolve_omni_auto_model(config)
+ assert resolved is not None
+ # and it really maps, rather than merely being a different name to fail on
+ assert resolve_model_class(resolved, config) is not None
+
+
[email protected]("config_name", ["Gemma3Config", "Qwen2VLConfig", "LlavaConfig"])
+def test_ordinary_vlms_are_never_rerouted(config_name):
+ """The resolver is behind `resolve_model_class(...) is None`, so a model
+ that resolves today must never reach it."""
+ config_class = getattr(transformers, config_name, None)
+ if config_class is None:
+ pytest.skip(f"this transformers has no {config_name}")
+ assert resolve_model_class(IMAGE_TEXT_CLASS, config_class()) is not None
+
+
+def test_nothing_matching_returns_None_rather_than_a_concrete_class():
+ """Returning the concrete class the checkpoint names is WRONG: it is in no
+ auto mapping, so it leaves the processor set and downgrades to AutoTokenizer."""
+ assert _resolve_omni_auto_model(transformers.LlamaConfig()) is None
+ assert _resolve_omni_auto_model(object()) is None
+
+
+def test_every_class_the_resolver_can_return_takes_a_processor():
+ """The resolver's answer decides processor selection, so every possible
+ answer must be in the set that selects AutoProcessor."""
+ import unsloth.models.loader as loader
+
+ processor_classes = _multimodal_auto_classes()
+ for name in loader._OMNI_AUTO_CLASS_NAMES:
+ auto_class = getattr(transformers, name, None)
+ if auto_class is None:
+ continue
+ assert auto_class in processor_classes, name
+
+
+def test_speech_seq2seq_is_not_treated_as_a_vision_model():
+ """Including it made loading Whisper call
+ _construct_vlm_processor_fallback("openai/whisper-tiny", "whisper"): an image
+ processor build for an audio model, whose WhisperProcessor has none."""
+ speech = getattr(transformers, "AutoModelForSpeechSeq2Seq", None)
+ if speech is None:
+ pytest.skip("this transformers has no AutoModelForSpeechSeq2Seq")
+ assert speech not in _multimodal_auto_classes()
+
+
+def test_image_text_class_is_still_in_the_processor_set():
+ assert IMAGE_TEXT_CLASS in _multimodal_auto_classes()
+
+
+class _CannotAnswer:
+ def get_input_embeddings(self):
+ raise NotImplementedError("composite model, no single embedding")
+
+
+class _WrongSignature:
+ def get_input_embeddings(self, input_ids): # remote code does this
+ raise AssertionError("must not be reached")
+
+
+class _Normal:
+ def get_input_embeddings(self):
+ return "embeddings"
+
+
+ "model, expected",
+ [
+ (_CannotAnswer(), None), # transformers 5 base impl raises
+ (_WrongSignature(), None), # TypeError, not AttributeError
+ (object(), None), # method absent entirely
+ (_Normal(), "embeddings"),
+ ],
+)
+def test_embeddings_or_none(model, expected):
+ """`hasattr` does not answer the question; only calling does."""
+ assert _embeddings_or_none(model, "get_input_embeddings") == expected
+
+
+class _NoEmbeddings:
+ """A model that cannot answer for its embeddings, like Qwen3-Omni."""
+
+ def get_input_embeddings(self):
+ raise NotImplementedError("composite model, no single embedding")
+
+ def get_output_embeddings(self):
+ raise NotImplementedError("composite model, no single embedding")
+
+
[email protected]("requested", [True, "auto"])
+def test_offload_embedding_declines_a_model_it_cannot_inspect(requested, capsys):
+ """Returning the explicit True is WRONG: the caller then calls
+ get_input_embeddings() unguarded, failing the load over a VRAM optimisation
+ that cannot be applied anyway."""
+ from unsloth.models.vision import _resolve_offload_embedding
+
+ assert _resolve_offload_embedding(_NoEmbeddings(), requested) is False
+ printed = capsys.readouterr().out
+ if requested == "auto":
+ # the default declines silently: nobody asked for it
+ assert "Not offloading embeddings" not in printed
+ else:
+ assert "Not offloading embeddings" in printed
+
+
+def test_omni_reaches_the_vllm_guard_rather_than_the_language_model_path():
+ """Without the needs_processor term, is_vlm_config is False for an omni
+ checkpoint (its vision config hides under thinker_config), which skips the
+ guard and calls load_vllm with is_vision_model=False for an unsupported model."""
+ from unsloth.models.vision import VLLM_SUPPORTED_VLM
+
+ config = _omni_config()
+ assert not hasattr(config, "vision_config"), "premise: vision lives under thinker_config"
+ assert "qwen3_omni_moe" not in VLLM_SUPPORTED_VLM, "premise: vLLM does not support it"
+
+ # what is_vlm_config now computes for it
+ resolved = _resolve_omni_auto_model(config)
+ is_vlm = resolved in [IMAGE_TEXT_CLASS]
+ needs_processor = is_vlm or resolved in _multimodal_auto_classes()
+ is_vlm_config = is_vlm or needs_processor or hasattr(config, "vision_config")
+ assert is_vlm_config, "must reach the fast_inference guard"
+
+
+def _tiny(cls):
+ from transformers import LlamaConfig
+
+ return cls(
+ LlamaConfig(
+ hidden_size = 4, num_hidden_layers = 1, num_attention_heads = 1,
+ vocab_size = 8, intermediate_size = 8,
+ )
+ )
+
+
+def _pretrained_base():
+ import torch.nn as nn
+ from transformers import LlamaConfig, PreTrainedModel
+
+ class Base(PreTrainedModel):
+ config_class = LlamaConfig
+
+ def __init__(self, config):
+ super().__init__(config)
+ self.embed = nn.Embedding(8, 4)
+
+ return Base
+
+
+def test_a_getter_that_fails_internally_is_not_read_as_a_bad_signature():
+ """Skipping it would cost the module its input-gradient hook, so under PEFT
+ with gradient checkpointing backward fails or the adapter trains on nothing."""
+ Base = _pretrained_base()
+
+ class RaisesInside(Base):
+ def get_input_embeddings(self):
+ raise TypeError("genuine bug inside a valid zero-arg getter")
+
+ with pytest.raises(TypeError, match = "genuine bug inside"):
+ _tiny(RaisesInside).enable_input_require_grads()
+
+
+def test_a_getter_that_cannot_take_zero_arguments_is_skipped():
+ """stepfun-ai/Step-3.7-Flash declares get_input_embeddings(self, input_ids)."""
+ Base = _pretrained_base()
+
+ class WrongSignature(Base):
+ def get_input_embeddings(self, input_ids):
+ raise AssertionError("must not be reached")
+
+ _tiny(WrongSignature).enable_input_require_grads() # must not raise
+
+
+def test_a_normal_model_still_gets_its_hook():
+ """The negative control: neither guard may swallow the ordinary case."""
+ Base = _pretrained_base()
+
+ class Normal(Base):
+ def get_input_embeddings(self):
+ return self.embed
+
+ model = _tiny(Normal)
+ model.enable_input_require_grads()
+ assert model.embed._forward_hooks, "the input-gradient hook must be registered"
diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py
index cd7d77672ef..79ce03c52f8 100644
--- a/unsloth/import_fixes.py
+++ b/unsloth/import_fixes.py
@@ -2005,10 +2005,22 @@ def patch_enable_input_require_grads():
):
continue
+ getter = module.get_input_embeddings
try:
- input_embeddings = module.get_input_embeddings()
+ inspect.signature(getter).bind()
+ except TypeError:
+ # Remote code may declare get_input_embeddings(self, input_ids)
+ # (stepfun-ai/Step-3.7-Flash). Asked by binding, not by calling: a
+ # TypeError from INSIDE a working getter must propagate, not cost the
+ # module its input-gradient hook.
+ continue
+ except (ValueError, AttributeError):
+ pass # no introspectable signature; undecidable, so just call it
+
+ try:
+ input_embeddings = getter()
except NotImplementedError:
- # Vision models may not implement get_input_embeddings (GLM V4.6 skips only self.visual).
+ # transformers 5 gives every PreTrainedModel a base impl that raises.
continue
if input_embeddings is None:
diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py
index 47eae3cdcc9..2108cedd6c4 100644
--- a/unsloth/models/loader.py
+++ b/unsloth/models/loader.py
@@ -226,6 +226,40 @@ def _has_sequence_classification_architecture(config):
return any(str(arch).endswith("ForSequenceClassification") for arch in architectures)
+# Most to least specific, so a config mapped under several gets its own family's
+# class. Every name must also be in vision.py's _multimodal_auto_classes(), since
+# the class picked here decides processor selection (asserted by
+# test_every_class_the_resolver_can_return_takes_a_processor).
+_OMNI_AUTO_CLASS_NAMES = (
+ "AutoModelForImageTextToText",
+ "AutoModelForTextToWaveform",
+)
+
+
+def _resolve_omni_auto_model(model_config):
+ """A multimodal auto class that really maps this config, or None.
+
+ Qwen3-Omni names Qwen3OmniMoeForConditionalGeneration so it reads as a VLM,
+ but transformers registers qwen3_omni_moe only under
+ AutoModelForTextToWaveform, and asking a class with no mapping is a hard
+ load failure, not a fallback.
+ """
+ import transformers
+
+ for name in _OMNI_AUTO_CLASS_NAMES:
+ auto_class = getattr(transformers, name, None)
+ if auto_class is None:
+ continue
+ try:
+ if resolve_model_class(auto_class, model_config) is not None:
+ return auto_class
+ except Exception:
+ continue
+ # Falling back to the concrete class the checkpoint names is WRONG: it is in no
+ # auto mapping, so it leaves the processor set and downgrades to AutoTokenizer.
+ return None
+
+
def _get_user_task_config_attrs(user_config):
if user_config is None:
return {}
@@ -1874,6 +1908,10 @@ class FastModel(FastBaseModel):
auto_model = AutoModel
else:
auto_model = AutoModelForVision2Seq
+ # Only when the image-text class has no mapping, so anything that
+ # resolves today keeps the class it resolves to now.
+ if resolve_model_class(auto_model, model_config) is None:
+ auto_model = _resolve_omni_auto_model(model_config) or auto_model
else:
auto_model = AutoModelForCausalLM
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index 1668084f452..0b6d99aa2bc 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -22,6 +22,49 @@ try:
AutoModelForVision2Seq = AutoModelForImageTextToText
except:
from transformers import AutoModelForVision2Seq
+
+
+def _embeddings_or_none(model, getter):
+ """Call `model.<getter>()`, or None when the model cannot answer.
+
+ `hasattr` is not the question: transformers 5 defines the method on every
+ PreTrainedModel with a base impl that raises, so a composite checkpoint
+ passes the hasattr check and then raises. Ask by calling.
+ """
+ fn = getattr(model, getter, None)
+ if fn is None:
+ return None
+ try:
+ return fn()
+ except Exception:
+ return None
+
+
+def _multimodal_auto_classes():
+ """Auto classes whose models need a processor rather than a tokenizer.
+
+ Processor SELECTION only, never `is_vlm`: `is_vlm` also arms the
+ image-processor repair path, and a Whisper processor legitimately has no
+ `image_processor`, so widening `is_vlm` made loading Whisper try to build an
+ image processor for an audio model.
+ """
+ import transformers
+
+ # Looked up, not referenced: which names exist varies (4.51.3 has both, 5.5.0
+ # dropped AutoModelForVision2Seq), so only the alias above is always bound.
+ classes = [AutoModelForVision2Seq]
+ # AutoModelForSpeechSeq2Seq is deliberately absent: Whisper already reaches
+ # AutoProcessor through is_whisper, so adding it would move a cell for nothing.
+ for name in (
+ "AutoModelForImageTextToText",
+ "AutoModelForTextToWaveform",
+ ):
+ extra = getattr(transformers, name, None)
+ if extra is not None and extra not in classes:
+ classes.append(extra)
+ return classes
+
+
from ..kernels import (
post_patch_loss_function,
)
@@ -554,8 +597,10 @@ def _resolve_offload_embedding(model, offload_embedding):
model.get_output_embeddings() if hasattr(model, "get_output_embeddings") else None
)
except Exception:
- # Cannot inspect it, so leave an explicit request alone and decline the default.
- return False if automatic else offload_embedding
+ # Honouring an explicit request here is WRONG: the caller goes straight on
+ # to call get_input_embeddings() unguarded, turning an inapplicable VRAM
+ # optimisation into a failed load.
+ return _decline("its embeddings cannot be inspected.")
if _embeddings_are_tied(in_embed, out_embed):
return _decline("this model ties embed_tokens to lm_head, so offloading saves no VRAM.")
if _embedding_dispatch_device(in_embed) is not None:
@@ -1262,10 +1307,20 @@ class FastBaseModel:
]:
auto_model = AutoModelForCausalLM
is_vlm = auto_model in [AutoModelForVision2Seq, AutoModelForImageTextToText]
- # A repo-code VLM may register only AutoModel / AutoModelForCausalLM (DeepSeek-OCR, Nemotron-VL), so auto_model is not a VLM class though the config is a vision model. Keep is_vlm for processor selection, but treat it as a VLM on the vLLM path so a vision_config model is never silently loaded as text-only.
- is_vlm_config = is_vlm or (not text_only and hasattr(auto_config, "vision_config"))
is_whisper = whisper_language is not None and whisper_task is not None
- auto_processor = AutoProcessor if (is_vlm or is_whisper) else AutoTokenizer
+ # Audio and omni classes need a processor but are NOT image models, so they
+ # must not widen is_vlm, which arms the image-processor repair path below.
+ needs_processor = is_vlm or auto_model in _multimodal_auto_classes()
+ # A repo-code VLM may register only AutoModel / AutoModelForCausalLM (DeepSeek-OCR, Nemotron-VL), so auto_model is not a VLM class though the config is a vision model. Keep is_vlm for processor selection, but treat it as a VLM on the vLLM path so a vision_config model is never silently loaded as text-only.
+ is_vlm_config = (
+ is_vlm
+ # An omni checkpoint hides its vision config under thinker_config, so the
+ # hasattr below misses it and fast_inference would enter the vLLM
+ # language-model path with is_vision_model=False.
+ or needs_processor
+ or (not text_only and hasattr(auto_config, "vision_config"))
+ )
+ auto_processor = AutoProcessor if (needs_processor or is_whisper) else AutoTokenizer
model_type_arch = model_types[0]
if model_type_arch == "siglip":
@@ -2612,12 +2667,8 @@ class FastBaseModel:
if hasattr(module, "gradient_checkpointing"):
module.gradient_checkpointing = False
- if hasattr(model, "get_input_embeddings"):
- embeddings = model.get_input_embeddings()
- if hasattr(embeddings, "training"):
- embeddings.training = False
- if hasattr(model, "get_output_embeddings"):
- embeddings = model.get_output_embeddings()
+ for _getter in ("get_input_embeddings", "get_output_embeddings"):
+ embeddings = _embeddings_or_none(model, _getter)
if hasattr(embeddings, "training"):
embeddings.training = False
# Restore use_cache values that prepare_model_for_training disabled for gradient checkpointing (older unsloth_zoo has no restore helper).
@@ -2671,12 +2722,8 @@ class FastBaseModel:
if hasattr(module, "gradient_checkpointing"):
module.gradient_checkpointing = use_gradient_checkpointing
- if hasattr(model, "get_input_embeddings"):
- embeddings = model.get_input_embeddings()
- if hasattr(embeddings, "training"):
- embeddings.training = True
- if hasattr(model, "get_output_embeddings"):
- embeddings = model.get_output_embeddings()
+ for _getter in ("get_input_embeddings", "get_output_embeddings"):
+ embeddings = _embeddings_or_none(model, _getter)
if hasattr(embeddings, "training"):
embeddings.training = True
# Re-disable use_cache if prepare_model_for_training had disabled it and for_inference restored it; the record only exists after a disable.