Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Resolve the auto class for omni checkpoints, and guard embedding lookups (#11467)
Daniel Han3945bde1a168
diff --git a/tests/test_for_training_generation_flag.py b/tests/test_for_training_generation_flag.pyindex 03ec864fa5a..490c0e205ea 100644--- a/tests/test_for_training_generation_flag.py+++ b/tests/test_for_training_generation_flag.py@@ -19,6 +19,27 @@ class _Namespace(dict):return getattr(builtins, name, None)+def _helpers_used_by(method, tree):+ """Module-level functions the lifted method calls, so they run for real.++ Without this they fall to `_Namespace.__missing__` and become None, and the+ first call raises `TypeError: 'NoneType' object is not callable`: the test+ then fails for a reason that has nothing to do with what it asserts.+ """+ defined = {+ node.name: node+ for node in tree.body+ if isinstance(node, ast.FunctionDef) and not node.decorator_list+ }+ wanted, pending = {}, [method]+ while pending:+ for node in ast.walk(pending.pop()):+ if isinstance(node, ast.Name) and node.id in defined and node.id not in wanted:+ wanted[node.id] = defined[node.id]+ pending.append(defined[node.id])+ return list(wanted.values())++def _for_training(module, class_name):path = Path(__file__).parents[1] / "unsloth" / "models" / moduletree = ast.parse(path.read_text(encoding = "utf-8"))@@ -31,7 +52,7 @@ def _for_training(module, class_name):if isinstance(node, ast.FunctionDef) and node.name == "for_training")method.decorator_list = []- compiled = ast.Module(body = [method], type_ignores = [])+ compiled = ast.Module(body = _helpers_used_by(method, tree) + [method], type_ignores = [])namespace = _Namespace(os = os)exec(compile(ast.fix_missing_locations(compiled), str(path), "exec"), namespace)return namespace["for_training"]diff --git a/tests/test_offload_tied_autodisable.py b/tests/test_offload_tied_autodisable.pyindex c1b2a3fd569..3b3720b4a47 100644--- a/tests/test_offload_tied_autodisable.py+++ b/tests/test_offload_tied_autodisable.py@@ -143,10 +143,11 @@ def test_tied_model_disables_offload_instead_of_raising():assert resolve(_tied_model(), True) is False-def test_opaque_model_leaves_request_alone():- # Cannot inspect it, so do not guess, and do not crash.+def test_opaque_model_disables_offload():+ # Used to return the request unchanged, which is the one answer that crashes: the caller+ # acts on True by calling get_input_embeddings() unguarded, the same call that raised here.with _as_platform("posix"):- assert resolve(_Opaque(), True) is True+ assert resolve(_Opaque(), True) is Falsedef test_wsl_and_windows_disable_offload():diff --git a/tests/test_omni_auto_class_resolution.py b/tests/test_omni_auto_class_resolution.pynew file mode 100644index 00000000000..e154b7f36f0--- /dev/null+++ b/tests/test_omni_auto_class_resolution.py@@ -0,0 +1,233 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Auto-class resolution for omni checkpoints, and what it must NOT move.++`Qwen/Qwen3-Omni-30B-A3B-Instruct` names `Qwen3OmniMoeForConditionalGeneration`+and so reads as a VLM, but transformers registers `qwen3_omni_moe` only under+`AutoModelForTextToWaveform`. Every other auto class raises "Unrecognized+configuration class" on it, which is a hard load failure before the weights are+touched.++The resolver is consulted ONLY when the already-chosen class has no mapping, so+these tests spend as much effort on the models that must keep taking exactly+the branch they take today as on the one that changes.+"""++import pytest++transformers = pytest.importorskip("transformers")++from unsloth.models.loader import ( # noqa: E402+ _resolve_omni_auto_model,+ resolve_model_class,+)+from unsloth.models.vision import ( # noqa: E402+ _embeddings_or_none,+ _multimodal_auto_classes,+)++try:+ from transformers import AutoModelForImageTextToText as IMAGE_TEXT_CLASS+except ImportError: # transformers 4.x+ from transformers import AutoModelForVision2Seq as IMAGE_TEXT_CLASS+++def _omni_config():+ config_class = getattr(transformers, "Qwen3OmniMoeConfig", None)+ if config_class is None:+ pytest.skip("this transformers has no Qwen3-Omni")+ return config_class()+++def test_the_defect_is_real_on_this_transformers():+ """Without this, the test below would prove nothing on a build that maps it."""+ config = _omni_config()+ assert resolve_model_class(IMAGE_TEXT_CLASS, config) is None+++def test_omni_resolves_to_the_class_its_family_registered():+ config = _omni_config()+ resolved = _resolve_omni_auto_model(config)+ assert resolved is not None+ # and it really maps, rather than merely being a different name to fail on+ assert resolve_model_class(resolved, config) is not None++[email protected]("config_name", ["Gemma3Config", "Qwen2VLConfig", "LlavaConfig"])+def test_ordinary_vlms_are_never_rerouted(config_name):+ """The resolver is behind `resolve_model_class(...) is None`, so a model+ that resolves today must never reach it."""+ config_class = getattr(transformers, config_name, None)+ if config_class is None:+ pytest.skip(f"this transformers has no {config_name}")+ assert resolve_model_class(IMAGE_TEXT_CLASS, config_class()) is not None+++def test_nothing_matching_returns_None_rather_than_a_concrete_class():+ """Returning the concrete class the checkpoint names is WRONG: it is in no+ auto mapping, so it leaves the processor set and downgrades to AutoTokenizer."""+ assert _resolve_omni_auto_model(transformers.LlamaConfig()) is None+ assert _resolve_omni_auto_model(object()) is None+++def test_every_class_the_resolver_can_return_takes_a_processor():+ """The resolver's answer decides processor selection, so every possible+ answer must be in the set that selects AutoProcessor."""+ import unsloth.models.loader as loader++ processor_classes = _multimodal_auto_classes()+ for name in loader._OMNI_AUTO_CLASS_NAMES:+ auto_class = getattr(transformers, name, None)+ if auto_class is None:+ continue+ assert auto_class in processor_classes, name+++def test_speech_seq2seq_is_not_treated_as_a_vision_model():+ """Including it made loading Whisper call+ _construct_vlm_processor_fallback("openai/whisper-tiny", "whisper"): an image+ processor build for an audio model, whose WhisperProcessor has none."""+ speech = getattr(transformers, "AutoModelForSpeechSeq2Seq", None)+ if speech is None:+ pytest.skip("this transformers has no AutoModelForSpeechSeq2Seq")+ assert speech not in _multimodal_auto_classes()+++def test_image_text_class_is_still_in_the_processor_set():+ assert IMAGE_TEXT_CLASS in _multimodal_auto_classes()+++class _CannotAnswer:+ def get_input_embeddings(self):+ raise NotImplementedError("composite model, no single embedding")+++class _WrongSignature:+ def get_input_embeddings(self, input_ids): # remote code does this+ raise AssertionError("must not be reached")+++class _Normal:+ def get_input_embeddings(self):+ return "embeddings"+++ "model, expected",+ [+ (_CannotAnswer(), None), # transformers 5 base impl raises+ (_WrongSignature(), None), # TypeError, not AttributeError+ (object(), None), # method absent entirely+ (_Normal(), "embeddings"),+ ],+)+def test_embeddings_or_none(model, expected):+ """`hasattr` does not answer the question; only calling does."""+ assert _embeddings_or_none(model, "get_input_embeddings") == expected+++class _NoEmbeddings:+ """A model that cannot answer for its embeddings, like Qwen3-Omni."""++ def get_input_embeddings(self):+ raise NotImplementedError("composite model, no single embedding")++ def get_output_embeddings(self):+ raise NotImplementedError("composite model, no single embedding")++[email protected]("requested", [True, "auto"])+def test_offload_embedding_declines_a_model_it_cannot_inspect(requested, capsys):+ """Returning the explicit True is WRONG: the caller then calls+ get_input_embeddings() unguarded, failing the load over a VRAM optimisation+ that cannot be applied anyway."""+ from unsloth.models.vision import _resolve_offload_embedding++ assert _resolve_offload_embedding(_NoEmbeddings(), requested) is False+ printed = capsys.readouterr().out+ if requested == "auto":+ # the default declines silently: nobody asked for it+ assert "Not offloading embeddings" not in printed+ else:+ assert "Not offloading embeddings" in printed+++def test_omni_reaches_the_vllm_guard_rather_than_the_language_model_path():+ """Without the needs_processor term, is_vlm_config is False for an omni+ checkpoint (its vision config hides under thinker_config), which skips the+ guard and calls load_vllm with is_vision_model=False for an unsupported model."""+ from unsloth.models.vision import VLLM_SUPPORTED_VLM++ config = _omni_config()+ assert not hasattr(config, "vision_config"), "premise: vision lives under thinker_config"+ assert "qwen3_omni_moe" not in VLLM_SUPPORTED_VLM, "premise: vLLM does not support it"++ # what is_vlm_config now computes for it+ resolved = _resolve_omni_auto_model(config)+ is_vlm = resolved in [IMAGE_TEXT_CLASS]+ needs_processor = is_vlm or resolved in _multimodal_auto_classes()+ is_vlm_config = is_vlm or needs_processor or hasattr(config, "vision_config")+ assert is_vlm_config, "must reach the fast_inference guard"+++def _tiny(cls):+ from transformers import LlamaConfig++ return cls(+ LlamaConfig(+ hidden_size = 4, num_hidden_layers = 1, num_attention_heads = 1,+ vocab_size = 8, intermediate_size = 8,+ )+ )+++def _pretrained_base():+ import torch.nn as nn+ from transformers import LlamaConfig, PreTrainedModel++ class Base(PreTrainedModel):+ config_class = LlamaConfig++ def __init__(self, config):+ super().__init__(config)+ self.embed = nn.Embedding(8, 4)++ return Base+++def test_a_getter_that_fails_internally_is_not_read_as_a_bad_signature():+ """Skipping it would cost the module its input-gradient hook, so under PEFT+ with gradient checkpointing backward fails or the adapter trains on nothing."""+ Base = _pretrained_base()++ class RaisesInside(Base):+ def get_input_embeddings(self):+ raise TypeError("genuine bug inside a valid zero-arg getter")++ with pytest.raises(TypeError, match = "genuine bug inside"):+ _tiny(RaisesInside).enable_input_require_grads()+++def test_a_getter_that_cannot_take_zero_arguments_is_skipped():+ """stepfun-ai/Step-3.7-Flash declares get_input_embeddings(self, input_ids)."""+ Base = _pretrained_base()++ class WrongSignature(Base):+ def get_input_embeddings(self, input_ids):+ raise AssertionError("must not be reached")++ _tiny(WrongSignature).enable_input_require_grads() # must not raise+++def test_a_normal_model_still_gets_its_hook():+ """The negative control: neither guard may swallow the ordinary case."""+ Base = _pretrained_base()++ class Normal(Base):+ def get_input_embeddings(self):+ return self.embed++ model = _tiny(Normal)+ model.enable_input_require_grads()+ assert model.embed._forward_hooks, "the input-gradient hook must be registered"diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.pyindex cd7d77672ef..79ce03c52f8 100644--- a/unsloth/import_fixes.py+++ b/unsloth/import_fixes.py@@ -2005,10 +2005,22 @@ def patch_enable_input_require_grads():):continue+ getter = module.get_input_embeddingstry:- input_embeddings = module.get_input_embeddings()+ inspect.signature(getter).bind()+ except TypeError:+ # Remote code may declare get_input_embeddings(self, input_ids)+ # (stepfun-ai/Step-3.7-Flash). Asked by binding, not by calling: a+ # TypeError from INSIDE a working getter must propagate, not cost the+ # module its input-gradient hook.+ continue+ except (ValueError, AttributeError):+ pass # no introspectable signature; undecidable, so just call it++ try:+ input_embeddings = getter()except NotImplementedError:- # Vision models may not implement get_input_embeddings (GLM V4.6 skips only self.visual).+ # transformers 5 gives every PreTrainedModel a base impl that raises.continueif input_embeddings is None:diff --git a/unsloth/models/loader.py b/unsloth/models/loader.pyindex 47eae3cdcc9..2108cedd6c4 100644--- a/unsloth/models/loader.py+++ b/unsloth/models/loader.py@@ -226,6 +226,40 @@ def _has_sequence_classification_architecture(config):return any(str(arch).endswith("ForSequenceClassification") for arch in architectures)+# Most to least specific, so a config mapped under several gets its own family's+# class. Every name must also be in vision.py's _multimodal_auto_classes(), since+# the class picked here decides processor selection (asserted by+# test_every_class_the_resolver_can_return_takes_a_processor).+_OMNI_AUTO_CLASS_NAMES = (+ "AutoModelForImageTextToText",+ "AutoModelForTextToWaveform",+)+++def _resolve_omni_auto_model(model_config):+ """A multimodal auto class that really maps this config, or None.++ Qwen3-Omni names Qwen3OmniMoeForConditionalGeneration so it reads as a VLM,+ but transformers registers qwen3_omni_moe only under+ AutoModelForTextToWaveform, and asking a class with no mapping is a hard+ load failure, not a fallback.+ """+ import transformers++ for name in _OMNI_AUTO_CLASS_NAMES:+ auto_class = getattr(transformers, name, None)+ if auto_class is None:+ continue+ try:+ if resolve_model_class(auto_class, model_config) is not None:+ return auto_class+ except Exception:+ continue+ # Falling back to the concrete class the checkpoint names is WRONG: it is in no+ # auto mapping, so it leaves the processor set and downgrades to AutoTokenizer.+ return None++def _get_user_task_config_attrs(user_config):if user_config is None:return {}@@ -1874,6 +1908,10 @@ class FastModel(FastBaseModel):auto_model = AutoModelelse:auto_model = AutoModelForVision2Seq+ # Only when the image-text class has no mapping, so anything that+ # resolves today keeps the class it resolves to now.+ if resolve_model_class(auto_model, model_config) is None:+ auto_model = _resolve_omni_auto_model(model_config) or auto_modelelse:auto_model = AutoModelForCausalLMdiff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex 1668084f452..0b6d99aa2bc 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -22,6 +22,49 @@ try:AutoModelForVision2Seq = AutoModelForImageTextToTextexcept:from transformers import AutoModelForVision2Seq+++def _embeddings_or_none(model, getter):+ """Call `model.<getter>()`, or None when the model cannot answer.++ `hasattr` is not the question: transformers 5 defines the method on every+ PreTrainedModel with a base impl that raises, so a composite checkpoint+ passes the hasattr check and then raises. Ask by calling.+ """+ fn = getattr(model, getter, None)+ if fn is None:+ return None+ try:+ return fn()+ except Exception:+ return None+++def _multimodal_auto_classes():+ """Auto classes whose models need a processor rather than a tokenizer.++ Processor SELECTION only, never `is_vlm`: `is_vlm` also arms the+ image-processor repair path, and a Whisper processor legitimately has no+ `image_processor`, so widening `is_vlm` made loading Whisper try to build an+ image processor for an audio model.+ """+ import transformers++ # Looked up, not referenced: which names exist varies (4.51.3 has both, 5.5.0+ # dropped AutoModelForVision2Seq), so only the alias above is always bound.+ classes = [AutoModelForVision2Seq]+ # AutoModelForSpeechSeq2Seq is deliberately absent: Whisper already reaches+ # AutoProcessor through is_whisper, so adding it would move a cell for nothing.+ for name in (+ "AutoModelForImageTextToText",+ "AutoModelForTextToWaveform",+ ):+ extra = getattr(transformers, name, None)+ if extra is not None and extra not in classes:+ classes.append(extra)+ return classes++from ..kernels import (post_patch_loss_function,)@@ -554,8 +597,10 @@ def _resolve_offload_embedding(model, offload_embedding):model.get_output_embeddings() if hasattr(model, "get_output_embeddings") else None)except Exception:- # Cannot inspect it, so leave an explicit request alone and decline the default.- return False if automatic else offload_embedding+ # Honouring an explicit request here is WRONG: the caller goes straight on+ # to call get_input_embeddings() unguarded, turning an inapplicable VRAM+ # optimisation into a failed load.+ return _decline("its embeddings cannot be inspected.")if _embeddings_are_tied(in_embed, out_embed):return _decline("this model ties embed_tokens to lm_head, so offloading saves no VRAM.")if _embedding_dispatch_device(in_embed) is not None:@@ -1262,10 +1307,20 @@ class FastBaseModel:]:auto_model = AutoModelForCausalLMis_vlm = auto_model in [AutoModelForVision2Seq, AutoModelForImageTextToText]- # A repo-code VLM may register only AutoModel / AutoModelForCausalLM (DeepSeek-OCR, Nemotron-VL), so auto_model is not a VLM class though the config is a vision model. Keep is_vlm for processor selection, but treat it as a VLM on the vLLM path so a vision_config model is never silently loaded as text-only.- is_vlm_config = is_vlm or (not text_only and hasattr(auto_config, "vision_config"))is_whisper = whisper_language is not None and whisper_task is not None- auto_processor = AutoProcessor if (is_vlm or is_whisper) else AutoTokenizer+ # Audio and omni classes need a processor but are NOT image models, so they+ # must not widen is_vlm, which arms the image-processor repair path below.+ needs_processor = is_vlm or auto_model in _multimodal_auto_classes()+ # A repo-code VLM may register only AutoModel / AutoModelForCausalLM (DeepSeek-OCR, Nemotron-VL), so auto_model is not a VLM class though the config is a vision model. Keep is_vlm for processor selection, but treat it as a VLM on the vLLM path so a vision_config model is never silently loaded as text-only.+ is_vlm_config = (+ is_vlm+ # An omni checkpoint hides its vision config under thinker_config, so the+ # hasattr below misses it and fast_inference would enter the vLLM+ # language-model path with is_vision_model=False.+ or needs_processor+ or (not text_only and hasattr(auto_config, "vision_config"))+ )+ auto_processor = AutoProcessor if (needs_processor or is_whisper) else AutoTokenizermodel_type_arch = model_types[0]if model_type_arch == "siglip":@@ -2612,12 +2667,8 @@ class FastBaseModel:if hasattr(module, "gradient_checkpointing"):module.gradient_checkpointing = False- if hasattr(model, "get_input_embeddings"):- embeddings = model.get_input_embeddings()- if hasattr(embeddings, "training"):- embeddings.training = False- if hasattr(model, "get_output_embeddings"):- embeddings = model.get_output_embeddings()+ for _getter in ("get_input_embeddings", "get_output_embeddings"):+ embeddings = _embeddings_or_none(model, _getter)if hasattr(embeddings, "training"):embeddings.training = False# Restore use_cache values that prepare_model_for_training disabled for gradient checkpointing (older unsloth_zoo has no restore helper).@@ -2671,12 +2722,8 @@ class FastBaseModel:if hasattr(module, "gradient_checkpointing"):module.gradient_checkpointing = use_gradient_checkpointing- if hasattr(model, "get_input_embeddings"):- embeddings = model.get_input_embeddings()- if hasattr(embeddings, "training"):- embeddings.training = True- if hasattr(model, "get_output_embeddings"):- embeddings = model.get_output_embeddings()+ for _getter in ("get_input_embeddings", "get_output_embeddings"):+ embeddings = _embeddings_or_none(model, _getter)if hasattr(embeddings, "training"):embeddings.training = True# Re-disable use_cache if prepare_model_for_training had disabled it and for_inference restored it; the record only exists after a disable.