Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Load a VLM through its native image-text class when the repo's auto_map class is untrusted (#11613)
Daniel Han6eef21ffa61b
diff --git a/tests/test_untrusted_vlm_native_class.py b/tests/test_untrusted_vlm_native_class.pynew file mode 100644index 00000000000..b41771211b9--- /dev/null+++ b/tests/test_untrusted_vlm_native_class.py@@ -0,0 +1,100 @@+# SPDX-License-Identifier: AGPL-3.0-only+"""Untrusted FastModel load of a VLM whose auto_map names only AutoModelForCausalLM (Step-3.7-Flash)+must build the native image-text class instead of refusing on remote code."""++import json++import pytest+import torch+from real_accelerator import has_real_accelerator+++step3p7 = pytest.importorskip("transformers.models.step3p7.modeling_step3p7")+++def _write_tokenizer(path, vocab_size):+ from tokenizers import Tokenizer, models, pre_tokenizers+ from transformers import PreTrainedTokenizerFast++ vocab = {f"t{i}": i for i in range(vocab_size)}+ tok = Tokenizer(models.WordLevel(vocab = vocab, unk_token = "t0"))+ tok.pre_tokenizer = pre_tokenizers.Whitespace()+ PreTrainedTokenizerFast(+ tokenizer_object = tok, unk_token = "t0", pad_token = "t1", eos_token = "t2"+ ).save_pretrained(path)+++def _tiny_step3p7(path, auto_map):+ from transformers.models.step3p7.configuration_step3p7 import Step3p7Config++ vocab = 64+ config = Step3p7Config(+ text_config = dict(+ vocab_size = vocab,+ hidden_size = 64,+ intermediate_size = 128,+ num_hidden_layers = 2,+ num_attention_heads = 2,+ num_key_value_heads = 1,+ head_dim = 32,+ n_routed_experts = 4,+ num_experts_per_tok = 2,+ moe_intermediate_size = 32,+ share_expert_dim = 32,+ layer_types = ["full_attention", "full_attention"],+ mlp_layer_types = ["dense", "sparse"],+ max_position_embeddings = 256,+ ),+ vision_config = dict(width = 32, layers = 1, heads = 2, image_size = 28, patch_size = 14),+ )+ torch.manual_seed(0)+ model = step3p7.Step3p7ForConditionalGeneration(config).to(torch.bfloat16)+ model.save_pretrained(path)+ cfg = json.loads((path / "config.json").read_text(encoding = "utf-8"))+ # Module never shipped: building the repo class fails loudly.+ cfg["auto_map"] = {+ name: "modeling_step3p7.Step3p7ForConditionalGeneration" for name in auto_map+ }+ (path / "config.json").write_text(json.dumps(cfg), encoding = "utf-8")+ _write_tokenizer(path, vocab)+ return path+++def test_native_class_lookup():+ from transformers import AutoModelForCausalLM, AutoModelForImageTextToText+ from transformers.models.llama.configuration_llama import LlamaConfig+ from transformers.models.step3p7.configuration_step3p7 import Step3p7Config++ loader = pytest.importorskip("unsloth.models.loader")+ _config_has_native_class = loader._config_has_native_class++ assert _config_has_native_class(AutoModelForImageTextToText, Step3p7Config())+ assert not _config_has_native_class(AutoModelForCausalLM, Step3p7Config())+ assert _config_has_native_class(AutoModelForCausalLM, LlamaConfig())+ assert not _config_has_native_class(None, LlamaConfig())++[email protected](not has_real_accelerator(), reason = "import unsloth needs an accelerator")+# Kimi-K2.5 also names its class under AutoModel; native AutoModel is the headless backbone.+ "auto_map",+ [("AutoModelForCausalLM",), ("AutoModel", "AutoModelForCausalLM")],+ ids = ["causal_lm_only", "auto_model_and_causal_lm"],+)+def test_untrusted_load_builds_the_native_image_text_class(tmp_path, auto_map):+ import unsloth # noqa: F401+ from unsloth import FastModel++ path = _tiny_step3p7(tmp_path / "tiny_step3p7", auto_map)+ model, _ = FastModel.from_pretrained(+ str(path),+ max_seq_length = 64,+ dtype = torch.bfloat16,+ load_in_4bit = False,+ load_in_16bit = True,+ trust_remote_code = False,+ text_only = True,+ )+ assert type(model).__module__ == step3p7.__name__+ assert type(model).__name__ == "Step3p7ForConditionalGeneration"diff --git a/unsloth/models/loader.py b/unsloth/models/loader.pyindex 49727b1f069..6dd2dccb91e 100644--- a/unsloth/models/loader.py+++ b/unsloth/models/loader.py@@ -267,6 +267,14 @@ _OMNI_AUTO_CLASS_NAMES = ()+def _config_has_native_class(auto_class, config):+ """True when transformers itself maps ``type(config)`` in ``auto_class`` (no repo code needed)."""+ try:+ return auto_class is not None and type(config) in auto_class._model_mapping+ except Exception:+ return False++def _resolve_omni_auto_model(model_config):"""A multimodal auto class that really maps this config, or None.@@ -1933,6 +1941,24 @@ class FastModel(FastBaseModel):_auto_map = getattr(model_config, "auto_map", {}) or {}_vlm_class_name = AutoModelForVision2Seq.__name___has_vlm_class = _vlm_class_name in _auto_map+ # Untrusted: keep only auto_map entries transformers builds natively (Step-3.7: step3p7 is image-text).+ if not trust_remote_code:+ import transformers as _transformers++ _native_map = {+ _name: _ref+ for _name, _ref in _auto_map.items()+ if _config_has_native_class(+ getattr(_transformers, _name, None), model_config+ )+ }+ # Remote causal LM class: native AutoModel would be a headless backbone (Kimi-K2.5).+ if (+ "AutoModelForCausalLM" in _auto_map+ and "AutoModelForCausalLM" not in _native_map+ ):+ _native_map.pop("AutoModel", None)+ _auto_map = _native_mapif not _has_vlm_class and "AutoModelForCausalLM" in _auto_map:auto_model = AutoModelForCausalLMelif not _has_vlm_class and "AutoModel" in _auto_map: