Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Load a VLM through its native image-text class when the repo's auto_map class is untrusted (#11613)

Daniel Han6eef21ffa61b
diff --git a/tests/test_untrusted_vlm_native_class.py b/tests/test_untrusted_vlm_native_class.py
new file mode 100644
index 00000000000..b41771211b9
--- /dev/null
+++ b/tests/test_untrusted_vlm_native_class.py
@@ -0,0 +1,100 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+"""Untrusted FastModel load of a VLM whose auto_map names only AutoModelForCausalLM (Step-3.7-Flash)
+must build the native image-text class instead of refusing on remote code."""
+
+import json
+
+import pytest
+import torch
+from real_accelerator import has_real_accelerator
+
+
+step3p7 = pytest.importorskip("transformers.models.step3p7.modeling_step3p7")
+
+
+def _write_tokenizer(path, vocab_size):
+ from tokenizers import Tokenizer, models, pre_tokenizers
+ from transformers import PreTrainedTokenizerFast
+
+ vocab = {f"t{i}": i for i in range(vocab_size)}
+ tok = Tokenizer(models.WordLevel(vocab = vocab, unk_token = "t0"))
+ tok.pre_tokenizer = pre_tokenizers.Whitespace()
+ PreTrainedTokenizerFast(
+ tokenizer_object = tok, unk_token = "t0", pad_token = "t1", eos_token = "t2"
+ ).save_pretrained(path)
+
+
+def _tiny_step3p7(path, auto_map):
+ from transformers.models.step3p7.configuration_step3p7 import Step3p7Config
+
+ vocab = 64
+ config = Step3p7Config(
+ text_config = dict(
+ vocab_size = vocab,
+ hidden_size = 64,
+ intermediate_size = 128,
+ num_hidden_layers = 2,
+ num_attention_heads = 2,
+ num_key_value_heads = 1,
+ head_dim = 32,
+ n_routed_experts = 4,
+ num_experts_per_tok = 2,
+ moe_intermediate_size = 32,
+ share_expert_dim = 32,
+ layer_types = ["full_attention", "full_attention"],
+ mlp_layer_types = ["dense", "sparse"],
+ max_position_embeddings = 256,
+ ),
+ vision_config = dict(width = 32, layers = 1, heads = 2, image_size = 28, patch_size = 14),
+ )
+ torch.manual_seed(0)
+ model = step3p7.Step3p7ForConditionalGeneration(config).to(torch.bfloat16)
+ model.save_pretrained(path)
+ cfg = json.loads((path / "config.json").read_text(encoding = "utf-8"))
+ # Module never shipped: building the repo class fails loudly.
+ cfg["auto_map"] = {
+ name: "modeling_step3p7.Step3p7ForConditionalGeneration" for name in auto_map
+ }
+ (path / "config.json").write_text(json.dumps(cfg), encoding = "utf-8")
+ _write_tokenizer(path, vocab)
+ return path
+
+
+def test_native_class_lookup():
+ from transformers import AutoModelForCausalLM, AutoModelForImageTextToText
+ from transformers.models.llama.configuration_llama import LlamaConfig
+ from transformers.models.step3p7.configuration_step3p7 import Step3p7Config
+
+ loader = pytest.importorskip("unsloth.models.loader")
+ _config_has_native_class = loader._config_has_native_class
+
+ assert _config_has_native_class(AutoModelForImageTextToText, Step3p7Config())
+ assert not _config_has_native_class(AutoModelForCausalLM, Step3p7Config())
+ assert _config_has_native_class(AutoModelForCausalLM, LlamaConfig())
+ assert not _config_has_native_class(None, LlamaConfig())
+
+
[email protected](not has_real_accelerator(), reason = "import unsloth needs an accelerator")
+# Kimi-K2.5 also names its class under AutoModel; native AutoModel is the headless backbone.
+ "auto_map",
+ [("AutoModelForCausalLM",), ("AutoModel", "AutoModelForCausalLM")],
+ ids = ["causal_lm_only", "auto_model_and_causal_lm"],
+)
+def test_untrusted_load_builds_the_native_image_text_class(tmp_path, auto_map):
+ import unsloth # noqa: F401
+ from unsloth import FastModel
+
+ path = _tiny_step3p7(tmp_path / "tiny_step3p7", auto_map)
+ model, _ = FastModel.from_pretrained(
+ str(path),
+ max_seq_length = 64,
+ dtype = torch.bfloat16,
+ load_in_4bit = False,
+ load_in_16bit = True,
+ trust_remote_code = False,
+ text_only = True,
+ )
+ assert type(model).__module__ == step3p7.__name__
+ assert type(model).__name__ == "Step3p7ForConditionalGeneration"
diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py
index 49727b1f069..6dd2dccb91e 100644
--- a/unsloth/models/loader.py
+++ b/unsloth/models/loader.py
@@ -267,6 +267,14 @@ _OMNI_AUTO_CLASS_NAMES = (
)
+def _config_has_native_class(auto_class, config):
+ """True when transformers itself maps ``type(config)`` in ``auto_class`` (no repo code needed)."""
+ try:
+ return auto_class is not None and type(config) in auto_class._model_mapping
+ except Exception:
+ return False
+
+
def _resolve_omni_auto_model(model_config):
"""A multimodal auto class that really maps this config, or None.
@@ -1933,6 +1941,24 @@ class FastModel(FastBaseModel):
_auto_map = getattr(model_config, "auto_map", {}) or {}
_vlm_class_name = AutoModelForVision2Seq.__name__
_has_vlm_class = _vlm_class_name in _auto_map
+ # Untrusted: keep only auto_map entries transformers builds natively (Step-3.7: step3p7 is image-text).
+ if not trust_remote_code:
+ import transformers as _transformers
+
+ _native_map = {
+ _name: _ref
+ for _name, _ref in _auto_map.items()
+ if _config_has_native_class(
+ getattr(_transformers, _name, None), model_config
+ )
+ }
+ # Remote causal LM class: native AutoModel would be a headless backbone (Kimi-K2.5).
+ if (
+ "AutoModelForCausalLM" in _auto_map
+ and "AutoModelForCausalLM" not in _native_map
+ ):
+ _native_map.pop("AutoModel", None)
+ _auto_map = _native_map
if not _has_vlm_class and "AutoModelForCausalLM" in _auto_map:
auto_model = AutoModelForCausalLM
elif not _has_vlm_class and "AutoModel" in _auto_map: