Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Load the repo AutoProcessor for AutoModel-only repo-code VLMs (#12037)

Daniel Han7c8a51f992fd
diff --git a/tests/test_repo_code_vlm_processor.py b/tests/test_repo_code_vlm_processor.py
new file mode 100644
index 00000000000..e29d82cd147
--- /dev/null
+++ b/tests/test_repo_code_vlm_processor.py
@@ -0,0 +1,169 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Processor selection for repo-code VLMs registering only AutoModel / AutoModelForCausalLM.
+
+Runs the selection code of vision.py with stub processor classes (no network, weights or GPU).
+"""
+
+import ast
+import importlib.util
+import types
+from pathlib import Path
+
+import pytest
+
+transformers = pytest.importorskip("transformers")
+
+VISION = Path(__file__).parents[1] / "unsloth" / "models" / "vision.py"
+if not VISION.exists(): # copied out of the repo: read the tree on PYTHONPATH
+ VISION = (
+ Path(importlib.util.find_spec("unsloth").submodule_search_locations[0])
+ / "models"
+ / "vision.py"
+ )
+_SELECTION_NAMES = {
+ "is_vlm",
+ "is_whisper",
+ "needs_processor",
+ "is_vlm_config",
+ "auto_processor",
+ "try_repo_processor",
+}
+
+
+def _from_pretrained():
+ tree = ast.parse(VISION.read_text(encoding = "utf-8"))
+ cls = next(n for n in tree.body if isinstance(n, ast.ClassDef) and n.name == "FastBaseModel")
+ return next(
+ n for n in cls.body if isinstance(n, ast.FunctionDef) and n.name == "from_pretrained"
+ )
+
+
+def _selection_nodes():
+ nodes = []
+ for node in _from_pretrained().body:
+ if isinstance(node, ast.Assign) and any(
+ getattr(t, "id", None) in _SELECTION_NAMES for t in node.targets
+ ):
+ nodes.append(node)
+ elif isinstance(node, ast.FunctionDef) and node.name == "_acquire_processor":
+ nodes.append(node)
+ return nodes
+
+
+class _Tokenizer:
+ padding_side = "left"
+
+
+class _Processor:
+ def __init__(self):
+ self.tokenizer = _Tokenizer()
+ self.image_processor = object()
+
+
+def _stub(result, calls, name):
+ def from_pretrained(path, **kwargs):
+ calls.append((name, kwargs))
+ if isinstance(result, Exception):
+ raise result
+ return result() if isinstance(result, type) else result
+
+ return type(name, (), {"from_pretrained": staticmethod(from_pretrained)})
+
+
+def _select(
+ auto_processor_result,
+ *,
+ has_vision_config = True,
+ text_only = False,
+ auto_model = transformers.AutoModelForCausalLM,
+):
+ calls = []
+ fake_processor = _stub(auto_processor_result, calls, "AutoProcessor")
+ fake_tokenizer = _stub(_Tokenizer, calls, "AutoTokenizer")
+ config = types.SimpleNamespace()
+ if has_vision_config:
+ config.vision_config = types.SimpleNamespace()
+ namespace = {
+ "auto_model": auto_model,
+ "auto_config": config,
+ "text_only": text_only,
+ "whisper_language": None,
+ "whisper_task": None,
+ "AutoModelForVision2Seq": getattr(
+ transformers, "AutoModelForVision2Seq", transformers.AutoModelForImageTextToText
+ ),
+ "AutoModelForImageTextToText": transformers.AutoModelForImageTextToText,
+ "_multimodal_auto_classes": lambda: (),
+ "AutoProcessor": fake_processor,
+ "AutoTokenizer": fake_tokenizer,
+ "tokenizer_name": "org/repo",
+ "token": None,
+ "trust_remote_code": True,
+ "kwargs": {},
+ "_tokenizer_revision": "abc123",
+ "model_type_arch": "remote_vlm",
+ "get_auto_processor": lambda *a, **k: None,
+ "_construct_vlm_processor_fallback": lambda *a, **k: (None, None),
+ "_is_offline_related_error": lambda e: False,
+ }
+ module = ast.Module(body = _selection_nodes(), type_ignores = [])
+ ast.fix_missing_locations(module)
+ exec(compile(module, str(VISION), "exec"), namespace)
+ tok, err = namespace["_acquire_processor"](False)
+ return tok, err, calls
+
+
+def test_repo_autoprocessor_is_used_for_automodel_only_vlm():
+ """Nemotron-3-Nano-Omni."""
+ tok, err, calls = _select(_Processor)
+ assert isinstance(tok, _Processor), f"got {type(tok).__name__}, calls={calls}"
+ assert hasattr(tok, "image_processor")
+ name, kwargs = calls[0]
+ assert name == "AutoProcessor"
+ assert kwargs["padding_side"] == "left"
+ assert kwargs["revision"] == "abc123"
+ assert kwargs["trust_remote_code"] is True
+
+
+def test_missing_autoprocessor_falls_back_to_tokenizer():
+ """DeepSeek-OCR."""
+ tok, err, calls = _select(ValueError("Unrecognized processing class"))
+ assert isinstance(tok, _Tokenizer)
+ assert err is None
+ assert [c[0] for c in calls][-1] == "AutoTokenizer"
+
+
+def test_autoprocessor_that_resolves_to_a_tokenizer_is_not_kept():
+ """Nemotron-Nano-VL."""
+ tok, err, calls = _select(_Tokenizer)
+ assert isinstance(tok, _Tokenizer)
+ assert [c[0] for c in calls][-1] == "AutoTokenizer"
+
+
+def test_automodel_only_vlm_uses_repo_autoprocessor():
+ """DeepSeek-OCR style auto_map with only AutoModel."""
+ tok, err, calls = _select(_Processor, auto_model = transformers.AutoModel)
+ assert isinstance(tok, _Processor)
+
+
+def test_sequence_classification_vlm_keeps_tokenizer():
+ """Gemma 3 with num_labels: native multimodal config, text classification head."""
+ tok, err, calls = _select(
+ _Processor, auto_model = transformers.AutoModelForSequenceClassification
+ )
+ assert isinstance(tok, _Tokenizer)
+ assert [c[0] for c in calls] == ["AutoTokenizer"]
+
+
+def test_text_model_never_tries_autoprocessor():
+ tok, err, calls = _select(_Processor, has_vision_config = False)
+ assert isinstance(tok, _Tokenizer)
+ assert [c[0] for c in calls] == ["AutoTokenizer"]
+
+
+def test_text_only_vlm_never_tries_autoprocessor():
+ tok, err, calls = _select(_Processor, text_only = True)
+ assert isinstance(tok, _Tokenizer)
+ assert [c[0] for c in calls] == ["AutoTokenizer"]
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index dc2397d85ad..6f81385f6fb 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -1873,6 +1873,12 @@ class FastBaseModel:
or (not text_only and hasattr(auto_config, "vision_config"))
)
auto_processor = AutoProcessor if (needs_processor or is_whisper) else AutoTokenizer
+ # Such repos may still ship an AutoProcessor (Nemotron-3-Nano-Omni); DeepSeek-OCR has none. Only the generic classes: Gemma 3 with num_labels must keep its tokenizer.
+ try_repo_processor = (
+ is_vlm_config
+ and auto_processor is AutoTokenizer
+ and getattr(auto_model, "__name__", "") in ("AutoModel", "AutoModelForCausalLM")
+ )
model_type_arch = model_types[0]
if model_type_arch == "siglip":
@@ -2606,16 +2612,33 @@ class FastBaseModel:
_tok = None
_err = _e
else:
+ _tok = None
+ if try_repo_processor:
+ try:
+ _tok = AutoProcessor.from_pretrained(
+ tokenizer_name,
+ padding_side = "left",
+ token = token,
+ trust_remote_code = trust_remote_code,
+ cache_dir = kwargs.get("cache_dir"),
+ local_files_only = lfo,
+ revision = _tokenizer_revision,
+ )
+ except Exception:
+ _tok = None
+ if not (hasattr(_tok, "image_processor") and hasattr(_tok, "tokenizer")):
+ _tok = None
try:
- _tok = auto_processor.from_pretrained(
- tokenizer_name,
- padding_side = "left",
- token = token,
- trust_remote_code = trust_remote_code,
- cache_dir = kwargs.get("cache_dir"),
- local_files_only = lfo,
- revision = _tokenizer_revision,
- )
+ if _tok is None:
+ _tok = auto_processor.from_pretrained(
+ tokenizer_name,
+ padding_side = "left",
+ token = token,
+ trust_remote_code = trust_remote_code,
+ cache_dir = kwargs.get("cache_dir"),
+ local_files_only = lfo,
+ revision = _tokenizer_revision,
+ )
except Exception as _e:
_err = _e
try: