Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Add Idefics3 support (Granite Docling VLM) (#4241)
Daniel Han757a3f6e886b
diff --git a/tests/test_idefics3_fast_inference_registration.py b/tests/test_idefics3_fast_inference_registration.pynew file mode 100644index 00000000000..141be35de9f--- /dev/null+++ b/tests/test_idefics3_fast_inference_registration.py@@ -0,0 +1,48 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++import ast+from pathlib import Path++VISION_PATH = Path(__file__).resolve().parents[1] / "unsloth" / "models" / "vision.py"+++def _vllm_supported_vlm():+ for node in ast.parse(VISION_PATH.read_text(encoding = "utf-8")).body:+ if isinstance(node, ast.Assign) and any(+ isinstance(t, ast.Name) and t.id == "VLLM_SUPPORTED_VLM" for t in node.targets+ ):+ return ast.literal_eval(node.value)+ raise AssertionError("VLLM_SUPPORTED_VLM not found")+++def test_idefics3_passes_the_fast_inference_gate():+ # granite-docling-258M reports model_types ["idefics3", "idefics3_vision"]+ supported = _vllm_supported_vlm()+ assert any(arch in supported for arch in ("idefics3", "idefics3_vision"))+ assert len(supported) == len(set(supported))+++def _probe(monkeypatch, layer_config):+ import unsloth_zoo.empty_model as empty_model+ from unsloth.models.vision import _zoo_supports_idefics3_fast_inference++ monkeypatch.setattr(empty_model, "get_model_layer_config", layer_config)+ return _zoo_supports_idefics3_fast_inference()+++def test_zoo_with_idefics3_templates_is_accepted(monkeypatch):+ config = {"standard_layers": {"model.text_model.layers.{kk}.self_attn.q_proj"}}+ assert _probe(monkeypatch, lambda: config) is True+++def test_older_zoo_is_refused_instead_of_crashing_in_conversion(monkeypatch):+ config = {"standard_layers": {"model.layers.{kk}.self_attn.q_proj"}}+ assert _probe(monkeypatch, lambda: config) is False+++def test_broken_zoo_probe_is_refused(monkeypatch):+ def boom():+ raise KeyError("standard_layers")++ assert _probe(monkeypatch, boom) is Falsediff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex e4e2b61ba23..79ac7d3f08b 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -693,7 +693,22 @@ VLLM_SUPPORTED_VLM = [# Qwen3.5 ships as Qwen3_5ForConditionalGeneration with a vision_config, so it# reaches this gate even for the text-only checkpoints."qwen3_5",+ "idefics3",]+++def _zoo_supports_idefics3_fast_inference():+ # unsloth_zoo ships separately; older releases crash converting Idefics3's model.text_model layout.+ try:+ from unsloth_zoo.empty_model import get_model_layer_config+ return (+ "model.text_model.layers.{kk}.self_attn.q_proj"+ in get_model_layer_config()["standard_layers"]+ )+ except Exception:+ return False++VLLM_NON_LORA_VLM = ["mllama",]@@ -2023,9 +2038,14 @@ class FastBaseModel:if is_vlm_config and fast_inference:if not any(arch in VLLM_SUPPORTED_VLM for arch in model_types):raise RuntimeError(- f"Unsloth: Fast inference is only supported for Language models and Qwen2.5-VL, Qwen3-VL, Gemma3, Mistral3 among vision models. "+ f"Unsloth: Fast inference is only supported for Language models and these vision model types: {', '.join(VLLM_SUPPORTED_VLM)}. "f"Found architectures: {', '.join(model_types)}!")+ if "idefics3" in model_types and not _zoo_supports_idefics3_fast_inference():+ raise RuntimeError(+ "Unsloth: Idefics3 fast_inference needs a newer unsloth_zoo. "+ "Please run `pip install --upgrade unsloth_zoo`."+ )if any(arch in VLLM_NON_LORA_VLM for arch in model_types):# mllama is still only in vllm v0, and vLLM V0 does not support LoRA on multimodal models. TODO: revisit once vLLM V1 supports Llama 3.2 (mllama).