Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Add Idefics3 support (Granite Docling VLM) (#4241)

Daniel Han757a3f6e886b
diff --git a/tests/test_idefics3_fast_inference_registration.py b/tests/test_idefics3_fast_inference_registration.py
new file mode 100644
index 00000000000..141be35de9f
--- /dev/null
+++ b/tests/test_idefics3_fast_inference_registration.py
@@ -0,0 +1,48 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+import ast
+from pathlib import Path
+
+VISION_PATH = Path(__file__).resolve().parents[1] / "unsloth" / "models" / "vision.py"
+
+
+def _vllm_supported_vlm():
+ for node in ast.parse(VISION_PATH.read_text(encoding = "utf-8")).body:
+ if isinstance(node, ast.Assign) and any(
+ isinstance(t, ast.Name) and t.id == "VLLM_SUPPORTED_VLM" for t in node.targets
+ ):
+ return ast.literal_eval(node.value)
+ raise AssertionError("VLLM_SUPPORTED_VLM not found")
+
+
+def test_idefics3_passes_the_fast_inference_gate():
+ # granite-docling-258M reports model_types ["idefics3", "idefics3_vision"]
+ supported = _vllm_supported_vlm()
+ assert any(arch in supported for arch in ("idefics3", "idefics3_vision"))
+ assert len(supported) == len(set(supported))
+
+
+def _probe(monkeypatch, layer_config):
+ import unsloth_zoo.empty_model as empty_model
+ from unsloth.models.vision import _zoo_supports_idefics3_fast_inference
+
+ monkeypatch.setattr(empty_model, "get_model_layer_config", layer_config)
+ return _zoo_supports_idefics3_fast_inference()
+
+
+def test_zoo_with_idefics3_templates_is_accepted(monkeypatch):
+ config = {"standard_layers": {"model.text_model.layers.{kk}.self_attn.q_proj"}}
+ assert _probe(monkeypatch, lambda: config) is True
+
+
+def test_older_zoo_is_refused_instead_of_crashing_in_conversion(monkeypatch):
+ config = {"standard_layers": {"model.layers.{kk}.self_attn.q_proj"}}
+ assert _probe(monkeypatch, lambda: config) is False
+
+
+def test_broken_zoo_probe_is_refused(monkeypatch):
+ def boom():
+ raise KeyError("standard_layers")
+
+ assert _probe(monkeypatch, boom) is False
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index e4e2b61ba23..79ac7d3f08b 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -693,7 +693,22 @@ VLLM_SUPPORTED_VLM = [
# Qwen3.5 ships as Qwen3_5ForConditionalGeneration with a vision_config, so it
# reaches this gate even for the text-only checkpoints.
"qwen3_5",
+ "idefics3",
]
+
+
+def _zoo_supports_idefics3_fast_inference():
+ # unsloth_zoo ships separately; older releases crash converting Idefics3's model.text_model layout.
+ try:
+ from unsloth_zoo.empty_model import get_model_layer_config
+ return (
+ "model.text_model.layers.{kk}.self_attn.q_proj"
+ in get_model_layer_config()["standard_layers"]
+ )
+ except Exception:
+ return False
+
+
VLLM_NON_LORA_VLM = [
"mllama",
]
@@ -2023,9 +2038,14 @@ class FastBaseModel:
if is_vlm_config and fast_inference:
if not any(arch in VLLM_SUPPORTED_VLM for arch in model_types):
raise RuntimeError(
- f"Unsloth: Fast inference is only supported for Language models and Qwen2.5-VL, Qwen3-VL, Gemma3, Mistral3 among vision models. "
+ f"Unsloth: Fast inference is only supported for Language models and these vision model types: {', '.join(VLLM_SUPPORTED_VLM)}. "
f"Found architectures: {', '.join(model_types)}!"
)
+ if "idefics3" in model_types and not _zoo_supports_idefics3_fast_inference():
+ raise RuntimeError(
+ "Unsloth: Idefics3 fast_inference needs a newer unsloth_zoo. "
+ "Please run `pip install --upgrade unsloth_zoo`."
+ )
if any(arch in VLLM_NON_LORA_VLM for arch in model_types):
# mllama is still only in vllm v0, and vLLM V0 does not support LoRA on multimodal models. TODO: revisit once vLLM V1 supports Llama 3.2 (mllama).