Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: let safetensors vision models see every image in a chat (#12787)
Nilayce74c2e51aac
diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.pyindex 34007175d17..cd1a398df98 100644--- a/studio/backend/core/inference/inference.py+++ b/studio/backend/core/inference/inference.py@@ -64,7 +64,11 @@ from core.inference.native_tool_tokens import (reasoning_control_tokens,stop_token_text,)-from core.inference.mlx_inference import _mlx_stop_cut, _mlx_stop_sequences+from core.inference.mlx_inference import (+ _mlx_stop_cut,+ _mlx_stop_sequences,+ _scaling_image_marker,+)from io import StringIOimport structlogfrom loggers import get_logger@@ -3317,6 +3321,11 @@ class InferenceBackend:except Exception:processes_images = processor is not None and hasattr(processor, "image_processor")chat_template_info["renders_image"] = bool(processes_images)+ # Not mlx-vlm's SINGLE_IMAGE_ONLY_MODELS: that list is its prompt builder's limit, while the+ # transformers processors for llava_next and mllama take one image per marker.+ chat_template_info["accepts_multiple_images"] = bool(processes_images) and bool(+ _scaling_image_marker(tokenizer, processor, self.models[model_name].get("model"))+ )if processes_images:processor_template = getattr(processor, "chat_template", None)# Narrowing the named-template list form away disables the image-turn override.diff --git a/studio/backend/tests/test_vision_client_tools.py b/studio/backend/tests/test_vision_client_tools.pyindex d467abb2ad7..50bf686a9f5 100644--- a/studio/backend/tests/test_vision_client_tools.py+++ b/studio/backend/tests/test_vision_client_tools.py@@ -474,6 +474,95 @@ def test_the_named_template_list_form_survives_the_mirror():assert mlx_backend.models["m"]["chat_template_info"]["processor_template"] == listed+_MARKS_EACH_IMAGE = (+ "{% for m in messages %}<|im_start|>{{ m['role'] }}\n"+ "{% if m['content'] is string %}{{ m['content'] }}{% else %}{% for c in m['content'] %}"+ "{% if c['type'] == 'image' %}<|vision_start|><|image_pad|><|vision_end|>"+ "{% elif c['type'] == 'text' %}{{ c['text'] }}{% endif %}{% endfor %}{% endif %}"+ "<|im_end|>\n{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}"+)++_MARKS_ONE_IMAGE = (+ "{% for m in messages %}<|user|>\n"+ "{% if m['content'] is string %}{{ m['content'] }}{% else %}"+ "{% if m['content'] | selectattr('type', 'equalto', 'image') | list %}<image>\n{% endif %}"+ "{% for c in m['content'] %}{% if c['type'] == 'text' %}{{ c['text'] }}{% endif %}{% endfor %}"+ "{% endif %}<|end|>\n{% endfor %}{% if add_generation_prompt %}<|assistant|>\n{% endif %}"+)+++class _TemplateProcessor:+ image_processor = object()+ all_special_tokens: list = []++ def __init__(self, chat_template, image_token):+ self.chat_template = chat_template+ self.image_token = image_token++ def apply_chat_template(+ self,+ messages,+ add_generation_prompt = False,+ **_kwargs,+ ):+ import jinja2+ return (+ jinja2.Environment()+ .from_string(self.chat_template)+ .render(messages = messages, add_generation_prompt = add_generation_prompt)+ )+++ "chat_template, image_token, model_type, served",+ [+ (_MARKS_EACH_IMAGE, "<|image_pad|>", "qwen2_5_vl", [2, 4]),+ (_MARKS_ONE_IMAGE, "<image>", "llava", [4]),+ (_MARKS_EACH_IMAGE, "<|image_pad|>", "mllama", [2, 4]),+ ],+ ids = ["marks each image", "marks one image", "mlx-vlm single-image model"],+)+def test_a_transformers_vision_model_is_served_every_image_its_template_can_mark(+ chat_template, image_token, model_type, served+):+ import base64+ import io+ import types++ from PIL import Image++ from models.inference import ChatMessage++ def picture(size):+ buffer = io.BytesIO()+ Image.new("RGB", (size, size), "white").save(buffer, format = "PNG")+ encoded = base64.b64encode(buffer.getvalue()).decode()+ return {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{encoded}"}}++ inf = _inference_module()+ loader = inf.InferenceBackend.__new__(inf.InferenceBackend)+ processor = _TemplateProcessor(chat_template, image_token)+ model = types.SimpleNamespace(config = types.SimpleNamespace(model_type = model_type))+ loader.models = {"vl": {"tokenizer": processor, "processor": processor, "model": model}}+ loader._load_chat_template_info("vl")++ _pytest = _shared_setup_1(__file__)+ backend, passthrough = _shared_setup_3()+ backend.models["sf-model"]["chat_template_info"] = loader.models["vl"]["chat_template_info"]+ payload = passthrough._request(+ messages = [+ ChatMessage(role = "user", content = [picture(2), {"type": "text", "text": "first"}]),+ ChatMessage(role = "assistant", content = "a white square"),+ ChatMessage(role = "user", content = [picture(4), {"type": "text", "text": "brighter?"}]),+ ],+ stream = False,+ )+ _shared_setup_2(_pytest, backend, passthrough, payload)++ call = backend.calls[0]+ assert [image.width for image in call["images"] or [call["image"]]] == served++def test_a_processor_body_that_cannot_advertise_empties_the_healing_catalog():"""The route profiles the mirrored processor body for image turns, so a body with notool handling at all must leave nothing authorized to heal."""