Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: serve every image an MLX conversation carries, not just the newest (#10970)

Long Yixing004a44734368
diff --git a/studio/backend/core/inference/chat_template_helpers.py b/studio/backend/core/inference/chat_template_helpers.py
index e770fb48b69..5dd45247d1d 100644
--- a/studio/backend/core/inference/chat_template_helpers.py
+++ b/studio/backend/core/inference/chat_template_helpers.py
@@ -2924,14 +2924,14 @@ def messages_with_attached_image(
system_prompt: str = "",
fallback_user_text: str = "",
structured_content: bool = False,
- image: bool = True,
+ image: int = 1,
video: bool = False,
) -> list:
"""The conversation to render for a turn that carries attached media.
- Prepends *system_prompt* as a leading system turn, then injects an ``{"type": "image"}`` or
- ``{"type": "video"}`` part into the LAST user turn and leaves every other turn -- assistant
- ``tool_calls`` and ``role="tool"`` results included -- exactly as the caller sent it.
+ Prepends *system_prompt* as a leading system turn, then injects *image* ``{"type": "image"}``
+ parts, or a ``{"type": "video"}`` part, into the LAST user turn and leaves every other turn --
+ assistant ``tool_calls`` and ``role="tool"`` results included -- exactly as the caller sent it.
Rebuilding from the newest user TEXT instead dropped the folded system instruction and the
tool history an OpenAI tool loop replays (#10092). Nothing the caller owns is mutated: callers
still read those dicts after generation, and a retry re-renders the same list.
@@ -2976,11 +2976,11 @@ def messages_with_attached_image(
("image", image, count_structured_images),
("video", video, count_structured_videos),
)
- if wanted
- and not any(
+ if not any(
isinstance(m, dict) and isinstance(m.get("content"), list) and counter(m["content"])
for m in conversation
)
+ for _ in range(int(wanted))
]
if not parts and not fallback_user_text:
return conversation
diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.py
index 15e088a0ec7..3f3824f967b 100644
--- a/studio/backend/core/inference/inference.py
+++ b/studio/backend/core/inference/inference.py
@@ -1234,18 +1234,21 @@ class InferenceBackend:
presence_penalty: float = 0.0,
tool_protocol_active: Optional[bool] = None,
stop: Optional[list] = None,
+ *,
+ images = None,
) -> Generator[str, None, None]:
"""Generate response for text or vision models (lock held by background thread).
``tools`` / ``enable_thinking`` / ``reasoning_effort`` / ``preserve_thinking`` are forwarded
into ``apply_chat_template`` so templates that understand them (Qwen3, Llama 3.1+, gpt-oss
harmony) advertise tool schemas and reasoning controls. ``presence_penalty`` matches the
- GGUF sampling path (0 disables it).
+ GGUF sampling path (0 disables it). ``images`` is the MLX backend's list spelling.
"""
yield from self._generate_chat_response_inner(
messages = messages,
system_prompt = system_prompt,
image = image,
+ images = images,
temperature = temperature,
top_p = top_p,
top_k = top_k,
@@ -1284,6 +1287,8 @@ class InferenceBackend:
presence_penalty: float = 0.0,
tool_protocol_active: Optional[bool] = None,
stop: Optional[list] = None,
+ *,
+ images = None,
) -> Generator[str, None, None]:
"""Inner generation logic.
@@ -1297,6 +1302,15 @@ class InferenceBackend:
if not self.active_model_name:
raise RuntimeError("No active model")
+ # Every generation entry point funnels through here, so the list collapses here.
+ if images:
+ if len(images) > 1:
+ raise ValueError(
+ "The transformers backend takes one image per request; "
+ f"{len(images)} were supplied."
+ )
+ image = images[0]
+
model_info = self.models[self.active_model_name]
is_vision = model_info.get("is_vision", False)
tokenizer = model_info.get("tokenizer") or model_info.get("processor")
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.py
index 652050408c0..334cfa1fc57 100644
--- a/studio/backend/core/inference/mlx_inference.py
+++ b/studio/backend/core/inference/mlx_inference.py
@@ -3,6 +3,7 @@
interface, using mlx-lm/mlx-vlm instead of torch/transformers for model loading and generation."""
import copy
+import difflib
import hashlib
import importlib
import os
@@ -842,6 +843,18 @@ _AUDIO_PROBE_MESSAGES = [{"role": "user", "content": "audio"}]
_IMAGE_PROBE_MESSAGES = [
{"role": "user", "content": [{"type": "image"}, {"type": "text", "text": "hi"}]}
]
+# A pair differing only in image count, so a diff isolates the marker; the question is distinctive.
+_MULTI_IMAGE_PROBE_TEXT = "what is in these"
+_MULTI_IMAGE_PROBE_MESSAGES = [
+ [
+ {
+ "role": "user",
+ "content": [{"type": "image"}] * count
+ + [{"type": "text", "text": _MULTI_IMAGE_PROBE_TEXT}],
+ }
+ ]
+ for count in (1, 2)
+]
_TEXT_PROBE_MESSAGES = [{"role": "user", "content": [{"type": "text", "text": "hi"}]}]
_VIDEO_PROBE_MESSAGES = [
{"role": "user", "content": [{"type": "video"}, {"type": "text", "text": "hi"}]}
@@ -1729,6 +1742,87 @@ def _image_placeholder(tokenizer, processor):
return None
+_IMAGE_TOKEN_ID_KEYS = ("image_token_index", "image_token_id", "boi_token_index")
+
+
+def _declared_image_tokens(
+ tokenizer,
+ processor,
+ model = None,
+):
+ """Config token ids too: granite-vision names its token only by id, on a bare tokenizer."""
+ named = []
+
+ def add(token):
+ if isinstance(token, str) and token and token not in named:
+ named.append(token)
+
+ for source in (processor, tokenizer):
+ for attr in ("image_token", "boi_token"):
+ add(getattr(source, attr, None))
+ config, _model_type = _mlx_vlm_model_config(model) if model is not None else (None, None)
+ for key in _IMAGE_TOKEN_ID_KEYS:
+ token_id = config.get(key) if isinstance(config, dict) else getattr(config, key, None)
+ for source in (getattr(processor, "tokenizer", None), tokenizer, processor):
+ try:
+ add(source.convert_ids_to_tokens(token_id))
+ except BaseException:
+ continue
+ break
+ return named
+
+
+def _scaling_image_marker(
+ tokenizer,
+ processor,
+ model = None,
+):
+ """The text this template writes once per image, or None -- a load must never fail on a probe.
+
+ Diffed rather than looked up by attribute name, which would only recognise families someone had
+ checked: Gemma 3 writes its ``boi_token``, Qwen-VL a three-token block. The block must repeat
+ once more for the second image, the question must survive (InternVL drops it), and the marker
+ must be a token the model names, so an override nothing binds pixels to cannot pass.
+ """
+ from core.inference.chat_template_helpers import apply_chat_template_for_generation
+
+ targets = _template_render_targets(tokenizer, processor)
+ if not targets:
+ return None
+ try:
+ one, two = (
+ apply_chat_template_for_generation(targets[0], probe)
+ for probe in _MULTI_IMAGE_PROBE_MESSAGES
+ )
+ if not isinstance(one, str) or not isinstance(two, str):
+ return None
+ if _MULTI_IMAGE_PROBE_TEXT not in one or _MULTI_IMAGE_PROBE_TEXT not in two:
+ return None
+ opcodes = difflib.SequenceMatcher(None, one, two, autojunk = False).get_opcodes()
+ inserted = [two[j1:j2] for tag, _i1, _i2, j1, j2 in opcodes if tag == "insert"]
+ if len(inserted) != 1 or any(tag in ("replace", "delete") for tag, *_rest in opcodes):
+ return None
+ marker = inserted[0]
+ if one.count(marker) != 1 or two.count(marker) != 2:
+ return None
+ named = _declared_image_tokens(tokenizer, processor, model)
+ if any(token in marker for token in named):
+ return marker
+ except BaseException:
+ return None
+ return None
+
+
+def _model_type_takes_several_images(model):
+ """Studio renders the model's own template, so mlx-vlm's deny-list has to be read here."""
+ try:
+ from mlx_vlm.prompt_utils import SINGLE_IMAGE_ONLY_MODELS
+ except BaseException:
+ return False
+ _config, model_type = _mlx_vlm_model_config(model)
+ return bool(model_type) and model_type not in SINGLE_IMAGE_ONLY_MODELS
+
+
def _image_marker_survives(
tokenizer,
processor,
@@ -2328,6 +2422,7 @@ class MLXInferenceBackend:
self._tokenizer = None
self._processor = None
self._is_vlm = False
+ self._multi_image_marker = None
self._config = {}
self._distributed_group = None
self._distributed_rank = 0
@@ -2828,6 +2923,12 @@ class MLXInferenceBackend:
_revoke_override_that_drops_image(
self._template_override, self._tokenizer, self._processor, image_placeholder
)
+ # After the override settles, so the probe measures the template generation will use.
+ self._multi_image_marker = (
+ _scaling_image_marker(self._tokenizer, self._processor, self._model)
+ if is_vision and _model_type_takes_several_images(self._model)
+ else None
+ )
# Released once the media checks are done: the pairs reference the tokenizer and processor, so keeping them
# would outlive the unload that nulls both.
self._template_override["restore"] = []
@@ -2931,6 +3032,7 @@ class MLXInferenceBackend:
# The body _generate_vlm renders an image turn with, not the tokenizer body.
"processor_template": None,
"renders_image": False,
+ "accepts_multiple_images": False,
}
from core.inference.chat_template_helpers import (
chat_render_target as _chat_render_target,
@@ -2944,6 +3046,9 @@ class MLXInferenceBackend:
if isinstance(_proc_tpl, (str, dict, list, tuple)) and _proc_tpl:
info["processor_template"] = _proc_tpl
info["renders_image"] = _proc is not None and bool(getattr(self, "_is_vlm", False))
+ info["accepts_multiple_images"] = info["renders_image"] and bool(
+ getattr(self, "_multi_image_marker", None)
+ )
try:
tpl = (
getattr(tok, "chat_template", None)
@@ -3127,6 +3232,8 @@ class MLXInferenceBackend:
# cannot tell that the wrappers below still have to survive decoding.
tool_protocol_active = None,
video = None,
+ *,
+ images = None,
) -> Generator[str, None, None]:
if self._model is None:
raise RuntimeError("No model loaded")
@@ -3139,8 +3246,15 @@ class MLXInferenceBackend:
# Reset so a failed run cannot surface stale stats.
self.last_generation_stats = None
+ images = list(images) if images else ([image] if image is not None else [])
+ if len(images) > 1 and not self._multi_image_marker:
+ raise ValueError(
+ f"'{self.active_model_name}' takes one image per request: its chat template does "
+ f"not mark {len(images)} images separately."
+ )
+
# Shared with the transformers vision path so both render the same turns (#10092).
- if self._is_vlm and (image is not None or video is not None):
+ if self._is_vlm and (images or video is not None):
# Processor templates want part lists, the tokenizer fallback wants strings.
from core.inference.chat_template_helpers import (
chat_render_target as _chat_render_target,
@@ -3153,7 +3267,7 @@ class MLXInferenceBackend:
messages,
system_prompt = system_prompt,
structured_content = _renders_via_processor,
- image = image is not None,
+ image = len(images or ()),
video = video is not None,
)
else:
@@ -3162,7 +3276,7 @@ class MLXInferenceBackend:
if self._is_vlm:
stream = self._generate_vlm(
full_messages,
- image,
+ images,
temperature,
top_p,
top_k,
@@ -3621,12 +3735,19 @@ class MLXInferenceBackend:
prompt = recovered_prompt
elif prompt_issue:
raise RuntimeError(f"VLM chat template returned {prompt_issue}.") from prompt_error
+ # mlx-vlm binds pixel_values[i] to the i-th image token: miscounted markers answer wrong.
+ if attached_images > 1 and self._multi_image_marker:
+ marked = prompt.count(self._multi_image_marker)
+ if marked != attached_images:
+ raise RuntimeError(
+ f"VLM chat template marked {marked} image(s) for {attached_images} attached."
+ )
return prompt, chat_target
def _generate_vlm(
self,
messages,
- image,
+ images,
temperature,
top_p,
top_k,
@@ -3651,7 +3772,7 @@ class MLXInferenceBackend:
):
from mlx_vlm import stream_generate as vlm_stream
- images = [image] if image is not None else None
+ images = list(images) if images else None
prompt, chat_target = self._render_vlm_prompt(
messages,
images,
@@ -3686,14 +3807,14 @@ class MLXInferenceBackend:
stopped = False
if max_new_tokens is None:
max_new_tokens = self._unset_generation_budget(prompt)
- if image is not None or video is not None:
+ if images or video is not None:
# Media expands past its placeholder (a clip by a frame each), so the counted
# prompt is short: cap at the default, under the cache window.
max_new_tokens = min(max_new_tokens, UNSET_GENERATION_BUDGET)
logger.info(
- "VLM generating: prompt_len=%d, has_image=%s, has_video=%s",
+ "VLM generating: prompt_len=%d, images=%d, has_video=%s",
len(prompt),
- image is not None,
+ len(images or ()),
video is not None,
)
# stream_generate forwards **kwargs into generate_step (which builds the sampler + logits_processors
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py
index 09bbd9a9ebe..9a2c6a4f187 100644
--- a/studio/backend/core/inference/orchestrator.py
+++ b/studio/backend/core/inference/orchestrator.py
@@ -200,6 +200,13 @@ def _summed_tool_loop_stats(total, turn):
return summed
+def _request_images(image, images):
+ """One request's images in render order; ``image`` is the older single-image spelling."""
+ if images:
+ return list(images)
+ return [image] if image is not None else []
+
+
def _mirrored_model_entry(model_info: dict, model_name: str) -> dict:
"""The parent's view of a model the worker holds. Measured or classified in the subprocess and
unrecoverable once the model lives there, so a field the worker sends and this does not copy
@@ -966,7 +973,7 @@ class InferenceOrchestrator:
def _build_generate_cmd(
self,
request_id: str,
- image_b64: Optional[str],
+ images_b64: list,
*,
messages: list = None,
system_prompt: str = "",
@@ -996,7 +1003,7 @@ class InferenceOrchestrator:
"request_id": request_id,
"messages": messages or [],
"system_prompt": system_prompt,
- "image_base64": image_b64,
+ "images_base64": images_b64,
"temperature": temperature,
"top_p": top_p,
"top_k": top_k,
@@ -1219,6 +1226,8 @@ class InferenceOrchestrator:
logit_bias: Optional[dict] = None,
stop: Optional[list] = None,
video: Optional[str] = None,
+ *,
+ images = None,
) -> Generator[str, None, None]:
"""Dispatched generation, sending the command without holding _gen_lock. Uses a per-request
mailbox for tokens so two compare-mode requests can be queued at once. The subprocess
@@ -1254,13 +1263,11 @@ class InferenceOrchestrator:
request_id = str(uuid.uuid4())
- image_b64 = None
- if image is not None:
- image_b64 = self._pil_to_base64(image)
+ images_b64 = [self._pil_to_base64(one) for one in _request_images(image, images)]
cmd = self._build_generate_cmd(
request_id,
- image_b64,
+ images_b64,
messages = messages,
system_prompt = system_prompt,
temperature = temperature,
@@ -2050,6 +2057,8 @@ class InferenceOrchestrator:
logit_bias: Optional[dict] = None,
stop: Optional[list] = None,
video: Optional[str] = None,
+ *,
+ images = None,
) -> Generator[str, None, None]:
"""Generate response, streaming tokens from subprocess. ``tools`` / ``enable_thinking`` /
``reasoning_effort`` / ``preserve_thinking`` are forwarded so the template can render
@@ -2061,6 +2070,7 @@ class InferenceOrchestrator:
messages = messages,
system_prompt = system_prompt,
image = image,
+ images = images,
temperature = temperature,
top_p = top_p,
top_k = top_k,
@@ -2304,6 +2314,8 @@ class InferenceOrchestrator:
logit_bias: Optional[dict] = None,
stop: Optional[list] = None,
video: Optional[str] = None,
+ *,
+ images = None,
) -> Generator[str, None, None]:
"""Inner generation logic: sends the command to the subprocess and yields tokens. Serialized
by _gen_lock (one generation at a time) so concurrent readers don't consume each other's
@@ -2328,10 +2340,10 @@ class InferenceOrchestrator:
if cancel_event is not None and cancel_event.is_set():
return
request_id = str(uuid.uuid4())
- image_b64 = self._pil_to_base64(image) if image is not None else None
+ images_b64 = [self._pil_to_base64(one) for one in _request_images(image, images)]
cmd = self._build_generate_cmd(
request_id,
- image_b64,
+ images_b64,
messages = messages,
system_prompt = system_prompt,
temperature = temperature,
diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.py
index da2223be4f8..52e5f9fa5d1 100644
--- a/studio/backend/core/inference/worker.py
+++ b/studio/backend/core/inference/worker.py
@@ -556,6 +556,7 @@ def _handle_load(backend, config: dict, resp_queue: Any) -> None:
# The IMAGE-turn body; the whitelist is the only way out.
"processor_template": _tpl_info.get("processor_template"),
"renders_image": _tpl_info.get("renders_image"),
+ "accepts_multiple_images": _tpl_info.get("accepts_multiple_images"),
}
except Exception as _tpl_exc:
logger.warning("chat_template_info forward failed: %s", _tpl_exc)
@@ -675,16 +676,16 @@ def _handle_generate(backend, cmd: dict, resp_queue: Any, cancel_event) -> None:
request_id = cmd.get("request_id", "")
try:
- image = None
- image_b64 = cmd.get("image_base64")
- if image_b64:
- image = _decode_image(image_b64)
- image = _resize_image(image)
+ images = [
+ _resize_image(_decode_image(encoded))
+ for encoded in cmd.get("images_base64") or ()
+ if encoded
+ ]
gen_kwargs = {
"messages": cmd["messages"],
"system_prompt": cmd.get("system_prompt", ""),
- "image": image,
+ "images": images,
"temperature": cmd.get("temperature", 0.7),
"top_p": cmd.get("top_p", 0.9),
"top_k": cmd.get("top_k", 40),
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index aef29eec7e7..de1fc6d8a8b 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -8203,9 +8203,41 @@ def _validate_image_base64(encoded: str) -> None:
image.load()
-async def _preflight_image_for_switch(image_preflight: dict, target_is_gguf: bool) -> None:
+def _local_target_may_take_several_images(load_path: Optional[str]) -> bool:
+ import json
+
+ from utils.hardware import DeviceType, get_device
+ try:
+ if get_device() != DeviceType.MLX:
+ return False
+ from mlx_vlm.prompt_utils import SINGLE_IMAGE_ONLY_MODELS
+
+ config_path = Path(os.path.expanduser(load_path or ""))
+ if config_path.is_dir():
+ config_path = config_path / "config.json"
+ if not config_path.is_file():
+ return False
+ with open(config_path, encoding = "utf-8") as f:
+ config = json.load(f)
+ model_type = config.get("model_type") if isinstance(config, dict) else None
+ return bool(model_type) and model_type not in SINGLE_IMAGE_ONLY_MODELS
+ except BaseException:
+ return False
+
+
+async def _preflight_image_for_switch(
+ image_preflight: dict,
+ target_is_gguf: bool,
+ target_takes_several: bool = False,
+) -> None:
"""Apply target-specific image checks before loading the resolved target."""
- if not target_is_gguf and image_preflight.get("multiple"):
+ serves_several = (
+ target_takes_several
+ and len(image_preflight.get("admitted", ())) > 1
+ and not image_preflight.get("unservable_alongside")
+ )
+ # Structural: one real image beside a payloadless URL is still a multi-image request.
+ if not target_is_gguf and image_preflight.get("multiple") and not serves_several:
raise HTTPException(
status_code = 400,
detail = (
@@ -8218,6 +8250,7 @@ async def _preflight_image_for_switch(image_preflight: dict, target_is_gguf: boo
status_code = 400,
detail = "Remote image URLs are not supported. Use a base64 data URL.",
)
+ # Only what the single-image path selects: an architecture guess must not refuse the rest.
encoded_images = (
image_preflight.get("b64s", ()) if target_is_gguf else (image_preflight.get("b64"),)
)
@@ -9609,7 +9642,12 @@ async def _maybe_auto_switch_model(
and resolved is not None
and (target_is_gguf or not require_audio_input)
):
- await _preflight_image_for_switch(image_preflight, target_is_gguf)
+ await _preflight_image_for_switch(
+ image_preflight,
+ target_is_gguf,
+ target_takes_several = not target_is_gguf
+ and await asyncio.to_thread(_local_target_may_take_several_images, target_id),
+ )
speech_cache_environment = None
speech_codec_path = None
if speech_type is not None:
@@ -17957,12 +17995,67 @@ async def get_api_monitor_entry(entry_id: str, current_subject: str = Depends(ge
def _decode_and_resize_image(backend, encoded: str):
- """Decode one request image and run Pillow resampling off the event loop."""
+ """Off the event loop. The decode is forced so a truncated picture is a bad request here, and
+ all but RGB/RGBA converts because CMYK, PA, F and LAB decode and none can be saved back."""
from PIL import Image
from io import BytesIO
image_data = base64.b64decode(encoded)
- return backend.resize_image(Image.open(BytesIO(image_data)))
+ image = Image.open(BytesIO(image_data))
+ image.load()
+ # After the resize: converting first resamples interpolated RGB, a different picture.
+ image = _scaled_from_16_bit(backend.resize_image(image))
+ if image.mode not in ("RGB", "RGBA"):
+ image = image.convert("RGB")
+ return image
+
+
+# Serving a whole conversation's images made the retained set grow with the request, where one
+# image had always held one. resize_image caps each raster at 800x800, but a solid 800x800 PNG is
+# ~4.8 KB of base64 against a 1.92 MB raster, a 391x amplification, so a body small enough to pass
+# any transport limit still reaches gigabytes of resident pixels. No model in this path consumes
+# anything near this many images, so the cap costs real conversations nothing.
+_MAX_SERVED_IMAGES = 64
+
+
+async def _decode_request_images(
+ backend,
+ encoded_images,
+ decoded = None,
+ reject = None,
+):
+ """The images *encoded_images* names, in that order, reusing anything *decoded* already holds.
+
+ One at a time, because a decode is forced at full resolution before the resize shrinks it:
+ Pillow admits images up to about 179 million pixels, so decoding a conversation's images
+ together would hold that many rasters at once where one image has always held one.
+ """
+ # Occurrences, not distinct payloads: this helper reuses one PIL object per payload, but the
+ # worker decodes every entry of images_base64 (core/inference/worker.py:679-683), so a repeat
+ # costs a raster there whatever it costs here.
+ total = len(encoded_images)
+ if total > _MAX_SERVED_IMAGES:
+ detail = (
+ f"This request carries {total} images; at most {_MAX_SERVED_IMAGES} are "
+ "served per request. Send fewer, or split the conversation."
+ )
+ # Through the caller's reject, so an open API monitor row is failed rather than left
+ # reported as running.
+ raise (
+ reject(400, detail)
+ if reject is not None
+ else HTTPException(
+ status_code = 400,
+ detail = detail,
+ )
+ )
+ ready = dict(decoded or {})
+ images = []
+ for encoded in encoded_images:
+ if encoded not in ready:
+ ready[encoded] = await asyncio.to_thread(_decode_and_resize_image, backend, encoded)
+ images.append(ready[encoded])
+ return images
@router.post("/generate/stream")
@@ -21260,7 +21353,9 @@ def _inject_audio_part(messages: list[dict], audio_b64: str, audio_format: str)
return
-def _extract_content_parts(messages: list) -> tuple[str, list[dict], "Optional[str]"]:
+def _extract_content_parts(
+ messages: list, structured: bool = False
+) -> tuple[str, list[dict], "Union[Optional[str], list[str]]"]:
"""
Parse OpenAI-format messages into components the inference backend expects.
@@ -21269,14 +21364,16 @@ def _extract_content_parts(messages: list) -> tuple[str, list[dict], "Optional[s
Returns:
system_prompt: System message text (empty string if none).
- chat_messages: Non-system messages with content flattened to strings and
- assistant reasoning_content preserved.
- image_base64: Base64 of the most recent image found, or ``None``.
+ chat_messages: Non-system messages, reasoning_content preserved and content flattened
+ to strings -- or, when *structured*, parts with each served image cut
+ to ``{"type": "image"}`` in place.
+ images: The newest image's base64 -- or every served image's in document order.
"""
system_parts: list[str] = []
chat_messages: list[dict] = []
latest_image_b64: Optional[str] = None
latest_user_image_b64: Optional[str] = None
+ served_images: list[str] = []
for msg in messages:
# ── System / developer messages → extract as system_prompt ────────
@@ -21289,6 +21386,12 @@ def _extract_content_parts(messages: list) -> tuple[str, list[dict], "Optional[s
continue
# ── User / assistant messages ─────────────────────────
+ if structured and isinstance(msg.content, list):
+ parts, payloads = _conversation_with_image_markers([msg])
+ served_images.extend(payloads)
+ chat_messages.append(dict(parts[0]))
+ continue
+
combined_text: Optional[str] = None
if isinstance(msg.content, str):
# Plain string content - pass through
@@ -21342,7 +21445,7 @@ def _extract_content_parts(messages: list) -> tuple[str, list[dict], "Optional[s
return (
"\n\n".join(p for p in system_parts if p),
chat_messages,
- latest_user_image_b64 or latest_image_b64,
+ served_images if structured else (latest_user_image_b64 or latest_image_b64),
)
@@ -21457,6 +21560,101 @@ def _latest_user_image_payload(messages) -> "Optional[str]":
return None
+def _part_attr(part, name):
+ return part.get(name) if isinstance(part, dict) else getattr(part, name, None)
+
+
+def _image_part_payload(part) -> "Optional[str]":
+ """Shared, so extraction, the tool rebuild and the preflight agree on what is servable."""
+ if _part_attr(part, "type") != "image_url":
+ return None
+ image_url = _part_attr(part, "image_url")
+ url = image_url.get("url") if isinstance(image_url, dict) else getattr(image_url, "url", None)
+ if not isinstance(url, str) or not url.startswith("data:"):
+ return None
+ return url.partition(",")[2] or None
+
+
+def _conversation_with_image_markers(messages) -> tuple[list[dict], list[str]]:
+ """Markers stay where they stood: the backend binds pixel values to image tokens by position."""
+ rebuilt: list[dict] = []
+ payloads: list[str] = []
+ for message in messages or ():
+ plain = message if isinstance(message, dict) else message.model_dump(exclude_none = True)
+ content = plain.get("content")
+ if not isinstance(content, list):
+ rebuilt.append(plain)
+ continue
+ parts = []
+ for part in content:
+ if _part_attr(part, "type") == "image_url":
+ # User turns only: several templates leave an assistant's own image unmarked.
+ served = _image_part_payload(part) if plain.get("role") == "user" else None
+ if served is not None:
+ payloads.append(served)
+ parts.append({"type": "image"})
+ # Otherwise dropped: a part left here claims a marker nothing can bind.
+ continue
+ if _part_attr(part, "type") != "text":
+ # Text and image only, like _flatten_content_parts_for_local_template: a local
+ # template raises on anything else ("Only text and image blocks are supported in
+ # message content!" on mistral3), and mlx_inference re-raises that instead of
+ # recovering whenever the request carries tools or a reasoning knob, so an
+ # input_document earlier in the conversation would turn the turn into a 500.
+ continue
+ parts.append(part)
+ rebuilt.append({**plain, "content": parts})
+ return rebuilt, payloads
+
+
+def _serves_several_images(backend) -> bool:
+ entry = (getattr(backend, "models", None) or {}).get(
+ getattr(backend, "active_model_name", None), {}
+ )
+ return bool((entry.get("chat_template_info") or {}).get("accepts_multiple_images"))
+
+
+def _serve_legacy_image_on_the_newest_turn(
+ messages: list[dict], payloads: list[str], encoded: str
+) -> list[str]:
+ newest = len(messages)
+ for position in range(len(messages) - 1, -1, -1):
+ if messages[position].get("role") == "user":
+ newest = position
+ break
+
+ marker = {"type": "image"}
+ if newest == len(messages):
+ messages.append({"role": "user", "content": [marker]})
+ else:
+ content = messages[newest].get("content")
+ messages[newest] = {
+ **messages[newest],
+ "content": (
+ [*content, marker]
+ if isinstance(content, list)
+ else [{"type": "text", "text": content or ""}, marker]
+ ),
+ }
+ return [*payloads, encoded]
+
+
+def _newest_turn_shows_more_images_than_it_sends(messages, legacy_is_distinct: bool) -> bool:
+ """The shape the one-image-per-message refusal answers, rather than answering about the rest."""
+ for message in reversed(messages or ()):
+ role = message.get("role") if isinstance(message, dict) else message.role
+ if role != "user":
+ continue
+ content = message.get("content") if isinstance(message, dict) else message.content
+ if not isinstance(content, list):
+ return False
+ parts = [part for part in content if _part_attr(part, "type") == "image_url"]
+ if len(parts) + int(legacy_is_distinct) < 2:
+ return False
+ return any(_image_part_payload(part) is None for part in parts)
+ return False
+
+
def _local_image_payloads_from_messages(messages) -> list[str]:
"""All data-URL image payloads in OpenAI-format messages."""
encoded_images = []
@@ -23425,14 +23623,23 @@ async def produce_openai_chat_completions(
_image_b64 = _pre_parsed[2] or payload.image_base64
if _image_b64 is None and _local_image_payloads:
_image_b64 = _local_image_payloads[0]
+ # Read the way the render reads it; separate from b64s, which GGUF validates unchanged.
+ _admitted_payloads = _conversation_with_image_markers(payload.messages)[1]
+ if _legacy_image_distinct:
+ _admitted_payloads = [*_admitted_payloads, payload.image_base64]
_image_preflight = {
"b64": _image_b64,
"b64s": _local_image_payloads,
+ "admitted": _admitted_payloads,
"remote": _messages_have_remote_image(payload.messages),
"multiple": (
_images_on_turn + int(_legacy_image_distinct) > 1
or bool(_pre_parsed[2] and _legacy_image_distinct)
),
+ # Refused after the load whatever the target is, so refused before evicting for it.
+ "unservable_alongside": _newest_turn_shows_more_images_than_it_sends(
+ payload.messages, _legacy_image_distinct
+ ),
}
# Defer the resident claim: chat has several post-switch capability checks that can still
@@ -25886,6 +26093,23 @@ async def produce_openai_chat_completions(
_tracker.__exit__(None, None, None)
# ── Standard Unsloth path ─────────────────────────────────
+ # Re-derived, not reused from the pre-switch parse: an auto-switch may have changed models.
+ served_images: list[str] = []
+ images: list = []
+ if _serves_several_images(backend):
+ _, _msgs, _payloads = _extract_content_parts(payload.messages, structured = True)
+ _legacy_distinct = _legacy_image_is_distinct(payload)
+ # A legacy image the messages do not carry joins the newest user turn, as GGUF splices it.
+ if _legacy_distinct:
+ _payloads = _serve_legacy_image_on_the_newest_turn(
+ _msgs, _payloads, payload.image_base64
+ )
+ # Only above one image, so an ordinary single-image request renders exactly as before.
+ if len(_payloads) > 1 and not _newest_turn_shows_more_images_than_it_sends(
+ payload.messages, _legacy_distinct
+ ):
+ chat_messages, served_images = _msgs, _payloads
+
# Decode image (from content parts OR legacy field)
image_b64 = extracted_image_b64 or payload.image_base64
image = None
@@ -25912,8 +26136,9 @@ async def produce_openai_chat_completions(
# misses, a turn with no part while an earlier one has: selection is
# `extracted_image_b64 or image_base64`, so the legacy one goes unread.
_legacy_is_distinct = _legacy_image_is_distinct(payload)
- if images_on_turn + int(_legacy_is_distinct) > 1 or (
- extracted_image_b64 and _legacy_is_distinct
+ if not served_images and (
+ images_on_turn + int(_legacy_is_distinct) > 1
+ or (extracted_image_b64 and _legacy_is_distinct)
):
raise _reject(
400,
@@ -25923,7 +26148,9 @@ async def produce_openai_chat_completions(
),
)
- if image_b64:
+ if served_images:
+ images = await _decode_request_images(backend, served_images, reject = _reject)
+ elif image_b64:
image = await asyncio.to_thread(
_decode_and_resize_image,
backend,
@@ -26110,6 +26337,7 @@ async def produce_openai_chat_completions(
(_sf_tools_on or _sf_mcp_allowed)
and _sf_features.get("supports_tools", False)
and image is None
+ and not images
and _video_clip is None
and not _sf_is_gptoss
and _sf_tool_budget > 0
@@ -26611,6 +26839,7 @@ async def produce_openai_chat_completions(
messages = chat_messages,
system_prompt = system_prompt,
image = image,
+ images = images,
temperature = payload.temperature,
top_p = payload.top_p,
top_k = payload.top_k,
@@ -26642,14 +26871,15 @@ async def produce_openai_chat_completions(
# supports_tools=False falls through to plain relay (GGUF gate parity).
_sf_has_tool_msgs = any(m.role == "tool" or m.tool_calls for m in payload.messages)
# Resolved BEFORE the capability gate below, which classifies from this body (#10092).
+ _sf_has_image = image is not None or bool(images)
# A clip renders through the processor as an image does.
_sf_image_tpl = (
(_sf_model_info.get("chat_template_info") or {}).get("processor_template")
- if image is not None or _video_clip is not None
+ if _sf_has_image or _video_clip is not None
else None
)
# Differs from processor_template: a template-less processor still places the image.
- _sf_renders_image = image is not None and bool(
+ _sf_renders_image = _sf_has_image and bool(
(_sf_model_info.get("chat_template_info") or {}).get("renders_image")
)
if _sf_image_tpl is not None:
@@ -26669,7 +26899,7 @@ async def produce_openai_chat_completions(
# above withdraws the launcher default for exactly these requests, and
# recomputing here would hide that and drop the client catalog.
# An image or a clip rules out the server loop, so the passthrough takes the request (#10092).
- (not _sf_tools_on or ((image is not None or _video_clip is not None) and not _sf_use_tools))
+ (not _sf_tools_on or ((_sf_has_image or _video_clip is not None) and not _sf_use_tools))
and not _sf_use_tools
and not _sf_is_gptoss
and _sf_supports_tools
@@ -26730,19 +26960,32 @@ async def produce_openai_chat_completions(
# Re-derive from payload.messages so tool_calls / role="tool" history
# survives templating; fold system/developer into one leading system
# message (templates reject "developer") and clear prompt to avoid a dup.
- gen_kwargs["messages"] = _set_or_prepend_system_message(
- _structured_tool_history_for_local_template(
- _flatten_content_parts_for_local_template(
- # Not a llama-server body: the flatten below drops image parts.
- _openai_messages_for_passthrough(payload, normalize_images = False)
- )
- ),
- system_prompt,
- )
+ if served_images:
+ # One pass over the conversation this renders, so markers and payloads stay in step.
+ _sf_rebuilt, _sf_payloads = _conversation_with_image_markers(
+ _openai_messages_for_passthrough(payload, normalize_images = False)
+ )
+ # The same pictures the extraction already decoded, unless the rebuild dropped one.
+ gen_kwargs["images"] = await _decode_request_images(
+ backend, _sf_payloads, dict(zip(served_images, images)), reject = _reject
+ )
+ gen_kwargs["messages"] = _set_or_prepend_system_message(
+ _structured_tool_history_for_local_template(_sf_rebuilt), system_prompt
+ )
+ else:
+ gen_kwargs["messages"] = _set_or_prepend_system_message(
+ _structured_tool_history_for_local_template(
+ _flatten_content_parts_for_local_template(
+ # Not a llama-server body: the flatten below drops image parts.
+ _openai_messages_for_passthrough(payload, normalize_images = False)
+ )
+ ),
+ system_prompt,
+ )
# Mark the turn that owns the image so the newest-user-turn scan does not move an
# older picture onto a later question. Gated on _sf_renders_image, not on an image:
- # a text-template render must not be handed part lists (#10092).
- if _sf_renders_image:
+ # a text-template render must not be handed part lists (#10092). Kept markers need none.
+ if _sf_renders_image and not served_images:
_sf_image_ordinal = _user_ordinal_supplying_the_image(payload.messages)
if _sf_image_ordinal is not None:
gen_kwargs["messages"] = _mark_image_owner_turn(
@@ -31763,6 +32006,18 @@ def _select_anthropic_server_tools(
return [tool for tool in available if tool["function"]["name"] in selected_names]
+def _scaled_from_16_bit(image):
+ """8-bit values for a 16-bit source, which ``convert`` would otherwise read as 8-bit and clip
+ above 255, turning the picture nearly all white. The I;16 family only: plain "I" and "F"
+ declare no range, so a TIFF whose samples already sit in 0..255 would be scaled to black.
+ I;16B / I;16L reject point(), hence the normalisation to "I"."""
+ if not image.mode.startswith("I;16"):
+ return image
+ if image.mode != "I;16":
+ image = image.convert("I")
+ return image.point(lambda v: v * (1.0 / 257), mode = "L")
+
+
def _image_bytes_to_png_b64(raw: bytes) -> str:
"""Decode raw image bytes and re-encode to a base64-ascii PNG string.
@@ -31771,22 +32026,7 @@ def _image_bytes_to_png_b64(raw: bytes) -> str:
input; callers wrap the call in ``try`` -> HTTPException(400)."""
from PIL import Image
- img = Image.open(io.BytesIO(raw))
- # A 16-bit source carries 0..65535, but convert("RGB") reads those as 8-bit
- # and clips everything above 255, which turns the picture nearly all white.
- # Scale to 8 bits first, the way stb_image does when llama-server reads the
- # same file itself. I;16B / I;16L reject point(), so normalise them to "I".
- #
- # Only the I;16 family: plain "I" and "F" declare no range, and a 32-bit or
- # float TIFF whose samples already sit in 0..255 (or 0..1) would be scaled
- # to black. Those keep the straight convert("RGB"). A 16-bit PNG -- the
- # reachable case here, since this decodes pasted images -- opens as I;16 on
- # every Pillow this repo pins.
- if img.mode.startswith("I;16"):
- if img.mode != "I;16":
- img = img.convert("I")
- img = img.point(lambda v: v * (1.0 / 257), mode = "L")
- img = img.convert("RGB")
+ img = _scaled_from_16_bit(Image.open(io.BytesIO(raw))).convert("RGB")
buf = io.BytesIO()
img.save(buf, format = "PNG")
return base64.b64encode(buf.getvalue()).decode("ascii")
@@ -35303,16 +35543,12 @@ def _openai_messages_for_passthrough(payload, normalize_images: bool = True) ->
remote URLs are forwarded as-is. The vision guard lives in the callers,
which reject a non-vision model before the body is built.
- ``normalize_images=False`` is for callers that are not building a
- llama-server body: the local-template path flattens image parts away, and
- only reaches this helper when the turn has no decodable image at all, so
- re-encoding there can only turn an image it was already ignoring -- a
- payloadless ``data:`` URL -- into a 400.
+ ``normalize_images=False`` is for callers that decode these bytes themselves: re-encoding
+ converts to RGB, which the resize they apply would then resample.
When a client uses Unsloth's legacy ``image_base64`` top-level field, the
- image is re-encoded to PNG (llama-server's stb_image has limited format
- support) and spliced into the last user message as an OpenAI ``image_url``
- content part so vision + function-calling requests work transparently.
+ image is spliced into the last user message as an OpenAI ``image_url`` content part -- as PNG
+ while normalizing, since llama-server's stb_image is limited, byte-for-byte otherwise.
Only when it is a real second image, matching ``_openai_messages_for_gguf_chat`` and
what admission charges for. Studio echoes the current image into both spellings, so
@@ -35330,16 +35566,19 @@ def _openai_messages_for_passthrough(payload, normalize_images: bool = True) ->
if not _legacy_image_is_distinct(payload):
return messages
- try:
- raw = base64.b64decode(payload.image_base64)
- png_b64 = _image_bytes_to_png_b64(raw)
- except Exception:
- raise HTTPException(
- status_code = 400,
- detail = "Failed to process image.",
- )
-
- data_url = f"data:image/png;base64,{png_b64}"
+ if normalize_images:
+ try:
+ raw = base64.b64decode(payload.image_base64)
+ png_b64 = _image_bytes_to_png_b64(raw)
+ except Exception:
+ raise HTTPException(
+ status_code = 400,
+ detail = "Failed to process image.",
+ )
+ data_url = f"data:image/png;base64,{png_b64}"
+ else:
+ # As it arrived; the media type is never read back.
+ data_url = f"data:application/octet-stream;base64,{payload.image_base64}"
image_part = {"type": "image_url", "image_url": {"url": data_url}}
_splice_image_into_last_user(messages, image_part)
diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.py
index bc068847c1a..bf1340f51e6 100644
--- a/studio/backend/tests/test_mlx_inference_backend.py
+++ b/studio/backend/tests/test_mlx_inference_backend.py
@@ -908,7 +908,7 @@ def test_mlx_vlm_reemits_think_prefill_inside_adapter_context(
monkeypatch.setattr(
backend, "_release_vlm_snapshots", lambda: order.append("snapshots_released")
)
- args = ([{"role": "user", "content": [{"type": "image"}]}], object(), 0, 1, 0, 0, 1, 1, None)
+ args = ([{"role": "user", "content": [{"type": "image"}]}], [object()], 0, 1, 0, 0, 1, 1, None)
gen = backend._generate_vlm(*args, _adapter_state = False)
# First snapshot is the prefill alone, emitted after entering the adapter context.
@@ -984,7 +984,7 @@ def test_mlx_vlm_generation_selects_renderer_by_capability(monkeypatch):
backend = MLXInferenceBackend()
backend._model = SimpleNamespace(config = {"model_type": "deepseek_vl_v2"})
backend._processor = SimpleNamespace(tokenizer = SimpleNamespace())
- args = ([{"role": "user", "content": [{"type": "image"}]}], object(), 0, 1, 0, 0, 1, 1, None)
+ args = ([{"role": "user", "content": [{"type": "image"}]}], [object()], 0, 1, 0, 0, 1, 1, None)
tools = [{"function": {"name": "search"}}]
generator = backend._generate_vlm(*args, _adapter_state = False)
assert next(generator) == "ok"
@@ -3600,7 +3600,7 @@ def test_vlm_seed_rides_on_the_sampler_not_a_seed_kwarg(monkeypatch):
backend._processor = SimpleNamespace(chat_template = "template")
args = (
[{"role": "user", "content": [{"type": "image"}]}],
- object(),
+ [object()],
0.7,
0.9,
40,
@@ -4454,7 +4454,7 @@ def test_the_window_reaches_the_runtime_on_every_generation_route(monkeypatch):
next(
vlm._generate_vlm(
[{"role": "user", "content": [{"type": "image"}]}],
- object(),
+ [object()],
0,
1,
0,
@@ -4806,7 +4806,7 @@ def _run_spm_vlm_turn(
return list(
backend._generate_vlm(
[{"role": "user", "content": [{"type": "image"}]}],
- object(),
+ [object()],
0,
1,
0,
@@ -5493,7 +5493,7 @@ def _run_vlm_budget(
backend._is_vlm = True
backend._model = SimpleNamespace()
backend._processor = SimpleNamespace(tokenizer = backend._tokenizer)
- args = (messages, image, 0, 1, 0, 0, max_new_tokens, 1, None)
+ args = (messages, [image] if image is not None else None, 0, 1, 0, 0, max_new_tokens, 1, None)
list(backend._generate_vlm(*args, _adapter_state = False))
return seen["max_tokens"]
@@ -5839,7 +5839,7 @@ def test_mlx_vlm_a_video_turn_whose_render_is_unusable_is_refused_not_recovered(
with pytest.raises(RuntimeError, match = "for a video turn"):
list(
backend._generate_vlm(
- turn, object(), 0, 1, 0, 0, 1, 1, None, _adapter_state = False, video = _CLIP_B64
+ turn, [object()], 0, 1, 0, 0, 1, 1, None, _adapter_state = False, video = _CLIP_B64
)
)
@@ -6102,3 +6102,82 @@ def _mlx_reads_video_probe(monkeypatch):
lambda _t, _m, **_k: "<video> marked",
)
return _mlx_reads_video(SimpleNamespace(video_processor = object(), tokenizer = SimpleNamespace()))
+
+
+# ── Multi-image capability ────────────────────────────────────────────────
+
+_RENDERER = "core.inference.chat_template_helpers.apply_chat_template_for_generation"
+
+
+def _vlm_runtime(monkeypatch, marker = "<|image|>"):
+ """A backend already classified with ``marker`` as its image token, plus a stubbed mlx-vlm."""
+ from core.inference import mlx_inference
+
+ calls = []
+
+ def _render(_target, messages, **_kwargs):
+ def _flat(part):
+ return "<|image|>" if part.get("type") == "image" else part.get("text", "")
+
+ return "".join(
+ body if isinstance(body, str) else "".join(map(_flat, body or ()))
+ for body in (message.get("content") for message in messages)
+ )
+
+ def _vlm_stream(model, processor, prompt, images, **kwargs):
+ calls.append({"prompt": prompt, "images": images})
+ yield SimpleNamespace(text = "ok", prompt_tokens = 1, generation_tokens = 1)
+
+ mlx_vlm = types.ModuleType("mlx_vlm")
+ mlx_vlm.stream_generate = _vlm_stream
+ monkeypatch.setitem(sys.modules, "mlx_vlm", mlx_vlm)
+ monkeypatch.setattr(_RENDERER, _render)
+ monkeypatch.setattr(
+ mlx_inference, "_temporary_mlx_adapter_state", lambda *_a, **_k: contextlib.nullcontext()
+ )
+
+ backend = mlx_inference.MLXInferenceBackend()
+ backend._model = SimpleNamespace(config = {"model_type": "gemma4"})
+ backend._processor = SimpleNamespace(
+ chat_template = "template", tokenizer = SimpleNamespace(chat_template = "nested")
+ )
+ backend._is_vlm = True
+ backend._multi_image_marker = marker
+ return backend, calls
+
+
+def test_several_images_reach_the_runtime_in_the_order_they_were_attached(monkeypatch):
+ """Binding is positional, so a reordered list answers about the wrong picture, not an error."""
+ backend, calls = _vlm_runtime(monkeypatch)
+ first, second = object(), object()
+ history = [{"role": "user", "content": "ask"}, {"role": "user", "content": "compare"}]
+
+ assert list(
+ backend.generate_chat_response(history, images = [first, second], max_new_tokens = 4)
+ ) == ["ok"]
+ assert calls[-1]["images"] == [first, second]
+ assert calls[-1]["prompt"].count("<|image|>") == 2
+
+ only = [{"role": "user", "content": "describe"}]
+ list(backend.generate_chat_response(only, image = first, max_new_tokens = 4))
+ list(backend.generate_chat_response(only, images = [first], max_new_tokens = 4))
+ assert calls[-2] == calls[-1] == {"prompt": "<|image|>describe", "images": [first]}
+
+
+ "marker, render, error",
+ [
+ (None, None, "one image per request"),
+ ("<|image|>", "<|image|>only one", "marked 1 image"),
+ ],
+ ids = ["no marker was classified", "the render collapsed a marker"],
+)
+def test_a_render_that_cannot_bind_every_image_is_refused(monkeypatch, marker, render, error):
+ """One image is still taken either way; too few markers would describe one picture twice."""
+ backend, _calls = _vlm_runtime(monkeypatch, marker = marker)
+ if render is not None:
+ monkeypatch.setattr(_RENDERER, lambda *_a, **_k: render)
+
+ turn = [{"role": "user", "content": "compare"}]
+ with pytest.raises((ValueError, RuntimeError), match = error):
+ list(backend.generate_chat_response(turn, images = [object(), object()]))
diff --git a/studio/backend/tests/test_openai_multi_image_route.py b/studio/backend/tests/test_openai_multi_image_route.py
new file mode 100644
index 00000000000..abd13b2a929
--- /dev/null
+++ b/studio/backend/tests/test_openai_multi_image_route.py
@@ -0,0 +1,269 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Serving every image a user turn carried to a backend whose template can mark them.
+
+Flattening each turn to a string kept one base64 for the whole thread, so a second image on a turn
+was refused and an earlier turn's silently replaced by the newest.
+"""
+
+import base64
+import io
+import os
+import sys
+
+import pytest
+from fastapi import HTTPException
+from PIL import Image
+
+from models.inference import ChatMessage
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+import test_sf_client_tools_passthrough as passthrough # noqa: E402
+
+
+def _part(image, container = "PNG") -> dict:
+ buffer = io.BytesIO()
+ image.save(buffer, format = container)
+ encoded = base64.b64encode(buffer.getvalue()).decode()
+ return {"type": "image_url", "image_url": {"url": f"data:image/x;base64,{encoded}"}}
+
+
+def _sized(size: int) -> dict:
+ return _part(Image.new("RGB", (size, size), "white"))
+
+
+def _text(text: str) -> dict:
+ return {"type": "text", "text": text}
+
+
+_NOTHING_TO_SEND = {"type": "image_url", "image_url": {"url": "data:image/png;base64,"}}
+_ASK = _text("which is which?")
+# Small enough to skip resampling, so nothing but the route itself forces the decode.
+_WHOLE = _part(Image.new("RGB", (64, 64), "blue"), container = "JPEG")["image_url"]["url"]
+_TRUNCATED = {"type": "image_url", "image_url": {"url": _WHOLE[: len(_WHOLE) - 12]}}
+_ACROSS_TURNS = [
+ ChatMessage(role = "user", content = [_sized(2), _text("one")]),
+ ChatMessage(role = "assistant", content = [_sized(8), _text("mine")]),
+ ChatMessage(role = "user", content = [_text("two"), _sized(4)]),
+]
+
+
+def _markers(message) -> list[int]:
+ content = message.get("content")
+ if not isinstance(content, list):
+ return []
+ return [i for i, part in enumerate(content) if part.get("type") == "image"]
+
+
+def _call(
+ monkeypatch,
+ messages,
+ accepts_multiple_images = True,
+ **request_kwargs,
+):
+ backend = passthrough._ScriptedBackend(passthrough._fixed("an answer"))
+ backend.models["sf-model"].update(
+ is_vision = True,
+ chat_template_info = {
+ "template": "<tool_call> chatml",
+ "renders_image": True,
+ "accepts_multiple_images": accepts_multiple_images,
+ },
+ )
+ payload = passthrough._request(messages = messages, stream = False, **request_kwargs)
+ passthrough._call(payload, monkeypatch, backend)
+ return backend
+
+
+ "messages, sizes, markers",
+ [
+ ([ChatMessage(role = "user", content = [_sized(2), _sized(4), _ASK])], [2, 4], [[0, 1]]),
+ ([ChatMessage(role = "user", content = [_sized(4), _sized(2), _ASK])], [4, 2], [[0, 1]]),
+ (_ACROSS_TURNS, [2, 4], [[0], [], [1]]),
+ ],
+ ids = ["one turn", "one turn, reversed", "earlier turns, past an assistant's own image"],
+)
+def test_every_image_a_user_turn_carried_is_served_in_document_order_at_that_turn(
+ monkeypatch, messages, sizes, markers
+):
+ """An image on an earlier turn used to be replaced by the newest one, with no error at all."""
+ call = _call(monkeypatch, messages).calls[0]
+
+ assert [image.width for image in call["images"]] == sizes
+ assert call["image"] is None
+ # Per turn, because binding is positional: a marker on the wrong turn is the wrong picture.
+ assert [
+ _markers(message) for message in call["messages"] if message.get("role") != "system"
+ ] == markers
+
+
+ "content, accepts_multiple_images, detail",
+ [
+ ([_sized(2), _sized(4), _ASK], False, "one image per message"),
+ ([_sized(2), _NOTHING_TO_SEND, _ASK], True, "one image per message"),
+ ([_TRUNCATED], True, ""),
+ ],
+ ids = ["a backend that published one image", "an image part with nothing to send", "truncated"],
+)
+def test_what_cannot_be_served_is_answered_as_a_bad_request(
+ monkeypatch, content, accepts_multiple_images, detail
+):
+ """A part yielding no payload is still an image to the renderer, so the turn stays a 400."""
+ turn = [ChatMessage(role = "user", content = content)]
+ with pytest.raises(HTTPException) as refusal:
+ _call(monkeypatch, turn, accepts_multiple_images = accepts_multiple_images)
+
+ assert refusal.value.status_code == 400
+ assert detail in refusal.value.detail
+
+
+ "mode, container", [("CMYK", "JPEG"), ("PA", "TIFF"), ("LAB", "TIFF"), ("RGBA", "TIFF")]
+)
+def test_an_image_reaches_the_backend_in_a_mode_it_can_be_handed_over_in(
+ monkeypatch, mode, container
+):
+ """The first three reopen in a mode PNG cannot write and must convert; RGBA must not."""
+ part = _part(Image.new(mode, (16, 16)), container = container)
+
+ delivered = _call(monkeypatch, [ChatMessage(role = "user", content = [part])]).calls[0]["image"]
+
+ assert delivered.mode == ("RGBA" if mode == "RGBA" else "RGB")
+ # The boundary itself, not a claim about which modes it takes.
+ delivered.save(io.BytesIO(), format = "PNG")
+
+
+def test_each_image_is_decoded_once_for_a_request_that_rebuilds_the_conversation(monkeypatch):
+ """A client-tool request renders a rebuilt conversation, which used to decode the whole set a
+ second time -- twice the work and both full sets resident, on exactly the largest requests."""
+ from routes import inference as route
+
+ decoded = []
+ real = route._decode_and_resize_image
+
+ def _watch(backend, encoded):
+ decoded.append(encoded)
+ return real(backend, encoded)
+
+ monkeypatch.setattr(route, "_decode_and_resize_image", _watch)
+ turn = [ChatMessage(role = "user", content = [_sized(2), _sized(4), _sized(6), _ASK])]
+
+ _call(monkeypatch, turn, tools = [passthrough.LOOKUP_TOOL])
+
+ assert len(decoded) == len(set(decoded)) == 3
+
+
+def test_a_16_bit_image_keeps_its_levels_instead_of_clipping_to_white(monkeypatch):
+ """Converting reads 0..65535 as 8-bit, so everything above 255 lands on white."""
+ source = Image.new("I;16", (2, 2))
+ source.putdata([0, 20000, 40000, 65535])
+ turn = [ChatMessage(role = "user", content = [_part(source)])]
+
+ delivered = _call(monkeypatch, turn).calls[0]["image"]
+
+ assert [pixel[0] for pixel in delivered.convert("RGB").get_flattened_data()] == [
+ 0,
+ 77,
+ 155,
+ 255,
+ ]
+
+
+_PDF = {
+ "type": "input_document",
+ "file_data": "data:application/pdf;base64,JVBERi0xLjQK",
+ "filename": "spec.pdf",
+}
+
+
+def test_a_document_part_in_history_never_reaches_the_local_template(monkeypatch):
+ """Local templates take text and image only: mistral3's raises "Only text and image blocks
+ are supported in message content!", and mlx_inference re-raises that instead of recovering
+ whenever the request carries tools or a reasoning knob, so the turn 500s."""
+ messages = [
+ ChatMessage(role = "user", content = [_text("here is the spec"), _PDF, _sized(2)]),
+ ChatMessage(role = "assistant", content = "Got it."),
+ ChatMessage(role = "user", content = [_sized(4), _ASK]),
+ ]
+ call = _call(monkeypatch, messages).calls[0]
+
+ kept = [
+ part.get("type")
+ for message in call["messages"]
+ if isinstance(message.get("content"), list)
+ for part in message["content"]
+ ]
+ assert "input_document" not in kept
+ # Dropping the document must not drop the text beside it, nor either picture.
+ assert [image.width for image in call["images"]] == [2, 4]
+ assert "here is the spec" in str(call["messages"])
+
+
+def test_a_request_beyond_the_image_budget_is_refused_before_decoding(monkeypatch):
+ """Each retained raster is up to 1.92 MB while a solid 800x800 PNG is ~4.8 KB of base64, so
+ an unbounded list reaches gigabytes from a body small enough to pass any transport limit."""
+ from routes import inference as inference_route
+
+ monkeypatch.setattr(inference_route, "_MAX_SERVED_IMAGES", 3)
+ decoded = []
+ real = inference_route._decode_and_resize_image
+
+ def counted(backend, encoded):
+ decoded.append(encoded)
+ return real(backend, encoded)
+
+ monkeypatch.setattr(inference_route, "_decode_and_resize_image", counted)
+ content = [_part(Image.new("RGB", (8 + i, 8), "white")) for i in range(4)] + [_ASK]
+ with pytest.raises(HTTPException) as exc:
+ _call(monkeypatch, [ChatMessage(role = "user", content = content)])
+ assert exc.value.status_code == 400
+ assert "carries 4 images" in str(exc.value.detail)
+ assert "at most 3 are served per request" in str(exc.value.detail)
+ # Refused before allocating any of them, which is the point.
+ assert decoded == []
+
+
+def test_a_request_at_the_image_budget_is_served(monkeypatch):
+ from routes import inference as inference_route
+
+ monkeypatch.setattr(inference_route, "_MAX_SERVED_IMAGES", 3)
+ content = [_part(Image.new("RGB", (8 + i, 8), "white")) for i in range(3)] + [_ASK]
+ call = _call(monkeypatch, [ChatMessage(role = "user", content = content)]).calls[0]
+ assert [image.width for image in call["images"]] == [8, 9, 10]
+
+
+def test_the_image_budget_counts_repeats_not_distinct_payloads(monkeypatch):
+ """The worker decodes every entry of images_base64, so a payload repeated N times costs N
+ rasters there even though this route reuses one PIL object for all of them."""
+ from routes import inference as inference_route
+
+ monkeypatch.setattr(inference_route, "_MAX_SERVED_IMAGES", 3)
+ same = _part(Image.new("RGB", (8, 8), "white"))
+ content = [same, same, same, same, _ASK]
+ with pytest.raises(HTTPException) as exc:
+ _call(monkeypatch, [ChatMessage(role = "user", content = content)])
+ assert exc.value.status_code == 400
+ assert "carries 4 images" in str(exc.value.detail)
+
+
+def test_the_image_budget_refusal_goes_through_the_callers_reject(monkeypatch):
+ """_reject fails the open API monitor row before raising; a raw HTTPException would leave
+ it reported as running."""
+ import asyncio
+
+ from routes import inference as inference_route
+
+ monkeypatch.setattr(inference_route, "_MAX_SERVED_IMAGES", 1)
+ seen = []
+
+ def reject(status_code, detail):
+ seen.append((status_code, detail))
+ return HTTPException(status_code = status_code, detail = detail)
+
+ with pytest.raises(HTTPException):
+ asyncio.run(inference_route._decode_request_images(None, ["a", "b"], None, reject = reject))
+ assert len(seen) == 1 and seen[0][0] == 400