Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: convert pasted images when tools are turned on (#10094)

Nilay502c4d23776d
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 13563e463ac..7eb3c6529b8 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -24188,7 +24188,10 @@ async def produce_openai_chat_completions(
# message (templates reject "developer") and clear prompt to avoid a dup.
gen_kwargs["messages"] = _set_or_prepend_system_message(
_structured_tool_history_for_local_template(
- _flatten_content_parts_for_local_template(_openai_messages_for_passthrough(payload))
+ _flatten_content_parts_for_local_template(
+ # Not a llama-server body: the flatten below drops image parts.
+ _openai_messages_for_passthrough(payload, normalize_images = False)
+ )
),
system_prompt,
)
@@ -28963,25 +28966,38 @@ def _image_bytes_to_png_b64(raw: bytes) -> str:
input; callers wrap the call in ``try`` -> HTTPException(400)."""
from PIL import Image
- img = Image.open(io.BytesIO(raw)).convert("RGB")
+ img = Image.open(io.BytesIO(raw))
+ # A 16-bit source carries 0..65535, but convert("RGB") reads those as 8-bit
+ # and clips everything above 255, which turns the picture nearly all white.
+ # Scale to 8 bits first, the way stb_image does when llama-server reads the
+ # same file itself. I;16B / I;16L reject point(), so normalise them to "I".
+ #
+ # Only the I;16 family: plain "I" and "F" declare no range, and a 32-bit or
+ # float TIFF whose samples already sit in 0..255 (or 0..1) would be scaled
+ # to black. Those keep the straight convert("RGB"). A 16-bit PNG -- the
+ # reachable case here, since this decodes pasted images -- opens as I;16 on
+ # every Pillow this repo pins.
+ if img.mode.startswith("I;16"):
+ if img.mode != "I;16":
+ img = img.convert("I")
+ img = img.point(lambda v: v * (1.0 / 257), mode = "L")
+ img = img.convert("RGB")
buf = io.BytesIO()
img.save(buf, format = "PNG")
return base64.b64encode(buf.getvalue()).decode("ascii")
-def _normalize_anthropic_openai_images(openai_messages: list[dict], is_vision: bool) -> bool:
- """Enforce the vision guard on translated Anthropic messages and normalize
- any base64-data-URL ``image_url`` parts to PNG.
+def _normalize_openai_image_parts_to_png(openai_messages: list[dict], on_image = None) -> bool:
+ """Re-encode every base64-data-URL ``image_url`` part to PNG, in place.
- llama-server's stb_image only handles a few formats (JPEG/PNG/BMP/…);
- Anthropic clients commonly send JPEG or WebP, and Claude Code sends WebP.
- Re-encoding everything to PNG mirrors `_openai_messages_for_passthrough` /
- the GGUF branch of `/v1/chat/completions` so the two endpoints agree.
+ llama-server's stb_image only handles a few formats (JPEG/PNG/BMP/…), while
+ macOS and most browsers paste WebP and Claude Code sends WebP. Remote
+ (non-``data:``) URLs are forwarded as-is; llama-server will fetch (or fail)
+ per its own support matrix.
- Mutates ``openai_messages`` in place. Returns ``True`` when any image part
- was seen (so the caller can skip a second scan). Raises HTTPException(400)
- when images are present but the active model isn't a vision model, or when
- an image cannot be decoded.
+ ``on_image`` runs once per image part before conversion, so a caller can
+ apply its own guard. Returns ``True`` when any image part was seen. Raises
+ HTTPException(400) when an image cannot be decoded.
"""
has_image = False
for msg in openai_messages:
@@ -28989,20 +29005,16 @@ def _normalize_anthropic_openai_images(openai_messages: list[dict], is_vision: b
if not isinstance(content, list):
continue
for part in content:
- if part.get("type") != "image_url":
+ if not isinstance(part, dict) or part.get("type") != "image_url":
continue
has_image = True
- if not is_vision:
- raise HTTPException(
- status_code = 400,
- detail = "Image provided but current GGUF model does not support vision.",
- )
+ if on_image is not None:
+ on_image()
- url = (part.get("image_url") or {}).get("url", "")
+ image_url = part.get("image_url") or {}
+ url = image_url.get("url", "")
if not url.startswith("data:"):
- # Remote URLs are forwarded as-is; llama-server will
- # fetch (or fail) per its own support matrix.
continue
try:
@@ -29014,11 +29026,24 @@ def _normalize_anthropic_openai_images(openai_messages: list[dict], is_vision: b
status_code = 400,
detail = "Failed to process image.",
)
- part["image_url"] = {"url": f"data:image/png;base64,{png_b64}"}
+ # Only the url is re-encoded; `detail` is the client's request, not
+ # part of the encoding, and replacing the object would drop it.
+ image_url["url"] = f"data:image/png;base64,{png_b64}"
return has_image
+def _normalize_anthropic_openai_images(openai_messages: list[dict], is_vision: bool) -> bool:
+ def _guard():
+ if not is_vision:
+ raise HTTPException(
+ status_code = 400,
+ detail = "Image provided but current GGUF model does not support vision.",
+ )
+
+ return _normalize_openai_image_parts_to_png(openai_messages, on_image = _guard)
+
+
def _validate_anthropic_client_tools(tools) -> None:
# Reject malformed client tools before any model load, so an invalid request
# never evicts the loaded model. AnthropicTool relaxed name/input_schema to
@@ -32341,15 +32366,24 @@ def _splice_image_into_last_user(messages: list[dict], image_part: dict) -> None
messages.append({"role": "user", "content": [image_part]})
-def _openai_messages_for_passthrough(payload) -> list[dict]:
+def _openai_messages_for_passthrough(payload, normalize_images: bool = True) -> list[dict]:
"""Build OpenAI-format message dicts for the /v1/chat/completions
passthrough path.
``payload.messages`` are dumped through Pydantic (dropping unset optional
fields), so they're already standard OpenAI format -- including
``role="tool"`` tool-result messages and assistant messages carrying
- structured ``tool_calls``. Content-parts images already in the list are
- left untouched.
+ structured ``tool_calls``. Base64-data-URL images already in the list are
+ re-encoded to PNG exactly as ``_openai_messages_for_gguf_chat`` does, so
+ turning tools on does not change which formats llama-server can decode;
+ remote URLs are forwarded as-is. The vision guard lives in the callers,
+ which reject a non-vision model before the body is built.
+
+ ``normalize_images=False`` is for callers that are not building a
+ llama-server body: the local-template path flattens image parts away, and
+ only reaches this helper when the turn has no decodable image at all, so
+ re-encoding there can only turn an image it was already ignoring -- a
+ payloadless ``data:`` URL -- into a 400.
When a client uses Unsloth's legacy ``image_base64`` top-level field, the
image is re-encoded to PNG (llama-server's stb_image has limited format
@@ -32366,6 +32400,9 @@ def _openai_messages_for_passthrough(payload) -> list[dict]:
_drop_empty_assistant_sentinels([m.model_dump(exclude_none = True) for m in payload.messages])
)
+ if normalize_images:
+ _normalize_openai_image_parts_to_png(messages)
+
if not _legacy_image_is_distinct(payload):
return messages
diff --git a/studio/backend/tests/test_openai_tool_passthrough.py b/studio/backend/tests/test_openai_tool_passthrough.py
index 5b436b7ef62..9fb5c6effa1 100644
--- a/studio/backend/tests/test_openai_tool_passthrough.py
+++ b/studio/backend/tests/test_openai_tool_passthrough.py
@@ -63,7 +63,9 @@ from routes.inference import (
_normalize_openai_passthrough_sse_line,
_openai_compat_stream_stall_timeout,
_openai_llama_admission_capacity,
+ _image_bytes_to_png_b64,
_openai_messages_for_gguf_chat,
+ _openai_messages_for_passthrough,
_openai_passthrough_sse_line_terminal_state,
_openai_passthrough_upstream_headers,
_openai_passthrough_non_streaming,
@@ -3498,7 +3500,6 @@ class TestFriendlyErrorHttpx:
from routes.inference import ( # noqa: E402
_drop_empty_assistant_sentinels,
_openai_messages_for_gguf_chat,
- _openai_messages_for_passthrough,
)
@@ -10557,3 +10558,216 @@ def test_the_two_seed_helpers_agree_on_which_seeds_are_random():
payload = {}
_apply_seeded_llama_request(payload, value)
assert payload["cache_prompt"] is False, (seed, value)
+
+
+class TestPassthroughImageNormalization:
+ @staticmethod
+ def _data_url(fmt: str) -> str:
+ from io import BytesIO
+
+ from PIL import Image
+
+ buf = BytesIO()
+ Image.new("RGB", (2, 2), (0, 128, 255)).save(buf, format = fmt)
+ b64 = base64.b64encode(buf.getvalue()).decode("ascii")
+ return f"data:image/{fmt.lower()};base64,{b64}"
+
+ def _req(self, url: str, **kwargs):
+ return ChatCompletionRequest(
+ model = "default",
+ messages = [
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "what is this"},
+ {"type": "image_url", "image_url": {"url": url}},
+ ],
+ },
+ ],
+ **kwargs,
+ )
+
+ def test_webp_data_url_is_reencoded_to_png(self):
+ original = self._data_url("WEBP")
+ assert original.startswith("data:image/webp;base64,")
+
+ messages = _openai_messages_for_passthrough(self._req(original))
+
+ url = messages[0]["content"][1]["image_url"]["url"]
+ assert url.startswith("data:image/png;base64,")
+ assert url != original
+
+ def test_body_builder_forwards_png_with_tools_enabled(self):
+ req = self._req(
+ self._data_url("WEBP"),
+ tools = [
+ {
+ "type": "function",
+ "function": {"name": "noop", "parameters": {"type": "object"}},
+ }
+ ],
+ )
+
+ body = _build_openai_passthrough_body(req)
+
+ url = body["messages"][0]["content"][1]["image_url"]["url"]
+ assert url.startswith("data:image/png;base64,")
+
+ def test_requested_detail_survives_the_conversion(self):
+ req = ChatCompletionRequest(
+ model = "default",
+ messages = [
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "image_url",
+ "image_url": {"url": self._data_url("WEBP"), "detail": "high"},
+ },
+ ],
+ },
+ ],
+ )
+
+ part = _openai_messages_for_passthrough(req)[0]["content"][0]["image_url"]
+ assert part["url"].startswith("data:image/png;base64,")
+ assert part["detail"] == "high"
+
+ @staticmethod
+ def _gray16_ramp_png(width: int = 256) -> bytes:
+ from io import BytesIO
+
+ from PIL import Image
+
+ img = Image.new("I;16", (width, 2))
+ img.putdata([(x % width) * 257 for _ in range(2) for x in range(width)])
+ buf = BytesIO()
+ img.save(buf, format = "PNG")
+ return buf.getvalue()
+
+ def test_sixteen_bit_grayscale_keeps_its_levels(self):
+ # convert("RGB") reads a 16-bit source as 8-bit and clips: the whole ramp
+ # above 255 collapses to white. llama-server reading the same PNG itself
+ # scales instead, so re-encoding must not throw the picture away.
+ from io import BytesIO
+
+ from PIL import Image
+
+ raw = self._gray16_ramp_png()
+ out = base64.b64decode(_image_bytes_to_png_b64(raw))
+
+ row = [Image.open(BytesIO(out)).getpixel((x, 0))[0] for x in range(256)]
+ assert len(set(row)) == 256, f"levels collapsed to {len(set(row))}"
+ assert row.count(255) == 1, f"{row.count(255)} white pixels, expected 1"
+ assert row[0] == 0 and row[-1] == 255
+
+ def test_endian_tagged_sixteen_bit_mode_does_not_raise(self):
+ # I;16B rejects point() outright, and the caller turns any exception into
+ # a 400, so scaling without normalising the mode is worse than clipping.
+ # PNG cannot carry the tagged modes; a 16-bit TIFF reopens as I;16B.
+ from io import BytesIO
+
+ from PIL import Image
+
+ img = Image.new("I;16B", (256, 2))
+ img.putdata([(x % 256) * 257 for _ in range(2) for x in range(256)])
+ buf = BytesIO()
+ img.save(buf, format = "TIFF")
+ assert Image.open(BytesIO(buf.getvalue())).mode == "I;16B"
+
+ out = base64.b64decode(_image_bytes_to_png_b64(buf.getvalue()))
+ row = [Image.open(BytesIO(out)).getpixel((x, 0))[0] for x in range(256)]
+ assert len(set(row)) == 256, f"collapsed to {len(set(row))} levels"
+
+ def test_plain_int_mode_is_not_scaled(self):
+ # "I" and "F" declare no range. A 32-bit TIFF whose samples already sit
+ # in 0..255 must keep them: scaling it by 1/257 would black the picture
+ # out, which is worse than the clipping the scaling was added to fix.
+ from io import BytesIO
+
+ from PIL import Image
+
+ img = Image.new("I", (256, 2))
+ img.putdata([x % 256 for _ in range(2) for x in range(256)])
+ buf = BytesIO()
+ img.save(buf, format = "TIFF")
+ assert Image.open(BytesIO(buf.getvalue())).mode == "I"
+
+ out = base64.b64decode(_image_bytes_to_png_b64(buf.getvalue()))
+ row = [Image.open(BytesIO(out)).getpixel((x, 0))[0] for x in range(256)]
+ assert row == list(range(256)), f"0..255 ramp altered: max={max(row)}"
+
+ def test_eight_bit_images_are_unchanged_by_the_scaling_branch(self):
+ from io import BytesIO
+
+ from PIL import Image
+ for mode, colour in (("RGB", (10, 20, 30)), ("RGBA", (10, 20, 30, 255)), ("L", 77)):
+ buf = BytesIO()
+ Image.new(mode, (4, 4), colour).save(buf, format = "PNG")
+
+ out = base64.b64decode(_image_bytes_to_png_b64(buf.getvalue()))
+ px = Image.open(BytesIO(out)).getpixel((0, 0))
+ assert px == ((77, 77, 77) if mode == "L" else (10, 20, 30)), (mode, px)
+
+ def test_remote_url_is_forwarded_unchanged(self):
+ messages = _openai_messages_for_passthrough(self._req("https://x.example/a.webp"))
+ assert messages[0]["content"][1]["image_url"]["url"] == "https://x.example/a.webp"
+
+ def test_local_template_caller_leaves_a_payloadless_data_url_alone(self):
+ # The safetensors/MLX client-tools path only gets here when the turn has no
+ # decodable image, and flattens image parts away straight after. A payloadless
+ # data URL was ignored with a warning before; it must not become a 400.
+ req = self._req("data:image/png;base64,")
+
+ messages = _openai_messages_for_passthrough(req, normalize_images = False)
+
+ assert messages[0]["content"][1]["image_url"]["url"] == "data:image/png;base64,"
+
+ def test_the_local_template_call_site_opts_out_of_normalization(self):
+ # The guard that keeps the re-encode on llama-server bodies only. Reverting it
+ # brings the 400 above back to the safetensors/MLX client-tools path.
+ import inspect
+
+ import routes.inference as inference_mod
+
+ src = inspect.getsource(inference_mod)
+ _, _, after = src.partition("_flatten_content_parts_for_local_template(")
+ assert after, "the local-template call site moved"
+ assert "normalize_images = False" in after[:300]
+
+ def test_undecodable_data_url_raises_400(self):
+ with pytest.raises(HTTPException) as exc:
+ _openai_messages_for_passthrough(self._req("data:image/webp;base64,!!!nope!!!"))
+ assert exc.value.status_code == 400
+
+ def test_echoed_legacy_image_is_not_spliced_twice(self):
+ from io import BytesIO
+
+ from PIL import Image
+
+ buf = BytesIO()
+ Image.new("RGB", (2, 2), (1, 2, 3)).save(buf, format = "WEBP")
+ b64 = base64.b64encode(buf.getvalue()).decode("ascii")
+
+ req = ChatCompletionRequest(
+ model = "default",
+ image_base64 = b64,
+ messages = [
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "what is this"},
+ {
+ "type": "image_url",
+ "image_url": {"url": f"data:image/webp;base64,{b64}"},
+ },
+ ],
+ },
+ ],
+ )
+
+ messages = _openai_messages_for_passthrough(req)
+
+ parts = [p for p in messages[0]["content"] if p.get("type") == "image_url"]
+ assert len(parts) == 1
+ assert parts[0]["image_url"]["url"].startswith("data:image/png;base64,")