Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: convert pasted images when tools are turned on (#10094)
Nilay502c4d23776d
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 13563e463ac..7eb3c6529b8 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -24188,7 +24188,10 @@ async def produce_openai_chat_completions(# message (templates reject "developer") and clear prompt to avoid a dup.gen_kwargs["messages"] = _set_or_prepend_system_message(_structured_tool_history_for_local_template(- _flatten_content_parts_for_local_template(_openai_messages_for_passthrough(payload))+ _flatten_content_parts_for_local_template(+ # Not a llama-server body: the flatten below drops image parts.+ _openai_messages_for_passthrough(payload, normalize_images = False)+ )),system_prompt,)@@ -28963,25 +28966,38 @@ def _image_bytes_to_png_b64(raw: bytes) -> str:input; callers wrap the call in ``try`` -> HTTPException(400)."""from PIL import Image- img = Image.open(io.BytesIO(raw)).convert("RGB")+ img = Image.open(io.BytesIO(raw))+ # A 16-bit source carries 0..65535, but convert("RGB") reads those as 8-bit+ # and clips everything above 255, which turns the picture nearly all white.+ # Scale to 8 bits first, the way stb_image does when llama-server reads the+ # same file itself. I;16B / I;16L reject point(), so normalise them to "I".+ #+ # Only the I;16 family: plain "I" and "F" declare no range, and a 32-bit or+ # float TIFF whose samples already sit in 0..255 (or 0..1) would be scaled+ # to black. Those keep the straight convert("RGB"). A 16-bit PNG -- the+ # reachable case here, since this decodes pasted images -- opens as I;16 on+ # every Pillow this repo pins.+ if img.mode.startswith("I;16"):+ if img.mode != "I;16":+ img = img.convert("I")+ img = img.point(lambda v: v * (1.0 / 257), mode = "L")+ img = img.convert("RGB")buf = io.BytesIO()img.save(buf, format = "PNG")return base64.b64encode(buf.getvalue()).decode("ascii")-def _normalize_anthropic_openai_images(openai_messages: list[dict], is_vision: bool) -> bool:- """Enforce the vision guard on translated Anthropic messages and normalize- any base64-data-URL ``image_url`` parts to PNG.+def _normalize_openai_image_parts_to_png(openai_messages: list[dict], on_image = None) -> bool:+ """Re-encode every base64-data-URL ``image_url`` part to PNG, in place.- llama-server's stb_image only handles a few formats (JPEG/PNG/BMP/…);- Anthropic clients commonly send JPEG or WebP, and Claude Code sends WebP.- Re-encoding everything to PNG mirrors `_openai_messages_for_passthrough` /- the GGUF branch of `/v1/chat/completions` so the two endpoints agree.+ llama-server's stb_image only handles a few formats (JPEG/PNG/BMP/…), while+ macOS and most browsers paste WebP and Claude Code sends WebP. Remote+ (non-``data:``) URLs are forwarded as-is; llama-server will fetch (or fail)+ per its own support matrix.- Mutates ``openai_messages`` in place. Returns ``True`` when any image part- was seen (so the caller can skip a second scan). Raises HTTPException(400)- when images are present but the active model isn't a vision model, or when- an image cannot be decoded.+ ``on_image`` runs once per image part before conversion, so a caller can+ apply its own guard. Returns ``True`` when any image part was seen. Raises+ HTTPException(400) when an image cannot be decoded."""has_image = Falsefor msg in openai_messages:@@ -28989,20 +29005,16 @@ def _normalize_anthropic_openai_images(openai_messages: list[dict], is_vision: bif not isinstance(content, list):continuefor part in content:- if part.get("type") != "image_url":+ if not isinstance(part, dict) or part.get("type") != "image_url":continuehas_image = True- if not is_vision:- raise HTTPException(- status_code = 400,- detail = "Image provided but current GGUF model does not support vision.",- )+ if on_image is not None:+ on_image()- url = (part.get("image_url") or {}).get("url", "")+ image_url = part.get("image_url") or {}+ url = image_url.get("url", "")if not url.startswith("data:"):- # Remote URLs are forwarded as-is; llama-server will- # fetch (or fail) per its own support matrix.continuetry:@@ -29014,11 +29026,24 @@ def _normalize_anthropic_openai_images(openai_messages: list[dict], is_vision: bstatus_code = 400,detail = "Failed to process image.",)- part["image_url"] = {"url": f"data:image/png;base64,{png_b64}"}+ # Only the url is re-encoded; `detail` is the client's request, not+ # part of the encoding, and replacing the object would drop it.+ image_url["url"] = f"data:image/png;base64,{png_b64}"return has_image+def _normalize_anthropic_openai_images(openai_messages: list[dict], is_vision: bool) -> bool:+ def _guard():+ if not is_vision:+ raise HTTPException(+ status_code = 400,+ detail = "Image provided but current GGUF model does not support vision.",+ )++ return _normalize_openai_image_parts_to_png(openai_messages, on_image = _guard)++def _validate_anthropic_client_tools(tools) -> None:# Reject malformed client tools before any model load, so an invalid request# never evicts the loaded model. AnthropicTool relaxed name/input_schema to@@ -32341,15 +32366,24 @@ def _splice_image_into_last_user(messages: list[dict], image_part: dict) -> Nonemessages.append({"role": "user", "content": [image_part]})-def _openai_messages_for_passthrough(payload) -> list[dict]:+def _openai_messages_for_passthrough(payload, normalize_images: bool = True) -> list[dict]:"""Build OpenAI-format message dicts for the /v1/chat/completionspassthrough path.``payload.messages`` are dumped through Pydantic (dropping unset optionalfields), so they're already standard OpenAI format -- including``role="tool"`` tool-result messages and assistant messages carrying- structured ``tool_calls``. Content-parts images already in the list are- left untouched.+ structured ``tool_calls``. Base64-data-URL images already in the list are+ re-encoded to PNG exactly as ``_openai_messages_for_gguf_chat`` does, so+ turning tools on does not change which formats llama-server can decode;+ remote URLs are forwarded as-is. The vision guard lives in the callers,+ which reject a non-vision model before the body is built.++ ``normalize_images=False`` is for callers that are not building a+ llama-server body: the local-template path flattens image parts away, and+ only reaches this helper when the turn has no decodable image at all, so+ re-encoding there can only turn an image it was already ignoring -- a+ payloadless ``data:`` URL -- into a 400.When a client uses Unsloth's legacy ``image_base64`` top-level field, theimage is re-encoded to PNG (llama-server's stb_image has limited format@@ -32366,6 +32400,9 @@ def _openai_messages_for_passthrough(payload) -> list[dict]:_drop_empty_assistant_sentinels([m.model_dump(exclude_none = True) for m in payload.messages]))+ if normalize_images:+ _normalize_openai_image_parts_to_png(messages)+if not _legacy_image_is_distinct(payload):return messagesdiff --git a/studio/backend/tests/test_openai_tool_passthrough.py b/studio/backend/tests/test_openai_tool_passthrough.pyindex 5b436b7ef62..9fb5c6effa1 100644--- a/studio/backend/tests/test_openai_tool_passthrough.py+++ b/studio/backend/tests/test_openai_tool_passthrough.py@@ -63,7 +63,9 @@ from routes.inference import (_normalize_openai_passthrough_sse_line,_openai_compat_stream_stall_timeout,_openai_llama_admission_capacity,+ _image_bytes_to_png_b64,_openai_messages_for_gguf_chat,+ _openai_messages_for_passthrough,_openai_passthrough_sse_line_terminal_state,_openai_passthrough_upstream_headers,_openai_passthrough_non_streaming,@@ -3498,7 +3500,6 @@ class TestFriendlyErrorHttpx:from routes.inference import ( # noqa: E402_drop_empty_assistant_sentinels,_openai_messages_for_gguf_chat,- _openai_messages_for_passthrough,)@@ -10557,3 +10558,216 @@ def test_the_two_seed_helpers_agree_on_which_seeds_are_random():payload = {}_apply_seeded_llama_request(payload, value)assert payload["cache_prompt"] is False, (seed, value)+++class TestPassthroughImageNormalization:+ @staticmethod+ def _data_url(fmt: str) -> str:+ from io import BytesIO++ from PIL import Image++ buf = BytesIO()+ Image.new("RGB", (2, 2), (0, 128, 255)).save(buf, format = fmt)+ b64 = base64.b64encode(buf.getvalue()).decode("ascii")+ return f"data:image/{fmt.lower()};base64,{b64}"++ def _req(self, url: str, **kwargs):+ return ChatCompletionRequest(+ model = "default",+ messages = [+ {+ "role": "user",+ "content": [+ {"type": "text", "text": "what is this"},+ {"type": "image_url", "image_url": {"url": url}},+ ],+ },+ ],+ **kwargs,+ )++ def test_webp_data_url_is_reencoded_to_png(self):+ original = self._data_url("WEBP")+ assert original.startswith("data:image/webp;base64,")++ messages = _openai_messages_for_passthrough(self._req(original))++ url = messages[0]["content"][1]["image_url"]["url"]+ assert url.startswith("data:image/png;base64,")+ assert url != original++ def test_body_builder_forwards_png_with_tools_enabled(self):+ req = self._req(+ self._data_url("WEBP"),+ tools = [+ {+ "type": "function",+ "function": {"name": "noop", "parameters": {"type": "object"}},+ }+ ],+ )++ body = _build_openai_passthrough_body(req)++ url = body["messages"][0]["content"][1]["image_url"]["url"]+ assert url.startswith("data:image/png;base64,")++ def test_requested_detail_survives_the_conversion(self):+ req = ChatCompletionRequest(+ model = "default",+ messages = [+ {+ "role": "user",+ "content": [+ {+ "type": "image_url",+ "image_url": {"url": self._data_url("WEBP"), "detail": "high"},+ },+ ],+ },+ ],+ )++ part = _openai_messages_for_passthrough(req)[0]["content"][0]["image_url"]+ assert part["url"].startswith("data:image/png;base64,")+ assert part["detail"] == "high"++ @staticmethod+ def _gray16_ramp_png(width: int = 256) -> bytes:+ from io import BytesIO++ from PIL import Image++ img = Image.new("I;16", (width, 2))+ img.putdata([(x % width) * 257 for _ in range(2) for x in range(width)])+ buf = BytesIO()+ img.save(buf, format = "PNG")+ return buf.getvalue()++ def test_sixteen_bit_grayscale_keeps_its_levels(self):+ # convert("RGB") reads a 16-bit source as 8-bit and clips: the whole ramp+ # above 255 collapses to white. llama-server reading the same PNG itself+ # scales instead, so re-encoding must not throw the picture away.+ from io import BytesIO++ from PIL import Image++ raw = self._gray16_ramp_png()+ out = base64.b64decode(_image_bytes_to_png_b64(raw))++ row = [Image.open(BytesIO(out)).getpixel((x, 0))[0] for x in range(256)]+ assert len(set(row)) == 256, f"levels collapsed to {len(set(row))}"+ assert row.count(255) == 1, f"{row.count(255)} white pixels, expected 1"+ assert row[0] == 0 and row[-1] == 255++ def test_endian_tagged_sixteen_bit_mode_does_not_raise(self):+ # I;16B rejects point() outright, and the caller turns any exception into+ # a 400, so scaling without normalising the mode is worse than clipping.+ # PNG cannot carry the tagged modes; a 16-bit TIFF reopens as I;16B.+ from io import BytesIO++ from PIL import Image++ img = Image.new("I;16B", (256, 2))+ img.putdata([(x % 256) * 257 for _ in range(2) for x in range(256)])+ buf = BytesIO()+ img.save(buf, format = "TIFF")+ assert Image.open(BytesIO(buf.getvalue())).mode == "I;16B"++ out = base64.b64decode(_image_bytes_to_png_b64(buf.getvalue()))+ row = [Image.open(BytesIO(out)).getpixel((x, 0))[0] for x in range(256)]+ assert len(set(row)) == 256, f"collapsed to {len(set(row))} levels"++ def test_plain_int_mode_is_not_scaled(self):+ # "I" and "F" declare no range. A 32-bit TIFF whose samples already sit+ # in 0..255 must keep them: scaling it by 1/257 would black the picture+ # out, which is worse than the clipping the scaling was added to fix.+ from io import BytesIO++ from PIL import Image++ img = Image.new("I", (256, 2))+ img.putdata([x % 256 for _ in range(2) for x in range(256)])+ buf = BytesIO()+ img.save(buf, format = "TIFF")+ assert Image.open(BytesIO(buf.getvalue())).mode == "I"++ out = base64.b64decode(_image_bytes_to_png_b64(buf.getvalue()))+ row = [Image.open(BytesIO(out)).getpixel((x, 0))[0] for x in range(256)]+ assert row == list(range(256)), f"0..255 ramp altered: max={max(row)}"++ def test_eight_bit_images_are_unchanged_by_the_scaling_branch(self):+ from io import BytesIO++ from PIL import Image+ for mode, colour in (("RGB", (10, 20, 30)), ("RGBA", (10, 20, 30, 255)), ("L", 77)):+ buf = BytesIO()+ Image.new(mode, (4, 4), colour).save(buf, format = "PNG")++ out = base64.b64decode(_image_bytes_to_png_b64(buf.getvalue()))+ px = Image.open(BytesIO(out)).getpixel((0, 0))+ assert px == ((77, 77, 77) if mode == "L" else (10, 20, 30)), (mode, px)++ def test_remote_url_is_forwarded_unchanged(self):+ messages = _openai_messages_for_passthrough(self._req("https://x.example/a.webp"))+ assert messages[0]["content"][1]["image_url"]["url"] == "https://x.example/a.webp"++ def test_local_template_caller_leaves_a_payloadless_data_url_alone(self):+ # The safetensors/MLX client-tools path only gets here when the turn has no+ # decodable image, and flattens image parts away straight after. A payloadless+ # data URL was ignored with a warning before; it must not become a 400.+ req = self._req("data:image/png;base64,")++ messages = _openai_messages_for_passthrough(req, normalize_images = False)++ assert messages[0]["content"][1]["image_url"]["url"] == "data:image/png;base64,"++ def test_the_local_template_call_site_opts_out_of_normalization(self):+ # The guard that keeps the re-encode on llama-server bodies only. Reverting it+ # brings the 400 above back to the safetensors/MLX client-tools path.+ import inspect++ import routes.inference as inference_mod++ src = inspect.getsource(inference_mod)+ _, _, after = src.partition("_flatten_content_parts_for_local_template(")+ assert after, "the local-template call site moved"+ assert "normalize_images = False" in after[:300]++ def test_undecodable_data_url_raises_400(self):+ with pytest.raises(HTTPException) as exc:+ _openai_messages_for_passthrough(self._req("data:image/webp;base64,!!!nope!!!"))+ assert exc.value.status_code == 400++ def test_echoed_legacy_image_is_not_spliced_twice(self):+ from io import BytesIO++ from PIL import Image++ buf = BytesIO()+ Image.new("RGB", (2, 2), (1, 2, 3)).save(buf, format = "WEBP")+ b64 = base64.b64encode(buf.getvalue()).decode("ascii")++ req = ChatCompletionRequest(+ model = "default",+ image_base64 = b64,+ messages = [+ {+ "role": "user",+ "content": [+ {"type": "text", "text": "what is this"},+ {+ "type": "image_url",+ "image_url": {"url": f"data:image/webp;base64,{b64}"},+ },+ ],+ },+ ],+ )++ messages = _openai_messages_for_passthrough(req)++ parts = [p for p in messages[0]["content"] if p.get("type") == "image_url"]+ assert len(parts) == 1+ assert parts[0]["image_url"]["url"].startswith("data:image/png;base64,")