Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Serve the decoded pixels on a multi-image turn, not the base64 they arrived as (#11320)

Daniel Han6448afca0069
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 3af42c342d0..067d24148d3 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -26711,11 +26711,19 @@ async def produce_openai_chat_completions(
# the same history again from the raw payload: a retained picture can be a 40
# megapixel raster, and it is decoded once, not once per promotion.
_sf_mcp_decode_cache: dict = {}
+ # The DECODED pictures, not the base64 they arrived as. Promotion treats a caller
+ # payload as opaque -- it zips it onto a marker by identity and deletes it by index,
+ # never reading it -- and the sink it fills is what generation sends, ahead of the
+ # decoded list (`sf_mcp_images or images`). Handing it the raw base64 therefore threw
+ # the decode away: _decode_and_resize_image forces the load, scales a 16-bit raster
+ # down from 65535 and converts CMYK/PA/F/LAB to RGB, while the worker's own decode
+ # does none of that, so a 16-bit PNG reached the model clipped to white. Falls back
+ # to the payloads when nothing was decoded, which keeps the positions aligned.
chat_messages, sf_mcp_images = await _promote_local_mcp_images_async(
chat_messages,
vision = bool(_sf_model_info.get("is_vision")),
decode_cache = _sf_mcp_decode_cache,
- caller_images = served_images,
+ caller_images = images or served_images,
)
_sf_tpl = (_sf_model_info.get("chat_template_info") or {}).get("template")
# Resolve the tool policy BEFORE the protocol is classified: the template
diff --git a/studio/backend/tests/test_openai_multi_image_route.py b/studio/backend/tests/test_openai_multi_image_route.py
index abd13b2a929..c8ab3221c05 100644
--- a/studio/backend/tests/test_openai_multi_image_route.py
+++ b/studio/backend/tests/test_openai_multi_image_route.py
@@ -173,6 +173,35 @@ def test_a_16_bit_image_keeps_its_levels_instead_of_clipping_to_white(monkeypatc
]
+def _as_image(delivered):
+ """A served entry, whether it rides as pixels or as the base64 they came in."""
+ if isinstance(delivered, str):
+ return Image.open(io.BytesIO(base64.b64decode(delivered)))
+ return delivered
+
+
+def test_a_16_bit_image_on_a_multi_image_turn_keeps_its_levels_too(monkeypatch):
+ """The single-image path had this guard and the multi-image one did not, which is how the
+ route came to hand generation the caller's raw base64 instead of what it had just decoded.
+ Only `_decode_and_resize_image` scales 0..65535 down; the worker's own decode does not, so
+ the picture arrived clipped to white with every structural assertion still green.
+
+ Asserted on the pixels rather than on the type, so the claim survives a transport that
+ carries base64 again -- what must not come back is the lost conversion.
+ """
+ source = Image.new("I;16", (2, 2))
+ source.putdata([0, 20000, 40000, 65535])
+ turn = [ChatMessage(role = "user", content = [_part(source), _sized(4), _ASK])]
+
+ served = _call(monkeypatch, turn).calls[0]["images"]
+
+ assert len(served) == 2
+ levels = _as_image(served[0]).convert("RGB")
+ # getpixel, not the flattened read the single-image test uses: that one is spelled
+ # getdata() on Pillow 10 and get_flattened_data() on 12, and this runs on both.
+ assert [levels.getpixel(at)[0] for at in ((0, 0), (1, 0), (0, 1), (1, 1))] == [0, 77, 155, 255]
+
+
_PDF = {
"type": "input_document",
"file_data": "data:application/pdf;base64,JVBERi0xLjQK",