Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: show the model a phone photo the right way up (#12241)

Nilaybe52b45c9d21
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 01ec92befa8..622af0b4b7b 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -133,6 +133,7 @@ from .diffusion_memory import (
vae_is_sliced,
)
from .diffusion_torchao_patches import install_torchao_int_mm_patch
+from .image_orientation import exif_upright
from .media_decode_phase import decode_phase
from .diffusion_speed import (
SPEED_DEFAULT,
@@ -538,32 +539,7 @@ def decode_b64_image(
)
img.load()
# Below the size guard because the transpose needs the pixels the guard exists to not read.
- # Read the EXIF block directly, not via getexif(), which also recovers orientation from XMP
- # and ImageMagick profiles that Chromium ignores, and may have cached one during open().
- exif = Image.Exif()
- try:
- if img.info.get("exif"):
- exif.load(img.info["exif"])
- except Exception: # noqa: BLE001 - a malformed block leaves it unrotated, as the preview
- pass
- # Chromium does not apply orientation to WebP but WebKit does, so macOS desktop keeps a
- # known divergence: Studio ships on Tauri against three engines and no rule fits them all.
- # TIFF needs no branch: its plugin applies the IFD orientation during load().
- orientation = None if img.format == "WEBP" else exif.get(0x0112)
- method = {
- 2: Image.Transpose.FLIP_LEFT_RIGHT,
- 3: Image.Transpose.ROTATE_180,
- 4: Image.Transpose.FLIP_TOP_BOTTOM,
- 5: Image.Transpose.TRANSPOSE,
- 6: Image.Transpose.ROTATE_270,
- 7: Image.Transpose.TRANSVERSE,
- 8: Image.Transpose.ROTATE_90,
- }.get(orientation)
- if method is not None:
- img = img.transpose(method)
- # Drop the sources it could be read from again, so a re-save cannot re-apply it.
- for consumed in ("exif", "XML:com.adobe.xmp", "xmp", "Raw profile type exif"):
- img.info.pop(consumed, None)
+ img = exif_upright(img)
except ValueError:
raise # the size guard's own message; don't wrap it as a decode error
except Exception as exc: # noqa: BLE001 - surfaced as a 400 to the client
diff --git a/studio/backend/core/inference/image_orientation.py b/studio/backend/core/inference/image_orientation.py
new file mode 100644
index 00000000000..8d2fb05127b
--- /dev/null
+++ b/studio/backend/core/inference/image_orientation.py
@@ -0,0 +1,37 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+from typing import Any
+
+
+def exif_upright(image: Any) -> Any:
+ """*image* turned per its EXIF orientation, or *image* itself when nothing turns it."""
+ from PIL import Image
+
+ # Read the EXIF block directly, not via getexif(), which also recovers orientation from XMP
+ # and ImageMagick profiles that Chromium ignores, and may have cached one during open().
+ exif = Image.Exif()
+ try:
+ if image.info.get("exif"):
+ exif.load(image.info["exif"])
+ except Exception: # noqa: BLE001 - a malformed block leaves it unrotated, as the preview
+ return image
+ # Chromium does not apply orientation to WebP but WebKit does, so macOS desktop keeps a
+ # known divergence: Studio ships on Tauri against three engines and no rule fits them all.
+ # TIFF needs no branch: its plugin applies the IFD orientation during load().
+ method = {
+ 2: Image.Transpose.FLIP_LEFT_RIGHT,
+ 3: Image.Transpose.ROTATE_180,
+ 4: Image.Transpose.FLIP_TOP_BOTTOM,
+ 5: Image.Transpose.TRANSPOSE,
+ 6: Image.Transpose.ROTATE_270,
+ 7: Image.Transpose.TRANSVERSE,
+ 8: Image.Transpose.ROTATE_90,
+ }.get(None if image.format == "WEBP" else exif.get(0x0112))
+ if method is None:
+ return image
+ upright = image.transpose(method)
+ # Drop the sources it could be read from again, so a re-save cannot re-apply it.
+ for consumed in ("exif", "XML:com.adobe.xmp", "xmp", "Raw profile type exif"):
+ upright.info.pop(consumed, None)
+ return upright
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 2a567075a3d..e520a0bf13e 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -3736,6 +3736,7 @@ from state.tool_approvals import resolve_tool_decision, tool_decision_is_pending
from core.inference.model_ids import display_model_name, model_id_matches, public_model_id
from core.inference.api_monitor import api_monitor
+from core.inference.image_orientation import exif_upright
from core.inference.llama_http import nonstreaming_client
from core.inference.mcp_images import (
MAX_MODEL_IMAGES as _MCP_MAX_MODEL_IMAGES,
@@ -19686,6 +19687,7 @@ def _decode_and_resize_image(backend, encoded: str):
image_data = base64.b64decode(encoded)
image = Image.open(BytesIO(image_data))
image.load()
+ image = exif_upright(image)
# After the resize: converting first resamples interpolated RGB, a different picture.
image = _scaled_from_16_bit(backend.resize_image(image))
if image.mode not in ("RGB", "RGBA"):
@@ -35213,13 +35215,30 @@ def _stb_reads_png(raw: bytes) -> bool:
def _llama_image_data_url(raw: bytes) -> str:
"""Preserve PNG and JPEG bytes stb_image reads; convert other images to PNG.
+ An image whose EXIF orientation turns it is re-encoded upright: a JPEG stays JPEG with its
+ source quantization tables and subsampling, anything else becomes PNG.
+
Avoid inflating photos while still rejecting corrupt images with Pillow:
stb_image silently accepts some truncated JPEGs. Callers map failures to HTTP 400.
"""
- from PIL import Image
+ from PIL import Image, JpegImagePlugin
with Image.open(io.BytesIO(raw)) as img:
img.load()
+ upright = exif_upright(img)
+ if upright is not img:
+ buf = io.BytesIO()
+ if raw.startswith(b"\xff\xd8") and _stb_reads_jpeg(raw):
+ upright.save(
+ buf,
+ format = "JPEG",
+ # A list: Pillow keeps only dict keys 0..n-1, and some encoders number tables from 1.
+ qtables = [img.quantization[key] for key in sorted(img.quantization)],
+ subsampling = JpegImagePlugin.get_sampling(img),
+ )
+ return f"data:image/jpeg;base64,{base64.b64encode(buf.getvalue()).decode('ascii')}"
+ _scaled_from_16_bit(upright).convert("RGB").save(buf, format = "PNG")
+ return f"data:image/png;base64,{base64.b64encode(buf.getvalue()).decode('ascii')}"
if raw.startswith(_PNG_SIGNATURE) and _stb_reads_png(raw):
return f"data:image/png;base64,{base64.b64encode(raw).decode('ascii')}"
if raw.startswith(b"\xff\xd8") and _stb_reads_jpeg(raw):
diff --git a/studio/backend/tests/test_chat_image_exif_orientation.py b/studio/backend/tests/test_chat_image_exif_orientation.py
new file mode 100644
index 00000000000..2bd78202d3d
--- /dev/null
+++ b/studio/backend/tests/test_chat_image_exif_orientation.py
@@ -0,0 +1,151 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import base64
+import os
+import sys
+from io import BytesIO
+from types import SimpleNamespace
+
+_backend = os.path.join(os.path.dirname(__file__), "..")
+sys.path.insert(0, _backend)
+
+import pytest
+from PIL import Image, JpegImagePlugin
+
+from routes.inference import _decode_and_resize_image, _normalize_openai_image_parts_for_llama
+
+RED, GREEN, BLUE, YELLOW = (220, 20, 20), (20, 200, 20), (20, 20, 220), (220, 200, 20)
+
+_AS_DISPLAYED = {
+ 1: ((300, 100), ["red", "green", "blue", "yellow"]),
+ 2: ((300, 100), ["green", "red", "yellow", "blue"]),
+ 3: ((300, 100), ["yellow", "blue", "green", "red"]),
+ 4: ((300, 100), ["blue", "yellow", "red", "green"]),
+ 5: ((100, 300), ["red", "blue", "green", "yellow"]),
+ 6: ((100, 300), ["blue", "red", "yellow", "green"]),
+ 7: ((100, 300), ["yellow", "green", "blue", "red"]),
+ 8: ((100, 300), ["green", "yellow", "red", "blue"]),
+}
+
+
+def _photo(
+ orientation,
+ fmt = "JPEG",
+ subsampling = 0,
+) -> bytes:
+ img = Image.new("RGB", (300, 100))
+ img.paste(RED, (0, 0, 150, 50))
+ img.paste(GREEN, (150, 0, 300, 50))
+ img.paste(BLUE, (0, 50, 150, 100))
+ img.paste(YELLOW, (150, 50, 300, 100))
+ exif = img.getexif()
+ if orientation is not None:
+ exif[0x0112] = orientation
+ buf = BytesIO()
+ extra = {"quality": 95, "subsampling": subsampling} if fmt == "JPEG" else {}
+ img.save(buf, format = fmt, exif = exif, **extra)
+ return buf.getvalue()
+
+
+def _quadrants(img) -> list:
+ img = img.convert("RGB")
+ w, h = img.size
+ names = {RED: "red", GREEN: "green", BLUE: "blue", YELLOW: "yellow"}
+
+ def at(fx, fy):
+ px = img.getpixel((int(w * fx), int(h * fy)))
+ return names[min(names, key = lambda c: sum((a - b) ** 2 for a, b in zip(c, px)))]
+
+ return [at(0.25, 0.25), at(0.75, 0.25), at(0.25, 0.75), at(0.75, 0.75)]
+
+
+def _sent_to_llama(raw: bytes, mime = "image/jpeg") -> str:
+ url = f"data:{mime};base64," + base64.b64encode(raw).decode("ascii")
+ part = {"type": "image_url", "image_url": {"url": url}}
+ _normalize_openai_image_parts_for_llama([{"role": "user", "content": [part]}])
+ return part["image_url"]["url"]
+
+
[email protected]("orientation", sorted(_AS_DISPLAYED))
+def test_llama_server_gets_a_tagged_jpeg_the_way_the_chat_shows_it(orientation):
+ head, b64 = _sent_to_llama(_photo(orientation)).split(",", 1)
+ assert head == "data:image/jpeg;base64"
+ sent = Image.open(BytesIO(base64.b64decode(b64)))
+ assert sent.format == "JPEG"
+ size, quadrants = _AS_DISPLAYED[orientation]
+ assert sent.size == size
+ assert _quadrants(sent) == quadrants
+ assert sent.getexif().get(0x0112) in (None, 1)
+
+
[email protected]("orientation", [None, 1])
+def test_an_upright_jpeg_still_reaches_llama_server_byte_for_byte(orientation):
+ raw = _photo(orientation)
+ assert _sent_to_llama(raw) == "data:image/jpeg;base64," + base64.b64encode(raw).decode()
+
+
+def test_llama_server_gets_a_tagged_png_upright():
+ head, b64 = _sent_to_llama(_photo(6, "PNG"), "image/png").split(",", 1)
+ assert head == "data:image/png;base64"
+ sent = Image.open(BytesIO(base64.b64decode(b64)))
+ assert sent.size == (100, 300)
+ assert _quadrants(sent) == _AS_DISPLAYED[6][1]
+
+
+def test_a_tagged_png_is_decoded_once(monkeypatch):
+ opened = []
+ real_open = Image.open
+
+ def counting_open(*args, **kwargs):
+ opened.append(args)
+ return real_open(*args, **kwargs)
+
+ monkeypatch.setattr(Image, "open", counting_open)
+ _sent_to_llama(_photo(6, "PNG"), "image/png")
+ assert len(opened) == 1
+
+
[email protected]("subsampling", [0, 1, 2])
+def test_a_tagged_jpeg_keeps_its_quality_and_colour_detail(subsampling):
+ raw = _photo(6, subsampling = subsampling)
+ head, b64 = _sent_to_llama(raw).split(",", 1)
+ assert head == "data:image/jpeg;base64"
+ sent = Image.open(BytesIO(base64.b64decode(b64)))
+ assert sent.quantization == Image.open(BytesIO(raw)).quantization
+ assert JpegImagePlugin.get_sampling(sent) == subsampling
+
+
+def test_a_tagged_jpeg_whose_tables_start_at_one_is_still_accepted():
+ img = Image.new("L", (300, 100))
+ exif = img.getexif()
+ exif[0x0112] = 6
+ buf = BytesIO()
+ img.save(buf, format = "JPEG", exif = exif, quality = 95)
+ raw = bytearray(buf.getvalue())
+ raw[raw.index(b"\xff\xdb") + 4] = 1 # DQT defines table 1
+ raw[raw.index(b"\xff\xc0") + 12] = 1 # SOF0's one component reads it
+ head, b64 = _sent_to_llama(bytes(raw)).split(",", 1)
+ assert head == "data:image/jpeg;base64"
+ sent = Image.open(BytesIO(base64.b64decode(b64)))
+ assert sent.size == (100, 300)
+ assert list(sent.quantization.values()) == list(Image.open(BytesIO(raw)).quantization.values())
+
+
+def test_a_webp_tag_is_skipped_the_way_chromium_skips_it():
+ head, b64 = _sent_to_llama(_photo(6, "WEBP"), "image/webp").split(",", 1)
+ assert head == "data:image/png;base64"
+ sent = Image.open(BytesIO(base64.b64decode(b64)))
+ assert sent.size == (300, 100)
+ assert _quadrants(sent) == _AS_DISPLAYED[1][1]
+
+
[email protected]("orientation", [3, 6, 8])
+def test_the_transformers_and_mlx_path_decodes_a_tagged_jpeg_upright(orientation):
+ backend = SimpleNamespace(resize_image = lambda img: img)
+ encoded = base64.b64encode(_photo(orientation)).decode("ascii")
+ img = _decode_and_resize_image(backend, encoded)
+ size, quadrants = _AS_DISPLAYED[orientation]
+ assert img.size == size
+ assert _quadrants(img) == quadrants
+ assert img.getexif().get(0x0112) in (None, 1)