Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: let GGUF vision models see dark text in transparent images (#12310)

Nilayf66bb106fd70
diff --git a/studio/backend/core/inference/mcp_images.py b/studio/backend/core/inference/mcp_images.py
index 5d072d960c2..cd774291b5e 100644
--- a/studio/backend/core/inference/mcp_images.py
+++ b/studio/backend/core/inference/mcp_images.py
@@ -320,14 +320,14 @@ def png_payloads_per_result(
def flattened_rgb(image):
- """RGB with any transparency composited onto white, not simply dropped.
+ """RGB with any transparency composited onto white (black for light ink), not dropped.
``convert("RGB")`` keeps whatever colour sits UNDER the alpha, and a tool that
never painted a background leaves that black -- so a transparent screenshot's
dark text or line art converts to black on black and the model is handed a
blank rectangle. Only images that actually carry alpha take the composite.
"""
- from PIL import Image
+ from PIL import Image, ImageChops, ImageStat
# Match routes/inference.py's _image_bytes_to_png_b64: scale declared I;16
# values to 8-bit before RGB conversion clips them. I;16B/I;16L must pass
@@ -336,14 +336,19 @@ def flattened_rgb(image):
if image.mode != "I;16":
image = image.convert("I")
image = image.point(lambda v: v * (1.0 / 257), mode = "L")
- has_alpha = image.mode in ("RGBA", "LA", "PA") or (
- image.mode == "P" and "transparency" in image.info
- )
+ # A 16-bit tRNS key would match the wrong 8-bit samples; drop it, as convert("RGB") did.
+ image.info.pop("transparency", None)
+ # "transparency" also keys a colour out of L / RGB PNGs (tRNS), not just P.
+ has_alpha = image.mode in ("RGBA", "LA", "PA") or "transparency" in image.info
if not has_alpha:
return image.convert("RGB")
- rgba = image.convert("RGBA")
- canvas = Image.new("RGB", rgba.size, (255, 255, 255))
- canvas.paste(rgba, mask = rgba.getchannel("A"))
+ rgba = image if image.mode == "RGBA" else image.convert("RGBA")
+ alpha = rgba.getchannel("A")
+ # Alpha-weighted: light ink (dark-mode logos, white text) goes onto black, not white.
+ ink = ImageStat.Stat(ImageChops.multiply(rgba.convert("L"), alpha)).sum[0]
+ light = 255 * ink > 128 * ImageStat.Stat(alpha).sum[0] > 0
+ canvas = Image.new("RGB", rgba.size, (0, 0, 0) if light else (255, 255, 255))
+ canvas.paste(rgba, mask = alpha)
return canvas
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index d22631c2570..2898bc8e4da 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -35646,7 +35646,7 @@ def _image_bytes_to_png_b64(raw: bytes) -> str:
"""Convert image bytes to base64 PNG for formats llama-server cannot decode."""
from PIL import Image
- img = _scaled_from_16_bit(Image.open(io.BytesIO(raw))).convert("RGB")
+ img = _mcp_flattened_rgb(Image.open(io.BytesIO(raw)))
buf = io.BytesIO()
img.save(buf, format = "PNG")
return base64.b64encode(buf.getvalue()).decode("ascii")
@@ -35733,6 +35733,17 @@ def _llama_image_data_url(raw: bytes) -> str:
with Image.open(io.BytesIO(raw)) as img:
img.load()
upright = exif_upright(img)
+ # 16-bit tRNS keys cannot survive the 8-bit scaling, so those pass through as before.
+ if img.has_transparency_data and not img.mode.startswith("I;16"):
+ # Alpha band only: no RGBA copy of opaque screenshots.
+ bands = img.getbands()
+ alpha = img.getchannel("A") if "A" in bands else img.convert("RGBA").getchannel("A")
+ if alpha.getextrema()[0] < 255:
+ del alpha
+ buf = io.BytesIO()
+ # Reuse this decode: a second one doubles peak memory on large images.
+ _mcp_flattened_rgb(upright).save(buf, format = "PNG")
+ return f"data:image/png;base64,{base64.b64encode(buf.getvalue()).decode('ascii')}"
if upright is not img:
buf = io.BytesIO()
if raw.startswith(b"\xff\xd8") and _stb_reads_jpeg(raw):
@@ -35744,7 +35755,7 @@ def _llama_image_data_url(raw: bytes) -> str:
subsampling = JpegImagePlugin.get_sampling(img),
)
return f"data:image/jpeg;base64,{base64.b64encode(buf.getvalue()).decode('ascii')}"
- _scaled_from_16_bit(upright).convert("RGB").save(buf, format = "PNG")
+ _mcp_flattened_rgb(upright).save(buf, format = "PNG")
return f"data:image/png;base64,{base64.b64encode(buf.getvalue()).decode('ascii')}"
if raw.startswith(_PNG_SIGNATURE) and _stb_reads_png(raw):
return f"data:image/png;base64,{base64.b64encode(raw).decode('ascii')}"
diff --git a/studio/backend/tests/test_llama_image_passthrough.py b/studio/backend/tests/test_llama_image_passthrough.py
index 998b567960e..e2962da34cc 100644
--- a/studio/backend/tests/test_llama_image_passthrough.py
+++ b/studio/backend/tests/test_llama_image_passthrough.py
@@ -113,6 +113,55 @@ def test_other_formats_are_still_reencoded_to_png(raw):
assert Image.open(BytesIO(out)).mode == "RGB"
+ "fmt, mode, kw",
+ [
+ ("PNG", "RGBA", {}),
+ ("PNG", "LA", {}),
+ ("WEBP", "RGBA", {"lossless": True, "exact": True}),
+ ("GIF", "RGBA", {}),
+ ],
+ ids = ["png", "grey-alpha-png", "webp", "gif"],
+)
+def test_dark_strokes_on_a_transparent_background_stay_visible(fmt, mode, kw):
+ img = Image.new("RGBA", (64, 48), (0, 0, 0, 0))
+ img.paste((0, 0, 0, 255), (8, 20, 56, 28))
+ head, out = _split(_llama_image_data_url(_encode(img.convert(mode), fmt, **kw)))
+ assert head == "data:image/png;base64"
+ stb_view = Image.open(BytesIO(out)).convert("RGBA").convert("RGB")
+ assert stb_view.getpixel((2, 2)) == (255, 255, 255)
+ assert max(stb_view.getpixel((32, 24))) < 64
+
+
+def test_light_strokes_on_a_transparent_background_stay_visible():
+ img = Image.new("RGBA", (64, 48), (0, 0, 0, 0))
+ img.paste((255, 255, 255, 255), (8, 20, 56, 28))
+ head, out = _split(_llama_image_data_url(_encode(img, "PNG")))
+ stb_view = Image.open(BytesIO(out)).convert("RGBA").convert("RGB")
+ assert stb_view.getpixel((2, 2)) == (0, 0, 0)
+ assert min(stb_view.getpixel((32, 24))) > 192
+
+
+def test_a_faint_light_halo_does_not_black_out_dark_content():
+ img = Image.new("RGBA", (64, 48), (255, 255, 255, 3))
+ img.paste((0, 0, 0, 255), (8, 20, 56, 28))
+ head, out = _split(_llama_image_data_url(_encode(img, "PNG")))
+ stb_view = Image.open(BytesIO(out)).convert("RGBA").convert("RGB")
+ assert min(stb_view.getpixel((2, 2))) > 192
+ assert max(stb_view.getpixel((32, 24))) < 64
+
+
[email protected]("mode, key", [("RGB", (0, 0, 0)), ("L", 0)], ids = ["rgb", "grey"])
+def test_colour_keyed_png_background_is_composited(mode, key):
+ img = Image.new(mode, (64, 48), key)
+ img.paste((16,) * len(img.getbands()), (8, 20, 56, 28))
+ head, out = _split(_llama_image_data_url(_encode(img, "PNG", transparency = key)))
+ assert head == "data:image/png;base64"
+ stb_view = Image.open(BytesIO(out)).convert("RGBA").convert("RGB")
+ assert stb_view.getpixel((2, 2)) == (255, 255, 255)
+ assert max(stb_view.getpixel((32, 24))) < 64
+
+
def test_jpeg_frames_stb_rejects_are_not_passed_through():
raw = _encode(_photo(), "JPEG")
assert _stb_reads_jpeg(raw)
@@ -197,3 +246,11 @@ def test_legacy_image_base64_jpeg_is_forwarded_unchanged():
messages = _openai_messages_for_passthrough(payload, vision = True)
parts = [p for p in messages[-1]["content"] if p.get("type") == "image_url"]
assert [p["image_url"]["url"] for p in parts] == [f"data:image/jpeg;base64,{b64}"]
+
+
+def test_sixteen_bit_colour_keyed_png_still_passes_through():
+ img = Image.new("I;16", (16, 16), 30000)
+ img.putpixel((0, 0), 100)
+ raw = _encode(img, "PNG", transparency = 100)
+ head, out = _split(_llama_image_data_url(raw))
+ assert (head, out) == ("data:image/png;base64", raw)