Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: let GGUF vision models see dark text in transparent images (#12310)
Nilayf66bb106fd70
diff --git a/studio/backend/core/inference/mcp_images.py b/studio/backend/core/inference/mcp_images.pyindex 5d072d960c2..cd774291b5e 100644--- a/studio/backend/core/inference/mcp_images.py+++ b/studio/backend/core/inference/mcp_images.py@@ -320,14 +320,14 @@ def png_payloads_per_result(def flattened_rgb(image):- """RGB with any transparency composited onto white, not simply dropped.+ """RGB with any transparency composited onto white (black for light ink), not dropped.``convert("RGB")`` keeps whatever colour sits UNDER the alpha, and a tool thatnever painted a background leaves that black -- so a transparent screenshot'sdark text or line art converts to black on black and the model is handed ablank rectangle. Only images that actually carry alpha take the composite."""- from PIL import Image+ from PIL import Image, ImageChops, ImageStat# Match routes/inference.py's _image_bytes_to_png_b64: scale declared I;16# values to 8-bit before RGB conversion clips them. I;16B/I;16L must pass@@ -336,14 +336,19 @@ def flattened_rgb(image):if image.mode != "I;16":image = image.convert("I")image = image.point(lambda v: v * (1.0 / 257), mode = "L")- has_alpha = image.mode in ("RGBA", "LA", "PA") or (- image.mode == "P" and "transparency" in image.info- )+ # A 16-bit tRNS key would match the wrong 8-bit samples; drop it, as convert("RGB") did.+ image.info.pop("transparency", None)+ # "transparency" also keys a colour out of L / RGB PNGs (tRNS), not just P.+ has_alpha = image.mode in ("RGBA", "LA", "PA") or "transparency" in image.infoif not has_alpha:return image.convert("RGB")- rgba = image.convert("RGBA")- canvas = Image.new("RGB", rgba.size, (255, 255, 255))- canvas.paste(rgba, mask = rgba.getchannel("A"))+ rgba = image if image.mode == "RGBA" else image.convert("RGBA")+ alpha = rgba.getchannel("A")+ # Alpha-weighted: light ink (dark-mode logos, white text) goes onto black, not white.+ ink = ImageStat.Stat(ImageChops.multiply(rgba.convert("L"), alpha)).sum[0]+ light = 255 * ink > 128 * ImageStat.Stat(alpha).sum[0] > 0+ canvas = Image.new("RGB", rgba.size, (0, 0, 0) if light else (255, 255, 255))+ canvas.paste(rgba, mask = alpha)return canvasdiff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex d22631c2570..2898bc8e4da 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -35646,7 +35646,7 @@ def _image_bytes_to_png_b64(raw: bytes) -> str:"""Convert image bytes to base64 PNG for formats llama-server cannot decode."""from PIL import Image- img = _scaled_from_16_bit(Image.open(io.BytesIO(raw))).convert("RGB")+ img = _mcp_flattened_rgb(Image.open(io.BytesIO(raw)))buf = io.BytesIO()img.save(buf, format = "PNG")return base64.b64encode(buf.getvalue()).decode("ascii")@@ -35733,6 +35733,17 @@ def _llama_image_data_url(raw: bytes) -> str:with Image.open(io.BytesIO(raw)) as img:img.load()upright = exif_upright(img)+ # 16-bit tRNS keys cannot survive the 8-bit scaling, so those pass through as before.+ if img.has_transparency_data and not img.mode.startswith("I;16"):+ # Alpha band only: no RGBA copy of opaque screenshots.+ bands = img.getbands()+ alpha = img.getchannel("A") if "A" in bands else img.convert("RGBA").getchannel("A")+ if alpha.getextrema()[0] < 255:+ del alpha+ buf = io.BytesIO()+ # Reuse this decode: a second one doubles peak memory on large images.+ _mcp_flattened_rgb(upright).save(buf, format = "PNG")+ return f"data:image/png;base64,{base64.b64encode(buf.getvalue()).decode('ascii')}"if upright is not img:buf = io.BytesIO()if raw.startswith(b"\xff\xd8") and _stb_reads_jpeg(raw):@@ -35744,7 +35755,7 @@ def _llama_image_data_url(raw: bytes) -> str:subsampling = JpegImagePlugin.get_sampling(img),)return f"data:image/jpeg;base64,{base64.b64encode(buf.getvalue()).decode('ascii')}"- _scaled_from_16_bit(upright).convert("RGB").save(buf, format = "PNG")+ _mcp_flattened_rgb(upright).save(buf, format = "PNG")return f"data:image/png;base64,{base64.b64encode(buf.getvalue()).decode('ascii')}"if raw.startswith(_PNG_SIGNATURE) and _stb_reads_png(raw):return f"data:image/png;base64,{base64.b64encode(raw).decode('ascii')}"diff --git a/studio/backend/tests/test_llama_image_passthrough.py b/studio/backend/tests/test_llama_image_passthrough.pyindex 998b567960e..e2962da34cc 100644--- a/studio/backend/tests/test_llama_image_passthrough.py+++ b/studio/backend/tests/test_llama_image_passthrough.py@@ -113,6 +113,55 @@ def test_other_formats_are_still_reencoded_to_png(raw):assert Image.open(BytesIO(out)).mode == "RGB"+ "fmt, mode, kw",+ [+ ("PNG", "RGBA", {}),+ ("PNG", "LA", {}),+ ("WEBP", "RGBA", {"lossless": True, "exact": True}),+ ("GIF", "RGBA", {}),+ ],+ ids = ["png", "grey-alpha-png", "webp", "gif"],+)+def test_dark_strokes_on_a_transparent_background_stay_visible(fmt, mode, kw):+ img = Image.new("RGBA", (64, 48), (0, 0, 0, 0))+ img.paste((0, 0, 0, 255), (8, 20, 56, 28))+ head, out = _split(_llama_image_data_url(_encode(img.convert(mode), fmt, **kw)))+ assert head == "data:image/png;base64"+ stb_view = Image.open(BytesIO(out)).convert("RGBA").convert("RGB")+ assert stb_view.getpixel((2, 2)) == (255, 255, 255)+ assert max(stb_view.getpixel((32, 24))) < 64+++def test_light_strokes_on_a_transparent_background_stay_visible():+ img = Image.new("RGBA", (64, 48), (0, 0, 0, 0))+ img.paste((255, 255, 255, 255), (8, 20, 56, 28))+ head, out = _split(_llama_image_data_url(_encode(img, "PNG")))+ stb_view = Image.open(BytesIO(out)).convert("RGBA").convert("RGB")+ assert stb_view.getpixel((2, 2)) == (0, 0, 0)+ assert min(stb_view.getpixel((32, 24))) > 192+++def test_a_faint_light_halo_does_not_black_out_dark_content():+ img = Image.new("RGBA", (64, 48), (255, 255, 255, 3))+ img.paste((0, 0, 0, 255), (8, 20, 56, 28))+ head, out = _split(_llama_image_data_url(_encode(img, "PNG")))+ stb_view = Image.open(BytesIO(out)).convert("RGBA").convert("RGB")+ assert min(stb_view.getpixel((2, 2))) > 192+ assert max(stb_view.getpixel((32, 24))) < 64++[email protected]("mode, key", [("RGB", (0, 0, 0)), ("L", 0)], ids = ["rgb", "grey"])+def test_colour_keyed_png_background_is_composited(mode, key):+ img = Image.new(mode, (64, 48), key)+ img.paste((16,) * len(img.getbands()), (8, 20, 56, 28))+ head, out = _split(_llama_image_data_url(_encode(img, "PNG", transparency = key)))+ assert head == "data:image/png;base64"+ stb_view = Image.open(BytesIO(out)).convert("RGBA").convert("RGB")+ assert stb_view.getpixel((2, 2)) == (255, 255, 255)+ assert max(stb_view.getpixel((32, 24))) < 64++def test_jpeg_frames_stb_rejects_are_not_passed_through():raw = _encode(_photo(), "JPEG")assert _stb_reads_jpeg(raw)@@ -197,3 +246,11 @@ def test_legacy_image_base64_jpeg_is_forwarded_unchanged():messages = _openai_messages_for_passthrough(payload, vision = True)parts = [p for p in messages[-1]["content"] if p.get("type") == "image_url"]assert [p["image_url"]["url"] for p in parts] == [f"data:image/jpeg;base64,{b64}"]+++def test_sixteen_bit_colour_keyed_png_still_passes_through():+ img = Image.new("I;16", (16, 16), 30000)+ img.putpixel((0, 0), 100)+ raw = _encode(img, "PNG", transparency = 100)+ head, out = _split(_llama_image_data_url(raw))+ assert (head, out) == ("data:image/png;base64", raw)