Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: read a remote image URL on safetensors and MLX models instead of ignoring it (#12243)
Nilay1783a35e2c36
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 0cc3661f50d..7dcdbdac5cd 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -8845,12 +8845,11 @@ async def _preflight_image_for_switch(" GGUF build of it, which accepts several."),)- # Non-GGUF backends only receive decoded base64 images.- if not target_is_gguf and image_preflight.get("remote") and image_preflight.get("b64") is None:- raise HTTPException(- status_code = 400,- detail = "Remote image URLs are not supported. Use a base64 data URL.",- )+ # The images the model reads are fetched after the load, so a scheme it refuses is refused now.+ for scheme in image_preflight.get("fetch_schemes", {}).get(target_takes_several, ()):+ rejection = _remote_image_scheme_rejection(scheme) if scheme else None+ if not target_is_gguf and rejection is not None:+ raise HTTPException(status_code = rejection[0], detail = rejection[1])# Only what the single-image path selects: an architecture guess must not refuse the rest.encoded_images = (image_preflight.get("b64s", ()) if target_is_gguf else (image_preflight.get("b64"),)@@ -26218,26 +26217,40 @@ async def produce_openai_chat_completions(_images_on_turn = _images_in_last_user_message(payload.messages)_legacy_image_distinct = _legacy_image_is_distinct(payload)_local_image_payloads = _request_local_image_payloads(payload)- _image_b64 = _pre_parsed[2] or payload.image_base64- if _image_b64 is None and _local_image_payloads:+ # A remote image the model reads is decoded after its fetch; no older image stands in for it.+ _selected_image = _served_image_part(payload.messages)+ _selects_remote = _remote_image_part(_selected_image)+ _image_b64 = (+ (payload.image_base64 if _legacy_image_distinct else None)+ if _selects_remote+ else _pre_parsed[2] or payload.image_base64+ )+ if _image_b64 is None and _local_image_payloads and not _selects_remote:_image_b64 = _local_image_payloads[0]- # Read the way the render reads it; separate from b64s, which GGUF validates unchanged.- _admitted_payloads = _conversation_with_image_markers(payload.messages)[1]+ # Read as the render reads it after the fetch; b64s stays as sent for GGUF to validate.+ _served_messages = _remote_images_as_served(payload.messages)+ _admitted_payloads = _conversation_with_image_markers(_served_messages)[1]if _legacy_image_distinct:_admitted_payloads = [*_admitted_payloads, payload.image_base64]_image_preflight = {"b64": _image_b64,"b64s": _local_image_payloads,"admitted": _admitted_payloads,- "remote": _messages_have_remote_image(payload.messages),"multiple": (_images_on_turn + int(_legacy_image_distinct) > 1- or bool(_pre_parsed[2] and _legacy_image_distinct)+ or bool((_pre_parsed[2] or _selects_remote) and _legacy_image_distinct)),# Refused after the load whatever the target is, so refused before evicting for it."unservable_alongside": _newest_turn_shows_more_images_than_it_sends(- payload.messages, _legacy_image_distinct+ _served_messages, _legacy_image_distinct),+ "fetch_schemes": {+ several: [+ _image_url_scheme(part.image_url.url)+ for part in _served_remote_parts(payload.messages, several)+ ]+ for several in (False, True)+ },}# Defer the resident claim: chat has several post-switch capability checks that can still@@ -26937,6 +26950,21 @@ async def produce_openai_chat_completions(finally:_tracker.__exit__(None, None, None)+ if not using_gguf and _messages_have_remote_image(+ m for m in payload.messages if m.role not in ("system", "developer")+ ):+ if not model_info.get("is_vision"):+ raise _reject(+ 400, "Image provided but current model is text-only. Load a vision model."+ )+ try:+ await asyncio.to_thread(+ _inline_served_remote_images, payload, _serves_several_images(backend)+ )+ except HTTPException as exc:+ raise _reject(exc.status_code, exc.detail)+ _pre_parsed = None+# ── Parse messages (handles multimodal content parts) ─────# Reuse the pre-hook parse when auto-switch did it, else parse now.if _pre_parsed is not None:@@ -35259,7 +35287,7 @@ _REMOTE_IMAGE_REQUEST_DEADLINE_S = 60.0_REMOTE_IMAGE_FETCH_REFUSAL = ("Could not fetch the remote image URL. Send the image as a base64 data URL instead.")-# Token counting renders a fixed media marker and charges a flat per-image allowance.+# Stands in for a remote image in token counts and in the switch preflight's image checks._COUNT_IMAGE_PLACEHOLDER = ("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1Pe""AAAADElEQVR4nGNgYGAAAAAEAAH2FzhVAAAAAElFTkSuQmCC"@@ -35319,6 +35347,29 @@ def _placeholder_remote_images_for_count(openai_messages: list[dict]) -> None:image_url["url"] = _COUNT_IMAGE_PLACEHOLDER+def _remote_images_as_served(messages) -> list[dict]:+ def _fetchable(part) -> bool:+ url = (+ (part.get("image_url") or {}).get("url", "") if part.get("type") == "image_url" else ""+ )+ return _image_url_scheme(url) == "https"++ served = []+ for message in messages or ():+ plain = message if isinstance(message, dict) else message.model_dump(exclude_none = True)+ content = plain.get("content")+ if isinstance(content, list):+ content = [+ {**part, "image_url": {"url": _COUNT_IMAGE_PLACEHOLDER}}+ if _fetchable(part)+ else part+ for part in content+ ]+ plain = {**plain, "content": content}+ served.append(plain)+ return served++def _inline_remote_image_url(url: str, scheme: str, budget_bytes: int, deadline: float) -> tuple[str, int]:@@ -35333,7 +35384,6 @@ def _inline_remote_image_url(url,"image/png",max_bytes = min(_REMOTE_IMAGE_MAX_BYTES, budget_bytes),- label = "llama-server image fetch",deadline = deadline,require_image_content_type = False,)@@ -35386,6 +35436,60 @@ def _inline_request_remote_images(payload) -> None:part.image_url.url = fetches.inline(part.image_url.url)+def _served_image_part(messages):+ """The image part _extract_content_parts selects, counting a remote URL as fetched."""+ latest = latest_user = None+ for message in messages:+ if message.role in ("system", "developer") or not isinstance(message.content, list):+ continue+ part = next(+ (+ p+ for p in message.content+ if isinstance(p, ImageContentPart)+ and (+ p.image_url.url.partition(",")[2]+ if p.image_url.url.startswith("data:")+ else _image_url_scheme(p.image_url.url)+ )+ ),+ None,+ )+ if part is not None:+ latest = part+ if message.role == "user":+ latest_user = part+ return latest_user or latest+++def _remote_image_part(part) -> bool:+ return part is not None and not part.image_url.url.startswith("data:")+++def _served_remote_parts(messages, several: bool) -> list:+ # Only what the renderers read: the image _extract_content_parts selects, and on a+ # multi-image model every user turn's (_conversation_with_image_markers).+ selected = _served_image_part(messages)+ return [+ part+ for message in messages+ if isinstance(message.content, list)+ for part in message.content+ if isinstance(part, ImageContentPart)+ and _remote_image_part(part)+ and (part is selected or (several and message.role == "user"))+ ]+++def _inline_served_remote_images(payload, several: bool) -> None:+ images_on_turn = _images_in_last_user_message(payload.messages)+ if not several and images_on_turn + int(_legacy_image_is_distinct(payload)) > 1:+ return # The caller refuses it as "one image per message".+ fetches = _RemoteImageFetches()+ for part in _served_remote_parts(payload.messages, several):+ part.image_url.url = fetches.inline(part.image_url.url)++def _normalize_openai_image_parts_for_llama(openai_messages: list[dict], on_image = None) -> bool:"""Fetch and normalize image URLs for llama-server in place.diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.pyindex e66dc30f408..472fe237925 100644--- a/studio/backend/tests/test_openai_auto_switch.py+++ b/studio/backend/tests/test_openai_auto_switch.py@@ -11045,39 +11045,130 @@ def _wire_image_switch_target(monkeypatch, *, target_is_gguf):return backend, recorder+def _chat_image_request(*urls):+ return _chat_request(+ model = "org/B-GGUF",+ messages = [+ ChatMessage(+ role = "user",+ content = [+ ImageContentPart(type = "image_url", image_url = ImageUrl(url = url)) for url in urls+ ],+ )+ ],+ )+++def test_chat_rejects_an_empty_data_url_before_non_gguf_switch(monkeypatch):+ backend, recorder = _wire_image_switch_target(monkeypatch, target_is_gguf = False)+ payload = _chat_image_request("data:image/png;base64,")++ with pytest.raises(HTTPException) as exc:+ asyncio.run(inference_route.openai_chat_completions(payload, object(), "tester"))++ assert exc.value.status_code == 400+ assert exc.value.detail == "Failed to decode image"+ assert recorder.calls == []+ assert backend.model_identifier == "org/A-GGUF"++[email protected]("count", [1, 2])+def test_chat_loads_a_non_gguf_target_for_remote_images(monkeypatch, count):+ # The loaded model fetches the URLs as a resident one does, so the switch is not refused.+ _, recorder = _wire_image_switch_target(monkeypatch, target_is_gguf = False)+ monkeypatch.setattr(inference_route, "_local_target_may_take_several_images", lambda *_a: True)+ recorder.fail = True+ urls = [f"https://example.com/{index}.png" for index in range(count)]++ with pytest.raises(HTTPException) as exc:+ asyncio.run(+ inference_route.openai_chat_completions(_chat_image_request(*urls), object(), "tester")+ )++ assert exc.value.detail == "load failed"+ assert len(recorder.calls) == 1++@pytest.mark.parametrize(- ("url", "detail"),+ ("turns", "detail"),[- ("data:image/png;base64,", "Failed to decode image"),+ ([["http://example.com/0.png"]], "Unsupported image URL scheme ('http:')"),(- "https://example.com/image.png",- "Remote image URLs are not supported. Use a base64 data URL.",+ [["https://example.com/0.png", "http://example.com/1.png"]],+ "one image per message",+ ),+ (+ [["http://example.com/0.png"], ["https://example.com/1.png"]],+ "Unsupported image URL scheme ('http:')",),],- ids = ["empty data url", "remote url"],+ ids = ["alone", "beside https", "earlier turn"],)-def test_chat_rejects_unsupported_openai_images_before_non_gguf_switch(monkeypatch, url, detail):+def test_chat_refuses_an_unfetchable_scheme_before_non_gguf_switch(monkeypatch, turns, detail):+ backend, recorder = _wire_image_switch_target(monkeypatch, target_is_gguf = False)+ monkeypatch.setattr(inference_route, "_local_target_may_take_several_images", lambda *_a: True)+ payload = _chat_request(+ model = "org/B-GGUF",+ messages = [_chat_image_request(*urls).messages[0] for urls in turns],+ )++ with pytest.raises(HTTPException) as exc:+ asyncio.run(inference_route.openai_chat_completions(payload, object(), "tester"))++ assert detail in exc.value.detail+ assert recorder.calls == []+ assert backend.model_identifier == "org/A-GGUF"+++def test_chat_leaves_an_unread_older_image_to_the_loaded_model(monkeypatch):+ # The newer remote image is the one the model reads, so the older one is not validated.+ _, recorder = _wire_image_switch_target(monkeypatch, target_is_gguf = False)+ recorder.fail = True+ turns = ["data:image/png;base64,Zm9v", "https://example.com/0.png"]+ payload = _chat_request(+ model = "org/B-GGUF",+ messages = [_chat_image_request(url).messages[0] for url in turns],+ )++ with pytest.raises(HTTPException) as exc:+ asyncio.run(inference_route.openai_chat_completions(payload, object(), "tester"))++ assert exc.value.detail == "load failed"+ assert len(recorder.calls) == 1+++def test_chat_validates_a_legacy_image_beside_a_remote_one_before_non_gguf_switch(monkeypatch):+ backend, recorder = _wire_image_switch_target(monkeypatch, target_is_gguf = False)+ monkeypatch.setattr(inference_route, "_local_target_may_take_several_images", lambda *_a: True)+ payload = _chat_image_request("https://example.com/0.png")+ payload.image_base64 = "Zm9v"++ with pytest.raises(HTTPException) as exc:+ asyncio.run(inference_route.openai_chat_completions(payload, object(), "tester"))++ assert exc.value.detail == "Failed to decode image"+ assert recorder.calls == []+ assert backend.model_identifier == "org/A-GGUF"+++def test_chat_refuses_a_remote_image_beside_a_legacy_one_before_non_gguf_switch(monkeypatch):backend, recorder = _wire_image_switch_target(monkeypatch, target_is_gguf = False)+ reply = _chat_image_request("https://example.com/0.png").messages[0]+ reply.role = "assistant"payload = _chat_request(model = "org/B-GGUF",messages = [- ChatMessage(- role = "user",- content = [- ImageContentPart(- type = "image_url",- image_url = ImageUrl(url = url),- )- ],- )+ ChatMessage(role = "user", content = "hi"),+ reply,+ ChatMessage(role = "user", content = "and?"),],+ image_base64 = "aGVsbG8=",)with pytest.raises(HTTPException) as exc:asyncio.run(inference_route.openai_chat_completions(payload, object(), "tester"))- assert exc.value.status_code == 400- assert exc.value.detail == detail+ assert "one image per message" in exc.value.detailassert recorder.calls == []assert backend.model_identifier == "org/A-GGUF"diff --git a/studio/backend/tests/test_openai_tool_passthrough.py b/studio/backend/tests/test_openai_tool_passthrough.pyindex e925f213463..38f324b1192 100644--- a/studio/backend/tests/test_openai_tool_passthrough.py+++ b/studio/backend/tests/test_openai_tool_passthrough.py@@ -1138,7 +1138,7 @@ class TestChatCompletionRequestToolFields:assert monitor.active_count() == 0def test_an_unreadable_part_alone_is_still_answered(self, monkeypatch):- """Control: one remote image and no top-level image is not a multi-image+ """Control: one payloadless image and no top-level image is not a multi-imagecall, and clients relying on that text answer must keep it."""monitor = ApiMonitor(max_entries = 3)client, backend = self._standard_vision_client(monkeypatch, monitor)@@ -1150,7 +1150,7 @@ class TestChatCompletionRequestToolFields:"role": "user","content": [{"type": "text", "text": "x"},- {"type": "image_url", "image_url": {"url": "https://e.com/a.png"}},+ {"type": "image_url", "image_url": {"url": "data:image/png;base64,"}},],}]@@ -1384,9 +1384,10 @@ class TestChatCompletionRequestToolFields:assert backend.generated[0]["image"] is not Noneassert monitor.active_count() == 0- def test_a_single_undecodable_image_is_left_alone(self, monkeypatch):- """One remote image is not a multi-image call. Clients that pass a remote- URL today get a text answer, and this guard must not turn that into a 400."""+ def test_an_unfetchable_remote_image_closes_the_monitor_row(self, monkeypatch):+ import core.inference.external_provider as ep++ monkeypatch.setattr(ep, "safe_fetch_remote_image_sync", lambda *_a, **_k: None)monitor = ApiMonitor(max_entries = 3)client, backend = self._standard_vision_client(monkeypatch, monitor)resp = client.post(@@ -1407,9 +1408,9 @@ class TestChatCompletionRequestToolFields:},)- assert resp.status_code == 200- assert len(backend.generated) == 1- assert backend.generated[0]["image"] is None+ assert resp.status_code == 400+ assert "Could not fetch the remote image URL" in resp.text+ assert backend.generated == []assert monitor.active_count() == 0def test_a_text_only_model_closes_the_monitor_row(self, monkeypatch):diff --git a/studio/backend/tests/test_remote_image_url_fetch_behaviour.py b/studio/backend/tests/test_remote_image_url_fetch_behaviour.pyindex 57ede0dcca2..f7dda71bce3 100644--- a/studio/backend/tests/test_remote_image_url_fetch_behaviour.py+++ b/studio/backend/tests/test_remote_image_url_fetch_behaviour.py@@ -27,6 +27,7 @@ from auth.authentication import get_current_subject # noqa: E402import core.inference.external_provider as external_provider # noqa: E402import routes.inference as inference_route # noqa: E402+from . import test_sf_client_tools_passthrough as safetensors # noqa: E402from .llama_backend_double import FakeLlamaCppBackend # noqa: E402_METADATA = "http://169.254.169.254/latest/meta-data/"@@ -359,6 +360,128 @@ class TestRefusalsAreRealRefusals:assert r.status_code == 400, r.text+class TestSafetensorsAndMlxFetchToo:+ def _post(+ self,+ monkeypatch,+ body = None,+ *,+ is_vision = True,+ several = False,+ ):+ backend = safetensors._ScriptedBackend(safetensors._fixed("a cat"))+ backend.models["sf-model"]["is_vision"] = is_vision+ backend.models["sf-model"]["chat_template_info"]["accepts_multiple_images"] = several+ client = _client(monkeypatch, safetensors._llama_stub())+ safetensors._install(monkeypatch, backend)+ body = body or _chat_body("https://images.example/cat.webp", model = "sf-model")+ return backend, client.post("/v1/chat/completions", json = body)++ @staticmethod+ def _fetch_all_but_old(monkeypatch):+ fetched = []++ def _fetch(url, *_a, **_k):+ fetched.append(url)+ return None if "old" in url else ("image/webp", _webp_b64())++ monkeypatch.setattr(external_provider, "safe_fetch_remote_image_sync", _fetch)+ return fetched++ @staticmethod+ def _forbid_fetch(monkeypatch):+ def _never(*_a, **_k):+ raise AssertionError("this request must not wait on a fetch")++ monkeypatch.setattr(external_provider, "safe_fetch_remote_image_sync", _never)++ @staticmethod+ def _turn(url, role = "user"):+ return {**_chat_body(url, model = "sf-model")["messages"][0], "role": role}++ def test_the_fetched_image_reaches_the_model(self, monkeypatch):+ monkeypatch.setattr(+ external_provider,+ "safe_fetch_remote_image_sync",+ lambda *_a, **_k: ("image/webp", _webp_b64()),+ )+ backend, r = self._post(monkeypatch)++ assert r.status_code == 200, r.text+ assert backend.calls[0]["image"] is not None+ assert backend.calls[0]["image"].size == (2, 2)++ def test_a_failed_fetch_refuses_rather_than_dropping_the_image(self, monkeypatch):+ monkeypatch.setattr(+ external_provider, "safe_fetch_remote_image_sync", lambda *_a, **_k: None+ )+ backend, r = self._post(monkeypatch)++ assert r.status_code == 400, r.text+ assert "Could not fetch the remote image URL" in r.text+ assert backend.calls == []++ @pytest.mark.parametrize("later", ["user", "assistant"])+ def test_only_the_image_a_single_image_model_reads_is_fetched(self, monkeypatch, later):+ fetched = self._fetch_all_but_old(monkeypatch)+ old = self._turn("https://images.example/old.webp")+ body = _chat_body("https://images.example/new.webp", model = "sf-model")+ if later == "user":+ body["messages"][:0] = [old, {"role": "assistant", "content": "ok"}]+ else:+ # A user's own image outranks a newer assistant one.+ body["messages"] += [{**old, "role": "assistant"}, {"role": "user", "content": "and?"}]+ backend, r = self._post(monkeypatch, body)++ assert r.status_code == 200, r.text+ assert fetched == ["https://images.example/new.webp"]+ assert backend.calls[0]["image"].size == (2, 2)++ def test_a_multi_image_model_fetches_only_user_turn_images(self, monkeypatch):+ fetched = self._fetch_all_but_old(monkeypatch)+ body = _chat_body("https://images.example/b.webp", model = "sf-model")+ body["messages"][:0] = [+ self._turn("https://images.example/a.webp"),+ self._turn("https://images.example/old.webp", "assistant"),+ ]+ backend, r = self._post(monkeypatch, body, several = True)++ assert r.status_code == 200, r.text+ assert fetched == ["https://images.example/a.webp", "https://images.example/b.webp"]+ assert len(backend.calls[0]["images"]) == 2++ @pytest.mark.parametrize("legacy", [False, True], ids = ["two_parts", "part_and_legacy"])+ def test_two_images_on_a_single_image_model_refuse_before_any_fetch(self, monkeypatch, legacy):+ self._forbid_fetch(monkeypatch)+ urls = ["https://images.example/a.webp", "https://images.example/b.webp"]+ body = _chat_body(*urls[: 1 if legacy else 2], model = "sf-model")+ if legacy:+ body["image_base64"] = _webp_b64()+ backend, r = self._post(monkeypatch, body)++ assert r.status_code == 400, r.text+ assert "one image per message" in r.text+ assert backend.calls == []++ @pytest.mark.parametrize("role", ["system", "developer"])+ def test_a_system_or_developer_image_does_not_refuse_a_text_only_model(self, monkeypatch, role):+ self._forbid_fetch(monkeypatch)+ body = _chat_body(model = "sf-model")+ body["messages"][:0] = [self._turn("https://images.example/logo.webp", role)]+ backend, r = self._post(monkeypatch, body, is_vision = False)++ assert r.status_code == 200, r.text+ assert len(backend.calls) == 1++ def test_a_text_only_model_refuses_before_any_fetch(self, monkeypatch):+ self._forbid_fetch(monkeypatch)+ backend, r = self._post(monkeypatch, is_vision = False)++ assert r.status_code == 400, r.text+ assert "text-only" in r.text+ assert backend.calls == []++class TestBudget:def test_the_per_request_byte_budget_bounds_a_long_thread(self, monkeypatch):b64 = _webp_b64()diff --git a/studio/backend/tests/test_sf_client_tools_passthrough.py b/studio/backend/tests/test_sf_client_tools_passthrough.pyindex 324cac6f9c7..ef04704f8c4 100644--- a/studio/backend/tests/test_sf_client_tools_passthrough.py+++ b/studio/backend/tests/test_sf_client_tools_passthrough.py@@ -1220,7 +1220,7 @@ def test_forced_tool_choice_narrows_templated_tools(monkeypatch):def test_multimodal_content_parts_flattened_for_local_template(monkeypatch):- # Remote image URLs leave image=None, so content arrives as a part LIST:+ # An image part with no payload leaves image=None, so content arrives as a part LIST:# text parts are kept, the image part dropped.backend = _ScriptedBackend(_fixed(_CALL_XML))payload = _request(@@ -1231,7 +1231,7 @@ def test_multimodal_content_parts_flattened_for_local_template(monkeypatch):{"type": "text", "text": "what is this?"},{"type": "image_url",- "image_url": {"url": "https://example.com/cat.png"},+ "image_url": {"url": "data:image/png;base64,"},},],)