Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: read a remote image URL on safetensors and MLX models instead of ignoring it (#12243)

Nilay1783a35e2c36
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 0cc3661f50d..7dcdbdac5cd 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -8845,12 +8845,11 @@ async def _preflight_image_for_switch(
" GGUF build of it, which accepts several."
),
)
- # Non-GGUF backends only receive decoded base64 images.
- if not target_is_gguf and image_preflight.get("remote") and image_preflight.get("b64") is None:
- raise HTTPException(
- status_code = 400,
- detail = "Remote image URLs are not supported. Use a base64 data URL.",
- )
+ # The images the model reads are fetched after the load, so a scheme it refuses is refused now.
+ for scheme in image_preflight.get("fetch_schemes", {}).get(target_takes_several, ()):
+ rejection = _remote_image_scheme_rejection(scheme) if scheme else None
+ if not target_is_gguf and rejection is not None:
+ raise HTTPException(status_code = rejection[0], detail = rejection[1])
# Only what the single-image path selects: an architecture guess must not refuse the rest.
encoded_images = (
image_preflight.get("b64s", ()) if target_is_gguf else (image_preflight.get("b64"),)
@@ -26218,26 +26217,40 @@ async def produce_openai_chat_completions(
_images_on_turn = _images_in_last_user_message(payload.messages)
_legacy_image_distinct = _legacy_image_is_distinct(payload)
_local_image_payloads = _request_local_image_payloads(payload)
- _image_b64 = _pre_parsed[2] or payload.image_base64
- if _image_b64 is None and _local_image_payloads:
+ # A remote image the model reads is decoded after its fetch; no older image stands in for it.
+ _selected_image = _served_image_part(payload.messages)
+ _selects_remote = _remote_image_part(_selected_image)
+ _image_b64 = (
+ (payload.image_base64 if _legacy_image_distinct else None)
+ if _selects_remote
+ else _pre_parsed[2] or payload.image_base64
+ )
+ if _image_b64 is None and _local_image_payloads and not _selects_remote:
_image_b64 = _local_image_payloads[0]
- # Read the way the render reads it; separate from b64s, which GGUF validates unchanged.
- _admitted_payloads = _conversation_with_image_markers(payload.messages)[1]
+ # Read as the render reads it after the fetch; b64s stays as sent for GGUF to validate.
+ _served_messages = _remote_images_as_served(payload.messages)
+ _admitted_payloads = _conversation_with_image_markers(_served_messages)[1]
if _legacy_image_distinct:
_admitted_payloads = [*_admitted_payloads, payload.image_base64]
_image_preflight = {
"b64": _image_b64,
"b64s": _local_image_payloads,
"admitted": _admitted_payloads,
- "remote": _messages_have_remote_image(payload.messages),
"multiple": (
_images_on_turn + int(_legacy_image_distinct) > 1
- or bool(_pre_parsed[2] and _legacy_image_distinct)
+ or bool((_pre_parsed[2] or _selects_remote) and _legacy_image_distinct)
),
# Refused after the load whatever the target is, so refused before evicting for it.
"unservable_alongside": _newest_turn_shows_more_images_than_it_sends(
- payload.messages, _legacy_image_distinct
+ _served_messages, _legacy_image_distinct
),
+ "fetch_schemes": {
+ several: [
+ _image_url_scheme(part.image_url.url)
+ for part in _served_remote_parts(payload.messages, several)
+ ]
+ for several in (False, True)
+ },
}
# Defer the resident claim: chat has several post-switch capability checks that can still
@@ -26937,6 +26950,21 @@ async def produce_openai_chat_completions(
finally:
_tracker.__exit__(None, None, None)
+ if not using_gguf and _messages_have_remote_image(
+ m for m in payload.messages if m.role not in ("system", "developer")
+ ):
+ if not model_info.get("is_vision"):
+ raise _reject(
+ 400, "Image provided but current model is text-only. Load a vision model."
+ )
+ try:
+ await asyncio.to_thread(
+ _inline_served_remote_images, payload, _serves_several_images(backend)
+ )
+ except HTTPException as exc:
+ raise _reject(exc.status_code, exc.detail)
+ _pre_parsed = None
+
# ── Parse messages (handles multimodal content parts) ─────
# Reuse the pre-hook parse when auto-switch did it, else parse now.
if _pre_parsed is not None:
@@ -35259,7 +35287,7 @@ _REMOTE_IMAGE_REQUEST_DEADLINE_S = 60.0
_REMOTE_IMAGE_FETCH_REFUSAL = (
"Could not fetch the remote image URL. Send the image as a base64 data URL instead."
)
-# Token counting renders a fixed media marker and charges a flat per-image allowance.
+# Stands in for a remote image in token counts and in the switch preflight's image checks.
_COUNT_IMAGE_PLACEHOLDER = (
"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1Pe"
"AAAADElEQVR4nGNgYGAAAAAEAAH2FzhVAAAAAElFTkSuQmCC"
@@ -35319,6 +35347,29 @@ def _placeholder_remote_images_for_count(openai_messages: list[dict]) -> None:
image_url["url"] = _COUNT_IMAGE_PLACEHOLDER
+def _remote_images_as_served(messages) -> list[dict]:
+ def _fetchable(part) -> bool:
+ url = (
+ (part.get("image_url") or {}).get("url", "") if part.get("type") == "image_url" else ""
+ )
+ return _image_url_scheme(url) == "https"
+
+ served = []
+ for message in messages or ():
+ plain = message if isinstance(message, dict) else message.model_dump(exclude_none = True)
+ content = plain.get("content")
+ if isinstance(content, list):
+ content = [
+ {**part, "image_url": {"url": _COUNT_IMAGE_PLACEHOLDER}}
+ if _fetchable(part)
+ else part
+ for part in content
+ ]
+ plain = {**plain, "content": content}
+ served.append(plain)
+ return served
+
+
def _inline_remote_image_url(
url: str, scheme: str, budget_bytes: int, deadline: float
) -> tuple[str, int]:
@@ -35333,7 +35384,6 @@ def _inline_remote_image_url(
url,
"image/png",
max_bytes = min(_REMOTE_IMAGE_MAX_BYTES, budget_bytes),
- label = "llama-server image fetch",
deadline = deadline,
require_image_content_type = False,
)
@@ -35386,6 +35436,60 @@ def _inline_request_remote_images(payload) -> None:
part.image_url.url = fetches.inline(part.image_url.url)
+def _served_image_part(messages):
+ """The image part _extract_content_parts selects, counting a remote URL as fetched."""
+ latest = latest_user = None
+ for message in messages:
+ if message.role in ("system", "developer") or not isinstance(message.content, list):
+ continue
+ part = next(
+ (
+ p
+ for p in message.content
+ if isinstance(p, ImageContentPart)
+ and (
+ p.image_url.url.partition(",")[2]
+ if p.image_url.url.startswith("data:")
+ else _image_url_scheme(p.image_url.url)
+ )
+ ),
+ None,
+ )
+ if part is not None:
+ latest = part
+ if message.role == "user":
+ latest_user = part
+ return latest_user or latest
+
+
+def _remote_image_part(part) -> bool:
+ return part is not None and not part.image_url.url.startswith("data:")
+
+
+def _served_remote_parts(messages, several: bool) -> list:
+ # Only what the renderers read: the image _extract_content_parts selects, and on a
+ # multi-image model every user turn's (_conversation_with_image_markers).
+ selected = _served_image_part(messages)
+ return [
+ part
+ for message in messages
+ if isinstance(message.content, list)
+ for part in message.content
+ if isinstance(part, ImageContentPart)
+ and _remote_image_part(part)
+ and (part is selected or (several and message.role == "user"))
+ ]
+
+
+def _inline_served_remote_images(payload, several: bool) -> None:
+ images_on_turn = _images_in_last_user_message(payload.messages)
+ if not several and images_on_turn + int(_legacy_image_is_distinct(payload)) > 1:
+ return # The caller refuses it as "one image per message".
+ fetches = _RemoteImageFetches()
+ for part in _served_remote_parts(payload.messages, several):
+ part.image_url.url = fetches.inline(part.image_url.url)
+
+
def _normalize_openai_image_parts_for_llama(openai_messages: list[dict], on_image = None) -> bool:
"""Fetch and normalize image URLs for llama-server in place.
diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.py
index e66dc30f408..472fe237925 100644
--- a/studio/backend/tests/test_openai_auto_switch.py
+++ b/studio/backend/tests/test_openai_auto_switch.py
@@ -11045,39 +11045,130 @@ def _wire_image_switch_target(monkeypatch, *, target_is_gguf):
return backend, recorder
+def _chat_image_request(*urls):
+ return _chat_request(
+ model = "org/B-GGUF",
+ messages = [
+ ChatMessage(
+ role = "user",
+ content = [
+ ImageContentPart(type = "image_url", image_url = ImageUrl(url = url)) for url in urls
+ ],
+ )
+ ],
+ )
+
+
+def test_chat_rejects_an_empty_data_url_before_non_gguf_switch(monkeypatch):
+ backend, recorder = _wire_image_switch_target(monkeypatch, target_is_gguf = False)
+ payload = _chat_image_request("data:image/png;base64,")
+
+ with pytest.raises(HTTPException) as exc:
+ asyncio.run(inference_route.openai_chat_completions(payload, object(), "tester"))
+
+ assert exc.value.status_code == 400
+ assert exc.value.detail == "Failed to decode image"
+ assert recorder.calls == []
+ assert backend.model_identifier == "org/A-GGUF"
+
+
[email protected]("count", [1, 2])
+def test_chat_loads_a_non_gguf_target_for_remote_images(monkeypatch, count):
+ # The loaded model fetches the URLs as a resident one does, so the switch is not refused.
+ _, recorder = _wire_image_switch_target(monkeypatch, target_is_gguf = False)
+ monkeypatch.setattr(inference_route, "_local_target_may_take_several_images", lambda *_a: True)
+ recorder.fail = True
+ urls = [f"https://example.com/{index}.png" for index in range(count)]
+
+ with pytest.raises(HTTPException) as exc:
+ asyncio.run(
+ inference_route.openai_chat_completions(_chat_image_request(*urls), object(), "tester")
+ )
+
+ assert exc.value.detail == "load failed"
+ assert len(recorder.calls) == 1
+
+
@pytest.mark.parametrize(
- ("url", "detail"),
+ ("turns", "detail"),
[
- ("data:image/png;base64,", "Failed to decode image"),
+ ([["http://example.com/0.png"]], "Unsupported image URL scheme ('http:')"),
(
- "https://example.com/image.png",
- "Remote image URLs are not supported. Use a base64 data URL.",
+ [["https://example.com/0.png", "http://example.com/1.png"]],
+ "one image per message",
+ ),
+ (
+ [["http://example.com/0.png"], ["https://example.com/1.png"]],
+ "Unsupported image URL scheme ('http:')",
),
],
- ids = ["empty data url", "remote url"],
+ ids = ["alone", "beside https", "earlier turn"],
)
-def test_chat_rejects_unsupported_openai_images_before_non_gguf_switch(monkeypatch, url, detail):
+def test_chat_refuses_an_unfetchable_scheme_before_non_gguf_switch(monkeypatch, turns, detail):
+ backend, recorder = _wire_image_switch_target(monkeypatch, target_is_gguf = False)
+ monkeypatch.setattr(inference_route, "_local_target_may_take_several_images", lambda *_a: True)
+ payload = _chat_request(
+ model = "org/B-GGUF",
+ messages = [_chat_image_request(*urls).messages[0] for urls in turns],
+ )
+
+ with pytest.raises(HTTPException) as exc:
+ asyncio.run(inference_route.openai_chat_completions(payload, object(), "tester"))
+
+ assert detail in exc.value.detail
+ assert recorder.calls == []
+ assert backend.model_identifier == "org/A-GGUF"
+
+
+def test_chat_leaves_an_unread_older_image_to_the_loaded_model(monkeypatch):
+ # The newer remote image is the one the model reads, so the older one is not validated.
+ _, recorder = _wire_image_switch_target(monkeypatch, target_is_gguf = False)
+ recorder.fail = True
+ turns = ["data:image/png;base64,Zm9v", "https://example.com/0.png"]
+ payload = _chat_request(
+ model = "org/B-GGUF",
+ messages = [_chat_image_request(url).messages[0] for url in turns],
+ )
+
+ with pytest.raises(HTTPException) as exc:
+ asyncio.run(inference_route.openai_chat_completions(payload, object(), "tester"))
+
+ assert exc.value.detail == "load failed"
+ assert len(recorder.calls) == 1
+
+
+def test_chat_validates_a_legacy_image_beside_a_remote_one_before_non_gguf_switch(monkeypatch):
+ backend, recorder = _wire_image_switch_target(monkeypatch, target_is_gguf = False)
+ monkeypatch.setattr(inference_route, "_local_target_may_take_several_images", lambda *_a: True)
+ payload = _chat_image_request("https://example.com/0.png")
+ payload.image_base64 = "Zm9v"
+
+ with pytest.raises(HTTPException) as exc:
+ asyncio.run(inference_route.openai_chat_completions(payload, object(), "tester"))
+
+ assert exc.value.detail == "Failed to decode image"
+ assert recorder.calls == []
+ assert backend.model_identifier == "org/A-GGUF"
+
+
+def test_chat_refuses_a_remote_image_beside_a_legacy_one_before_non_gguf_switch(monkeypatch):
backend, recorder = _wire_image_switch_target(monkeypatch, target_is_gguf = False)
+ reply = _chat_image_request("https://example.com/0.png").messages[0]
+ reply.role = "assistant"
payload = _chat_request(
model = "org/B-GGUF",
messages = [
- ChatMessage(
- role = "user",
- content = [
- ImageContentPart(
- type = "image_url",
- image_url = ImageUrl(url = url),
- )
- ],
- )
+ ChatMessage(role = "user", content = "hi"),
+ reply,
+ ChatMessage(role = "user", content = "and?"),
],
+ image_base64 = "aGVsbG8=",
)
with pytest.raises(HTTPException) as exc:
asyncio.run(inference_route.openai_chat_completions(payload, object(), "tester"))
- assert exc.value.status_code == 400
- assert exc.value.detail == detail
+ assert "one image per message" in exc.value.detail
assert recorder.calls == []
assert backend.model_identifier == "org/A-GGUF"
diff --git a/studio/backend/tests/test_openai_tool_passthrough.py b/studio/backend/tests/test_openai_tool_passthrough.py
index e925f213463..38f324b1192 100644
--- a/studio/backend/tests/test_openai_tool_passthrough.py
+++ b/studio/backend/tests/test_openai_tool_passthrough.py
@@ -1138,7 +1138,7 @@ class TestChatCompletionRequestToolFields:
assert monitor.active_count() == 0
def test_an_unreadable_part_alone_is_still_answered(self, monkeypatch):
- """Control: one remote image and no top-level image is not a multi-image
+ """Control: one payloadless image and no top-level image is not a multi-image
call, and clients relying on that text answer must keep it."""
monitor = ApiMonitor(max_entries = 3)
client, backend = self._standard_vision_client(monkeypatch, monitor)
@@ -1150,7 +1150,7 @@ class TestChatCompletionRequestToolFields:
"role": "user",
"content": [
{"type": "text", "text": "x"},
- {"type": "image_url", "image_url": {"url": "https://e.com/a.png"}},
+ {"type": "image_url", "image_url": {"url": "data:image/png;base64,"}},
],
}
]
@@ -1384,9 +1384,10 @@ class TestChatCompletionRequestToolFields:
assert backend.generated[0]["image"] is not None
assert monitor.active_count() == 0
- def test_a_single_undecodable_image_is_left_alone(self, monkeypatch):
- """One remote image is not a multi-image call. Clients that pass a remote
- URL today get a text answer, and this guard must not turn that into a 400."""
+ def test_an_unfetchable_remote_image_closes_the_monitor_row(self, monkeypatch):
+ import core.inference.external_provider as ep
+
+ monkeypatch.setattr(ep, "safe_fetch_remote_image_sync", lambda *_a, **_k: None)
monitor = ApiMonitor(max_entries = 3)
client, backend = self._standard_vision_client(monkeypatch, monitor)
resp = client.post(
@@ -1407,9 +1408,9 @@ class TestChatCompletionRequestToolFields:
},
)
- assert resp.status_code == 200
- assert len(backend.generated) == 1
- assert backend.generated[0]["image"] is None
+ assert resp.status_code == 400
+ assert "Could not fetch the remote image URL" in resp.text
+ assert backend.generated == []
assert monitor.active_count() == 0
def test_a_text_only_model_closes_the_monitor_row(self, monkeypatch):
diff --git a/studio/backend/tests/test_remote_image_url_fetch_behaviour.py b/studio/backend/tests/test_remote_image_url_fetch_behaviour.py
index 57ede0dcca2..f7dda71bce3 100644
--- a/studio/backend/tests/test_remote_image_url_fetch_behaviour.py
+++ b/studio/backend/tests/test_remote_image_url_fetch_behaviour.py
@@ -27,6 +27,7 @@ from auth.authentication import get_current_subject # noqa: E402
import core.inference.external_provider as external_provider # noqa: E402
import routes.inference as inference_route # noqa: E402
+from . import test_sf_client_tools_passthrough as safetensors # noqa: E402
from .llama_backend_double import FakeLlamaCppBackend # noqa: E402
_METADATA = "http://169.254.169.254/latest/meta-data/"
@@ -359,6 +360,128 @@ class TestRefusalsAreRealRefusals:
assert r.status_code == 400, r.text
+class TestSafetensorsAndMlxFetchToo:
+ def _post(
+ self,
+ monkeypatch,
+ body = None,
+ *,
+ is_vision = True,
+ several = False,
+ ):
+ backend = safetensors._ScriptedBackend(safetensors._fixed("a cat"))
+ backend.models["sf-model"]["is_vision"] = is_vision
+ backend.models["sf-model"]["chat_template_info"]["accepts_multiple_images"] = several
+ client = _client(monkeypatch, safetensors._llama_stub())
+ safetensors._install(monkeypatch, backend)
+ body = body or _chat_body("https://images.example/cat.webp", model = "sf-model")
+ return backend, client.post("/v1/chat/completions", json = body)
+
+ @staticmethod
+ def _fetch_all_but_old(monkeypatch):
+ fetched = []
+
+ def _fetch(url, *_a, **_k):
+ fetched.append(url)
+ return None if "old" in url else ("image/webp", _webp_b64())
+
+ monkeypatch.setattr(external_provider, "safe_fetch_remote_image_sync", _fetch)
+ return fetched
+
+ @staticmethod
+ def _forbid_fetch(monkeypatch):
+ def _never(*_a, **_k):
+ raise AssertionError("this request must not wait on a fetch")
+
+ monkeypatch.setattr(external_provider, "safe_fetch_remote_image_sync", _never)
+
+ @staticmethod
+ def _turn(url, role = "user"):
+ return {**_chat_body(url, model = "sf-model")["messages"][0], "role": role}
+
+ def test_the_fetched_image_reaches_the_model(self, monkeypatch):
+ monkeypatch.setattr(
+ external_provider,
+ "safe_fetch_remote_image_sync",
+ lambda *_a, **_k: ("image/webp", _webp_b64()),
+ )
+ backend, r = self._post(monkeypatch)
+
+ assert r.status_code == 200, r.text
+ assert backend.calls[0]["image"] is not None
+ assert backend.calls[0]["image"].size == (2, 2)
+
+ def test_a_failed_fetch_refuses_rather_than_dropping_the_image(self, monkeypatch):
+ monkeypatch.setattr(
+ external_provider, "safe_fetch_remote_image_sync", lambda *_a, **_k: None
+ )
+ backend, r = self._post(monkeypatch)
+
+ assert r.status_code == 400, r.text
+ assert "Could not fetch the remote image URL" in r.text
+ assert backend.calls == []
+
+ @pytest.mark.parametrize("later", ["user", "assistant"])
+ def test_only_the_image_a_single_image_model_reads_is_fetched(self, monkeypatch, later):
+ fetched = self._fetch_all_but_old(monkeypatch)
+ old = self._turn("https://images.example/old.webp")
+ body = _chat_body("https://images.example/new.webp", model = "sf-model")
+ if later == "user":
+ body["messages"][:0] = [old, {"role": "assistant", "content": "ok"}]
+ else:
+ # A user's own image outranks a newer assistant one.
+ body["messages"] += [{**old, "role": "assistant"}, {"role": "user", "content": "and?"}]
+ backend, r = self._post(monkeypatch, body)
+
+ assert r.status_code == 200, r.text
+ assert fetched == ["https://images.example/new.webp"]
+ assert backend.calls[0]["image"].size == (2, 2)
+
+ def test_a_multi_image_model_fetches_only_user_turn_images(self, monkeypatch):
+ fetched = self._fetch_all_but_old(monkeypatch)
+ body = _chat_body("https://images.example/b.webp", model = "sf-model")
+ body["messages"][:0] = [
+ self._turn("https://images.example/a.webp"),
+ self._turn("https://images.example/old.webp", "assistant"),
+ ]
+ backend, r = self._post(monkeypatch, body, several = True)
+
+ assert r.status_code == 200, r.text
+ assert fetched == ["https://images.example/a.webp", "https://images.example/b.webp"]
+ assert len(backend.calls[0]["images"]) == 2
+
+ @pytest.mark.parametrize("legacy", [False, True], ids = ["two_parts", "part_and_legacy"])
+ def test_two_images_on_a_single_image_model_refuse_before_any_fetch(self, monkeypatch, legacy):
+ self._forbid_fetch(monkeypatch)
+ urls = ["https://images.example/a.webp", "https://images.example/b.webp"]
+ body = _chat_body(*urls[: 1 if legacy else 2], model = "sf-model")
+ if legacy:
+ body["image_base64"] = _webp_b64()
+ backend, r = self._post(monkeypatch, body)
+
+ assert r.status_code == 400, r.text
+ assert "one image per message" in r.text
+ assert backend.calls == []
+
+ @pytest.mark.parametrize("role", ["system", "developer"])
+ def test_a_system_or_developer_image_does_not_refuse_a_text_only_model(self, monkeypatch, role):
+ self._forbid_fetch(monkeypatch)
+ body = _chat_body(model = "sf-model")
+ body["messages"][:0] = [self._turn("https://images.example/logo.webp", role)]
+ backend, r = self._post(monkeypatch, body, is_vision = False)
+
+ assert r.status_code == 200, r.text
+ assert len(backend.calls) == 1
+
+ def test_a_text_only_model_refuses_before_any_fetch(self, monkeypatch):
+ self._forbid_fetch(monkeypatch)
+ backend, r = self._post(monkeypatch, is_vision = False)
+
+ assert r.status_code == 400, r.text
+ assert "text-only" in r.text
+ assert backend.calls == []
+
+
class TestBudget:
def test_the_per_request_byte_budget_bounds_a_long_thread(self, monkeypatch):
b64 = _webp_b64()
diff --git a/studio/backend/tests/test_sf_client_tools_passthrough.py b/studio/backend/tests/test_sf_client_tools_passthrough.py
index 324cac6f9c7..ef04704f8c4 100644
--- a/studio/backend/tests/test_sf_client_tools_passthrough.py
+++ b/studio/backend/tests/test_sf_client_tools_passthrough.py
@@ -1220,7 +1220,7 @@ def test_forced_tool_choice_narrows_templated_tools(monkeypatch):
def test_multimodal_content_parts_flattened_for_local_template(monkeypatch):
- # Remote image URLs leave image=None, so content arrives as a part LIST:
+ # An image part with no payload leaves image=None, so content arrives as a part LIST:
# text parts are kept, the image part dropped.
backend = _ScriptedBackend(_fixed(_CALL_XML))
payload = _request(
@@ -1231,7 +1231,7 @@ def test_multimodal_content_parts_flattened_for_local_template(monkeypatch):
{"type": "text", "text": "what is this?"},
{
"type": "image_url",
- "image_url": {"url": "https://example.com/cat.png"},
+ "image_url": {"url": "data:image/png;base64,"},
},
],
)