Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: stop tool images breaking chats on text-only connection models (#17561)

Nilayded777ab91d7
diff --git a/studio/backend/core/inference/external_provider.py b/studio/backend/core/inference/external_provider.py
index f8cf9672161..a83c2585d93 100644
--- a/studio/backend/core/inference/external_provider.py
+++ b/studio/backend/core/inference/external_provider.py
@@ -84,6 +84,7 @@ _GEMINI_RESTRICTED_GENERATION = re.compile(r"^gemini-(?:3\.[78]-flash|4-argon)(?
_served_windows: dict[tuple[str, str, str, str], tuple[float, Optional[int]]] = {}
_served_templates: dict[tuple[str, str, str], tuple[float, Optional[str]]] = {}
_served_versions: dict[tuple[str, str], tuple[float, bool]] = {}
+_served_vision: dict[tuple[str, str, str, str], tuple[float, Optional[bool]]] = {}
# outlives the TTL: Ollama unloads an idle model after keep_alive and reloads it at the same window.
_ollama_last_windows: dict[tuple[str, str, str, str], int] = {}
@@ -7734,6 +7735,51 @@ class ExternalProviderClient:
_served_templates[key] = (now + _SERVED_WINDOW_TTL_S, template)
return template
+ async def _read_served_vision(self, model: str) -> Optional[bool]:
+ if self.provider_type == "ollama":
+ response = await _client().post(
+ _native_server_path(self.base_url, "/api/show"),
+ json = {"model": model},
+ headers = self._auth_headers(),
+ timeout = _SERVED_WINDOW_TIMEOUT_S,
+ )
+ response.raise_for_status()
+ body = response.json()
+ names = body.get("capabilities") if isinstance(body, dict) else None
+ return "vision" in names if isinstance(names, list) else None
+ response = await _client().get(
+ f"{self.base_url.removesuffix('/v1')}/props",
+ params = {"model": model},
+ headers = self._auth_headers(),
+ timeout = _SERVED_WINDOW_TIMEOUT_S,
+ )
+ response.raise_for_status()
+ body = response.json()
+ modalities = body.get("modalities") if isinstance(body, dict) else None
+ vision = modalities.get("vision") if isinstance(modalities, dict) else None
+ return vision if isinstance(vision, bool) else None
+
+ async def served_model_reads_images(self, model: str) -> Optional[bool]:
+ if self.provider_type not in ("ollama", "llama_cpp"):
+ return None
+ now = time.monotonic()
+ key = (self.provider_type, self.base_url, self._credential_digest(), model)
+ cached = _served_vision.get(key)
+ if cached is not None and cached[0] > now:
+ return cached[1]
+ vision = None
+ try:
+ vision = await self._read_served_vision(model)
+ except httpx.ConnectError as exc:
+ logger.info("No served vision flag from %s: %s", self.provider_type, type(exc).__name__)
+ return None
+ except (httpx.HTTPError, ValueError) as exc:
+ logger.info("No served vision flag from %s: %s", self.provider_type, type(exc).__name__)
+ for stale in [k for k, (expiry, _) in _served_vision.items() if expiry <= now]:
+ del _served_vision[stale]
+ _served_vision[key] = (now + _SERVED_WINDOW_TTL_S, vision)
+ return vision
+
@staticmethod
def _parse_gemini_models(payload: Any) -> list[dict[str, Any]]:
"""map Gemini models to OpenAI entries and exclude advertised embedding-only models."""
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 7a9d0553d29..499659f7b3e 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -29683,6 +29683,27 @@ async def _proxy_to_external_provider(
_supports_vision = (
managed.supports_vision if managed is not None else _pinfo.get("supports_vision", False)
)
+ client = await asyncio.to_thread(
+ ExternalProviderClient,
+ provider_type = provider_type,
+ base_url = base_url,
+ api_key = api_key,
+ api_type = api_type,
+ **({"reasoning_config": reasoning_config} if reasoning_config is not None else {}),
+ **({"extra_headers": extra_headers} if extra_headers else {}),
+ **({"managed_loopback": True} if managed is not None else {}),
+ )
+ # Metadata preflights (served context window, template) run before streaming starts.
+ if extra_headers:
+ client.set_header_context(payload.thread_id, model)
+ _takes_tool_images = _external_takes_mcp_images(provider_type, _supports_vision, model, _pinfo)
+ if (
+ _takes_tool_images
+ and provider_type in ("ollama", "llama_cpp")
+ and (studio_tool_loop or _messages_mention_mcp_images(payload.messages))
+ ):
+ # An unread flag keeps the image: a vision model must not lose it to a silent server.
+ _takes_tool_images = await client.served_model_reads_images(model) is not False
_external_promoted_parts: list = []
chat_messages = await _build_external_messages_async(
payload.messages,
@@ -29692,9 +29713,7 @@ async def _proxy_to_external_provider(
base_url = base_url,
api_type = api_type,
promoted_out = _external_promoted_parts,
- promote_mcp_images = _external_takes_mcp_images(
- provider_type, _supports_vision, model, _pinfo
- ),
+ promote_mcp_images = _takes_tool_images,
model = model,
preserve_thinking = payload.preserve_thinking,
)
@@ -29712,19 +29731,6 @@ async def _proxy_to_external_provider(
subject = current_subject,
)
- client = await asyncio.to_thread(
- ExternalProviderClient,
- provider_type = provider_type,
- base_url = base_url,
- api_key = api_key,
- api_type = api_type,
- **({"reasoning_config": reasoning_config} if reasoning_config is not None else {}),
- **({"extra_headers": extra_headers} if extra_headers else {}),
- **({"managed_loopback": True} if managed is not None else {}),
- )
- # Metadata preflights (served context window, template) run before streaming starts.
- if extra_headers:
- client.set_header_context(payload.thread_id, model)
_non_stream_custom_responses = (
provider_type == "custom" and api_type == "responses" and payload.stream is False
)
@@ -29745,9 +29751,7 @@ async def _proxy_to_external_provider(
if studio_tool_loop:
external_studio_tools = await _select_request_tools(
payload,
- supports_vision = _external_takes_mcp_images(
- provider_type, _supports_vision, model, _pinfo
- ),
+ supports_vision = _takes_tool_images,
tools_on = _effective_enable_tools(payload) is True,
mcp_allowed = bool(payload.mcp_enabled),
ui_events = _ui_events,
@@ -29953,9 +29957,7 @@ async def _proxy_to_external_provider(
model = model,
tool_choice = payload.tool_choice,
continue_final_message = _continue_final_message(payload),
- supports_vision = _external_takes_mcp_images(
- provider_type, _supports_vision, model, _pinfo
- ),
+ supports_vision = _takes_tool_images,
promoted_image_parts = tuple(_external_promoted_parts),
),
policy = ToolLoopPolicy(
diff --git a/studio/backend/tests/test_connection_tool_images.py b/studio/backend/tests/test_connection_tool_images.py
new file mode 100644
index 00000000000..487e09e2244
--- /dev/null
+++ b/studio/backend/tests/test_connection_tool_images.py
@@ -0,0 +1,174 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+import pytest
+from starlette.requests import Request
+
+from core.inference import external_provider as ep_mod
+from core.inference.mcp_images import SENTINEL
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from test_external_auto_compaction import _proxy # noqa: E402
+from test_external_provider_sampling_over_the_wire import _Handler # noqa: E402
+
+_PNG_B64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="
+
+
+class _CapabilityHandler(_Handler):
+ show: dict | None = None
+ props: dict | None = None
+ probes: list[str] = []
+
+ def _json(self, body: dict | None) -> None:
+ raw = json.dumps(body if body is not None else {"error": "not found"}).encode()
+ self.send_response(200 if body is not None else 404)
+ self.send_header("Content-Type", "application/json")
+ self.send_header("Content-Length", str(len(raw)))
+ self.end_headers()
+ self.wfile.write(raw)
+
+ def do_GET(self) -> None: # noqa: N802
+ type(self).probes.append(self.path)
+ self._json(self.props if self.path.startswith("/props") else None)
+
+ def do_POST(self) -> None: # noqa: N802
+ if self.path != "/api/show":
+ return super().do_POST()
+ type(self).probes.append(self.path)
+ self.rfile.read(int(self.headers.get("Content-Length") or 0))
+ self._json(self.show)
+
+
+def _server(**attrs):
+ return type("Capability", (_CapabilityHandler,), {"probes": [], **attrs})
+
+
[email protected](autouse = True)
+def _nothing_remembered(monkeypatch):
+ monkeypatch.setattr(ep_mod, "_served_vision", {}, raising = False)
+
+
+_TEXT_ONLY = {
+ "ollama": {"show": {"capabilities": ["completion", "tools", "thinking"]}},
+ "llama_cpp": {"props": {"modalities": {"vision": False, "audio": False}}},
+}
+_VISION = {
+ "ollama": {"show": {"capabilities": ["completion", "vision", "tools"]}},
+ "llama_cpp": {"props": {"modalities": {"vision": True, "audio": False}}},
+}
+_SILENT = {
+ "ollama": {"show": {"details": {"family": "qwen3"}}},
+ "llama_cpp": {"props": {"default_generation_settings": {"n_ctx": 8192}}},
+}
+
+
+def _chat_with_a_viewed_image() -> list[dict]:
+ call = {"id": "call_0", "type": "function"}
+ call["function"] = {"name": "view_image", "arguments": '{"path": "chart.png"}'}
+ envelope = json.dumps([{"data": _PNG_B64, "mimeType": "image/png"}])
+ return [
+ {"role": "user", "content": "make a bar chart and check it"},
+ {"role": "assistant", "content": "", "tool_calls": [call]},
+ {
+ "role": "tool",
+ "tool_call_id": "call_0",
+ "name": "view_image",
+ "content": f"Opened image.\n{SENTINEL}{envelope}",
+ },
+ {"role": "assistant", "content": "The chart is saved."},
+ {"role": "user", "content": "thanks! what is 2+2?"},
+ ]
+
+
+def _image_parts(sent: dict) -> list[dict]:
+ return [
+ part
+ for message in sent["messages"]
+ if isinstance(message.get("content"), list)
+ for part in message["content"]
+ if part.get("type") == "image_url"
+ ]
+
+
+def _tool_names(sent: dict) -> set[str]:
+ return {tool["function"]["name"] for tool in sent.get("tools") or []}
+
+
+def connected(monkeypatch):
+ async def is_disconnected(self) -> bool:
+ return False
+
+ monkeypatch.setattr(Request, "is_disconnected", is_disconnected)
+
+
[email protected]("provider_type", ["ollama", "llama_cpp"])
+def test_a_text_only_connection_model_gets_the_tool_result_without_its_image(provider_type):
+ _, sent = _proxy(
+ provider_type, _chat_with_a_viewed_image(), handler = _server(**_TEXT_ONLY[provider_type])
+ )
+ assert not _image_parts(sent)
+ assert SENTINEL not in json.dumps(sent)
+ [result] = [m for m in sent["messages"] if m["role"] == "tool"]
+ assert result["content"] == "Opened image."
+ assert sent["messages"][-1]["content"] == "thanks! what is 2+2?"
+
+
[email protected]("provider_type", ["ollama", "llama_cpp"])
+def test_a_vision_connection_model_still_gets_the_tool_image(provider_type):
+ _, sent = _proxy(
+ provider_type, _chat_with_a_viewed_image(), handler = _server(**_VISION[provider_type])
+ )
+ [image] = _image_parts(sent)
+ assert image["image_url"]["url"].startswith("data:image/")
+ assert SENTINEL not in json.dumps(sent)
+
+
[email protected]("provider_type", ["ollama", "llama_cpp"])
[email protected]("answer", ["silent", "missing"])
+def test_a_server_that_does_not_report_vision_keeps_the_tool_image(provider_type, answer):
+ handler = _server(**_SILENT[provider_type]) if answer == "silent" else _server()
+ _, sent = _proxy(provider_type, _chat_with_a_viewed_image(), handler = handler)
+ assert len(_image_parts(sent)) == 1
+
+
[email protected]("provider_type", ["ollama", "llama_cpp"])
+def test_view_image_is_offered_only_to_a_connection_model_that_reads_images(
+ provider_type, connected
+):
+ tools = {"enable_tools": True, "enabled_tools": ["python", "view_image"]}
+ tools["permission_mode"] = "off"
+ question = [{"role": "user", "content": "plot the populations"}]
+ _, text_only = _proxy(
+ provider_type, question, handler = _server(**_TEXT_ONLY[provider_type]), **tools
+ )
+ _, vision = _proxy(provider_type, question, handler = _server(**_VISION[provider_type]), **tools)
+ assert _tool_names(text_only) == {"python"}
+ assert _tool_names(vision) == {"python", "view_image"}
+
+
+def test_a_chat_without_tools_or_tool_images_does_not_ask_the_server():
+ handler = _server(**_TEXT_ONLY["ollama"])
+ _proxy("ollama", [{"role": "user", "content": "hello"}], handler = handler)
+ assert handler.probes == []
+
+
+def test_the_model_asked_about_is_the_one_chatting():
+ class Show(_CapabilityHandler):
+ asked: list[dict] = []
+
+ def do_POST(self) -> None: # noqa: N802
+ if self.path != "/api/show":
+ return super().do_POST()
+ raw = self.rfile.read(int(self.headers.get("Content-Length") or 0))
+ type(self).asked.append(json.loads(raw))
+ self._json({"capabilities": ["completion"]})
+
+ _proxy("ollama", _chat_with_a_viewed_image(), handler = Show, model = "qwen3:8b")
+ assert Show.asked == [{"model": "qwen3:8b"}]
diff --git a/studio/backend/tests/test_external_hosted_tool_passthrough.py b/studio/backend/tests/test_external_hosted_tool_passthrough.py
index bf11340b448..48f9a4c6866 100644
--- a/studio/backend/tests/test_external_hosted_tool_passthrough.py
+++ b/studio/backend/tests/test_external_hosted_tool_passthrough.py
@@ -48,6 +48,9 @@ class FakeExternalClient:
async def served_chat_template(self, model):
return FakeExternalClient.served_template
+ async def served_model_reads_images(self, model):
+ return None
+
async def close(self):
return None
diff --git a/studio/backend/tests/test_external_hosted_tool_selection.py b/studio/backend/tests/test_external_hosted_tool_selection.py
index 0165939ca25..a461186633e 100644
--- a/studio/backend/tests/test_external_hosted_tool_selection.py
+++ b/studio/backend/tests/test_external_hosted_tool_selection.py
@@ -34,6 +34,9 @@ class _FakeExternalClient:
async def served_chat_template(self, model):
return None
+ async def served_model_reads_images(self, model):
+ return None
+
async def close(self):
return None