Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: stop tool images breaking chats on text-only connection models (#17561)
Nilayded777ab91d7
diff --git a/studio/backend/core/inference/external_provider.py b/studio/backend/core/inference/external_provider.pyindex f8cf9672161..a83c2585d93 100644--- a/studio/backend/core/inference/external_provider.py+++ b/studio/backend/core/inference/external_provider.py@@ -84,6 +84,7 @@ _GEMINI_RESTRICTED_GENERATION = re.compile(r"^gemini-(?:3\.[78]-flash|4-argon)(?_served_windows: dict[tuple[str, str, str, str], tuple[float, Optional[int]]] = {}_served_templates: dict[tuple[str, str, str], tuple[float, Optional[str]]] = {}_served_versions: dict[tuple[str, str], tuple[float, bool]] = {}+_served_vision: dict[tuple[str, str, str, str], tuple[float, Optional[bool]]] = {}# outlives the TTL: Ollama unloads an idle model after keep_alive and reloads it at the same window._ollama_last_windows: dict[tuple[str, str, str, str], int] = {}@@ -7734,6 +7735,51 @@ class ExternalProviderClient:_served_templates[key] = (now + _SERVED_WINDOW_TTL_S, template)return template+ async def _read_served_vision(self, model: str) -> Optional[bool]:+ if self.provider_type == "ollama":+ response = await _client().post(+ _native_server_path(self.base_url, "/api/show"),+ json = {"model": model},+ headers = self._auth_headers(),+ timeout = _SERVED_WINDOW_TIMEOUT_S,+ )+ response.raise_for_status()+ body = response.json()+ names = body.get("capabilities") if isinstance(body, dict) else None+ return "vision" in names if isinstance(names, list) else None+ response = await _client().get(+ f"{self.base_url.removesuffix('/v1')}/props",+ params = {"model": model},+ headers = self._auth_headers(),+ timeout = _SERVED_WINDOW_TIMEOUT_S,+ )+ response.raise_for_status()+ body = response.json()+ modalities = body.get("modalities") if isinstance(body, dict) else None+ vision = modalities.get("vision") if isinstance(modalities, dict) else None+ return vision if isinstance(vision, bool) else None++ async def served_model_reads_images(self, model: str) -> Optional[bool]:+ if self.provider_type not in ("ollama", "llama_cpp"):+ return None+ now = time.monotonic()+ key = (self.provider_type, self.base_url, self._credential_digest(), model)+ cached = _served_vision.get(key)+ if cached is not None and cached[0] > now:+ return cached[1]+ vision = None+ try:+ vision = await self._read_served_vision(model)+ except httpx.ConnectError as exc:+ logger.info("No served vision flag from %s: %s", self.provider_type, type(exc).__name__)+ return None+ except (httpx.HTTPError, ValueError) as exc:+ logger.info("No served vision flag from %s: %s", self.provider_type, type(exc).__name__)+ for stale in [k for k, (expiry, _) in _served_vision.items() if expiry <= now]:+ del _served_vision[stale]+ _served_vision[key] = (now + _SERVED_WINDOW_TTL_S, vision)+ return vision+@staticmethoddef _parse_gemini_models(payload: Any) -> list[dict[str, Any]]:"""map Gemini models to OpenAI entries and exclude advertised embedding-only models."""diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 7a9d0553d29..499659f7b3e 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -29683,6 +29683,27 @@ async def _proxy_to_external_provider(_supports_vision = (managed.supports_vision if managed is not None else _pinfo.get("supports_vision", False))+ client = await asyncio.to_thread(+ ExternalProviderClient,+ provider_type = provider_type,+ base_url = base_url,+ api_key = api_key,+ api_type = api_type,+ **({"reasoning_config": reasoning_config} if reasoning_config is not None else {}),+ **({"extra_headers": extra_headers} if extra_headers else {}),+ **({"managed_loopback": True} if managed is not None else {}),+ )+ # Metadata preflights (served context window, template) run before streaming starts.+ if extra_headers:+ client.set_header_context(payload.thread_id, model)+ _takes_tool_images = _external_takes_mcp_images(provider_type, _supports_vision, model, _pinfo)+ if (+ _takes_tool_images+ and provider_type in ("ollama", "llama_cpp")+ and (studio_tool_loop or _messages_mention_mcp_images(payload.messages))+ ):+ # An unread flag keeps the image: a vision model must not lose it to a silent server.+ _takes_tool_images = await client.served_model_reads_images(model) is not False_external_promoted_parts: list = []chat_messages = await _build_external_messages_async(payload.messages,@@ -29692,9 +29713,7 @@ async def _proxy_to_external_provider(base_url = base_url,api_type = api_type,promoted_out = _external_promoted_parts,- promote_mcp_images = _external_takes_mcp_images(- provider_type, _supports_vision, model, _pinfo- ),+ promote_mcp_images = _takes_tool_images,model = model,preserve_thinking = payload.preserve_thinking,)@@ -29712,19 +29731,6 @@ async def _proxy_to_external_provider(subject = current_subject,)- client = await asyncio.to_thread(- ExternalProviderClient,- provider_type = provider_type,- base_url = base_url,- api_key = api_key,- api_type = api_type,- **({"reasoning_config": reasoning_config} if reasoning_config is not None else {}),- **({"extra_headers": extra_headers} if extra_headers else {}),- **({"managed_loopback": True} if managed is not None else {}),- )- # Metadata preflights (served context window, template) run before streaming starts.- if extra_headers:- client.set_header_context(payload.thread_id, model)_non_stream_custom_responses = (provider_type == "custom" and api_type == "responses" and payload.stream is False)@@ -29745,9 +29751,7 @@ async def _proxy_to_external_provider(if studio_tool_loop:external_studio_tools = await _select_request_tools(payload,- supports_vision = _external_takes_mcp_images(- provider_type, _supports_vision, model, _pinfo- ),+ supports_vision = _takes_tool_images,tools_on = _effective_enable_tools(payload) is True,mcp_allowed = bool(payload.mcp_enabled),ui_events = _ui_events,@@ -29953,9 +29957,7 @@ async def _proxy_to_external_provider(model = model,tool_choice = payload.tool_choice,continue_final_message = _continue_final_message(payload),- supports_vision = _external_takes_mcp_images(- provider_type, _supports_vision, model, _pinfo- ),+ supports_vision = _takes_tool_images,promoted_image_parts = tuple(_external_promoted_parts),),policy = ToolLoopPolicy(diff --git a/studio/backend/tests/test_connection_tool_images.py b/studio/backend/tests/test_connection_tool_images.pynew file mode 100644index 00000000000..487e09e2244--- /dev/null+++ b/studio/backend/tests/test_connection_tool_images.py@@ -0,0 +1,174 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++from __future__ import annotations++import json+import sys+from pathlib import Path++import pytest+from starlette.requests import Request++from core.inference import external_provider as ep_mod+from core.inference.mcp_images import SENTINEL++sys.path.insert(0, str(Path(__file__).resolve().parent))+from test_external_auto_compaction import _proxy # noqa: E402+from test_external_provider_sampling_over_the_wire import _Handler # noqa: E402++_PNG_B64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="+++class _CapabilityHandler(_Handler):+ show: dict | None = None+ props: dict | None = None+ probes: list[str] = []++ def _json(self, body: dict | None) -> None:+ raw = json.dumps(body if body is not None else {"error": "not found"}).encode()+ self.send_response(200 if body is not None else 404)+ self.send_header("Content-Type", "application/json")+ self.send_header("Content-Length", str(len(raw)))+ self.end_headers()+ self.wfile.write(raw)++ def do_GET(self) -> None: # noqa: N802+ type(self).probes.append(self.path)+ self._json(self.props if self.path.startswith("/props") else None)++ def do_POST(self) -> None: # noqa: N802+ if self.path != "/api/show":+ return super().do_POST()+ type(self).probes.append(self.path)+ self.rfile.read(int(self.headers.get("Content-Length") or 0))+ self._json(self.show)+++def _server(**attrs):+ return type("Capability", (_CapabilityHandler,), {"probes": [], **attrs})++[email protected](autouse = True)+def _nothing_remembered(monkeypatch):+ monkeypatch.setattr(ep_mod, "_served_vision", {}, raising = False)+++_TEXT_ONLY = {+ "ollama": {"show": {"capabilities": ["completion", "tools", "thinking"]}},+ "llama_cpp": {"props": {"modalities": {"vision": False, "audio": False}}},+}+_VISION = {+ "ollama": {"show": {"capabilities": ["completion", "vision", "tools"]}},+ "llama_cpp": {"props": {"modalities": {"vision": True, "audio": False}}},+}+_SILENT = {+ "ollama": {"show": {"details": {"family": "qwen3"}}},+ "llama_cpp": {"props": {"default_generation_settings": {"n_ctx": 8192}}},+}+++def _chat_with_a_viewed_image() -> list[dict]:+ call = {"id": "call_0", "type": "function"}+ call["function"] = {"name": "view_image", "arguments": '{"path": "chart.png"}'}+ envelope = json.dumps([{"data": _PNG_B64, "mimeType": "image/png"}])+ return [+ {"role": "user", "content": "make a bar chart and check it"},+ {"role": "assistant", "content": "", "tool_calls": [call]},+ {+ "role": "tool",+ "tool_call_id": "call_0",+ "name": "view_image",+ "content": f"Opened image.\n{SENTINEL}{envelope}",+ },+ {"role": "assistant", "content": "The chart is saved."},+ {"role": "user", "content": "thanks! what is 2+2?"},+ ]+++def _image_parts(sent: dict) -> list[dict]:+ return [+ part+ for message in sent["messages"]+ if isinstance(message.get("content"), list)+ for part in message["content"]+ if part.get("type") == "image_url"+ ]+++def _tool_names(sent: dict) -> set[str]:+ return {tool["function"]["name"] for tool in sent.get("tools") or []}+++def connected(monkeypatch):+ async def is_disconnected(self) -> bool:+ return False++ monkeypatch.setattr(Request, "is_disconnected", is_disconnected)++[email protected]("provider_type", ["ollama", "llama_cpp"])+def test_a_text_only_connection_model_gets_the_tool_result_without_its_image(provider_type):+ _, sent = _proxy(+ provider_type, _chat_with_a_viewed_image(), handler = _server(**_TEXT_ONLY[provider_type])+ )+ assert not _image_parts(sent)+ assert SENTINEL not in json.dumps(sent)+ [result] = [m for m in sent["messages"] if m["role"] == "tool"]+ assert result["content"] == "Opened image."+ assert sent["messages"][-1]["content"] == "thanks! what is 2+2?"++[email protected]("provider_type", ["ollama", "llama_cpp"])+def test_a_vision_connection_model_still_gets_the_tool_image(provider_type):+ _, sent = _proxy(+ provider_type, _chat_with_a_viewed_image(), handler = _server(**_VISION[provider_type])+ )+ [image] = _image_parts(sent)+ assert image["image_url"]["url"].startswith("data:image/")+ assert SENTINEL not in json.dumps(sent)++[email protected]("provider_type", ["ollama", "llama_cpp"])[email protected]("answer", ["silent", "missing"])+def test_a_server_that_does_not_report_vision_keeps_the_tool_image(provider_type, answer):+ handler = _server(**_SILENT[provider_type]) if answer == "silent" else _server()+ _, sent = _proxy(provider_type, _chat_with_a_viewed_image(), handler = handler)+ assert len(_image_parts(sent)) == 1++[email protected]("provider_type", ["ollama", "llama_cpp"])+def test_view_image_is_offered_only_to_a_connection_model_that_reads_images(+ provider_type, connected+):+ tools = {"enable_tools": True, "enabled_tools": ["python", "view_image"]}+ tools["permission_mode"] = "off"+ question = [{"role": "user", "content": "plot the populations"}]+ _, text_only = _proxy(+ provider_type, question, handler = _server(**_TEXT_ONLY[provider_type]), **tools+ )+ _, vision = _proxy(provider_type, question, handler = _server(**_VISION[provider_type]), **tools)+ assert _tool_names(text_only) == {"python"}+ assert _tool_names(vision) == {"python", "view_image"}+++def test_a_chat_without_tools_or_tool_images_does_not_ask_the_server():+ handler = _server(**_TEXT_ONLY["ollama"])+ _proxy("ollama", [{"role": "user", "content": "hello"}], handler = handler)+ assert handler.probes == []+++def test_the_model_asked_about_is_the_one_chatting():+ class Show(_CapabilityHandler):+ asked: list[dict] = []++ def do_POST(self) -> None: # noqa: N802+ if self.path != "/api/show":+ return super().do_POST()+ raw = self.rfile.read(int(self.headers.get("Content-Length") or 0))+ type(self).asked.append(json.loads(raw))+ self._json({"capabilities": ["completion"]})++ _proxy("ollama", _chat_with_a_viewed_image(), handler = Show, model = "qwen3:8b")+ assert Show.asked == [{"model": "qwen3:8b"}]diff --git a/studio/backend/tests/test_external_hosted_tool_passthrough.py b/studio/backend/tests/test_external_hosted_tool_passthrough.pyindex bf11340b448..48f9a4c6866 100644--- a/studio/backend/tests/test_external_hosted_tool_passthrough.py+++ b/studio/backend/tests/test_external_hosted_tool_passthrough.py@@ -48,6 +48,9 @@ class FakeExternalClient:async def served_chat_template(self, model):return FakeExternalClient.served_template+ async def served_model_reads_images(self, model):+ return None+async def close(self):return Nonediff --git a/studio/backend/tests/test_external_hosted_tool_selection.py b/studio/backend/tests/test_external_hosted_tool_selection.pyindex 0165939ca25..a461186633e 100644--- a/studio/backend/tests/test_external_hosted_tool_selection.py+++ b/studio/backend/tests/test_external_hosted_tool_selection.py@@ -34,6 +34,9 @@ class _FakeExternalClient:async def served_chat_template(self, model):return None+ async def served_model_reads_images(self, model):+ return None+async def close(self):return None