Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: compact long chats on an Ollama connection to its loaded window (#17457)
Nilaye550e39fcb3d
diff --git a/studio/backend/core/inference/external_provider.py b/studio/backend/core/inference/external_provider.pyindex 563828aeab9..d05932aac41 100644--- a/studio/backend/core/inference/external_provider.py+++ b/studio/backend/core/inference/external_provider.py@@ -75,15 +75,17 @@ def _json_object_refused_for_schema(# launch-time windows are absent from catalogues; custom covers unregistered self-hosted servers.-_SERVED_WINDOW_PROVIDERS = frozenset({"vllm", "llama_cpp", "custom"})+_SERVED_WINDOW_PROVIDERS = frozenset({"vllm", "llama_cpp", "custom", "ollama"})# refresh within one or two turns to detect server restarts._SERVED_WINDOW_TTL_S = 60.0_SERVED_WINDOW_TIMEOUT_S = 5.0_GEMINI_MAX_MODEL_PAGES = 20_GEMINI_RESTRICTED_GENERATION = re.compile(r"^gemini-(?:3\.[78]-flash|4-argon)(?:-|$)")-_served_windows: dict[tuple[str, str, str], tuple[float, Optional[int]]] = {}+_served_windows: dict[tuple[str, str, str, str], tuple[float, Optional[int]]] = {}_served_templates: dict[tuple[str, str, str], tuple[float, Optional[str]]] = {}_served_versions: dict[tuple[str, str], tuple[float, bool]] = {}+# outlives the TTL: Ollama unloads an idle model after keep_alive and reloads it at the same window.+_ollama_last_windows: dict[tuple[str, str, str, str], int] = {}def _positive_int(value: Any) -> Optional[int]:@@ -196,6 +198,12 @@ def _append_provider_path(base_url: str, endpoint: str) -> str:return urlunsplit((parts.scheme, parts.netloc, path, query, parts.fragment))+def _native_server_path(base_url: str, endpoint: str) -> str:+ parts = urlsplit(base_url)+ root = urlunsplit(parts._replace(path = parts.path.rstrip("/").removesuffix("/v1")))+ return _append_provider_path(root, endpoint)++def caches_at_the_last_block(provider_type: Optional[str], model: Optional[str], enable_prompt_caching: Optional[bool]) -> bool:@@ -7553,7 +7561,27 @@ class ExternalProviderClient:token = page.get("nextPageToken")return {**data, "models": models}+ async def _read_ollama_loaded_window(self, model: str) -> Optional[int]:+ response = await _client().get(+ _native_server_path(self.base_url, "/api/ps"),+ headers = self._auth_headers(),+ timeout = _SERVED_WINDOW_TIMEOUT_S,+ )+ response.raise_for_status()+ body = response.json()+ loaded = body.get("models") if isinstance(body, dict) else None+ # /api/ps always names the tag; a bare name means :latest.+ wanted = (model if ":" in model.rsplit("/", 1)[-1] else f"{model}:latest").casefold()+ for entry in loaded if isinstance(loaded, list) else []:+ if isinstance(entry, dict) and wanted in {+ str(entry.get(k) or "").casefold() for k in ("name", "model")+ }:+ return _positive_int(entry.get("context_length"))+ return None+async def _read_served_window(self, model: str) -> Optional[int]:+ if self.provider_type == "ollama":+ return await self._read_ollama_loaded_window(model)_, entries = await self._models_payload(_SERVED_WINDOW_TIMEOUT_S)by_id = {e["id"]: e for e in entries if isinstance(e.get("id"), str)}entry = by_id.get(model) or next(@@ -7617,11 +7645,11 @@ class ExternalProviderClient:if self.provider_type not in _SERVED_WINDOW_PROVIDERS:return Nonenow = time.monotonic()- key = (self.base_url, self._credential_digest(), model)+ key = (self.provider_type, self.base_url, self._credential_digest(), model)cached = _served_windows.get(key)if cached is not None and cached[0] > now:return cached[1]- window = None+ window, failed = None, Falsetry:# both reads share one deadline because the first token waits for them.window = await asyncio.wait_for(@@ -7630,9 +7658,16 @@ class ExternalProviderClient:except httpx.ConnectError as exc:# retry a down or restarting server next turn instead of caching the miss for a minute.logger.info("No served context window from %s: %s", self.provider_type, exc)- return None+ return _ollama_last_windows.get(key)except (httpx.HTTPError, ValueError, asyncio.TimeoutError) as exc:logger.info("No served context window from %s: %r", self.provider_type, exc)+ failed = True+ if self.provider_type == "ollama":+ if window is not None:+ _ollama_last_windows[key] = window+ elif key in _ollama_last_windows or not failed:+ # /api/ps lists a model only while it is loaded.+ return _ollama_last_windows.get(key)for stale in [k for k, (expiry, _) in _served_windows.items() if expiry <= now]:del _served_windows[stale]_served_windows[key] = (now + _SERVED_WINDOW_TTL_S, window)diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 32e216f4001..a9bcf5d3895 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -29770,18 +29770,25 @@ async def _proxy_to_external_provider(if (managed is Noneand payload.context_overflow == "truncate_oldest"- and not payload.compaction_threshold+ and (provider_type == "ollama" or not payload.compaction_threshold)and not _provider_compacts):# a self-hosted server reports its startup window when the catalog has none.- _served_window = payload.context_window or await client.served_context_window(model)- if _served_window:+ _loaded_window = None+ if provider_type == "ollama" or not payload.context_window:+ # Ollama loads its own num_ctx, usually far below the catalogued window.+ _loaded_window = await client.served_context_window(model)+ _served_window = min(filter(None, (payload.context_window, _loaded_window)), default = None)+ if _served_window and (_loaded_window or not payload.compaction_threshold):+ _threshold = min(+ _EXTERNAL_COMPACTION_THRESHOLD_MAX,+ int(_served_window * (1 - _EXTERNAL_COMPACTION_HEADROOM)),+ )payload = payload.model_copy(update = {"context_window": _served_window,"compaction_threshold": min(- _EXTERNAL_COMPACTION_THRESHOLD_MAX,- int(_served_window * (1 - _EXTERNAL_COMPACTION_HEADROOM)),+ payload.compaction_threshold or _threshold, _threshold),})diff --git a/studio/backend/tests/test_external_auto_compaction.py b/studio/backend/tests/test_external_auto_compaction.pyindex 4e2a6d6794b..819a1fbcb8d 100644--- a/studio/backend/tests/test_external_auto_compaction.py+++ b/studio/backend/tests/test_external_auto_compaction.py@@ -407,6 +407,7 @@ def test_external_fit_reserves_tool_schemas():class _ServedWindowHandler(_Handler):models: list[dict] = []props: dict | None = None+ ps: list[dict] | None = Nonekey: str | None = Nonegets: list[str] = []@@ -418,6 +419,8 @@ class _ServedWindowHandler(_Handler):status, body = 200, {"object": "list", "data": self.models}elif self.path.startswith("/props") and self.props is not None:status, body = 200, self.props+ elif self.path == "/api/ps" and self.ps is not None:+ status, body = 200, {"models": self.ps}else:status, body = 404, {"error": "not found"}raw = json.dumps(body).encode()@@ -431,6 +434,7 @@ class _ServedWindowHandler(_Handler):def _served(**attrs):ep_mod._served_windows.clear()ep_mod._served_templates.clear()+ ep_mod._ollama_last_windows.clear()return type("Served", (_ServedWindowHandler,), {"gets": [], **attrs})@@ -568,6 +572,171 @@ def test_a_hosted_provider_is_not_probed_for_a_window():assert server.gets == []+def _ollama(context_length: int = 4_096):+ loaded = {"name": "qwen3:1.7b", "model": "qwen3:1.7b", "context_length": context_length}+ return _served(ps = [loaded])+++def test_an_ollama_connection_compacts_to_the_window_ollama_loaded():+ chunks, sent = _proxy(+ "ollama",+ _long_chat(),+ handler = _ollama(),+ model = "qwen3:1.7b",+ context_overflow = "truncate_oldest",+ compaction_threshold = 30_720,+ context_window = 40_960,+ )+ assert len(sent["messages"]) < len(_long_chat())+ assert sent["messages"][-1]["content"] == "latest question"+ assert _fits(sent, 4_096)+ [truncation] = _truncations(chunks)+ assert truncation["fits"] is True+++def test_an_ollama_window_above_the_catalogued_one_does_not_raise_it():+ _, sent = _proxy(+ "ollama",+ _long_chat(),+ handler = _ollama(262_144),+ model = "qwen3:1.7b",+ context_overflow = "truncate_oldest",+ compaction_threshold = 6_144,+ context_window = 8_192,+ )+ assert len(sent["messages"]) < len(_long_chat())+ assert _fits(sent, 8_192)+++def test_an_ollama_model_not_loaded_yet_keeps_the_catalogued_window():+ idle = _served(ps = [])+ chunks, sent = _proxy(+ "ollama",+ _long_chat(),+ handler = idle,+ model = "qwen3:1.7b",+ context_overflow = "truncate_oldest",+ compaction_threshold = 30_720,+ context_window = 40_960,+ )+ assert len(sent["messages"]) == len(_long_chat())+ assert _truncations(chunks) == []+ assert idle.gets == ["/api/ps"]+++def test_an_ollama_model_is_read_again_once_it_has_loaded():+ server = _served(ps = [])+ windows: list = []++ with _Server(handler = server) as upstream:++ async def go() -> None:+ client = ep_mod.ExternalProviderClient(+ provider_type = "ollama", base_url = upstream.base_url, api_key = None+ )+ windows.append(await client.served_context_window("qwen3:1.7b"))+ server.ps = [{"name": "qwen3:1.7b", "context_length": 4_096}]+ windows.append(await client.served_context_window("qwen3:1.7b"))+ windows.append(await client.served_context_window("qwen3:1.7b"))++ _run(go)+ assert windows == [None, 4_096, 4_096]+ assert server.gets == ["/api/ps", "/api/ps"]+++def test_an_idle_ollama_model_keeps_the_window_it_last_loaded(monkeypatch):+ monkeypatch.setattr(ep_mod, "_SERVED_WINDOW_TTL_S", 0.0)+ server = _ollama()+ windows: list = []++ with _Server(handler = server) as upstream:++ async def go() -> None:+ client = ep_mod.ExternalProviderClient(+ provider_type = "ollama", base_url = upstream.base_url, api_key = None+ )+ windows.append(await client.served_context_window("qwen3:1.7b"))+ server.ps = []+ # another connection's refresh purges expired entries from the TTL cache.+ ep_mod._served_windows.clear()+ windows.append(await client.served_context_window("qwen3:1.7b"))++ _run(go)+ assert windows == [4_096, 4_096]+ assert server.gets == ["/api/ps", "/api/ps"]+++def test_a_failed_ollama_read_keeps_the_window_it_last_loaded(monkeypatch):+ monkeypatch.setattr(ep_mod, "_SERVED_WINDOW_TTL_S", 0.0)+ server = _ollama()+ windows: list = []++ with _Server(handler = server) as upstream:++ async def go() -> None:+ client = ep_mod.ExternalProviderClient(+ provider_type = "ollama", base_url = upstream.base_url, api_key = None+ )+ windows.append(await client.served_context_window("qwen3:1.7b"))+ server.ps = None+ windows.append(await client.served_context_window("qwen3:1.7b"))++ _run(go)+ assert windows == [4_096, 4_096]+++def test_a_custom_miss_on_the_same_server_does_not_hide_ollamas_window():+ server = _ollama()+ windows: list = []++ with _Server(handler = server) as upstream:++ async def go() -> None:+ for provider_type in ("custom", "ollama"):+ client = ep_mod.ExternalProviderClient(+ provider_type = provider_type, base_url = upstream.base_url, api_key = None+ )+ windows.append(await client.served_context_window("qwen3:1.7b"))++ _run(go)+ assert windows == [None, 4_096]+++ ("base", "probe"),+ [+ ("http://h:11434/v1", "http://h:11434/api/ps"),+ ("http://h:11434", "http://h:11434/api/ps"),+ ("https://gw.example/ollama/v1?tenant=a", "https://gw.example/ollama/api/ps?tenant=a"),+ ],+)+def test_the_ollama_probe_keeps_the_base_path_and_query(base, probe):+ assert ep_mod._native_server_path(base, "/api/ps") == probe+++def test_a_bare_ollama_model_name_matches_its_latest_tag():+ server = _served(+ ps = [{"name": "qwen3:latest", "model": "qwen3:latest", "context_length": 4_096}]+ )+ _, sent = _proxy(+ "ollama",+ _long_chat(),+ handler = server,+ model = "qwen3",+ context_overflow = "truncate_oldest",+ compaction_threshold = 30_720,+ context_window = 40_960,+ )+ assert _fits(sent, 4_096)+++def test_auto_compact_off_never_asks_ollama_for_its_window():+ server = _ollama()+ _, sent = _proxy("ollama", _long_chat(), handler = server, model = "qwen3:1.7b")+ assert len(sent["messages"]) == len(_long_chat())+ assert server.gets == []++def test_a_tool_enabled_llama_server_chat_compacts_to_the_reported_window(monkeypatch):async def connected(self) -> bool:return False