Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: compact long chats on an Ollama connection to its loaded window (#17457)

Nilaye550e39fcb3d
diff --git a/studio/backend/core/inference/external_provider.py b/studio/backend/core/inference/external_provider.py
index 563828aeab9..d05932aac41 100644
--- a/studio/backend/core/inference/external_provider.py
+++ b/studio/backend/core/inference/external_provider.py
@@ -75,15 +75,17 @@ def _json_object_refused_for_schema(
# launch-time windows are absent from catalogues; custom covers unregistered self-hosted servers.
-_SERVED_WINDOW_PROVIDERS = frozenset({"vllm", "llama_cpp", "custom"})
+_SERVED_WINDOW_PROVIDERS = frozenset({"vllm", "llama_cpp", "custom", "ollama"})
# refresh within one or two turns to detect server restarts.
_SERVED_WINDOW_TTL_S = 60.0
_SERVED_WINDOW_TIMEOUT_S = 5.0
_GEMINI_MAX_MODEL_PAGES = 20
_GEMINI_RESTRICTED_GENERATION = re.compile(r"^gemini-(?:3\.[78]-flash|4-argon)(?:-|$)")
-_served_windows: dict[tuple[str, str, str], tuple[float, Optional[int]]] = {}
+_served_windows: dict[tuple[str, str, str, str], tuple[float, Optional[int]]] = {}
_served_templates: dict[tuple[str, str, str], tuple[float, Optional[str]]] = {}
_served_versions: dict[tuple[str, str], tuple[float, bool]] = {}
+# outlives the TTL: Ollama unloads an idle model after keep_alive and reloads it at the same window.
+_ollama_last_windows: dict[tuple[str, str, str, str], int] = {}
def _positive_int(value: Any) -> Optional[int]:
@@ -196,6 +198,12 @@ def _append_provider_path(base_url: str, endpoint: str) -> str:
return urlunsplit((parts.scheme, parts.netloc, path, query, parts.fragment))
+def _native_server_path(base_url: str, endpoint: str) -> str:
+ parts = urlsplit(base_url)
+ root = urlunsplit(parts._replace(path = parts.path.rstrip("/").removesuffix("/v1")))
+ return _append_provider_path(root, endpoint)
+
+
def caches_at_the_last_block(
provider_type: Optional[str], model: Optional[str], enable_prompt_caching: Optional[bool]
) -> bool:
@@ -7553,7 +7561,27 @@ class ExternalProviderClient:
token = page.get("nextPageToken")
return {**data, "models": models}
+ async def _read_ollama_loaded_window(self, model: str) -> Optional[int]:
+ response = await _client().get(
+ _native_server_path(self.base_url, "/api/ps"),
+ headers = self._auth_headers(),
+ timeout = _SERVED_WINDOW_TIMEOUT_S,
+ )
+ response.raise_for_status()
+ body = response.json()
+ loaded = body.get("models") if isinstance(body, dict) else None
+ # /api/ps always names the tag; a bare name means :latest.
+ wanted = (model if ":" in model.rsplit("/", 1)[-1] else f"{model}:latest").casefold()
+ for entry in loaded if isinstance(loaded, list) else []:
+ if isinstance(entry, dict) and wanted in {
+ str(entry.get(k) or "").casefold() for k in ("name", "model")
+ }:
+ return _positive_int(entry.get("context_length"))
+ return None
+
async def _read_served_window(self, model: str) -> Optional[int]:
+ if self.provider_type == "ollama":
+ return await self._read_ollama_loaded_window(model)
_, entries = await self._models_payload(_SERVED_WINDOW_TIMEOUT_S)
by_id = {e["id"]: e for e in entries if isinstance(e.get("id"), str)}
entry = by_id.get(model) or next(
@@ -7617,11 +7645,11 @@ class ExternalProviderClient:
if self.provider_type not in _SERVED_WINDOW_PROVIDERS:
return None
now = time.monotonic()
- key = (self.base_url, self._credential_digest(), model)
+ key = (self.provider_type, self.base_url, self._credential_digest(), model)
cached = _served_windows.get(key)
if cached is not None and cached[0] > now:
return cached[1]
- window = None
+ window, failed = None, False
try:
# both reads share one deadline because the first token waits for them.
window = await asyncio.wait_for(
@@ -7630,9 +7658,16 @@ class ExternalProviderClient:
except httpx.ConnectError as exc:
# retry a down or restarting server next turn instead of caching the miss for a minute.
logger.info("No served context window from %s: %s", self.provider_type, exc)
- return None
+ return _ollama_last_windows.get(key)
except (httpx.HTTPError, ValueError, asyncio.TimeoutError) as exc:
logger.info("No served context window from %s: %r", self.provider_type, exc)
+ failed = True
+ if self.provider_type == "ollama":
+ if window is not None:
+ _ollama_last_windows[key] = window
+ elif key in _ollama_last_windows or not failed:
+ # /api/ps lists a model only while it is loaded.
+ return _ollama_last_windows.get(key)
for stale in [k for k, (expiry, _) in _served_windows.items() if expiry <= now]:
del _served_windows[stale]
_served_windows[key] = (now + _SERVED_WINDOW_TTL_S, window)
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 32e216f4001..a9bcf5d3895 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -29770,18 +29770,25 @@ async def _proxy_to_external_provider(
if (
managed is None
and payload.context_overflow == "truncate_oldest"
- and not payload.compaction_threshold
+ and (provider_type == "ollama" or not payload.compaction_threshold)
and not _provider_compacts
):
# a self-hosted server reports its startup window when the catalog has none.
- _served_window = payload.context_window or await client.served_context_window(model)
- if _served_window:
+ _loaded_window = None
+ if provider_type == "ollama" or not payload.context_window:
+ # Ollama loads its own num_ctx, usually far below the catalogued window.
+ _loaded_window = await client.served_context_window(model)
+ _served_window = min(filter(None, (payload.context_window, _loaded_window)), default = None)
+ if _served_window and (_loaded_window or not payload.compaction_threshold):
+ _threshold = min(
+ _EXTERNAL_COMPACTION_THRESHOLD_MAX,
+ int(_served_window * (1 - _EXTERNAL_COMPACTION_HEADROOM)),
+ )
payload = payload.model_copy(
update = {
"context_window": _served_window,
"compaction_threshold": min(
- _EXTERNAL_COMPACTION_THRESHOLD_MAX,
- int(_served_window * (1 - _EXTERNAL_COMPACTION_HEADROOM)),
+ payload.compaction_threshold or _threshold, _threshold
),
}
)
diff --git a/studio/backend/tests/test_external_auto_compaction.py b/studio/backend/tests/test_external_auto_compaction.py
index 4e2a6d6794b..819a1fbcb8d 100644
--- a/studio/backend/tests/test_external_auto_compaction.py
+++ b/studio/backend/tests/test_external_auto_compaction.py
@@ -407,6 +407,7 @@ def test_external_fit_reserves_tool_schemas():
class _ServedWindowHandler(_Handler):
models: list[dict] = []
props: dict | None = None
+ ps: list[dict] | None = None
key: str | None = None
gets: list[str] = []
@@ -418,6 +419,8 @@ class _ServedWindowHandler(_Handler):
status, body = 200, {"object": "list", "data": self.models}
elif self.path.startswith("/props") and self.props is not None:
status, body = 200, self.props
+ elif self.path == "/api/ps" and self.ps is not None:
+ status, body = 200, {"models": self.ps}
else:
status, body = 404, {"error": "not found"}
raw = json.dumps(body).encode()
@@ -431,6 +434,7 @@ class _ServedWindowHandler(_Handler):
def _served(**attrs):
ep_mod._served_windows.clear()
ep_mod._served_templates.clear()
+ ep_mod._ollama_last_windows.clear()
return type("Served", (_ServedWindowHandler,), {"gets": [], **attrs})
@@ -568,6 +572,171 @@ def test_a_hosted_provider_is_not_probed_for_a_window():
assert server.gets == []
+def _ollama(context_length: int = 4_096):
+ loaded = {"name": "qwen3:1.7b", "model": "qwen3:1.7b", "context_length": context_length}
+ return _served(ps = [loaded])
+
+
+def test_an_ollama_connection_compacts_to_the_window_ollama_loaded():
+ chunks, sent = _proxy(
+ "ollama",
+ _long_chat(),
+ handler = _ollama(),
+ model = "qwen3:1.7b",
+ context_overflow = "truncate_oldest",
+ compaction_threshold = 30_720,
+ context_window = 40_960,
+ )
+ assert len(sent["messages"]) < len(_long_chat())
+ assert sent["messages"][-1]["content"] == "latest question"
+ assert _fits(sent, 4_096)
+ [truncation] = _truncations(chunks)
+ assert truncation["fits"] is True
+
+
+def test_an_ollama_window_above_the_catalogued_one_does_not_raise_it():
+ _, sent = _proxy(
+ "ollama",
+ _long_chat(),
+ handler = _ollama(262_144),
+ model = "qwen3:1.7b",
+ context_overflow = "truncate_oldest",
+ compaction_threshold = 6_144,
+ context_window = 8_192,
+ )
+ assert len(sent["messages"]) < len(_long_chat())
+ assert _fits(sent, 8_192)
+
+
+def test_an_ollama_model_not_loaded_yet_keeps_the_catalogued_window():
+ idle = _served(ps = [])
+ chunks, sent = _proxy(
+ "ollama",
+ _long_chat(),
+ handler = idle,
+ model = "qwen3:1.7b",
+ context_overflow = "truncate_oldest",
+ compaction_threshold = 30_720,
+ context_window = 40_960,
+ )
+ assert len(sent["messages"]) == len(_long_chat())
+ assert _truncations(chunks) == []
+ assert idle.gets == ["/api/ps"]
+
+
+def test_an_ollama_model_is_read_again_once_it_has_loaded():
+ server = _served(ps = [])
+ windows: list = []
+
+ with _Server(handler = server) as upstream:
+
+ async def go() -> None:
+ client = ep_mod.ExternalProviderClient(
+ provider_type = "ollama", base_url = upstream.base_url, api_key = None
+ )
+ windows.append(await client.served_context_window("qwen3:1.7b"))
+ server.ps = [{"name": "qwen3:1.7b", "context_length": 4_096}]
+ windows.append(await client.served_context_window("qwen3:1.7b"))
+ windows.append(await client.served_context_window("qwen3:1.7b"))
+
+ _run(go)
+ assert windows == [None, 4_096, 4_096]
+ assert server.gets == ["/api/ps", "/api/ps"]
+
+
+def test_an_idle_ollama_model_keeps_the_window_it_last_loaded(monkeypatch):
+ monkeypatch.setattr(ep_mod, "_SERVED_WINDOW_TTL_S", 0.0)
+ server = _ollama()
+ windows: list = []
+
+ with _Server(handler = server) as upstream:
+
+ async def go() -> None:
+ client = ep_mod.ExternalProviderClient(
+ provider_type = "ollama", base_url = upstream.base_url, api_key = None
+ )
+ windows.append(await client.served_context_window("qwen3:1.7b"))
+ server.ps = []
+ # another connection's refresh purges expired entries from the TTL cache.
+ ep_mod._served_windows.clear()
+ windows.append(await client.served_context_window("qwen3:1.7b"))
+
+ _run(go)
+ assert windows == [4_096, 4_096]
+ assert server.gets == ["/api/ps", "/api/ps"]
+
+
+def test_a_failed_ollama_read_keeps_the_window_it_last_loaded(monkeypatch):
+ monkeypatch.setattr(ep_mod, "_SERVED_WINDOW_TTL_S", 0.0)
+ server = _ollama()
+ windows: list = []
+
+ with _Server(handler = server) as upstream:
+
+ async def go() -> None:
+ client = ep_mod.ExternalProviderClient(
+ provider_type = "ollama", base_url = upstream.base_url, api_key = None
+ )
+ windows.append(await client.served_context_window("qwen3:1.7b"))
+ server.ps = None
+ windows.append(await client.served_context_window("qwen3:1.7b"))
+
+ _run(go)
+ assert windows == [4_096, 4_096]
+
+
+def test_a_custom_miss_on_the_same_server_does_not_hide_ollamas_window():
+ server = _ollama()
+ windows: list = []
+
+ with _Server(handler = server) as upstream:
+
+ async def go() -> None:
+ for provider_type in ("custom", "ollama"):
+ client = ep_mod.ExternalProviderClient(
+ provider_type = provider_type, base_url = upstream.base_url, api_key = None
+ )
+ windows.append(await client.served_context_window("qwen3:1.7b"))
+
+ _run(go)
+ assert windows == [None, 4_096]
+
+
+ ("base", "probe"),
+ [
+ ("http://h:11434/v1", "http://h:11434/api/ps"),
+ ("http://h:11434", "http://h:11434/api/ps"),
+ ("https://gw.example/ollama/v1?tenant=a", "https://gw.example/ollama/api/ps?tenant=a"),
+ ],
+)
+def test_the_ollama_probe_keeps_the_base_path_and_query(base, probe):
+ assert ep_mod._native_server_path(base, "/api/ps") == probe
+
+
+def test_a_bare_ollama_model_name_matches_its_latest_tag():
+ server = _served(
+ ps = [{"name": "qwen3:latest", "model": "qwen3:latest", "context_length": 4_096}]
+ )
+ _, sent = _proxy(
+ "ollama",
+ _long_chat(),
+ handler = server,
+ model = "qwen3",
+ context_overflow = "truncate_oldest",
+ compaction_threshold = 30_720,
+ context_window = 40_960,
+ )
+ assert _fits(sent, 4_096)
+
+
+def test_auto_compact_off_never_asks_ollama_for_its_window():
+ server = _ollama()
+ _, sent = _proxy("ollama", _long_chat(), handler = server, model = "qwen3:1.7b")
+ assert len(sent["messages"]) == len(_long_chat())
+ assert server.gets == []
+
+
def test_a_tool_enabled_llama_server_chat_compacts_to_the_reported_window(monkeypatch):
async def connected(self) -> bool:
return False