Skip to content
Python

perf(models): ask the local model servers at once, with a short loopback connect

Devika Verma079939f63592
diff --git a/coworker/providers/local_server.py b/coworker/providers/local_server.py
index 94093b8f..5204bbef 100644
--- a/coworker/providers/local_server.py
+++ b/coworker/providers/local_server.py
@@ -33,6 +33,7 @@ from __future__ import annotations
import threading
import time
from typing import Any, Optional
+from urllib.parse import urlparse
import httpx
@@ -65,6 +66,23 @@ def v1_base(name: str, base_url: Optional[str]) -> str:
return base if base.endswith("/v1") else base + "/v1"
+LOOPBACK = {"localhost", "127.0.0.1", "::1"}
+LOOPBACK_CONNECT_SECONDS = 0.3
+
+
+def probe_timeout(url: str, read: float) -> httpx.Timeout:
+ """The timeout of a first request to a local model server. On Windows a connection to a
+ loopback port nobody listens on is refused only after about 2 s (and `localhost` tries
+ ::1, then 127.0.0.1), so the Models & Keys page waited 2-15 s on servers that are not
+ running (T3-2). A server on this machine accepts at once, so the connect wait is short
+ for loopback; a server elsewhere keeps the whole `read` budget."""
+ try:
+ host = (urlparse(url).hostname or "").lower()
+ except ValueError:
+ host = ""
+ return httpx.Timeout(read, connect=LOOPBACK_CONNECT_SECONDS) if host in LOOPBACK else httpx.Timeout(read)
+
+
def _headers(api_key: Optional[str]) -> dict[str, str]:
key = (api_key or "").strip()
return {"Authorization": f"Bearer {key}"} if key else {}
@@ -72,7 +90,8 @@ def _headers(api_key: Optional[str]) -> dict[str, str]:
def alive(name: str, base_url: Optional[str], api_key: Optional[str] = None) -> bool:
try:
- return httpx.get(v1_base(name, base_url) + "/models", headers=_headers(api_key), timeout=0.8).status_code == 200
+ url = v1_base(name, base_url) + "/models"
+ return httpx.get(url, headers=_headers(api_key), timeout=probe_timeout(url, 0.8)).status_code == 200
except Exception: # noqa: BLE001 - a probe never raises
return False
@@ -135,7 +154,7 @@ def context_window_for(model: str) -> Optional[int]:
def _fetch(name: str, base: str, api_key: Optional[str]) -> list[dict[str, Any]]:
headers = _headers(api_key)
try:
- listing = httpx.get(base + "/models", headers=headers, timeout=4.0).json()
+ listing = httpx.get(base + "/models", headers=headers, timeout=probe_timeout(base, 4.0)).json()
except (httpx.HTTPError, ValueError, OSError):
return []
items = listing.get("data") if isinstance(listing, dict) else None
diff --git a/coworker/providers/ollama_facts.py b/coworker/providers/ollama_facts.py
index 0938c4c5..fe2c4f06 100644
--- a/coworker/providers/ollama_facts.py
+++ b/coworker/providers/ollama_facts.py
@@ -61,21 +61,30 @@ def forget_all() -> None:
def _fetch(base: str) -> list[dict[str, Any]]:
# Module-level httpx calls, so a test can stand the server in with two fakes.
try:
- tags = httpx.get(base + "/api/tags", timeout=4.0).json()
+ from .local_server import probe_timeout
+
+ tags = httpx.get(base + "/api/tags", timeout=probe_timeout(base, 4.0)).json()
except (httpx.HTTPError, ValueError, OSError):
return []
- rows = []
- for item in tags.get("models") or []:
- name = item.get("name")
- if not name:
- continue
- show: dict[str, Any] = {}
+ items = [item for item in tags.get("models") or [] if item.get("name")]
+
+ def show_of(name: str) -> dict[str, Any]:
try:
show = httpx.post(base + "/api/show", json={"model": name}, timeout=4.0).json()
except (httpx.HTTPError, ValueError, OSError):
- show = {}
- rows.append(_row(name, item, show if isinstance(show, dict) else {}))
- return rows
+ return {}
+ return show if isinstance(show, dict) else {}
+
+ # One /api/show per model, asked at the same time (T3-2: one after another, five
+ # models took about 4 s of the Models & Keys page's wait).
+ if len(items) > 1:
+ from concurrent.futures import ThreadPoolExecutor
+
+ with ThreadPoolExecutor(max_workers=min(4, len(items)), thread_name_prefix="ollama-show") as pool:
+ shows = list(pool.map(show_of, [item["name"] for item in items]))
+ else:
+ shows = [show_of(item["name"]) for item in items]
+ return [_row(item["name"], item, show) for item, show in zip(items, shows)]
def _row(name: str, tag: dict[str, Any], show: dict[str, Any]) -> dict[str, Any]:
diff --git a/coworker/server/manager.py b/coworker/server/manager.py
index fad31df5..3d5c60eb 100644
--- a/coworker/server/manager.py
+++ b/coworker/server/manager.py
@@ -4203,7 +4203,9 @@ class SessionManager:
non-secret field values (e.g. the Ollama base URL) ARE returned so the form can prefill.
"""
out: list[dict[str, Any]] = []
- for d in provider_descriptors(self.secrets):
+ descriptors = provider_descriptors(self.secrets)
+ self._warm_local_probes([d.name for d in descriptors if d.kind == "local"])
+ for d in descriptors:
profile = self.secrets.get(f"provider:{d.name}") or {}
configured = descriptor_configured(d, profile)
values = {
@@ -4382,6 +4384,28 @@ class SessionManager:
"siliconflow": ["zai-org/GLM-5.2"],
}
+ def _warm_local_probes(self, names: list[str]) -> None:
+ """Ask the local model servers at the same time (T3-2): each answer is cached (30 s
+ for liveness, 60 s for the models), so the loop in get_providers reads the caches
+ instead of waiting on each server in turn. Best-effort: a probe that fails here is
+ tried again, alone, by the loop."""
+ if len(names) < 2:
+ return
+ from concurrent.futures import ThreadPoolExecutor
+
+ def probe(name: str) -> None:
+ try:
+ self._suggested_models(name)
+ if name == "ollama":
+ self._ollama_alive()
+ else:
+ self._local_server_alive(name)
+ except Exception: # noqa: BLE001 - the loop asks again and shows what it gets
+ pass
+
+ with ThreadPoolExecutor(max_workers=min(8, len(names)), thread_name_prefix="provider-probe") as pool:
+ list(pool.map(probe, names))
+
def _suggested_models(self, name: str) -> list[str]:
"""Bare model-name suggestions for the 'add model' form (datalist), per provider.
Ollama → live `/api/tags` (best-effort); everyone else → the curated matrix,
@@ -4688,7 +4712,9 @@ class SessionManager:
try:
import httpx
- alive = httpx.get(base + "/api/tags", timeout=0.8).status_code == 200
+ from ..providers.local_server import probe_timeout
+
+ alive = httpx.get(base + "/api/tags", timeout=probe_timeout(base, 0.8)).status_code == 200
except Exception:
alive = False
self._ollama_alive_cache = (now, alive)
@@ -4787,14 +4813,17 @@ class SessionManager:
from ..providers import local_server
now = time.monotonic()
- cache = getattr(self, "_local_alive_cache", None) or {}
+ # One dict, updated in place: get_providers asks several servers at once (T3-2), and
+ # each thread putting back its own copy lost the others' answers.
+ cache = getattr(self, "_local_alive_cache", None)
+ if cache is None:
+ cache = self.__dict__.setdefault("_local_alive_cache", {})
hit = cache.get(name)
if hit and now - hit[0] < 30:
return hit[1]
profile = self.secrets.get(f"provider:{name}") or {}
alive = local_server.alive(name, profile.get("base_url"), profile.get("api_key"))
cache[name] = (now, alive)
- self._local_alive_cache = cache
return alive
def _local_server_models(self, name: str) -> list[str]:
diff --git a/tests/test_providers_page_speed.py b/tests/test_providers_page_speed.py
new file mode 100644
index 00000000..c8775a62
--- /dev/null
+++ b/tests/test_providers_page_speed.py
@@ -0,0 +1,63 @@
+"""T3-2: Models & Keys waited 2-15 s on local model servers that are not running. On Windows a
+refused loopback connection takes about 2 s, `localhost` tries two addresses, and the servers
+were asked one after another. Now the connect wait is short on loopback, the servers are asked
+at the same time, and every answer stays in the caches."""
+
+from __future__ import annotations
+
+import threading
+import time
+
+from coworker.providers import local_server
+from coworker.server import SessionManager
+
+
+def test_a_loopback_server_gets_a_short_connect_wait_and_others_the_whole_budget():
+ for url in ("http://localhost:8080/v1", "http://127.0.0.1:11434", "http://[::1]:8000/v1"):
+ t = local_server.probe_timeout(url, 4.0)
+ assert t.connect == local_server.LOOPBACK_CONNECT_SECONDS and t.read == 4.0, url
+ far = local_server.probe_timeout("http://gpu-box.lan:8000/v1", 4.0)
+ assert far.connect == 4.0 and far.read == 4.0
+ assert local_server.probe_timeout("not a url", 0.8).connect == 0.8
+
+
+def test_the_local_servers_are_asked_at_once_and_every_answer_is_kept(tmp_path, monkeypatch):
+ monkeypatch.setenv("COWORKER_STATE_DIR", str(tmp_path / "state"))
+ manager = SessionManager(workspace=tmp_path)
+ names = list(local_server.LOCAL_SERVERS)
+ running = {"now": 0, "most": 0}
+ lock = threading.Lock()
+
+ def slow(*_a, **_k):
+ with lock:
+ running["now"] += 1
+ running["most"] = max(running["most"], running["now"])
+ time.sleep(0.2)
+ with lock:
+ running["now"] -= 1
+ return False
+
+ alive_calls: list[str] = []
+
+ def alive(name, base_url, api_key=None):
+ alive_calls.append(name)
+ return slow()
+
+ monkeypatch.setattr(local_server, "alive", alive)
+ monkeypatch.setattr(local_server, "model_facts", lambda *a, **k: slow() or [])
+ monkeypatch.setattr(manager, "_ollama_alive", lambda: False)
+ monkeypatch.setattr(manager, "_ollama_models", lambda: [])
+ started = time.monotonic()
+ manager._warm_local_probes(["ollama", *names])
+ took = time.monotonic() - started
+ assert running["most"] > 1 # asked at the same time
+ assert took < 0.2 * 2 * len(names) / 2 # far from one after another
+ # every server's answer was kept, so get_providers asks none of them again
+ assert sorted(alive_calls) == sorted(names)
+ assert set(manager._local_alive_cache) == set(names)
+ alive_calls.clear()
+ rows = {r["name"]: r for r in manager.get_providers()}
+ assert alive_calls == []
+ assert all(rows[n]["alive"] is False for n in names if n in rows)
+ if hasattr(manager, "observability"):
+ manager.observability.close()