Python
perf(models): ask the local model servers at once, with a short loopback connect
Devika Verma079939f63592
diff --git a/coworker/providers/local_server.py b/coworker/providers/local_server.pyindex 94093b8f..5204bbef 100644--- a/coworker/providers/local_server.py+++ b/coworker/providers/local_server.py@@ -33,6 +33,7 @@ from __future__ import annotationsimport threadingimport timefrom typing import Any, Optional+from urllib.parse import urlparseimport httpx@@ -65,6 +66,23 @@ def v1_base(name: str, base_url: Optional[str]) -> str:return base if base.endswith("/v1") else base + "/v1"+LOOPBACK = {"localhost", "127.0.0.1", "::1"}+LOOPBACK_CONNECT_SECONDS = 0.3+++def probe_timeout(url: str, read: float) -> httpx.Timeout:+ """The timeout of a first request to a local model server. On Windows a connection to a+ loopback port nobody listens on is refused only after about 2 s (and `localhost` tries+ ::1, then 127.0.0.1), so the Models & Keys page waited 2-15 s on servers that are not+ running (T3-2). A server on this machine accepts at once, so the connect wait is short+ for loopback; a server elsewhere keeps the whole `read` budget."""+ try:+ host = (urlparse(url).hostname or "").lower()+ except ValueError:+ host = ""+ return httpx.Timeout(read, connect=LOOPBACK_CONNECT_SECONDS) if host in LOOPBACK else httpx.Timeout(read)++def _headers(api_key: Optional[str]) -> dict[str, str]:key = (api_key or "").strip()return {"Authorization": f"Bearer {key}"} if key else {}@@ -72,7 +90,8 @@ def _headers(api_key: Optional[str]) -> dict[str, str]:def alive(name: str, base_url: Optional[str], api_key: Optional[str] = None) -> bool:try:- return httpx.get(v1_base(name, base_url) + "/models", headers=_headers(api_key), timeout=0.8).status_code == 200+ url = v1_base(name, base_url) + "/models"+ return httpx.get(url, headers=_headers(api_key), timeout=probe_timeout(url, 0.8)).status_code == 200except Exception: # noqa: BLE001 - a probe never raisesreturn False@@ -135,7 +154,7 @@ def context_window_for(model: str) -> Optional[int]:def _fetch(name: str, base: str, api_key: Optional[str]) -> list[dict[str, Any]]:headers = _headers(api_key)try:- listing = httpx.get(base + "/models", headers=headers, timeout=4.0).json()+ listing = httpx.get(base + "/models", headers=headers, timeout=probe_timeout(base, 4.0)).json()except (httpx.HTTPError, ValueError, OSError):return []items = listing.get("data") if isinstance(listing, dict) else Nonediff --git a/coworker/providers/ollama_facts.py b/coworker/providers/ollama_facts.pyindex 0938c4c5..fe2c4f06 100644--- a/coworker/providers/ollama_facts.py+++ b/coworker/providers/ollama_facts.py@@ -61,21 +61,30 @@ def forget_all() -> None:def _fetch(base: str) -> list[dict[str, Any]]:# Module-level httpx calls, so a test can stand the server in with two fakes.try:- tags = httpx.get(base + "/api/tags", timeout=4.0).json()+ from .local_server import probe_timeout++ tags = httpx.get(base + "/api/tags", timeout=probe_timeout(base, 4.0)).json()except (httpx.HTTPError, ValueError, OSError):return []- rows = []- for item in tags.get("models") or []:- name = item.get("name")- if not name:- continue- show: dict[str, Any] = {}+ items = [item for item in tags.get("models") or [] if item.get("name")]++ def show_of(name: str) -> dict[str, Any]:try:show = httpx.post(base + "/api/show", json={"model": name}, timeout=4.0).json()except (httpx.HTTPError, ValueError, OSError):- show = {}- rows.append(_row(name, item, show if isinstance(show, dict) else {}))- return rows+ return {}+ return show if isinstance(show, dict) else {}++ # One /api/show per model, asked at the same time (T3-2: one after another, five+ # models took about 4 s of the Models & Keys page's wait).+ if len(items) > 1:+ from concurrent.futures import ThreadPoolExecutor++ with ThreadPoolExecutor(max_workers=min(4, len(items)), thread_name_prefix="ollama-show") as pool:+ shows = list(pool.map(show_of, [item["name"] for item in items]))+ else:+ shows = [show_of(item["name"]) for item in items]+ return [_row(item["name"], item, show) for item, show in zip(items, shows)]def _row(name: str, tag: dict[str, Any], show: dict[str, Any]) -> dict[str, Any]:diff --git a/coworker/server/manager.py b/coworker/server/manager.pyindex fad31df5..3d5c60eb 100644--- a/coworker/server/manager.py+++ b/coworker/server/manager.py@@ -4203,7 +4203,9 @@ class SessionManager:non-secret field values (e.g. the Ollama base URL) ARE returned so the form can prefill."""out: list[dict[str, Any]] = []- for d in provider_descriptors(self.secrets):+ descriptors = provider_descriptors(self.secrets)+ self._warm_local_probes([d.name for d in descriptors if d.kind == "local"])+ for d in descriptors:profile = self.secrets.get(f"provider:{d.name}") or {}configured = descriptor_configured(d, profile)values = {@@ -4382,6 +4384,28 @@ class SessionManager:"siliconflow": ["zai-org/GLM-5.2"],}+ def _warm_local_probes(self, names: list[str]) -> None:+ """Ask the local model servers at the same time (T3-2): each answer is cached (30 s+ for liveness, 60 s for the models), so the loop in get_providers reads the caches+ instead of waiting on each server in turn. Best-effort: a probe that fails here is+ tried again, alone, by the loop."""+ if len(names) < 2:+ return+ from concurrent.futures import ThreadPoolExecutor++ def probe(name: str) -> None:+ try:+ self._suggested_models(name)+ if name == "ollama":+ self._ollama_alive()+ else:+ self._local_server_alive(name)+ except Exception: # noqa: BLE001 - the loop asks again and shows what it gets+ pass++ with ThreadPoolExecutor(max_workers=min(8, len(names)), thread_name_prefix="provider-probe") as pool:+ list(pool.map(probe, names))+def _suggested_models(self, name: str) -> list[str]:"""Bare model-name suggestions for the 'add model' form (datalist), per provider.Ollama → live `/api/tags` (best-effort); everyone else → the curated matrix,@@ -4688,7 +4712,9 @@ class SessionManager:try:import httpx- alive = httpx.get(base + "/api/tags", timeout=0.8).status_code == 200+ from ..providers.local_server import probe_timeout++ alive = httpx.get(base + "/api/tags", timeout=probe_timeout(base, 0.8)).status_code == 200except Exception:alive = Falseself._ollama_alive_cache = (now, alive)@@ -4787,14 +4813,17 @@ class SessionManager:from ..providers import local_servernow = time.monotonic()- cache = getattr(self, "_local_alive_cache", None) or {}+ # One dict, updated in place: get_providers asks several servers at once (T3-2), and+ # each thread putting back its own copy lost the others' answers.+ cache = getattr(self, "_local_alive_cache", None)+ if cache is None:+ cache = self.__dict__.setdefault("_local_alive_cache", {})hit = cache.get(name)if hit and now - hit[0] < 30:return hit[1]profile = self.secrets.get(f"provider:{name}") or {}alive = local_server.alive(name, profile.get("base_url"), profile.get("api_key"))cache[name] = (now, alive)- self._local_alive_cache = cachereturn alivedef _local_server_models(self, name: str) -> list[str]:diff --git a/tests/test_providers_page_speed.py b/tests/test_providers_page_speed.pynew file mode 100644index 00000000..c8775a62--- /dev/null+++ b/tests/test_providers_page_speed.py@@ -0,0 +1,63 @@+"""T3-2: Models & Keys waited 2-15 s on local model servers that are not running. On Windows a+refused loopback connection takes about 2 s, `localhost` tries two addresses, and the servers+were asked one after another. Now the connect wait is short on loopback, the servers are asked+at the same time, and every answer stays in the caches."""++from __future__ import annotations++import threading+import time++from coworker.providers import local_server+from coworker.server import SessionManager+++def test_a_loopback_server_gets_a_short_connect_wait_and_others_the_whole_budget():+ for url in ("http://localhost:8080/v1", "http://127.0.0.1:11434", "http://[::1]:8000/v1"):+ t = local_server.probe_timeout(url, 4.0)+ assert t.connect == local_server.LOOPBACK_CONNECT_SECONDS and t.read == 4.0, url+ far = local_server.probe_timeout("http://gpu-box.lan:8000/v1", 4.0)+ assert far.connect == 4.0 and far.read == 4.0+ assert local_server.probe_timeout("not a url", 0.8).connect == 0.8+++def test_the_local_servers_are_asked_at_once_and_every_answer_is_kept(tmp_path, monkeypatch):+ monkeypatch.setenv("COWORKER_STATE_DIR", str(tmp_path / "state"))+ manager = SessionManager(workspace=tmp_path)+ names = list(local_server.LOCAL_SERVERS)+ running = {"now": 0, "most": 0}+ lock = threading.Lock()++ def slow(*_a, **_k):+ with lock:+ running["now"] += 1+ running["most"] = max(running["most"], running["now"])+ time.sleep(0.2)+ with lock:+ running["now"] -= 1+ return False++ alive_calls: list[str] = []++ def alive(name, base_url, api_key=None):+ alive_calls.append(name)+ return slow()++ monkeypatch.setattr(local_server, "alive", alive)+ monkeypatch.setattr(local_server, "model_facts", lambda *a, **k: slow() or [])+ monkeypatch.setattr(manager, "_ollama_alive", lambda: False)+ monkeypatch.setattr(manager, "_ollama_models", lambda: [])+ started = time.monotonic()+ manager._warm_local_probes(["ollama", *names])+ took = time.monotonic() - started+ assert running["most"] > 1 # asked at the same time+ assert took < 0.2 * 2 * len(names) / 2 # far from one after another+ # every server's answer was kept, so get_providers asks none of them again+ assert sorted(alive_calls) == sorted(names)+ assert set(manager._local_alive_cache) == set(names)+ alive_calls.clear()+ rows = {r["name"]: r for r in manager.get_providers()}+ assert alive_calls == []+ assert all(rows[n]["alive"] is False for n in names if n in rows)+ if hasattr(manager, "observability"):+ manager.observability.close()