Python
"""llama.cpp, vLLM, LM Studio, Jan, Docker Model Runner and mlx-lm: servers the user
starts, reached through their OpenAI-compatible `/v1`. One module for all of them, since
OpenWorker treats them the same way and they differ only in what they report about
themselves.
- llama.cpp `llama-server`: `localhost:8080` by default; `/v1/models` lists the loaded
model (its id is the file path unless the server was started with `--alias`), with
`meta.n_ctx_train` for the trained window; `/props` carries the window the server was
started with (`default_generation_settings.n_ctx`), which cannot change while it runs.
Tool calls go through the model's chat template (`--jinja`, on by default).
- vLLM: `localhost:8000` by default, one model per server; `/v1/models` gives the id and
`max_model_len`. Tool calls work only when the server was started with
`--enable-auto-tool-choice --tool-call-parser <name>`; the harness cannot turn them on,
but it can find out, and say so, before a session breaks.
- LM Studio: `localhost:1234` by default (`lms server start` or the Developer tab); its
`/v1/models` lists every downloaded model, and its own `/api/v0/models` adds each one's
`max_context_length` and whether it is loaded. Tool calls go through the model's chat
template, as in llama.cpp.
- Jan: `127.0.0.1:1337` by default (Settings ▸ Local API Server ▸ Start Server, read from
docs.jan.ai/desktop/api-server 2026-10-26); a plain OpenAI listing, an optional key.
- Docker Model Runner: `localhost:12434/engines/v1` by default (TCP host access turned on
in Docker Desktop; docs.docker.com/ai/model-runner/api-reference, read 2026-10-26). The
`/engines` segment is part of the path, so v1_base adds it when the user types the root.
- mlx-lm: `127.0.0.1:8080` by default (`mlx_lm.server`, Apple silicon only; its server.py
read 2026-10-26) — the same port as llama.cpp, so the help says to move one of them.
None of them tells the harness what the machine is; that is local_machine.py. All may take
an API key; without one any value is accepted, and OpenWorker sends a placeholder.
"""
from __future__ import annotations
import threading
import time
from typing import Any, Optional
from urllib.parse import urlparse
import httpx
from .. import model_config
from . import local_machine
from .recommended import recommendation_for
LOCAL_SERVERS: dict[str, dict[str, str]] = {
"llamacpp": {"title": "llama.cpp", "default_base": "http://localhost:8080"},
"vllm": {"title": "vLLM", "default_base": "http://localhost:8000"},
"lmstudio": {"title": "LM Studio", "default_base": "http://localhost:1234"},
"jan": {"title": "Jan", "default_base": "http://127.0.0.1:1337"},
"docker": {"title": "Docker Model Runner", "default_base": "http://localhost:12434/engines"},
"mlx": {"title": "mlx-lm", "default_base": "http://127.0.0.1:8080"},
}
PLACEHOLDER_KEY = "none"
CACHE_SECONDS = 60.0
_cache: dict[tuple[str, str], tuple[float, list[dict[str, Any]]]] = {}
_lock = threading.Lock()
def v1_base(name: str, base_url: Optional[str]) -> str:
"""The server's `/v1` root from what the user typed: the root, or already `/v1`.
Docker Model Runner serves under `/engines/v1` (or `/engines/<engine>/v1`), so a bare
host:port gets the `/engines` segment too."""
base = (base_url or LOCAL_SERVERS[name]["default_base"]).strip().rstrip("/")
if name == "docker" and "/engines" not in base:
base += "/engines"
return base if base.endswith("/v1") else base + "/v1"
LOOPBACK = {"localhost", "127.0.0.1", "::1"}
LOOPBACK_CONNECT_SECONDS = 0.3
def probe_timeout(url: str, read: float) -> httpx.Timeout:
"""The timeout of a first request to a local model server. On Windows a connection to a
loopback port nobody listens on is refused only after about 2 s (and `localhost` tries
::1, then 127.0.0.1), so the Models & Keys page waited 2-15 s on servers that are not
running (T3-2). A server on this machine accepts at once, so the connect wait is short
for loopback; a server elsewhere keeps the whole `read` budget."""
try:
host = (urlparse(url).hostname or "").lower()
except ValueError:
host = ""
return httpx.Timeout(read, connect=LOOPBACK_CONNECT_SECONDS) if host in LOOPBACK else httpx.Timeout(read)
def _headers(api_key: Optional[str]) -> dict[str, str]:
key = (api_key or "").strip()
return {"Authorization": f"Bearer {key}"} if key else {}
def alive(name: str, base_url: Optional[str], api_key: Optional[str] = None) -> bool:
try:
url = v1_base(name, base_url) + "/models"
return httpx.get(url, headers=_headers(api_key), timeout=probe_timeout(url, 0.8)).status_code == 200
except Exception: # noqa: BLE001 - a probe never raises
return False
def model_facts(
name: str, base_url: Optional[str], api_key: Optional[str] = None, *, fresh: bool = False
) -> list[dict[str, Any]]:
"""One row per model the server serves (usually one). Empty when it does not answer."""
base = v1_base(name, base_url)
key = (name, base)
now = time.monotonic()
with _lock:
hit = _cache.get(key)
if hit and not fresh and now - hit[0] < CACHE_SECONDS:
return [dict(r) for r in hit[1]]
rows = _fetch(name, base, api_key)
with _lock:
_cache[key] = (now, rows)
return [dict(r) for r in rows]
def forget_all() -> None:
with _lock:
_cache.clear()
# The server each provider client was built against (registry._local_server), so a
# window lookup from the engine can ask the right server without a profile in hand.
_built: dict[str, tuple[str, Optional[str]]] = {}
def remember_server(name: str, base_url: Optional[str], api_key: Optional[str]) -> None:
_built[name] = (base_url or "", api_key)
def context_window_for(model: str) -> Optional[int]:
"""The window a served model runs at: from the last facts read, else read now from
the server the provider was built against. None when unknown or not a local-server
model."""
vendor, _, bare = model.partition(":")
if vendor not in LOCAL_SERVERS:
return None
def find(rows: list[dict[str, Any]]) -> Optional[int]:
for row in rows:
if row.get("name") == bare and row.get("context"):
return int(row["context"])
return None
with _lock:
for (name, _base), (_at, rows) in _cache.items():
if name == vendor:
found = find(rows)
if found:
return found
base_url, api_key = _built.get(vendor, ("", None))
return find(model_facts(vendor, base_url or None, api_key))
def _fetch(name: str, base: str, api_key: Optional[str]) -> list[dict[str, Any]]:
headers = _headers(api_key)
try:
listing = httpx.get(base + "/models", headers=headers, timeout=probe_timeout(base, 4.0)).json()
except (httpx.HTTPError, ValueError, OSError):
return []
items = listing.get("data") if isinstance(listing, dict) else None
if not isinstance(items, list):
return []
props: dict[str, Any] = {}
if name == "lmstudio":
props = _lmstudio_catalog(base, headers)
if name == "llamacpp":
try:
got = httpx.get(base[: -len("/v1")] + "/props", headers=headers, timeout=4.0).json()
props = got if isinstance(got, dict) else {}
except (httpx.HTTPError, ValueError, OSError):
props = {}
tools: Optional[bool] = None
if name == "vllm":
tools = _vllm_tools_on(base, headers, str((items[0] or {}).get("id") or "")) if items else None
elif name == "llamacpp":
template = str(props.get("chat_template") or "")
tools = ("tools" in template) if template else None
rows = []
for item in items:
if not isinstance(item, dict) or not item.get("id"):
continue
rows.append(_row(name, item, props, tools))
return rows
def _lmstudio_catalog(base: str, headers: dict[str, str]) -> dict[str, Any]:
"""LM Studio's own listing (`/api/v0/models`), keyed by id: the trained window
(`max_context_length`) and the load state the OpenAI-compatible listing leaves out.
Empty when the server does not answer it."""
try:
got = httpx.get(base[: -len("/v1")] + "/api/v0/models", headers=headers, timeout=4.0).json()
except (httpx.HTTPError, ValueError, OSError):
return {}
items = got.get("data") if isinstance(got, dict) else None
if not isinstance(items, list):
return {}
return {str(it["id"]): it for it in items if isinstance(it, dict) and it.get("id")}
def _vllm_tools_on(base: str, headers: dict[str, str], model: str) -> Optional[bool]:
"""Ask vLLM for a one-token reply with a tool offered. A server started without tool
calling refuses the request and names the flag; one with it answers."""
probe = {
"model": model,
"messages": [{"role": "user", "content": "hi"}],
"max_tokens": 1,
"tools": [{"type": "function", "function": {"name": "noop", "parameters": {"type": "object", "properties": {}}}}],
}
try:
resp = httpx.post(base + "/chat/completions", json=probe, headers=headers, timeout=30.0)
except (httpx.HTTPError, OSError):
return None
if resp.status_code == 200:
return True
text = resp.text.lower()
if "tool" in text and ("enable-auto-tool-choice" in text or "tool-call-parser" in text or "not supported" in text):
return False
return None
def _row(name: str, item: dict[str, Any], props: dict[str, Any], tools: Optional[bool]) -> dict[str, Any]:
served_id = str(item["id"])
model_id = f"{name}:{served_id}"
meta = item.get("meta") if isinstance(item.get("meta"), dict) else {}
if name == "llamacpp":
context = _int((props.get("default_generation_settings") or {}).get("n_ctx"))
context_max = _int(meta.get("n_ctx_train"))
size = _int(meta.get("size"))
elif name == "lmstudio":
# LM Studio sizes the window when it loads the model (JIT, on first request, by
# default); until then the trained window is the only figure it reports.
own = props.get(served_id) if isinstance(props.get(served_id), dict) else {}
context_max = _int(own.get("max_context_length"))
context = _int(own.get("loaded_context_length")) or context_max
size = None
else:
context = _int(item.get("max_model_len"))
context_max = context
size = None
rec = recommendation_for(model_id)
chosen = model_config.context_size_for(model_id)
# A chat template that reads `enable_thinking` has a thinking switch (Nemotron, Qwen3);
# the recommendation table says so for the models it knows.
template = str(props.get("chat_template") or "")
thinking = True if "enable_thinking" in template else (rec.thinking_available if rec else None)
return {
"model": model_id,
"name": served_id,
"size_bytes": size,
"tools": tools,
"thinking": thinking,
"vision": None,
"remote": False,
"parameter_size": None,
"quantization": None,
"context_max": context_max,
# The server fixes its window at start; a saved size only sizes compaction.
"context": chosen or context,
"context_from": "user" if chosen else "server",
"fit": local_machine.fit_for(size) if size else "unknown",
"recommendation": rec.name if rec else None,
}
def _int(value: Any) -> Optional[int]:
return int(value) if isinstance(value, (int, float)) and not isinstance(value, bool) and value > 0 else None
def thinking_as_template_kwargs(settings: dict[str, Any]) -> dict[str, Any]:
"""The thinking switch travels as Ollama's `think` field (model_config.py); llama.cpp,
vLLM, LM Studio and the other OpenAI-compatible local servers read it from the chat
template's `enable_thinking` instead. Rewrites one into the other and leaves everything
else alone."""
extra = settings.get("extra_body")
if not isinstance(extra, dict) or "think" not in extra:
return settings
extra = dict(extra)
think = extra.pop("think")
if isinstance(think, bool):
kwargs = dict(extra.get("chat_template_kwargs") or {})
kwargs["enable_thinking"] = think
extra["chat_template_kwargs"] = kwargs
out = dict(settings)
if extra:
out["extra_body"] = extra
else:
out.pop("extra_body", None)
return out