Skip to content
Python

llama.cpp and vLLM as providers

Devika Verma514c0beb379f
diff --git a/coworker/engine.py b/coworker/engine.py
index c787a9ba..a915fc2a 100644
--- a/coworker/engine.py
+++ b/coworker/engine.py
@@ -894,9 +894,10 @@ class TurnEngine:
if not cfg["context_window"]:
# Ollama's window is the num_ctx we send (providers/ollama_context.py),
# not the 128k guess — compact before that window context-shifts.
+ from .providers.local_server import context_window_for as _server_window
from .providers.ollama_context import context_window_for
- cfg["context_window"] = context_window_for(self.model)
+ cfg["context_window"] = context_window_for(self.model) or _server_window(self.model)
if not cfg["context_window"] and not self._warned_context_fallback:
# OPE-170: an unlisted model compacts on the 128k guess, which for a
# 1M-window model means compacting at a tenth of the window and
diff --git a/coworker/providers/capabilities.py b/coworker/providers/capabilities.py
index 2548c1b4..9c7e05e4 100644
--- a/coworker/providers/capabilities.py
+++ b/coworker/providers/capabilities.py
@@ -25,7 +25,7 @@ def capabilities_for(model: str) -> ModelCapabilities:
# Ollama (local) models vary widely and many fake/mishandle parallel tool calls — assume
# tools work (we only point at tool-capable models) but stay conservative otherwise.
# Vision is detected from common model naming conventions (-vl, vision, llava, etc.).
- if provider == "ollama":
+ if provider in ("ollama", "llamacpp", "vllm"):
_vision_patterns = ("-vl", "vision", "llava", "bakllava", "cogvlm", "minicpm-v")
has_vision = any(p in name for p in _vision_patterns)
return ModelCapabilities(
diff --git a/coworker/providers/local_server.py b/coworker/providers/local_server.py
new file mode 100644
index 00000000..3f87df26
--- /dev/null
+++ b/coworker/providers/local_server.py
@@ -0,0 +1,200 @@
+"""llama.cpp and vLLM: servers the user starts, reached through their OpenAI-compatible
+`/v1`. One module for both, since OpenWorker treats them the same way and the two
+differ only in what they report about themselves.
+
+- llama.cpp `llama-server`: `localhost:8080` by default; `/v1/models` lists the loaded
+ model (its id is the file path unless the server was started with `--alias`), with
+ `meta.n_ctx_train` for the trained window; `/props` carries the window the server was
+ started with (`default_generation_settings.n_ctx`), which cannot change while it runs.
+ Tool calls go through the model's chat template (`--jinja`, on by default).
+- vLLM: `localhost:8000` by default, one model per server; `/v1/models` gives the id and
+ `max_model_len`. Tool calls work only when the server was started with
+ `--enable-auto-tool-choice --tool-call-parser <name>`; the harness cannot turn them on,
+ but it can find out, and say so, before a session breaks.
+
+Neither tells the harness what the machine is; that is local_machine.py. Both may take an
+API key; without one any value is accepted, and OpenWorker sends a placeholder.
+"""
+
+from __future__ import annotations
+
+import threading
+import time
+from typing import Any, Optional
+
+import httpx
+
+from .. import model_config
+from . import local_machine
+from .recommended import recommendation_for
+
+LOCAL_SERVERS: dict[str, dict[str, str]] = {
+ "llamacpp": {"title": "llama.cpp", "default_base": "http://localhost:8080"},
+ "vllm": {"title": "vLLM", "default_base": "http://localhost:8000"},
+}
+PLACEHOLDER_KEY = "none"
+CACHE_SECONDS = 60.0
+
+_cache: dict[tuple[str, str], tuple[float, list[dict[str, Any]]]] = {}
+_lock = threading.Lock()
+
+
+def v1_base(name: str, base_url: Optional[str]) -> str:
+ """The server's `/v1` root from what the user typed: the root, or already `/v1`."""
+ base = (base_url or LOCAL_SERVERS[name]["default_base"]).strip().rstrip("/")
+ return base if base.endswith("/v1") else base + "/v1"
+
+
+def _headers(api_key: Optional[str]) -> dict[str, str]:
+ key = (api_key or "").strip()
+ return {"Authorization": f"Bearer {key}"} if key else {}
+
+
+def alive(name: str, base_url: Optional[str], api_key: Optional[str] = None) -> bool:
+ try:
+ return httpx.get(v1_base(name, base_url) + "/models", headers=_headers(api_key), timeout=0.8).status_code == 200
+ except Exception: # noqa: BLE001 - a probe never raises
+ return False
+
+
+def model_facts(
+ name: str, base_url: Optional[str], api_key: Optional[str] = None, *, fresh: bool = False
+) -> list[dict[str, Any]]:
+ """One row per model the server serves (usually one). Empty when it does not answer."""
+ base = v1_base(name, base_url)
+ key = (name, base)
+ now = time.monotonic()
+ with _lock:
+ hit = _cache.get(key)
+ if hit and not fresh and now - hit[0] < CACHE_SECONDS:
+ return [dict(r) for r in hit[1]]
+ rows = _fetch(name, base, api_key)
+ with _lock:
+ _cache[key] = (now, rows)
+ return [dict(r) for r in rows]
+
+
+def forget_all() -> None:
+ with _lock:
+ _cache.clear()
+
+
+# The server each provider client was built against (registry._local_server), so a
+# window lookup from the engine can ask the right server without a profile in hand.
+_built: dict[str, tuple[str, Optional[str]]] = {}
+
+
+def remember_server(name: str, base_url: Optional[str], api_key: Optional[str]) -> None:
+ _built[name] = (base_url or "", api_key)
+
+
+def context_window_for(model: str) -> Optional[int]:
+ """The window a served model runs at: from the last facts read, else read now from
+ the server the provider was built against. None when unknown or not a local-server
+ model."""
+ vendor, _, bare = model.partition(":")
+ if vendor not in LOCAL_SERVERS:
+ return None
+
+ def find(rows: list[dict[str, Any]]) -> Optional[int]:
+ for row in rows:
+ if row.get("name") == bare and row.get("context"):
+ return int(row["context"])
+ return None
+
+ with _lock:
+ for (name, _base), (_at, rows) in _cache.items():
+ if name == vendor:
+ found = find(rows)
+ if found:
+ return found
+ base_url, api_key = _built.get(vendor, ("", None))
+ return find(model_facts(vendor, base_url or None, api_key))
+
+
+def _fetch(name: str, base: str, api_key: Optional[str]) -> list[dict[str, Any]]:
+ headers = _headers(api_key)
+ try:
+ listing = httpx.get(base + "/models", headers=headers, timeout=4.0).json()
+ except (httpx.HTTPError, ValueError, OSError):
+ return []
+ items = listing.get("data") if isinstance(listing, dict) else None
+ if not isinstance(items, list):
+ return []
+ props: dict[str, Any] = {}
+ if name == "llamacpp":
+ try:
+ got = httpx.get(base[: -len("/v1")] + "/props", headers=headers, timeout=4.0).json()
+ props = got if isinstance(got, dict) else {}
+ except (httpx.HTTPError, ValueError, OSError):
+ props = {}
+ tools: Optional[bool] = None
+ if name == "vllm":
+ tools = _vllm_tools_on(base, headers, str((items[0] or {}).get("id") or "")) if items else None
+ elif name == "llamacpp":
+ template = str(props.get("chat_template") or "")
+ tools = ("tools" in template) if template else None
+ rows = []
+ for item in items:
+ if not isinstance(item, dict) or not item.get("id"):
+ continue
+ rows.append(_row(name, item, props, tools))
+ return rows
+
+
+def _vllm_tools_on(base: str, headers: dict[str, str], model: str) -> Optional[bool]:
+ """Ask vLLM for a one-token reply with a tool offered. A server started without tool
+ calling refuses the request and names the flag; one with it answers."""
+ probe = {
+ "model": model,
+ "messages": [{"role": "user", "content": "hi"}],
+ "max_tokens": 1,
+ "tools": [{"type": "function", "function": {"name": "noop", "parameters": {"type": "object", "properties": {}}}}],
+ }
+ try:
+ resp = httpx.post(base + "/chat/completions", json=probe, headers=headers, timeout=30.0)
+ except (httpx.HTTPError, OSError):
+ return None
+ if resp.status_code == 200:
+ return True
+ text = resp.text.lower()
+ if "tool" in text and ("enable-auto-tool-choice" in text or "tool-call-parser" in text or "not supported" in text):
+ return False
+ return None
+
+
+def _row(name: str, item: dict[str, Any], props: dict[str, Any], tools: Optional[bool]) -> dict[str, Any]:
+ served_id = str(item["id"])
+ model_id = f"{name}:{served_id}"
+ meta = item.get("meta") if isinstance(item.get("meta"), dict) else {}
+ if name == "llamacpp":
+ context = _int((props.get("default_generation_settings") or {}).get("n_ctx"))
+ context_max = _int(meta.get("n_ctx_train"))
+ size = _int(meta.get("size"))
+ else:
+ context = _int(item.get("max_model_len"))
+ context_max = context
+ size = None
+ rec = recommendation_for(model_id)
+ chosen = model_config.context_size_for(model_id)
+ return {
+ "model": model_id,
+ "name": served_id,
+ "size_bytes": size,
+ "tools": tools,
+ "thinking": rec.thinking_available if rec else None,
+ "vision": None,
+ "remote": False,
+ "parameter_size": None,
+ "quantization": None,
+ "context_max": context_max,
+ # The server fixes its window at start; a saved size only sizes compaction.
+ "context": chosen or context,
+ "context_from": "user" if chosen else "server",
+ "fit": local_machine.fit_for(size) if size else "unknown",
+ "recommendation": rec.name if rec else None,
+ }
+
+
+def _int(value: Any) -> Optional[int]:
+ return int(value) if isinstance(value, (int, float)) and not isinstance(value, bool) and value > 0 else None
diff --git a/coworker/providers/registry.py b/coworker/providers/registry.py
index 4efa9235..2a4e4c74 100644
--- a/coworker/providers/registry.py
+++ b/coworker/providers/registry.py
@@ -90,6 +90,9 @@ class ProviderDescriptor:
# (tokens in its `provider:<name>` profile) and the GUI renders connect/sign-out
# instead of fields. None → the usual key/field form.
auth: Optional[str] = None
+ # How the GUI groups it: "local" (a server on the user's own hardware), "subscription"
+ # (a plan signed into), or "api_key" (everything else).
+ kind: str = "api_key"
def to_dict(self) -> dict[str, Any]:
return {
@@ -100,6 +103,7 @@ class ProviderDescriptor:
"recommended_model": self.recommended_model,
"blurb": self.blurb,
"auth": self.auth,
+ "kind": self.kind,
}
@@ -205,6 +209,20 @@ def _build_ollama(profile: dict[str, Any], secrets: Any) -> ProviderClient:
return OpenAIProvider(api_key="ollama", base_url=base_url, http_client=ollama_http_client())
+def _local_server(name: str):
+ """Builder for llama.cpp and vLLM (providers/local_server.py): the user's own server,
+ reached through its `/v1`, with an optional key. Without a key the server takes any
+ value, so a placeholder goes out."""
+ from .local_server import PLACEHOLDER_KEY, remember_server, v1_base
+
+ def build(profile: dict[str, Any], secrets: Any) -> ProviderClient:
+ key = ((profile or {}).get("api_key") or "").strip() or PLACEHOLDER_KEY
+ remember_server(name, (profile or {}).get("base_url"), key if key != PLACEHOLDER_KEY else None)
+ return OpenAIProvider(api_key=key, base_url=v1_base(name, (profile or {}).get("base_url")))
+
+ return build
+
+
def _openai_compat(vendor: str, default_base_url: str, env_key: Optional[str] = None):
"""Builder factory for vendors reached through their OpenAI-compatible API (Z AI, DeepSeek,
Kimi, MiniMax, Qwen, xAI, Mistral). The key is resolved from the vendor's OWN profile (or its
@@ -366,6 +384,7 @@ DESCRIPTORS: list[ProviderDescriptor] = [
),
ProviderDescriptor(
name="openai-codex",
+ kind="subscription",
title="ChatGPT subscription",
needs_key=False,
fields=[],
@@ -690,6 +709,55 @@ DESCRIPTORS: list[ProviderDescriptor] = [
# Reliable native tool-calling + strong coding quality (verified). Pull with
# `ollama pull qwen3-coder:30b`.
recommended_model="qwen3-coder:30b",
+ kind="local",
+ ),
+ ProviderDescriptor(
+ name="llamacpp",
+ title="llama.cpp",
+ needs_key=False,
+ fields=[
+ ProviderField(
+ "base_url",
+ "Server address",
+ secret=False,
+ required=False,
+ placeholder="http://localhost:8080",
+ help="Where `llama-server` is listening. The /v1 path is added automatically.",
+ ),
+ ProviderField(
+ "api_key",
+ "API key (only if the server was started with one)",
+ secret=True,
+ required=False,
+ ),
+ ],
+ build=_local_server("llamacpp"),
+ blurb="A llama-server you started yourself, on this computer or another one.",
+ kind="local",
+ ),
+ ProviderDescriptor(
+ name="vllm",
+ title="vLLM",
+ needs_key=False,
+ fields=[
+ ProviderField(
+ "base_url",
+ "Server address",
+ secret=False,
+ required=False,
+ placeholder="http://localhost:8000",
+ help="Where `vllm serve` is listening. The /v1 path is added automatically.",
+ ),
+ ProviderField(
+ "api_key",
+ "API key (only if the server was started with one)",
+ secret=True,
+ required=False,
+ ),
+ ],
+ build=_local_server("vllm"),
+ blurb="A vLLM server, usually on a Linux machine with an NVIDIA GPU.",
+ kind="local",
),
]
@@ -976,6 +1044,11 @@ def verify_provider_key(
elif name == "ollama":
base = _normalize_ollama_url(base_url)
resp = httpx.get(base.rstrip("/") + "/models", timeout=timeout)
+ elif name in ("llamacpp", "vllm"):
+ from .local_server import v1_base
+
+ headers = {"Authorization": f"Bearer {key}"} if key else {}
+ resp = httpx.get(v1_base(name, base_url) + "/models", headers=headers, timeout=timeout)
elif name in ("ark", "ark-agent-plan-cn"):
default_base = next(
(f.default for f in d.fields if f.key == "base_url" and f.default), ""
diff --git a/coworker/server/manager.py b/coworker/server/manager.py
index e8c7d790..25e42837 100644
--- a/coworker/server/manager.py
+++ b/coworker/server/manager.py
@@ -4151,6 +4151,8 @@ class SessionManager:
topped up with the compat-vendor extras the matrix doesn't vouch for."""
if name == "ollama":
return [m.split(":", 1)[-1] for m in self._ollama_models()]
+ if name in ("llamacpp", "vllm"):
+ return [m.split(":", 1)[-1] for m in self._local_server_models(name)]
from ..providers.matrix import models_for_provider
return list(
@@ -4212,9 +4214,10 @@ class SessionManager:
if rec and rec in suggested:
# OpenAI models stay bare (the router's default); others carry their prefix.
added = rec if name == "openai" else f"{name}:{rec}"
- elif name == "ollama" and suggested:
- # Ollama offers whatever the user pulled — the recommended model may not be one.
- added = f"ollama:{suggested[0]}"
+ elif name in ("ollama", "llamacpp", "vllm") and suggested:
+ # A local server offers whatever the user loaded — the recommended model may
+ # not be one.
+ added = f"{name}:{suggested[0]}"
if added:
self.add_model(added)
if added and not self.model_selectable(self.model):
@@ -4415,16 +4418,49 @@ class SessionManager:
def local_model_facts(self, name: str) -> dict[str, Any]:
"""The models a local provider has, with size, tool support, thinking, context
- and how each sits on this machine. Ollama today; llama.cpp and vLLM report
- their own single model the same way once connected."""
- from ..providers import local_machine, ollama_facts
+ and how each sits on this machine. Ollama lists what it pulled; llama.cpp and
+ vLLM report the model they serve."""
+ from ..providers import local_server, ollama_facts
if name == "ollama":
profile = self.secrets.get("provider:ollama") or {}
rows = ollama_facts.model_facts(profile.get("base_url"), fresh=True)
return {"provider": name, "models": rows, "alive": self._ollama_alive()}
+ if name in local_server.LOCAL_SERVERS:
+ profile = self.secrets.get(f"provider:{name}") or {}
+ rows = local_server.model_facts(name, profile.get("base_url"), profile.get("api_key"), fresh=True)
+ return {"provider": name, "models": rows, "alive": self._local_server_alive(name)}
return {"provider": name, "models": [], "error": "no local model list for this provider"}
+ def _local_server_alive(self, name: str) -> bool:
+ """Best-effort liveness of a llama.cpp or vLLM server, cached 30 s like Ollama's."""
+ import time
+
+ from ..providers import local_server
+
+ now = time.monotonic()
+ cache = getattr(self, "_local_alive_cache", None) or {}
+ hit = cache.get(name)
+ if hit and now - hit[0] < 30:
+ return hit[1]
+ profile = self.secrets.get(f"provider:{name}") or {}
+ alive = local_server.alive(name, profile.get("base_url"), profile.get("api_key"))
+ cache[name] = (now, alive)
+ self._local_alive_cache = cache
+ return alive
+
+ def _local_server_models(self, name: str) -> list[str]:
+ """The model ids a llama.cpp or vLLM server serves, as `<name>:<id>`; empty when
+ it does not answer or was started without tool calling."""
+ from ..providers import local_server
+
+ profile = self.secrets.get(f"provider:{name}") or {}
+ return [
+ row["model"]
+ for row in local_server.model_facts(name, profile.get("base_url"), profile.get("api_key"))
+ if row.get("tools") is not False
+ ]
+
def system_facts(self) -> dict[str, Any]:
"""This machine, for the "Your system" section: processor, graphics, memory,
storage, and the largest model file that runs well here."""
@@ -4438,6 +4474,8 @@ class SessionManager:
provider = self._model_provider(model)
if provider == "ollama":
return self._ollama_alive()
+ if provider in ("llamacpp", "vllm"):
+ return self._local_server_alive(provider)
return self._provider_configured(provider)
def persona_models(self, persona_id: str) -> list[str]:
@@ -4669,6 +4707,9 @@ class SessionManager:
# Pulled Ollama models are offered live while Ollama answers — no need to add each one.
if self._ollama_alive():
selectable = list(dict.fromkeys([*selectable, *self._ollama_models()]))
+ for local in ("llamacpp", "vllm"):
+ if self.secrets.get(f"provider:{local}") and self._local_server_alive(local):
+ selectable = list(dict.fromkeys([*selectable, *self._local_server_models(local)]))
if self.model not in selectable:
selectable.insert(0, self.model)
from ..providers.matrix import model_context_windows, model_labels
diff --git a/docs/local-servers.md b/docs/local-servers.md
new file mode 100644
index 00000000..e904fe9e
--- /dev/null
+++ b/docs/local-servers.md
@@ -0,0 +1,59 @@
+# llama.cpp and vLLM
+
+OpenWorker can use a model served by [llama.cpp](https://github.com/ggml-org/llama.cpp)
+or [vLLM](https://github.com/vllm-project/vllm), on this computer or on another machine.
+Each is its own provider in Settings ▸ Models & Keys, under "On your own hardware", next
+to Ollama. Give it the server's address and, only if the server was started with one, its
+API key. OpenWorker reads the rest from the server.
+
+## llama.cpp
+
+Start `llama-server` with a model file. It serves one model:
+
+```bash
+llama-server -m model.gguf --alias my-model -c 65536 -ngl 99 --port 8080
+```
+
+- `--alias` names the model; without it the model's id is the file's path.
+- `-c` sets the context size. OpenWorker cannot change it while the server runs; the
+ settings page shows it and sizes auto-compaction to it.
+- Tool calling goes through the model's chat template (`--jinja`, on by default). Models
+ whose template has no tool support cannot do agent work.
+- A model file that Ollama downloaded can be served directly; the files under
+ `~/.ollama/models/blobs` are GGUF.
+
+The default address is `http://localhost:8080`.
+
+## vLLM
+
+vLLM runs on Linux with an NVIDIA GPU, so it is usually on another machine:
+
+```bash
+vllm serve nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 \
+ --enable-auto-tool-choice --tool-call-parser qwen3_coder --reasoning-parser nemotron_v3 \
+ --max-model-len 65536 --host 0.0.0.0 --port 8000
+```
+
+- `--enable-auto-tool-choice --tool-call-parser <name>` turns tool calling on. Without
+ them coworkers cannot work with the server; OpenWorker checks when it connects and says
+ so, with the options to add. The parser name depends on the model; the model's card on
+ Hugging Face gives it (the line above is NVIDIA's for Nemotron 3.5 Lightning).
+- `--max-model-len` sets the context size, shown in the settings page and used for
+ auto-compaction.
+- The model's id is its Hugging Face name.
+
+The default address is `http://localhost:8000`.
+
+## In the CLI
+
+The model id is `llamacpp:<id>` or `vllm:<id>`:
+
+```bash
+openworker run --model llamacpp:my-model --prompt "..."
+```
+
+## Per-model settings
+
+Longest reply, sampling, thinking and auto-compaction are set per model as for any other
+provider; see [model settings](model-settings.md). The context size of these two servers
+is set when the server starts, so OpenWorker shows it rather than setting it.
diff --git a/tests/test_local_server.py b/tests/test_local_server.py
new file mode 100644
index 00000000..57e1daa5
--- /dev/null
+++ b/tests/test_local_server.py
@@ -0,0 +1,113 @@
+"""llama.cpp and vLLM as providers (providers/local_server.py, registry.py): the user's
+own server through its /v1, what each reports about its model, and how the server's
+model lists, liveness and adoption treat them."""
+
+from __future__ import annotations
+
+import json
+
+import httpx
+import pytest
+
+from coworker import model_config
+from coworker.providers import local_machine, local_server
+from coworker.providers.capabilities import capabilities_for
+from coworker.providers.registry import get_descriptor, provider_descriptors
+
+
[email protected](autouse=True)
+def _state(tmp_path, monkeypatch):
+ monkeypatch.setenv("COWORKER_STATE_DIR", str(tmp_path / "state"))
+ local_server.forget_all()
+
+
+def _fake_http(monkeypatch, handler):
+ fake = httpx.Client(transport=httpx.MockTransport(handler))
+ monkeypatch.setattr(httpx, "get", lambda url, headers=None, timeout=None: fake.get(url, headers=headers))
+ monkeypatch.setattr(httpx, "post", lambda url, json=None, headers=None, timeout=None: fake.post(url, json=json, headers=headers))
+
+
+def test_descriptors_carry_a_kind_for_grouping():
+ kinds = {d.name: d.kind for d in provider_descriptors()}
+ assert kinds["ollama"] == kinds["llamacpp"] == kinds["vllm"] == "local"
+ assert kinds["openai-codex"] == "subscription"
+ assert kinds["openai"] == kinds["anthropic"] == "api_key"
+ for name in ("llamacpp", "vllm"):
+ d = get_descriptor(name)
+ assert d is not None and not d.needs_key
+ assert [f.key for f in d.fields] == ["base_url", "api_key"]
+ assert not any(f.required for f in d.fields)
+ assert capabilities_for(f"{name}:some/model").tools
+
+
+def test_v1_base_takes_the_root_or_an_explicit_v1():
+ assert local_server.v1_base("llamacpp", None) == "http://localhost:8080/v1"
+ assert local_server.v1_base("llamacpp", "http://box.local:9000/") == "http://box.local:9000/v1"
+ assert local_server.v1_base("vllm", "http://box.local:8000/v1") == "http://box.local:8000/v1"
+
+
+def test_the_builder_sends_a_placeholder_key_when_none_is_set():
+ client = get_descriptor("llamacpp").build({"base_url": "http://localhost:8080"}, None)
+ assert client._base_url == "http://localhost:8080/v1"
+ assert client._api_key == "none"
+ client = get_descriptor("vllm").build({"base_url": "http://gpu:8000", "api_key": "s3"}, None)
+ assert client._base_url == "http://gpu:8000/v1" and client._api_key == "s3"
+
+
+def test_llamacpp_facts_come_from_models_and_props(monkeypatch):
+ def handler(request: httpx.Request) -> httpx.Response:
+ if request.url.path == "/v1/models":
+ return httpx.Response(200, json={"data": [{"id": "/models/nemotron-3.5-lightning-30b-q4_k_m.gguf",
+ "meta": {"n_ctx_train": 1048576, "size": 25 * local_machine.GB}}]})
+ if request.url.path == "/props":
+ return httpx.Response(200, json={"default_generation_settings": {"n_ctx": 32768},
+ "chat_template": "{% if tools %}...{% endif %}"})
+ return httpx.Response(404)
+
+ _fake_http(monkeypatch, handler)
+ monkeypatch.setattr(local_machine, "model_memory_bytes", lambda: 96 * local_machine.GB)
+ (row,) = local_server.model_facts("llamacpp", "http://localhost:8080")
+ assert row["model"] == "llamacpp:/models/nemotron-3.5-lightning-30b-q4_k_m.gguf"
+ assert row["context"] == 32768 and row["context_from"] == "server"
+ assert row["context_max"] == 1048576 and row["tools"] is True
+ assert row["fit"] == "runs_well" and row["recommendation"] == "NVIDIA Nemotron 3.5 Lightning"
+ assert row["thinking"] is True # from the recommendation table
+ # Compaction sizes itself to the server's window.
+ assert local_server.context_window_for(row["model"]) == 32768
+ assert local_server.context_window_for("ollama:x") is None
+
+
+def test_vllm_facts_probe_whether_tool_calling_is_on(monkeypatch):
+ calls = []
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ if request.url.path == "/v1/models":
+ return httpx.Response(200, json={"data": [{"id": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4", "max_model_len": 131072}]})
+ if request.url.path == "/v1/chat/completions":
+ calls.append(json.loads(request.content))
+ return httpx.Response(400, json={"error": {"message": '"auto" tool choice requires --enable-auto-tool-choice and --tool-call-parser to be set'}})
+ return httpx.Response(404)
+
+ _fake_http(monkeypatch, handler)
+ (row,) = local_server.model_facts("vllm", "http://gpu:8000", "k")
+ assert row["tools"] is False and row["context"] == 131072
+ assert calls[0]["max_tokens"] == 1 and calls[0]["tools"]
+ assert local_server.alive("vllm", "http://gpu:8000", "k") is True
+
+
+def test_the_server_offers_and_adopts_a_local_servers_model(tmp_path, monkeypatch):
+ from coworker.server.manager import SessionManager
+
+ monkeypatch.delenv("OPENAI_API_KEY", raising=False)
+ rows = [{"model": "llamacpp:/m/a.gguf", "name": "/m/a.gguf", "tools": True, "context": 32768}]
+ monkeypatch.setattr(local_server, "model_facts", lambda name, base, key=None, fresh=False: rows)
+ monkeypatch.setattr(local_server, "alive", lambda name, base, key=None: True)
+ mgr = SessionManager(data_dir=tmp_path / "data")
+ out = mgr.set_provider("llamacpp", {"base_url": "http://localhost:8080"})
+ assert out["ok"]
+ settings = mgr.get_settings()
+ assert "llamacpp:/m/a.gguf" in settings["models"]
+ assert mgr.model_selectable("llamacpp:/m/a.gguf")
+ # A server started without tool calling offers nothing to the picker.
+ rows[0]["tools"] = False
+ assert mgr._local_server_models("llamacpp") == []