Skip to content
Python

Add Jan, Docker Model Runner and mlx-lm as local server providers

Rohit C Prasaddb6e0af7eb37
diff --git a/coworker/providers/capabilities.py b/coworker/providers/capabilities.py
index f9285fa5..88200dec 100644
--- a/coworker/providers/capabilities.py
+++ b/coworker/providers/capabilities.py
@@ -25,7 +25,9 @@ def capabilities_for(model: str) -> ModelCapabilities:
# Ollama (local) models vary widely and many fake/mishandle parallel tool calls — assume
# tools work (we only point at tool-capable models) but stay conservative otherwise.
# Vision is detected from common model naming conventions (-vl, vision, llava, etc.).
- if provider in ("ollama", "llamacpp", "vllm", "lmstudio"):
+ from .local_server import LOCAL_SERVERS
+
+ if provider == "ollama" or provider in LOCAL_SERVERS:
_vision_patterns = ("-vl", "vision", "llava", "bakllava", "cogvlm", "minicpm-v")
has_vision = any(p in name for p in _vision_patterns)
return ModelCapabilities(
diff --git a/coworker/providers/local_server.py b/coworker/providers/local_server.py
index 51f20375..94093b8f 100644
--- a/coworker/providers/local_server.py
+++ b/coworker/providers/local_server.py
@@ -1,6 +1,7 @@
-"""llama.cpp, vLLM and LM Studio: servers the user starts, reached through their
-OpenAI-compatible `/v1`. One module for all three, since OpenWorker treats them the same
-way and they differ only in what they report about themselves.
+"""llama.cpp, vLLM, LM Studio, Jan, Docker Model Runner and mlx-lm: servers the user
+starts, reached through their OpenAI-compatible `/v1`. One module for all of them, since
+OpenWorker treats them the same way and they differ only in what they report about
+themselves.
- llama.cpp `llama-server`: `localhost:8080` by default; `/v1/models` lists the loaded
model (its id is the file path unless the server was started with `--alias`), with
@@ -15,9 +16,16 @@ way and they differ only in what they report about themselves.
`/v1/models` lists every downloaded model, and its own `/api/v0/models` adds each one's
`max_context_length` and whether it is loaded. Tool calls go through the model's chat
template, as in llama.cpp.
-
-None of them tells the harness what the machine is; that is local_machine.py. Both may take an
-API key; without one any value is accepted, and OpenWorker sends a placeholder.
+- Jan: `127.0.0.1:1337` by default (Settings ▸ Local API Server ▸ Start Server, read from
+ docs.jan.ai/desktop/api-server 2026-10-26); a plain OpenAI listing, an optional key.
+- Docker Model Runner: `localhost:12434/engines/v1` by default (TCP host access turned on
+ in Docker Desktop; docs.docker.com/ai/model-runner/api-reference, read 2026-10-26). The
+ `/engines` segment is part of the path, so v1_base adds it when the user types the root.
+- mlx-lm: `127.0.0.1:8080` by default (`mlx_lm.server`, Apple silicon only; its server.py
+ read 2026-10-26) — the same port as llama.cpp, so the help says to move one of them.
+
+None of them tells the harness what the machine is; that is local_machine.py. All may take
+an API key; without one any value is accepted, and OpenWorker sends a placeholder.
"""
from __future__ import annotations
@@ -36,6 +44,9 @@ LOCAL_SERVERS: dict[str, dict[str, str]] = {
"llamacpp": {"title": "llama.cpp", "default_base": "http://localhost:8080"},
"vllm": {"title": "vLLM", "default_base": "http://localhost:8000"},
"lmstudio": {"title": "LM Studio", "default_base": "http://localhost:1234"},
+ "jan": {"title": "Jan", "default_base": "http://127.0.0.1:1337"},
+ "docker": {"title": "Docker Model Runner", "default_base": "http://localhost:12434/engines"},
+ "mlx": {"title": "mlx-lm", "default_base": "http://127.0.0.1:8080"},
}
PLACEHOLDER_KEY = "none"
CACHE_SECONDS = 60.0
@@ -45,8 +56,12 @@ _lock = threading.Lock()
def v1_base(name: str, base_url: Optional[str]) -> str:
- """The server's `/v1` root from what the user typed: the root, or already `/v1`."""
+ """The server's `/v1` root from what the user typed: the root, or already `/v1`.
+ Docker Model Runner serves under `/engines/v1` (or `/engines/<engine>/v1`), so a bare
+ host:port gets the `/engines` segment too."""
base = (base_url or LOCAL_SERVERS[name]["default_base"]).strip().rstrip("/")
+ if name == "docker" and "/engines" not in base:
+ base += "/engines"
return base if base.endswith("/v1") else base + "/v1"
@@ -234,8 +249,9 @@ def _int(value: Any) -> Optional[int]:
def thinking_as_template_kwargs(settings: dict[str, Any]) -> dict[str, Any]:
"""The thinking switch travels as Ollama's `think` field (model_config.py); llama.cpp,
- vLLM and LM Studio read it from the chat template's `enable_thinking` instead. Rewrites one into
- the other and leaves everything else alone."""
+ vLLM, LM Studio and the other OpenAI-compatible local servers read it from the chat
+ template's `enable_thinking` instead. Rewrites one into the other and leaves everything
+ else alone."""
extra = settings.get("extra_body")
if not isinstance(extra, dict) or "think" not in extra:
return settings
diff --git a/coworker/providers/registry.py b/coworker/providers/registry.py
index 32f39efa..225bbb16 100644
--- a/coworker/providers/registry.py
+++ b/coworker/providers/registry.py
@@ -26,6 +26,7 @@ from .anthropic_provider import AnthropicProvider
from .base import ProviderClient
from .bedrock_provider import BedrockProvider
from .gemini_provider import GeminiProvider
+from .local_server import LOCAL_SERVERS
from .openai_provider import OpenAIProvider
from .openai_responses import OpenAIResponsesProvider
from .vertex_provider import VertexProvider
@@ -214,7 +215,8 @@ def _build_ollama(profile: dict[str, Any], secrets: Any) -> ProviderClient:
def _local_server(name: str):
- """Builder for llama.cpp, vLLM and LM Studio (providers/local_server.py): the user's own server,
+ """Builder for the OpenAI-compatible local servers (providers/local_server.py: llama.cpp,
+ vLLM, LM Studio, Jan, Docker Model Runner, mlx-lm): the user's own server,
reached through its `/v1`, with an optional key. Without a key the server takes any
value, so a placeholder goes out."""
from .local_server import PLACEHOLDER_KEY, remember_server, thinking_as_template_kwargs, v1_base
@@ -936,6 +938,88 @@ DESCRIPTORS: list[ProviderDescriptor] = [
blurb="The LM Studio app's local server, with whatever models you downloaded there.",
kind="local",
),
+ # Three more OpenAI-compatible local servers (2026-10-26). Defaults read from each
+ # tool's own docs: docs.jan.ai/desktop/api-server (127.0.0.1:1337, /v1 prefix),
+ # docs.docker.com/ai/model-runner/api-reference (localhost:12434/engines/v1),
+ # mlx-lm's server.py (127.0.0.1:8080). No recommended model: each serves whatever the
+ # user loaded, and the picker lists that.
+ ProviderDescriptor(
+ name="jan",
+ title="Jan",
+ needs_key=False,
+ fields=[
+ ProviderField(
+ "base_url",
+ "Server address",
+ secret=False,
+ required=False,
+ placeholder="http://127.0.0.1:1337",
+ help="Where Jan's local API server is listening (Settings ▸ Local API Server ▸ "
+ "Start Server). The /v1 path is added automatically.",
+ ),
+ ProviderField(
+ "api_key",
+ "API key (only if you set one in Jan's server settings)",
+ secret=True,
+ required=False,
+ ),
+ ],
+ build=_local_server("jan"),
+ blurb="The Jan app's local API server, with the models you downloaded there.",
+ kind="local",
+ ),
+ ProviderDescriptor(
+ name="docker",
+ title="Docker Model Runner",
+ needs_key=False,
+ fields=[
+ ProviderField(
+ "base_url",
+ "Server address",
+ secret=False,
+ required=False,
+ placeholder="http://localhost:12434/engines",
+ help="Docker Model Runner's TCP address (enable it in Docker Desktop ▸ Settings ▸ "
+ "AI, or `docker desktop enable model-runner --tcp 12434`). The /engines/v1 path "
+ "is added automatically.",
+ ),
+ ProviderField(
+ "api_key",
+ "API key (only if the runner was started with one)",
+ secret=True,
+ required=False,
+ ),
+ ],
+ build=_local_server("docker"),
+ blurb="Models pulled with `docker model pull`, served by Docker Desktop.",
+ kind="local",
+ ),
+ ProviderDescriptor(
+ name="mlx",
+ title="mlx-lm",
+ needs_key=False,
+ fields=[
+ ProviderField(
+ "base_url",
+ "Server address",
+ secret=False,
+ required=False,
+ placeholder="http://127.0.0.1:8080",
+ help="Where `mlx_lm.server` is listening. It defaults to port 8080, the same as "
+ "llama.cpp — if you run both, start one on another port "
+ "(`mlx_lm.server --port 8081`). The /v1 path is added automatically.",
+ ),
+ ProviderField(
+ "api_key",
+ "API key (only if the server was started with one)",
+ secret=True,
+ required=False,
+ ),
+ ],
+ build=_local_server("mlx"),
+ blurb="An mlx-lm server on Apple silicon (`pip install mlx-lm`, then `mlx_lm.server --model <repo>`).",
+ kind="local",
+ ),
]
_BY_NAME = {d.name: d for d in DESCRIPTORS}
@@ -1319,7 +1403,7 @@ def verify_provider_key(
elif name == "ollama":
base = _normalize_ollama_url(base_url)
resp = httpx.get(base.rstrip("/") + "/models", timeout=timeout)
- elif name in ("llamacpp", "vllm", "lmstudio"):
+ elif name in LOCAL_SERVERS:
from .local_server import v1_base
headers = {"Authorization": f"Bearer {key}"} if key else {}
diff --git a/coworker/server/manager.py b/coworker/server/manager.py
index 8b75a8ea..85f508bb 100644
--- a/coworker/server/manager.py
+++ b/coworker/server/manager.py
@@ -91,6 +91,7 @@ from ..providers import (
provider_descriptors,
verify_provider_key,
)
+from ..providers.local_server import LOCAL_SERVERS
from ..providers.registry import CUSTOM_PROVIDERS_PROFILE
from ..secrets import SecretStore, state_dir
from ..sessions import SessionRecord, usage_totals
@@ -4361,7 +4362,7 @@ class SessionManager:
topped up with the compat-vendor extras the matrix doesn't vouch for."""
if name == "ollama":
return [m.split(":", 1)[-1] for m in self._ollama_models()]
- if name in ("llamacpp", "vllm", "lmstudio"):
+ if name in LOCAL_SERVERS:
return [m.split(":", 1)[-1] for m in self._local_server_models(name)]
from ..providers.matrix import models_for_provider
@@ -4430,7 +4431,7 @@ class SessionManager:
if rec and rec in suggested:
# OpenAI models stay bare (the router's default); others carry their prefix.
added = rec if name == "openai" else f"{name}:{rec}"
- elif name in ("ollama", "llamacpp", "vllm", "lmstudio") and suggested:
+ elif (name == "ollama" or name in LOCAL_SERVERS) and suggested:
# A local server offers whatever the user loaded — the recommended model may
# not be one.
added = f"{name}:{suggested[0]}"
@@ -4701,7 +4702,7 @@ class SessionManager:
return {"provider": name, "models": [], "error": "no local model list for this provider"}
def _local_server_alive(self, name: str) -> bool:
- """Best-effort liveness of a llama.cpp, vLLM or LM Studio server, cached 30 s like Ollama's."""
+ """Best-effort liveness of an OpenAI-compatible local server (local_server.LOCAL_SERVERS), cached 30 s like Ollama's."""
import time
from ..providers import local_server
@@ -4718,7 +4719,7 @@ class SessionManager:
return alive
def _local_server_models(self, name: str) -> list[str]:
- """The model ids a llama.cpp, vLLM or LM Studio server serves, as `<name>:<id>`; empty when
+ """The model ids a local server (local_server.LOCAL_SERVERS) serves, as `<name>:<id>`; empty when
it does not answer or was started without tool calling."""
from ..providers import local_server
@@ -4742,7 +4743,7 @@ class SessionManager:
provider = self._model_provider(model)
if provider == "ollama":
return self._ollama_alive()
- if provider in ("llamacpp", "vllm", "lmstudio"):
+ if provider in LOCAL_SERVERS:
return self._local_server_alive(provider)
return self._provider_configured(provider)
@@ -4975,7 +4976,7 @@ class SessionManager:
# Pulled Ollama models are offered live while Ollama answers — no need to add each one.
if self._ollama_alive():
selectable = list(dict.fromkeys([*selectable, *self._ollama_models()]))
- for local in ("llamacpp", "vllm", "lmstudio"):
+ for local in LOCAL_SERVERS:
if self.secrets.get(f"provider:{local}") and self._local_server_alive(local):
selectable = list(dict.fromkeys([*selectable, *self._local_server_models(local)]))
if self.model not in selectable:
diff --git a/docs/local-servers.md b/docs/local-servers.md
index f5a4addb..713e5e15 100644
--- a/docs/local-servers.md
+++ b/docs/local-servers.md
@@ -1,8 +1,10 @@
-# llama.cpp, vLLM and LM Studio
+# llama.cpp, vLLM, LM Studio, Jan, Docker Model Runner and mlx-lm
OpenWorker can use a model served by [llama.cpp](https://github.com/ggml-org/llama.cpp),
-[vLLM](https://github.com/vllm-project/vllm) or [LM Studio](https://lmstudio.ai), on this
-computer or on another machine. Each is its own provider in Settings ▸ Models & Keys, under "On your own hardware", next
+[vLLM](https://github.com/vllm-project/vllm), [LM Studio](https://lmstudio.ai),
+[Jan](https://jan.ai), [Docker Model Runner](https://docs.docker.com/ai/model-runner/) or
+[mlx-lm](https://github.com/ml-explore/mlx-lm), on this computer or on another machine.
+Each is its own provider in Settings ▸ Models & Keys, under "On your own hardware", next
to Ollama. Give it the server's address and, only if the server was started with one, its
API key. OpenWorker reads the rest from the server.
@@ -57,9 +59,50 @@ first request names it.
The default address is `http://localhost:1234`.
+## Jan
+
+Download a model in the Jan app, then go to Settings ▸ Local API Server, pick the model
+and click Start Server. Jan listens on `127.0.0.1:1337` and serves the OpenAI API under
+`/v1`.
+
+- A key is needed only if you set one in the server's configuration.
+
+The default address is `http://127.0.0.1:1337`.
+
+## Docker Model Runner
+
+Turn on Model Runner in Docker Desktop (Settings ▸ AI), with TCP host access, then pull a
+model:
+
+```bash
+docker model pull hf.co/<repo>:<quant>
+```
+
+- The OpenAI API is served under `/engines/v1` on port 12434; typing
+ `http://localhost:12434` or `http://localhost:12434/engines` both work.
+- The model's id is the name you pulled (`hf.co/...` or `ai/...`).
+
+The default address is `http://localhost:12434/engines`.
+
+## mlx-lm
+
+mlx-lm runs on Apple silicon:
+
+```bash
+pip install mlx-lm
+mlx_lm.server --model mlx-community/<repo>
+```
+
+- It listens on `127.0.0.1:8080`, the same port as llama.cpp. If you run both, start one
+ of them on another port, for example `mlx_lm.server --port 8081`, and type that address.
+- The model's id is the repo you started it with.
+
+The default address is `http://127.0.0.1:8080`.
+
## In the CLI
-The model id is `llamacpp:<id>`, `vllm:<id>` or `lmstudio:<id>`:
+The model id is `<provider>:<id>` — `llamacpp:<id>`, `vllm:<id>`, `lmstudio:<id>`,
+`jan:<id>`, `docker:<id>` or `mlx:<id>`:
```bash
openworker run --model llamacpp:my-model --prompt "..."
diff --git a/surfaces/gui/e2e/local-server-help.spec.ts b/surfaces/gui/e2e/local-server-help.spec.ts
index b839633b..d51001ba 100644
--- a/surfaces/gui/e2e/local-server-help.spec.ts
+++ b/surfaces/gui/e2e/local-server-help.spec.ts
@@ -10,6 +10,9 @@ const server = (name: string, title: string) => ({
for (const [name, title, says, link] of [
["llamacpp", "llama.cpp", "llama-server", "Install llama.cpp"],
["vllm", "vLLM", "NVIDIA GPU", "Install vLLM"],
+ ["jan", "Jan", "Local API Server", "Set up Jan's API server"],
+ ["docker", "Docker Model Runner", "docker model pull", "Set up Docker Model Runner"],
+ ["mlx", "mlx-lm", "--port 8081", "Set up mlx-lm"],
] as const) {
test(`${title}: its own help line, not Ollama's`, async ({ page }) => {
await page.route("**/v1/providers", (route) => route.fulfill({ json: [server(name, title)] }));
diff --git a/surfaces/gui/src/locales/en.json b/surfaces/gui/src/locales/en.json
index 0d82c75a..ad77da5d 100644
--- a/surfaces/gui/src/locales/en.json
+++ b/surfaces/gui/src/locales/en.json
@@ -1626,7 +1626,13 @@
"vllm_desc": "vLLM runs on Linux with an NVIDIA GPU. A key is needed only if the server was started with one.",
"install_vllm": "Install vLLM",
"lmstudio_desc": "The LM Studio app serves the models you downloaded there. A key is needed only if you turned one on in the app.",
- "install_lmstudio": "Install LM Studio"
+ "install_lmstudio": "Install LM Studio",
+ "jan_desc": "The Jan app serves the models you downloaded there once its local API server is on (Settings ▸ Local API Server ▸ Start Server). A key is needed only if you set one there.",
+ "install_jan": "Set up Jan's API server",
+ "docker_desc": "Docker Desktop serves models you pulled with `docker model pull hf.co/<repo>:<quant>`. Turn on Model Runner and its TCP access in Docker Desktop ▸ Settings ▸ AI. No key needed.",
+ "install_docker": "Set up Docker Model Runner",
+ "mlx_desc": "mlx-lm runs on Apple silicon: `pip install mlx-lm`, then `mlx_lm.server --model <repo>`. It uses port 8080 like llama.cpp — if you run both, start one on another port, e.g. `mlx_lm.server --port 8081`.",
+ "install_mlx": "Set up mlx-lm"
},
"workspace_trust": {
"title": "Trust this workspace’s commands?",
diff --git a/surfaces/gui/src/locales/zh.json b/surfaces/gui/src/locales/zh.json
index 04fee473..1d03f8c5 100644
--- a/surfaces/gui/src/locales/zh.json
+++ b/surfaces/gui/src/locales/zh.json
@@ -1606,7 +1606,13 @@
"vllm_desc": "vLLM 运行在带 NVIDIA GPU 的 Linux 上。仅当服务器启动时设置了密钥才需要填写。",
"install_vllm": "安装 vLLM",
"lmstudio_desc": "LM Studio 应用会提供你在其中下载的模型。仅当你在应用中开启了密钥才需要填写。",
- "install_lmstudio": "安装 LM Studio"
+ "install_lmstudio": "安装 LM Studio",
+ "jan_desc": "开启 Jan 的本地 API 服务器后(设置 ▸ Local API Server ▸ Start Server),Jan 应用会提供你在其中下载的模型。仅当你在那里设置了密钥才需要填写。",
+ "install_jan": "设置 Jan 的 API 服务器",
+ "docker_desc": "Docker Desktop 会提供你用 `docker model pull hf.co/<repo>:<quant>` 拉取的模型。请在 Docker Desktop ▸ Settings ▸ AI 中开启 Model Runner 及其 TCP 访问。无需密钥。",
+ "install_docker": "设置 Docker Model Runner",
+ "mlx_desc": "mlx-lm 运行在 Apple 芯片上:`pip install mlx-lm`,然后 `mlx_lm.server --model <repo>`。它与 llama.cpp 同样使用 8080 端口——如果两者都运行,请让其中一个使用其他端口,例如 `mlx_lm.server --port 8081`。",
+ "install_mlx": "设置 mlx-lm"
},
"workspace_trust": {
"title": "信任此工作区的命令?",
diff --git a/surfaces/gui/src/providers/ProviderSetup.tsx b/surfaces/gui/src/providers/ProviderSetup.tsx
index e7992287..460e41d9 100644
--- a/surfaces/gui/src/providers/ProviderSetup.tsx
+++ b/surfaces/gui/src/providers/ProviderSetup.tsx
@@ -29,6 +29,9 @@ const LOCAL_HELP: Record<string, { desc: string; link: string; url: string }> =
llamacpp: { desc: "provider.llamacpp_desc", link: "provider.install_llamacpp", url: "https://github.com/ggml-org/llama.cpp#quick-start" },
vllm: { desc: "provider.vllm_desc", link: "provider.install_vllm", url: "https://docs.vllm.ai/en/latest/getting_started/installation/" },
lmstudio: { desc: "provider.lmstudio_desc", link: "provider.install_lmstudio", url: "https://lmstudio.ai/download" },
+ jan: { desc: "provider.jan_desc", link: "provider.install_jan", url: "https://docs.jan.ai/desktop/api-server" },
+ docker: { desc: "provider.docker_desc", link: "provider.install_docker", url: "https://docs.docker.com/ai/model-runner/" },
+ mlx: { desc: "provider.mlx_desc", link: "provider.install_mlx", url: "https://github.com/ml-explore/mlx-lm/blob/main/mlx_lm/SERVER.md" },
};
export const KEY_HELP: Record<string, { url: string; label: string; labelKey?: string }> = {
diff --git a/surfaces/gui/src/providers/logos.ts b/surfaces/gui/src/providers/logos.ts
index 7e2995e9..a2087b60 100644
--- a/surfaces/gui/src/providers/logos.ts
+++ b/surfaces/gui/src/providers/logos.ts
@@ -87,6 +87,11 @@ export const PROVIDER_ORDER = [
"llamacpp",
"vllm",
"lmstudio",
+ // Jan, Docker Model Runner and mlx-lm carry no mark yet: the gallery shows the
+ // neutral monogram (see ProviderMark).
+ "jan",
+ "docker",
+ "mlx",
"bedrock",
"vertex",
"openrouter",
diff --git a/tests/test_local_server.py b/tests/test_local_server.py
index 889809ee..6bf632b9 100644
--- a/tests/test_local_server.py
+++ b/tests/test_local_server.py
@@ -1,5 +1,5 @@
-"""llama.cpp, vLLM and LM Studio as providers (providers/local_server.py, registry.py): the user's
-own server through its /v1, what each reports about its model, and how the server's
+"""The OpenAI-compatible local servers as providers (providers/local_server.py, registry.py:
+llama.cpp, vLLM, LM Studio, Jan, Docker Model Runner, mlx-lm): the user's own server through its /v1, what each reports about its model, and how the server's
model lists, liveness and adoption treat them."""
from __future__ import annotations
@@ -30,9 +30,10 @@ def _fake_http(monkeypatch, handler):
def test_descriptors_carry_a_kind_for_grouping():
kinds = {d.name: d.kind for d in provider_descriptors()}
assert kinds["ollama"] == kinds["llamacpp"] == kinds["vllm"] == kinds["lmstudio"] == "local"
+ assert kinds["jan"] == kinds["docker"] == kinds["mlx"] == "local"
assert kinds["openai-codex"] == "subscription"
assert kinds["openai"] == kinds["anthropic"] == "api_key"
- for name in ("llamacpp", "vllm", "lmstudio"):
+ for name in local_server.LOCAL_SERVERS:
d = get_descriptor(name)
assert d is not None and not d.needs_key
assert [f.key for f in d.fields] == ["base_url", "api_key"]
@@ -46,6 +47,16 @@ def test_v1_base_takes_the_root_or_an_explicit_v1():
assert local_server.v1_base("vllm", "http://box.local:8000/v1") == "http://box.local:8000/v1"
assert local_server.v1_base("lmstudio", None) == "http://localhost:1234/v1"
assert local_server.v1_base("lmstudio", "http://127.0.0.1:1234/v1/") == "http://127.0.0.1:1234/v1"
+ assert local_server.v1_base("jan", None) == "http://127.0.0.1:1337/v1"
+ assert local_server.v1_base("mlx", None) == "http://127.0.0.1:8080/v1"
+ assert local_server.v1_base("mlx", "http://127.0.0.1:8081") == "http://127.0.0.1:8081/v1"
+ # Docker Model Runner's API sits under /engines: the root, /engines and /engines/v1 all
+ # land on the same place, and a named engine path is kept.
+ assert local_server.v1_base("docker", None) == "http://localhost:12434/engines/v1"
+ assert local_server.v1_base("docker", "http://localhost:12434") == "http://localhost:12434/engines/v1"
+ assert local_server.v1_base("docker", "http://localhost:12434/engines") == "http://localhost:12434/engines/v1"
+ assert local_server.v1_base("docker", "http://localhost:12434/engines/v1") == "http://localhost:12434/engines/v1"
+ assert local_server.v1_base("docker", "http://box:12434/engines/llama.cpp/") == "http://box:12434/engines/llama.cpp/v1"
def test_the_builder_sends_a_placeholder_key_when_none_is_set():
@@ -56,6 +67,10 @@ def test_the_builder_sends_a_placeholder_key_when_none_is_set():
assert client._base_url == "http://gpu:8000/v1" and client._api_key == "s3"
client = get_descriptor("lmstudio").build({}, None)
assert client._base_url == "http://localhost:1234/v1" and client._api_key == "none"
+ client = get_descriptor("docker").build({"base_url": "http://localhost:12434"}, None)
+ assert client._base_url == "http://localhost:12434/engines/v1" and client._api_key == "none"
+ client = get_descriptor("jan").build({}, None)
+ assert client._base_url == "http://127.0.0.1:1337/v1"
def test_llamacpp_facts_come_from_models_and_props(monkeypatch):
@@ -124,6 +139,29 @@ def test_lmstudio_facts_survive_a_missing_own_listing(monkeypatch):
assert row["context"] is None and row["context_max"] is None
[email protected]("name,base,path", [
+ ("jan", "http://127.0.0.1:1337", "/v1/models"),
+ ("docker", "http://localhost:12434", "/engines/v1/models"),
+ ("mlx", "http://127.0.0.1:8081", "/v1/models"),
+])
+def test_plain_listing_servers_report_their_models(monkeypatch, name, base, path):
+ """Jan, Docker Model Runner and mlx-lm answer a plain OpenAI listing: one row per id,
+ routed under the provider's prefix, with no window claimed."""
+ seen = []
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ seen.append(request.url.path)
+ if request.url.path == path:
+ return httpx.Response(200, json={"data": [{"id": "org/some-model", "object": "model"}]})
+ return httpx.Response(404)
+
+ _fake_http(monkeypatch, handler)
+ (row,) = local_server.model_facts(name, base)
+ assert row["model"] == f"{name}:org/some-model" and row["name"] == "org/some-model"
+ assert row["context"] is None and row["tools"] is None
+ assert seen[0] == path
+
+
def test_vllm_facts_probe_whether_tool_calling_is_on(monkeypatch):
calls = []
@@ -166,7 +204,7 @@ def test_thinking_switch_reaches_llama_cpp_and_vllm_as_template_kwargs(monkeypat
sent = {}
monkeypatch.setattr(openai_provider.OpenAIProvider, "complete", lambda self, **kw: sent.update(kw) or "ok")
- for name in ("llamacpp", "vllm", "lmstudio"):
+ for name in local_server.LOCAL_SERVERS:
client = get_descriptor(name).build({}, None)
client.complete(model="m", messages=[], max_tokens=10, extra_body={"think": False})
assert sent["extra_body"] == {"chat_template_kwargs": {"enable_thinking": False}}