Skip to content

armgit / ODS

Public Imported from GitHub
0

ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.

feat(vllm): run an LLM_BACKEND=vllm install end to end (#8298)

Mike Bradleyd06bb8431e3e
diff --git a/ods/bin/model_switchboard/router_transport.py b/ods/bin/model_switchboard/router_transport.py
index 8d8deb16e..888b839b9 100644
--- a/ods/bin/model_switchboard/router_transport.py
+++ b/ods/bin/model_switchboard/router_transport.py
@@ -1,10 +1,13 @@
-"""Probe a host-native llama-server through this installation's router container.
+"""Probe a model server through this installation's router container.
A WSL host agent cannot reach a Windows loopback-only listener: WSL localhost
is not Windows localhost. Docker Desktop containers can, through
-``host.docker.internal``. This module runs one bounded stdlib request inside
-the owned, running model-router container, against exactly the
-``llama-server-default`` origin that the router itself serves inference from.
+``host.docker.internal``. The vLLM engine (``LLM_BACKEND=vllm``) publishes no
+host port at all; only containers on the ODS network reach it. This module
+runs one bounded stdlib request inside the owned, running model-router
+container, against exactly the origin of the router endpoint the caller names
+(``llama-server-default`` by default, ``vllm-default`` for vLLM), which is the
+origin the router itself serves inference from.
"""
from __future__ import annotations
@@ -18,6 +21,7 @@ from urllib.parse import urlsplit
ENDPOINT_ID = "llama-server-default"
+_ENDPOINT_IDS = frozenset({ENDPOINT_ID, "vllm-default"})
_LIMIT = 65536
_ANSWER_LIMIT = 2097152 # the worker's ANSWER_LIMIT for a probe exchange
_PATHS = frozenset({"/health", "/props", "/v1/models", "/v1/chat/completions", "/metrics"})
@@ -159,19 +163,20 @@ def _owned_router(install_dir: Path, project: str) -> str:
def request(install_dir: Path, origin: str, path: str,
payload: dict | None = None, api_key: str = "", timeout: float = 5,
- *, project: str = "ods") -> str:
+ *, project: str = "ods", endpoint_id: str = ENDPOINT_ID) -> str:
"""Return bounded HTTP text; never grant readiness or publish model state.
- ``origin`` is the router's ``llama-server-default`` base URL without a
- path; ``path`` is one of the proof and telemetry routes. An HTTP error
- status fails the request.
+ ``origin`` is the base URL, without a path, of the router endpoint
+ ``endpoint_id``; ``path`` is one of the proof and telemetry routes. An HTTP
+ error status fails the request.
"""
- return _send(install_dir, origin, path, payload, api_key, timeout, project, envelope=False)
+ return _send(install_dir, origin, path, payload, api_key, timeout, project,
+ envelope=False, endpoint_id=endpoint_id)
def exchange(install_dir: Path, origin: str, path: str,
payload: dict | None = None, api_key: str = "", timeout: float = 5,
- *, project: str = "ods") -> tuple[int, str]:
+ *, project: str = "ods", endpoint_id: str = ENDPOINT_ID) -> tuple[int, str]:
"""Return ``(http_status, text)``, error statuses included, for capability probes.
Same ownership, route and size rules as ``request``. A model's own error
@@ -179,7 +184,8 @@ def exchange(install_dir: Path, origin: str, path: str,
is evidence a probe classifies, so it comes back instead of failing.
Redirects are still refused.
"""
- raw = _send(install_dir, origin, path, payload, api_key, timeout, project, envelope=True)
+ raw = _send(install_dir, origin, path, payload, api_key, timeout, project,
+ envelope=True, endpoint_id=endpoint_id)
try:
envelope = json.loads(raw)
except ValueError as exc:
@@ -192,7 +198,9 @@ def exchange(install_dir: Path, origin: str, path: str,
def _send(install_dir: Path, origin: str, path: str, payload: dict | None, api_key: str,
- timeout: float, project: str, *, envelope: bool) -> str:
+ timeout: float, project: str, *, envelope: bool, endpoint_id: str) -> str:
+ if endpoint_id not in _ENDPOINT_IDS:
+ raise ValueError("Unsupported runtime endpoint")
parsed = urlsplit(origin)
if (parsed.scheme not in {"http", "https"} or not parsed.hostname
or parsed.username is not None or parsed.password is not None
@@ -210,7 +218,7 @@ def _send(install_dir: Path, origin: str, path: str, payload: dict | None, api_k
raise ValueError("Runtime proof timeout must be between 0 and 900 seconds")
if not re.fullmatch(r"[a-z0-9][a-z0-9_-]*", project):
raise ValueError("Invalid Compose project")
- message = dict(endpoint_id=ENDPOINT_ID, origin=origin, path=path, payload=payload,
+ message = dict(endpoint_id=endpoint_id, origin=origin, path=path, payload=payload,
api_key=api_key, timeout=timeout)
if envelope:
message["envelope"] = True
diff --git a/ods/bin/ods-host-agent.py b/ods/bin/ods-host-agent.py
index df73eec67..131dd8f8f 100755
--- a/ods/bin/ods-host-agent.py
+++ b/ods/bin/ods-host-agent.py
@@ -148,6 +148,15 @@ if _gguf_inspector_spec is None or _gguf_inspector_spec.loader is None:
_gguf_inspector = importlib.util.module_from_spec(_gguf_inspector_spec)
_gguf_inspector_spec.loader.exec_module(_gguf_inspector)
+# vLLM catalog entries ("runtime": "vllm"): the installer sets them up with
+# LLM_BACKEND=vllm; local model switching manages GGUF models only.
+_model_vllm_spec = importlib.util.spec_from_file_location(
+ "_ods_model_vllm", _MODEL_MEMORY_PATH.with_name("model_vllm.py"))
+if _model_vllm_spec is None or _model_vllm_spec.loader is None:
+ raise ImportError("Cannot load shared vLLM catalog policy")
+_model_vllm = importlib.util.module_from_spec(_model_vllm_spec)
+_model_vllm_spec.loader.exec_module(_model_vllm)
+
def _activation_architecture_refusal(target: Path, env: dict, *, windows_hosted: bool,
registered_runtime: dict | None) -> dict | None:
@@ -312,7 +321,7 @@ WINDOWS_WHISPER_CUDA_MIN_DRIVER_MAJOR = 575
# Distinct from CORE_SERVICE_IDS (which is the allowlist of known service IDs).
ALWAYS_ON_SERVICES: frozenset = frozenset({
"llama-server", "model-router", "remote-provider-egress", "remote-provider-ssh-tunnel",
- "open-webui", "dashboard", "dashboard-api",
+ "open-webui", "dashboard", "dashboard-api", "vllm-server",
})
USER_EXTENSIONS_DIR: Path = Path()
EXTENSIONS_DIR: Path = Path()
@@ -758,13 +767,57 @@ def _model_activation_public_failure(diagnosis: dict, healthy: bool) -> str:
# .env keys that decide whether inference is a container or a host-native
-# llama-server. Compose resolution must read them from installed state. The
-# LEMONADE_EXTERNAL selector stays one release for un-migrated installs.
+# llama-server, and LLM_BACKEND, which selects the vLLM engine overlay
+# (LLM_BACKEND=vllm). Compose resolution must read them from installed state.
+# The LEMONADE_EXTERNAL selector stays one release for un-migrated installs.
_HOST_LLM_COMPOSE_SELECTORS = (
"AMD_INFERENCE_RUNTIME", "AMD_INFERENCE_RUNTIME_MODE", "AMD_INFERENCE_LOCATION",
"AMD_INFERENCE_MANAGED", "ODS_HOST_LLM_TRANSPORT", "LEMONADE_EXTERNAL",
+ "LLM_BACKEND",
)
+# LLM_BACKEND=vllm (docker-compose.vllm.yml): vllm-server serves the one model
+# in its own container on the ODS network, without a host port, and the
+# in-stack llama-server is profiled out. Naming llama-server in a Compose
+# command would enable its profile again, so nothing here may do that.
+_VLLM_CONTAINER = "ods-vllm-server"
+_VLLM_ORIGIN = "http://vllm-server:8000"
+_VLLM_ENDPOINT_ID = "vllm-default"
+
+
+def _vllm_engine(env: dict) -> bool:
+ """Whether .env selects the vLLM engine (LLM_BACKEND=vllm)."""
+ return str(env.get("LLM_BACKEND") or "").strip().casefold() == "vllm"
+
+
+_ODS_HELP = "Need help? Ask on the ODS Discord: https://discord.gg/4ntNp9MAwC"
+_VLLM_LLAMA_SERVER_OFF = (
+ "llama-server is off on this install: the vLLM engine (vllm-server) serves the model. "
+ "Restart vllm-server instead, or run the ODS installer again to change the engine."
+)
+
+
+def _vllm_activation_refusal(model_id: str) -> dict:
+ """409 body for a model switch on an install the vLLM engine serves."""
+ return {
+ "error": "This install runs its model on the vLLM engine, and models cannot be switched here.",
+ "code": "vllm_runtime_installer_managed",
+ "hint": f"To use another model, run the ODS installer again and choose it. {_ODS_HELP}",
+ "requestedModelId": model_id,
+ }
+
+
+def _vllm_model_refusal(model: dict) -> dict:
+ """409 body for a vLLM catalog entry asked of llama-server's model lifecycle."""
+ return {
+ "error": f"{model.get('name') or model.get('id')} runs only on the vLLM engine, "
+ "which only the ODS installer sets up.",
+ "code": "vllm_model_installer_only",
+ "hint": ("It needs one NVIDIA datacenter Blackwell GPU. Run the ODS installer and "
+ f"choose the vLLM engine to use it. {_ODS_HELP}"),
+ "requestedModelId": model.get("id"),
+ }
+
_SWITCHBOARD_ROUTE_ENV_KEYS = (
"GPU_BACKEND",
@@ -2969,14 +3022,23 @@ def _switchboard_state_needs_current_env_verification(
def _catalog_model_for_current_env(env: dict) -> tuple[str, dict]:
- gguf_file = str(env.get("GGUF_FILE") or "").strip()
+ """The catalog id and entry the configured route serves.
+
+ Without a catalog entry: the configured model name and ``{}``.
+
+ A vLLM route (LLM_BACKEND=vllm) serves a vLLM entry, found by LLM_MODEL; a
+ GGUF_FILE left from an earlier llama-server install never names it. Every
+ other route serves a GGUF entry.
+ """
+ vllm = _vllm_engine(env)
+ gguf_file = "" if vllm else str(env.get("GGUF_FILE") or "").strip()
llm_model_name = str(env.get("LLM_MODEL") or "").strip()
try:
library = _load_model_library_records()
except RuntimeError:
library = []
for entry in library:
- if not isinstance(entry, dict):
+ if not isinstance(entry, dict) or _model_vllm.is_vllm_entry(entry) != vllm:
continue
entry_id = str(entry.get("id") or "")
entry_gguf = str(entry.get("gguf_file") or "")
@@ -2992,7 +3054,13 @@ def _catalog_model_for_current_env(env: dict) -> tuple[str, dict]:
def _initial_switchboard_backend(env: dict) -> tuple[str, str, str | None]:
- """Every managed runtime is upstream llama-server behind one endpoint."""
+ """The route's backend kind and router endpoint for the configured runtime.
+
+ LLM_BACKEND=vllm routes to vllm-server; every other managed runtime is
+ upstream llama-server behind the llama-server-default endpoint.
+ """
+ if _vllm_engine(env):
+ return "vllm", _VLLM_ENDPOINT_ID, None
return "llama-server", "llama-server-default", None
@@ -7602,6 +7670,9 @@ def docker_compose_recreate(service_ids: list[str], *, force_recreate: bool = Tr
ok, error = validate_core_recreate_ids(service_ids)
if not ok:
return False, error
+ if "llama-server" in service_ids and _vllm_engine(load_env(INSTALL_DIR / ".env")):
+ # Naming the service would enable its profile and start a second engine.
+ return False, _VLLM_LLAMA_SERVER_OFF
try:
flags = _core_recreate_compose_flags(resolve_compose_flags(), service_ids)
@@ -13103,6 +13174,11 @@ class AgentHandler(BaseHTTPRequestHandler):
if not isinstance(sid, str) or not SERVICE_ID_RE.fullmatch(sid):
json_response(self, 400, {"error": "Invalid service_id"})
return
+ if sid == "llama-server" and _vllm_engine(load_env(INSTALL_DIR / ".env")):
+ # A container left by an earlier llama-server install would start
+ # beside vLLM on the same GPU.
+ json_response(self, 409, {"error": _VLLM_LLAMA_SERVER_OFF, "code": "llama_server_off"})
+ return
has_container, restart_error = _service_has_docker_container(sid)
if not has_container:
@@ -13752,6 +13828,12 @@ class AgentHandler(BaseHTTPRequestHandler):
if model is None:
json_response(self, 404, {"error": "Unknown model"})
return
+ if _model_vllm.is_vllm_entry(model) or _vllm_engine(env):
+ # The probes read llama.cpp's /props and chat-template errors.
+ json_response(self, 409, {"error": "Model checks measure llama.cpp models; "
+ "a model on the vLLM engine is not checked",
+ "code": "vllm_not_profiled"})
+ return
if str(model.get("gguf_file") or "") != str(env.get("GGUF_FILE") or ""):
json_response(self, 409, {"error": "Only the running model can be checked; run it first", "code": "not_running"})
return
@@ -14684,6 +14766,12 @@ class AgentHandler(BaseHTTPRequestHandler):
"requestedModelId": model_id,
})
return
+ if _vllm_engine(persisted_env):
+ # Activation swaps llama-server's GGUF and rolls back by restoring
+ # it; the vLLM engine serves one installed snapshot instead, and
+ # starting llama-server here would run a second engine.
+ json_response(self, 409, _vllm_activation_refusal(model_id))
+ return
def local_gguf_model_from_id(raw_model_id: str) -> dict | None:
matching = []
@@ -14746,6 +14834,9 @@ class AgentHandler(BaseHTTPRequestHandler):
if model is None:
json_response(self, 404, {"error": f"Model '{model_id}' not found in library or local GGUF files"})
return
+ if _model_vllm.is_vllm_entry(model):
+ json_response(self, 409, _vllm_model_refusal(model))
+ return
gguf_file = model.get("gguf_file", "")
llm_model_name = model.get("llm_model_name", model_id)
@@ -16495,15 +16586,21 @@ def _runtime_uses_router_transport(env: dict) -> bool:
def _runtime_endpoint(env: dict) -> tuple[str, str]:
- """Return ``(origin, transport)`` of the configured llama-server runtime.
-
- Every managed runtime is upstream llama-server. ``transport`` is
- ``router`` when this agent runs in WSL and the server is the owned
- Windows task: WSL localhost is not Windows localhost, so proofs run in
- the owned model-router container against its ``llama-server-default``
- origin. Every other runtime is reached ``direct``: the container by
- Docker DNS or its published port, a host-native server on loopback.
+ """Return ``(origin, transport)`` of the configured model server.
+
+ ``transport`` is ``router`` when this agent runs in WSL and the server is
+ the owned Windows llama-server task: WSL localhost is not Windows
+ localhost, so proofs run in the owned model-router container against its
+ ``llama-server-default`` origin. The vLLM engine (LLM_BACKEND=vllm)
+ publishes no host port, so a host agent proves it the same way, against
+ the router's ``vllm-default`` origin. Every other runtime is reached
+ ``direct``: the container by Docker DNS or its published port, a
+ host-native server on loopback.
"""
+ if _vllm_engine(env):
+ if os.environ.get("ODS_HOST_INSTALL_DIR"):
+ return f"http://{_VLLM_CONTAINER}:8000", "direct"
+ return _VLLM_ORIGIN, "router"
if _runtime_uses_router_transport(env):
return _native_llm_container_origin(env), "router"
if _is_windows_host_llama_server(env):
@@ -16516,6 +16613,14 @@ def _runtime_endpoint(env: dict) -> tuple[str, str]:
return f"http://127.0.0.1:{env.get('OLLAMA_PORT') or '8080'}", "direct"
+def _runtime_router_endpoint(env: dict) -> dict:
+ """The router endpoint a router-transport proof names (keyword arguments).
+
+ The WSL route keeps the transport's default, ``llama-server-default``.
+ """
+ return {"endpoint_id": _VLLM_ENDPOINT_ID} if _vllm_engine(env) else {}
+
+
def _runtime_api_key(env: dict) -> str:
"""Bearer key of a Windows-owned llama-server; containers take none."""
if _runtime_uses_router_transport(env) or _is_windows_host_llama_server(env):
@@ -16542,6 +16647,7 @@ def _runtime_http(
if transport == "router":
return _router_transport_request(
INSTALL_DIR, origin, path, payload=payload, api_key=api_key, timeout=timeout,
+ **_runtime_router_endpoint(env),
)
command = ["curl", "-s", "--max-time", str(timeout)]
if payload is not None:
@@ -16599,6 +16705,7 @@ def _runtime_exchange(
if transport == "router":
return _router_transport_exchange(
INSTALL_DIR, origin, path, payload=payload, api_key=api_key, timeout=timeout,
+ **_runtime_router_endpoint(env),
)
headers = {"Content-Type": "application/json"}
if api_key:
@@ -16995,8 +17102,44 @@ def _profiled_capabilities(capabilities: object, traits: dict | None, model: dic
}
+def _container_health(container: str) -> str:
+ """Docker's health status of ``container`` (starting, healthy, unhealthy, none).
+
+ Empty when Docker cannot inspect it, for example when it does not exist.
+ """
+ result = subprocess.run(
+ ["docker", "inspect", "--type", "container", "--format",
+ "{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}", container],
+ capture_output=True, text=True, timeout=15,
+ )
+ return result.stdout.strip().casefold() if result.returncode == 0 else ""
+
+
+def _vllm_runtime_health(env: dict) -> str:
+ """Return ``ok``, ``loading`` or ``error`` for the vLLM engine.
+
+ vLLM's ``/health`` answers 200 with an empty body once the engine is
+ ready, and its server listens only from then on. A cold first start loads
+ the weights and compiles kernels for 3 to 15 minutes, during which Docker
+ reports the container ``starting`` (docker-compose.vllm.yml allows 60
+ minutes): that is ``loading``, not an error. Raises OSError when vLLM is
+ unreachable and its container is not starting.
+ """
+ try:
+ status, _body = _runtime_exchange(env, "/health", timeout=5)
+ except OSError:
+ if _container_health(_VLLM_CONTAINER) == "starting":
+ return "loading"
+ raise
+ if status == 200:
+ return "ok"
+ return "loading" if _container_health(_VLLM_CONTAINER) == "starting" else "error"
+
+
def _runtime_health(env: dict) -> str:
- """Return ``ok``, ``loading`` or ``error`` from llama-server ``/health``."""
+ """Return ``ok``, ``loading`` or ``error`` from the model server's ``/health``."""
+ if _vllm_engine(env):
+ return _vllm_runtime_health(env)
body = _runtime_http(env, "/health")
try:
data = json.loads(body or "{}")
@@ -17159,6 +17302,24 @@ def _llama_runtime_context_length(env: dict) -> int:
return _llama_runtime_props(env)[0]
+def _vllm_served_context_length(body: str, runtime_identity: str) -> int:
+ """Return the ``max_model_len`` vLLM's /v1/models reports for its served row.
+
+ vLLM has no /props; the row of the model it serves carries the context
+ its --max-model-len fixed.
+ """
+ try:
+ data = json.loads(body)
+ except (json.JSONDecodeError, TypeError):
+ return 0
+ rows = data.get("data") if isinstance(data, dict) else None
+ for row in rows if isinstance(rows, list) else ():
+ if isinstance(row, dict) and str(row.get("id") or "").strip() == runtime_identity:
+ value = row.get("max_model_len")
+ return value if type(value) is int and value > 0 else 0
+ return 0
+
+
def _llama_training_context_length(body: str, runtime_identity: str) -> int:
"""Return the GGUF training context llama.cpp reports for a loaded row."""
try:
@@ -17243,11 +17404,11 @@ def _completion_text(data: object, *, include_reasoning: bool = True) -> str:
if text.strip():
return text[:4096]
if include_reasoning:
- reasoning_content = (
- message.get("reasoning_content")
- if isinstance(message, dict)
- else choice.get("reasoning_content")
- )
+ source = message if isinstance(message, dict) else choice
+ reasoning_content = source.get("reasoning_content")
+ if not isinstance(reasoning_content, str):
+ # vLLM's reasoning parsers return the field as ``reasoning``.
+ reasoning_content = source.get("reasoning")
if isinstance(reasoning_content, str):
return reasoning_content[:4096]
if isinstance(content, str):
@@ -17275,7 +17436,14 @@ _COMPLETION_PROBE_THINKING_SECONDS = 120
def _completion_probe_payload(model_name: str, *, disable_thinking: bool,
- max_tokens: int = _COMPLETION_PROBE_TOKENS) -> dict:
+ max_tokens: int = _COMPLETION_PROBE_TOKENS,
+ thinking_off: dict | None = None) -> dict:
+ """The bounded readiness probe; ``thinking_off`` replaces enable_thinking=false.
+
+ A vLLM entry names the chat-template arguments that turn its thinking
+ down (catalog ``thinking_template_kwargs.off``); GLM's template has no
+ enable_thinking at all.
+ """
payload = {
"model": model_name,
"messages": [{
@@ -17288,10 +17456,21 @@ def _completion_probe_payload(model_name: str, *, disable_thinking: bool,
"temperature": 0,
}
if disable_thinking:
- payload["chat_template_kwargs"] = {"enable_thinking": False}
+ payload["chat_template_kwargs"] = (
+ dict(thinking_off) if thinking_off is not None else {"enable_thinking": False})
return payload
+def _vllm_probe_thinking_off(env: dict) -> dict | None:
+ """The configured vLLM entry's thinking-off template arguments, if any."""
+ if not _vllm_engine(env):
+ return None
+ _model_id, model = _catalog_model_for_current_env(env)
+ if not _model_vllm.is_vllm_entry(model):
+ return None
+ return dict(model["thinking_template_kwargs"]["off"])
+
+
def _reasoning_cut_short(response: object) -> bool:
"""The answer hit max_tokens inside its reasoning: thinking, but no visible text yet."""
choices = response.get("choices") if isinstance(response, dict) else None
@@ -17338,6 +17517,7 @@ def _chat_completion_ready(
base_url: str = "",
disable_thinking: bool = False,
require_visible_content: bool = False,
+ thinking_off: dict | None = None,
) -> bool:
"""Require a meaningful completion and, when requested, its model identity."""
prefix = "/" + api_prefix.strip("/")
@@ -17346,7 +17526,8 @@ def _chat_completion_ready(
def probe(max_tokens: int, seconds: int):
payload = json.dumps(_completion_probe_payload(
- model_name, disable_thinking=disable_thinking, max_tokens=max_tokens))
+ model_name, disable_thinking=disable_thinking, max_tokens=max_tokens,
+ thinking_off=thinking_off))
command = [
"curl", "-sf", "--max-time", str(seconds), "--max-filesize", "65536",
"-X", "POST", url,
@@ -17398,9 +17579,11 @@ def _runtime_completion_ready(
A reasoning-only answer does not prove the runtime is ready to serve
consumers, so every runtime family must return visible content from the
- same model it reported in /v1/models.
+ same model it reported in /v1/models. A vLLM entry turns its thinking down
+ with its own template arguments (``_vllm_probe_thinking_off``).
"""
origin, transport = _runtime_endpoint(env)
+ thinking_off = _vllm_probe_thinking_off(env)
expected = {
"expected_model_id": expected_model_id,
"expected_gguf_file": expected_gguf_file,
@@ -17410,13 +17593,14 @@ def _runtime_completion_ready(
return _chat_completion_ready(
"", "", model_name, "/v1", api_key=_runtime_api_key(env),
base_url=origin, disable_thinking=True, require_visible_content=True,
- **expected,
+ thinking_off=thinking_off, **expected,
)
try:
body = _runtime_http(
env,
"/v1/chat/completions",
- payload=_completion_probe_payload(model_name, disable_thinking=True),
+ payload=_completion_probe_payload(
+ model_name, disable_thinking=True, thinking_off=thinking_off),
timeout=30,
)
response = json.loads(body or "{}")
@@ -17425,7 +17609,8 @@ def _runtime_completion_ready(
env,
"/v1/chat/completions",
payload=_completion_probe_payload(
- model_name, disable_thinking=True, max_tokens=_COMPLETION_PROBE_THINKING_TOKENS),
+ model_name, disable_thinking=True, max_tokens=_COMPLETION_PROBE_THINKING_TOKENS,
+ thinking_off=thinking_off),
timeout=_COMPLETION_PROBE_THINKING_SECONDS,
)
response = json.loads(body or "{}")
@@ -17582,7 +17767,10 @@ def _wait_for_model_readiness(
requested model, ``/props`` serves its file with an ``n_ctx`` that meets
the request (llama.cpp may pad a custom size up to the next 256-cell
boundary), and one bounded completion with thinking disabled returns
- visible content from the same model.
+ visible content from the same model. The vLLM engine (LLM_BACKEND=vllm)
+ proves the same steps without ``/props``: its ``/v1/models`` row reports
+ the served context as ``max_model_len``, and the completion names the
+ exact id that row lists (vLLM refuses any other model name).
Legacy callers receive a boolean. Identity callers receive the concrete
runtime identity. Adapters receive identity, actual context, and proof time,
@@ -17608,6 +17796,7 @@ def _wait_for_model_readiness(
diagnosis = {}
not_ready = {} if return_proof else "" if return_identity else False
origin, transport = _runtime_endpoint(env)
+ vllm = _vllm_engine(env)
completion_model = llm_model_name or gguf_file
expected_context = _positive_int(env.get("CTX_SIZE") or env.get("MAX_CONTEXT"))
@@ -17671,7 +17860,10 @@ def _wait_for_model_readiness(
logger.info("Model readiness aborted: route env changed after probing %s", gguf_file)
return not_ready
if health == "loading":
- diagnosis["reason"] = "llama-server is still loading the model"
+ diagnosis["reason"] = (
+ "vLLM is still starting: a first start loads the weights and compiles "
+ "kernels, which can take 15 minutes" if vllm
+ else "llama-server is still loading the model")
runtime_identity = _llama_loaded_model_identity(
body,
model_id=model_id,
@@ -17679,7 +17871,11 @@ def _wait_for_model_readiness(
llm_model_name=llm_model_name,
) if body else ""
if runtime_identity:
- runtime_context, served_file, runtime_vision = _llama_runtime_props(env)
+ if vllm:
+ runtime_context = _vllm_served_context_length(body, runtime_identity)
+ served_file, runtime_vision = "", None
+ else:
+ runtime_context, served_file, runtime_vision = _llama_runtime_props(env)
if served_file and not _runtime_model_identity_matches(
served_file, gguf_file=gguf_file,
):
@@ -17710,7 +17906,7 @@ def _wait_for_model_readiness(
runtime_identity = ""
if runtime_identity and _runtime_completion_ready(
env,
- str(completion_model),
+ str(runtime_identity if vllm else completion_model),
expected_model_id=str(runtime_identity),
expected_gguf_file=gguf_file,
expected_llm_model_name=llm_model_name,
@@ -17878,6 +18074,7 @@ def _render_runtime_config(
remote_llm_transport: str = "",
remote_llm_base_url: str = "",
remote_llm_model: str = "",
+ llm_backend: str = "llama-server",
) -> bool:
renderer = install_dir / "scripts" / "render-runtime-configs.py"
if not renderer.exists():
@@ -17887,6 +18084,8 @@ def _render_runtime_config(
str(renderer),
"--surface",
surface,
+ "--llm-backend",
+ llm_backend,
"--switchboard-mode",
switchboard_mode,
"--ods-mode",
@@ -17968,6 +18167,11 @@ def _runtime_api_key_env(env: dict) -> str:
return "LLAMA_SERVER_API_KEY" if _runtime_api_key(env) else ""
+def _render_llm_backend(env: dict) -> str:
+ """The renderer's --llm-backend: the .env LLM_BACKEND, llama-server when unset."""
+ return str(env.get("LLM_BACKEND") or "").strip().lower() or "llama-server"
+
+
def _litellm_render_key(env: dict) -> str:
return str(env.get("LITELLM_KEY") or env.get("LITELLM_MASTER_KEY") or "")
@@ -17993,6 +18197,9 @@ def _render_model_router_runtime_configs(
"gpu_backend": env.get("GPU_BACKEND", "nvidia"),
"context_length": int(context_length),
"switchboard_mode": switchboard_mode,
+ # The installed backend, never this process's environment: vllm
+ # points model-router at vllm-server with the model's thinking map.
+ "llm_backend": _render_llm_backend(env),
}
surfaces = ["model-router-endpoints"]
if str(common["ods_mode"]).strip().lower() != "cloud":
diff --git a/ods/config/model-library.json b/ods/config/model-library.json
index 1806a478c..62ce71320 100644
--- a/ods/config/model-library.json
+++ b/ods/config/model-library.json
@@ -2832,6 +2832,8 @@
"name": "Qwen 3.8 Flash Next NVFP4 (vLLM)",
"family": "qwen",
"runtime": "vllm",
+ "tools": true,
+ "vision": true,
"hf_repo": "nvidia/Qwen3.8-Flash-Next-NVFP4",
"revision": "fc694b54fb0174e0913e6adf86691ef85a4ead47",
"source_url": "https://huggingface.co/nvidia/Qwen3.8-Flash-Next-NVFP4",
@@ -2846,6 +2848,12 @@
"min_gpu_mib": 158720,
"compute_capability": ["10.0", "10.3"],
"reasoning": "medium",
+ "thinking_template_kwargs": {
+ "off": {"enable_thinking": false},
+ "low": {"reasoning_effort": "low"},
+ "medium": {"reasoning_effort": "medium"},
+ "high": {"reasoning_effort": "xhigh"}
+ },
"tool_call_parser": "qwen3_coder",
"reasoning_parser": "qwen3",
"vllm_args": ["--quantization", "modelopt", "--max-model-len", "131072", "--max-num-seqs", "32", "--max-num-batched-tokens", "8192", "--enable-prefix-caching", "--enable-auto-tool-choice", "--tool-call-parser", "qwen3_coder", "--reasoning-parser", "qwen3", "--trust-remote-code", "--engram-config.cpu_offload=false", "--speculative-config.method=mtp", "--speculative-config.num_speculative_tokens=3"],
@@ -2891,6 +2899,19 @@
"name": "GLM 5.3 Flash NVFP4 (vLLM)",
"family": "glm",
"runtime": "vllm",
+ "tools": true,
+ "vision": true,
+ "app_compatibility": {
+ "pixel_agent": {
+ "status": "verified",
+ "label": "Pixel verified on a DGX Station (vLLM)",
+ "reason": "A fresh LLM_BACKEND=vllm install on Workstation06 (DGX Station GB300) served this model and passed Portal visitor rounds 5 and 7: a mood journal that remembers entries, built and checked in 48.6 s; a cited answer that matched GitHub; a CSV download; exact free GPU memory; a follow-up feature; a false bug report investigated without changing code; a three-page site checked page by page; and a clarifying question for an ambiguous request.",
+ "evidence": "Workstation06 ~/.local/state/claude-e2e/round5-glm-vllm.json and round7-glm-vllm.json; summarized on Osmantic/ODS#8298",
+ "recordedAt": "2026-10-27T22:05:00Z",
+ "productSha": "43143c59c1bb031394432af11c120d927e113d5b",
+ "hostScope": ["workstation06"]
+ }
+ },
"hf_repo": "nvidia/GLM-5.3-Flash-NVFP4",
"revision": "da920bb0b9f4a06727223a349e55468e38352348",
"source_url": "https://huggingface.co/nvidia/GLM-5.3-Flash-NVFP4",
@@ -2904,6 +2925,12 @@
"min_gpu_mib": 244736,
"compute_capability": ["10.0", "10.3"],
"reasoning": "medium",
+ "thinking_template_kwargs": {
+ "off": {"reasoning_effort": "low"},
+ "low": {"reasoning_effort": "low"},
+ "medium": {"reasoning_effort": "high"},
+ "high": {}
+ },
"tool_call_parser": "glm47",
"reasoning_parser": "glm47",
"vllm_args": ["--max-model-len", "131072", "--kv-cache-dtype", "fp8", "--max-num-seqs", "32", "--max-num-batched-tokens", "8192", "--enable-prefix-caching", "--enable-auto-tool-choice", "--tool-call-parser", "glm47", "--reasoning-parser", "glm47", "--trust-remote-code", "--speculative-config.method=mtp", "--speculative-config.num_speculative_tokens=5", "--no-enable-flashinfer-autotune"],
diff --git a/ods/extensions/services/dashboard-api/config.py b/ods/extensions/services/dashboard-api/config.py
index bce267a25..8e35b1b2b 100644
--- a/ods/extensions/services/dashboard-api/config.py
+++ b/ods/extensions/services/dashboard-api/config.py
@@ -321,6 +321,40 @@ def _apply_external_llm_service_override(
)
+VLLM_SERVICE_HOST = "vllm-server"
+VLLM_SERVICE_PORT = 8000
+VLLM_CONTAINER = "ods-vllm-server"
+
+
+def _apply_vllm_service_override(
+ services: dict[str, dict[str, Any]],
+ environment: Mapping[str, str] | None = None,
+) -> None:
+ """Route the LLM row's probes to the vLLM engine (LLM_BACKEND=vllm).
+
+ docker-compose.vllm.yml profiles the in-stack llama-server out and runs
+ vllm-server on the ODS network: /health answers 200 with an empty body once
+ the model is loaded, /v1/models lists the served id. As for a host-native
+ or external route, the row keeps the llama-server id that chat, readiness
+ and features key on; its container is ods-vllm-server, which the host
+ agent's Docker health snapshot reports (a cold start reads as starting),
+ and it publishes no host port to link to.
+ """
+ env = environment if environment is not None else os.environ
+ if str(env.get("LLM_BACKEND", "")).strip().lower() != "vllm":
+ return
+ service = services.get("llama-server")
+ if not service:
+ return
+ service["host"] = VLLM_SERVICE_HOST
+ service["port"] = VLLM_SERVICE_PORT
+ service["health"] = "/health"
+ service["container_name"] = VLLM_CONTAINER
+ service["name"] = "vLLM (LLM Inference)"
+ service["external_link"] = False
+ logger.info("vLLM engine selected; routing LLM probes to %s:%d", VLLM_SERVICE_HOST, VLLM_SERVICE_PORT)
+
+
def _read_env_value(key: str) -> str:
return (os.environ.get(key) or _read_env_from_file(key)).strip()
@@ -655,6 +689,7 @@ if not SERVICES:
LLM_BACKEND = normalize_llm_backend(os.environ.get("LLM_BACKEND", ""))
_apply_host_native_llm_service_override(SERVICES, GPU_BACKEND)
_apply_external_llm_service_override(SERVICES)
+_apply_vllm_service_override(SERVICES)
# --- Features ---
diff --git a/ods/extensions/services/dashboard-api/helpers.py b/ods/extensions/services/dashboard-api/helpers.py
index 0e8cb64e9..bd4311bb5 100644
--- a/ods/extensions/services/dashboard-api/helpers.py
+++ b/ods/extensions/services/dashboard-api/helpers.py
@@ -684,8 +684,17 @@ def _observe_live_output_slots(payload, sampled_at: float):
return round(sum(count - previous["counts"][key] for key, count in counts.items()) / elapsed, 1)
+# vLLM's counters (LLM_BACKEND=vllm) under the names the sampler reads. vLLM
+# exports no generation-seconds counter, so its throughput reads unavailable
+# while its token count still accumulates.
+_VLLM_COUNTERS = {
+ "vllm:generation_tokens_total": "tokens_predicted_total",
+ "vllm:num_requests_running": "requests_processing",
+}
+
+
def _parse_llama_prometheus(body: str) -> dict:
- """Read the llama.cpp counters the dashboard samples from a /metrics body."""
+ """Read the llama.cpp (or vLLM) counters the dashboard samples from a /metrics body."""
metrics: dict = {}
for line in body.split("\n"):
line = line.strip()
@@ -695,6 +704,12 @@ def _parse_llama_prometheus(body: str) -> dict:
if len(parts) < 2:
continue
metric_name = parts[0].split("{", 1)[0]
+ if metric_name in _VLLM_COUNTERS:
+ try:
+ metrics[_VLLM_COUNTERS[metric_name]] = float(parts[1])
+ except ValueError:
+ pass
+ continue
for counter in ("requests_processing", "tokens_predicted_total", "tokens_predicted_seconds_total"):
if metric_name.endswith(counter):
try:
@@ -735,7 +750,8 @@ async def _fetch_llama_metrics(model_hint: Optional[str] = None, counter_id: Opt
model_name = model_hint if model_hint is not None else (await get_loaded_model() or "")
url = f"http://{host}:{metrics_port}/metrics"
params = {"model": model_name} if model_name else {}
- slots_url = f"http://{host}:{metrics_port}/slots"
+ # /slots is llama.cpp's; vLLM has no per-slot output timing.
+ slots_url = None if LLM_BACKEND == "vllm" else f"http://{host}:{metrics_port}/slots"
client = await _get_httpx_client()
resp = await client.get(url, params=params)
resp.raise_for_status()
@@ -857,6 +873,8 @@ async def get_llama_context_size(model_hint: Optional[str] = None) -> Optional[i
return context if type(context) is int and context > 0 else None
if "llama-server" not in SERVICES:
return None
+ if LLM_BACKEND == "vllm":
+ return await _vllm_context_size(model_hint)
try:
host = SERVICES["llama-server"]["host"]
port = SERVICES["llama-server"]["port"]
@@ -873,12 +891,35 @@ async def get_llama_context_size(model_hint: Optional[str] = None) -> Optional[i
return None
+async def _vllm_context_size(model_hint: Optional[str]) -> Optional[int]:
+ """The context vLLM serves: ``max_model_len`` of its /v1/models row.
+
+ vLLM has no /props; its --max-model-len fixes the context, and the row of
+ the served model reports it.
+ """
+ try:
+ host = SERVICES["llama-server"]["host"]
+ port = SERVICES["llama-server"]["port"]
+ client = await _get_httpx_client()
+ resp = await client.get(f"http://{host}:{port}{_LLM_API_PREFIX}/models")
+ payload = resp.json()
+ except (httpx.HTTPError, httpx.TimeoutException, ValueError) as e:
+ logger.debug("vLLM context size unavailable: %s", e)
+ return None
+ rows = payload.get("data") if isinstance(payload, dict) else None
+ for row in rows if isinstance(rows, list) else ():
+ if isinstance(row, dict) and (model_hint is None or row.get("id") == model_hint):
+ context = row.get("max_model_len")
+ return context if type(context) is int and context > 0 else None
+ return None
+
+
async def get_llama_vision_support() -> Optional[bool]:
"""Whether the active llama-server loaded a vision projector.
llama-server reports it as ``/props`` ``modalities.vision``. None when that
- cannot be read: the owner's own server, an unreachable runtime, or a build
- without the field.
+ cannot be read: the owner's own server, an unreachable runtime, a build
+ without the field, or the vLLM engine, which reports no modalities.
"""
if _host_native_llm():
try:
@@ -886,7 +927,7 @@ async def get_llama_vision_support() -> Optional[bool]:
except (AgentClientError, OSError, ValueError):
return None
return vision if type(vision) is bool else None
- if LLM_BACKEND == "external" or "llama-server" not in SERVICES:
+ if LLM_BACKEND in {"external", "vllm"} or "llama-server" not in SERVICES:
return None
try:
host = SERVICES["llama-server"]["host"]
diff --git a/ods/extensions/services/dashboard-api/main.py b/ods/extensions/services/dashboard-api/main.py
index 9fd7f3a0a..601893a8d 100644
--- a/ods/extensions/services/dashboard-api/main.py
+++ b/ods/extensions/services/dashboard-api/main.py
@@ -42,7 +42,7 @@ from config import (
_detect_container_default_gateway, _running_inside_container,
_read_env_from_file,
get_env_file_read_error,
- normalize_ods_mode, read_live_env_value,
+ normalize_ods_mode, read_live_env_value, LLM_BACKEND,
)
from models import (
GPUInfo, ServiceStatus, DiskUsage, ModelInfo, BootstrapStatus,
@@ -320,12 +320,21 @@ def _build_readiness_payload(
llama_healthy = _service_is_healthy(service_statuses, "llama-server")
chat_ready = bool(llama_healthy and loaded_model and context_size)
+ # LLM_BACKEND=vllm: the row probes vllm-server (config.py). A cold start
+ # loads the model and compiles kernels for up to 15 minutes; a restart
+ # would only start that over, so none is suggested while it starts.
+ engine = "vllm-server" if LLM_BACKEND == "vllm" else "llama-server"
+ llm_row = _service_by_id(service_statuses, "llama-server")
+ engine_starting = engine == "vllm-server" and bool(llm_row and llm_row.startup_pending)
if chat_ready:
chat_detail = f"{loaded_model} loaded with {context_size} context"
elif bootstrap_info.active:
chat_detail = f"Full model is {_bootstrap_activity(bootstrap_info.phase)}; bootstrap mode may be limited"
+ elif engine_starting:
+ chat_detail = ("The vLLM engine is starting: a first start loads the model and compiles "
+ "kernels, which can take 15 minutes")
elif not llama_healthy:
- chat_detail = "llama-server is not healthy"
+ chat_detail = f"{engine} is not healthy"
elif not loaded_model:
chat_detail = "No loaded model reported by the inference server"
else:
@@ -337,7 +346,8 @@ def _build_readiness_payload(
ready=chat_ready,
status="ready" if chat_ready else "blocked",
detail=chat_detail,
- repair="ods restart llama-server" if not chat_ready and not bootstrap_info.active else None,
+ repair=(f"ods restart {engine}"
+ if not chat_ready and not bootstrap_info.active and not engine_starting else None),
))
webui_ready = _service_is_healthy(service_statuses, "open-webui")
diff --git a/ods/extensions/services/dashboard-api/model_vllm.py b/ods/extensions/services/dashboard-api/model_vllm.py
index 5067b3ac8..54af62e97 100644
--- a/ods/extensions/services/dashboard-api/model_vllm.py
+++ b/ods/extensions/services/dashboard-api/model_vllm.py
@@ -15,6 +15,13 @@ the vLLM engine (``.env`` ``LLM_BACKEND=vllm``), not llama.cpp. They carry no
compute capabilities;
* ``reasoning`` (the thinking default), plus ``tool_call_parser`` and
``reasoning_parser``, which name the same parsers as ``vllm_args``;
+* ``thinking_template_kwargs``: for each ODS thinking level (off, low,
+ medium, high) the ``chat_template_kwargs`` that select it in this model's
+ own chat template. Templates disagree (GLM-5.3 reads only
+ ``reasoning_effort`` low or high and thinks at "max" for any other value;
+ Qwen3.8 turns thinking off with ``enable_thinking`` and fails on an
+ unknown ``reasoning_effort``), so a level is never passed through
+ literally: model-router applies this map on the vllm route;
* ``selection``: 0 for every llama.cpp memory class, so no llama.cpp install
ever picks the entry, and a ``vllm`` priority for ``--llm-runtime vllm``.
@@ -55,6 +62,7 @@ _NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*")
# splitting, so they hold no whitespace, quotes or shell syntax.
_ARG = re.compile(r"[A-Za-z0-9_.:/=,+-]+")
_GPU_UUID = re.compile(r"[A-Za-z0-9][A-Za-z0-9-]*")
+_TEMPLATE_KWARG = re.compile(r"[a-z][a-z0-9_]{0,63}")
_GPU_MEMORY = re.compile(r"([0-9]+)(?:\s*MiB)?")
_FILE_KEYS = ({"path", "size", "sha256"}, {"path", "size", "git_blob_sha1"})
_SELECTION_KEYS = frozenset(LLAMA_MEMORY_CLASSES) | {VLLM_RUNTIME}
@@ -226,6 +234,28 @@ def _serving_problems(model: dict[str, Any]) -> list[str]:
return problems
+def _thinking_problems(value: Any) -> list[str]:
+ if not isinstance(value, dict) or set(value) != set(REASONING_LEVELS):
+ return [f"thinking_template_kwargs must map each of {', '.join(REASONING_LEVELS)} "
+ "to chat_template_kwargs"]
+ problems = []
+ for level in REASONING_LEVELS:
+ kwargs = value[level]
+ if not isinstance(kwargs, dict) or not all(
+ isinstance(key, str) and _TEMPLATE_KWARG.fullmatch(key)
+ and (isinstance(item, bool) or (isinstance(item, str) and _NAME.fullmatch(item)))
+ for key, item in kwargs.items()
+ ):
+ problems.append(f"thinking_template_kwargs.{level} must map template argument names "
+ "to booleans or plain words")
+ return problems
+
+
+def thinking_template_kwargs(model: dict[str, Any], level: str) -> dict[str, Any]:
+ """The ``chat_template_kwargs`` that select ODS thinking ``level`` in this model."""
+ return dict(model["thinking_template_kwargs"][level])
+
+
def _size_problems(model: dict[str, Any]) -> list[str]:
total = manifest_bytes(model)
problems = []
@@ -245,6 +275,7 @@ def entry_problems(model: Any) -> list[str]:
*_identity_problems(model),
*files,
*_serving_problems(model),
+ *_thinking_problems(model.get("thinking_template_kwargs")),
*_args_problems(model),
*_selection_problems(model.get("selection")),
*([] if files else _size_problems(model)),
diff --git a/ods/extensions/services/dashboard-api/performance_oracle.py b/ods/extensions/services/dashboard-api/performance_oracle.py
index 89177e5f9..eeafb0afb 100644
--- a/ods/extensions/services/dashboard-api/performance_oracle.py
+++ b/ods/extensions/services/dashboard-api/performance_oracle.py
@@ -29,7 +29,8 @@ from helpers import (
get_recorded_model_performance,
is_plausible_single_request_tps,
)
-from model_memory import memory_metadata, required_model_memory_gb
+from model_memory import memory_metadata, required_model_memory_gb, vllm_memory_estimate
+from model_vllm import is_vllm_entry
from model_selection import (
POLICY as _SHARED_SELECTOR_POLICY,
family_allowed as _shared_family_allowed,
@@ -1617,15 +1618,71 @@ def _measured_native_activation(model, path, data_root, install_dir, gpu_info, s
return None
+def vllm_model_row(raw: dict[str, Any], loaded_model: Optional[str],
+ context_length: Optional[int]) -> dict[str, Any]:
+ """The Models page row of the vLLM catalog entry a vLLM install serves.
+
+ LLM_BACKEND=vllm runs one installer-set snapshot (model_vllm.py): the
+ page shows its catalog name, size, context and GPU need, marks it
+ loaded once vLLM serves it (``served_model_name``) and otherwise
+ installed, and labels it managed by the engine (``source: runtime``,
+ ``runtime: vllm``) so it offers no download, switch or context change.
+ """
+ served = {normalize_key(raw.get(key)) for key in ("served_model_name", "llm_model_name", "id")}
+ is_loaded = bool(loaded_model) and normalize_key(loaded_model) in served
+ size_mb = float(raw.get("size_mb") or 0)
+ context = context_length if is_loaded and context_length else int(raw["context_length"])
+ return {
+ "id": raw["id"],
+ "name": raw.get("name") or raw["id"],
+ "gguf": None,
+ "ggufParts": None,
+ "downloadUrl": None,
+ "downloadSha256": None,
+ "llmModelName": raw.get("llm_model_name"),
+ "size": format_size(size_mb),
+ "sizeGb": round(size_mb / 1024, 1),
+ "vramRequired": round(int(raw["min_gpu_mib"]) / 1024, 1),
+ "estimatedRequired": vllm_memory_estimate(raw).device_gib,
+ # vLLM serves exactly the context its --max-model-len fixes.
+ "contextLength": context,
+ "maxContextLength": int(raw["context_length"]),
+ "specialty": raw.get("specialty") or "General",
+ "description": raw.get("description") or "",
+ "quantization": raw.get("quantization"),
+ "publisher": model_publisher({**raw, "source_repo": raw.get("hf_repo")}),
+ "metadata": {
+ "source": "runtime",
+ "runtime": "vllm",
+ "catalogSource": raw.get("source") or "ods",
+ "servedModelName": raw.get("served_model_name"),
+ "sourceRepo": raw.get("hf_repo"),
+ "sourceRevision": raw.get("revision"),
+ "sourceUrl": raw.get("source_url"),
+ "license": raw.get("license"),
+ "readable": False,
+ },
+ "appCompatibility": {},
+ "status": "loaded" if is_loaded else "downloaded",
+ "recommended": False,
+ "configured": True,
+ "fitsVram": True,
+ "activationSupport": None,
+ "fitsCurrentVram": None,
+ "performance": None,
+ }
+
+
def build_models_payload(gpu_info: Optional[GPUInfo], loaded_model: Optional[str], live_tps: float,
install_dir: str | Path, data_dir: str | Path | None = None,
context_length: Optional[int] = None,
catalog: list[dict[str, Any]] | None = None,
evidence: list[dict[str, Any]] | None = None,
downloaded_files_override: dict[str, Any] | None = None) -> dict[str, Any]:
+ source_catalog = catalog or load_model_catalog(install_dir)
catalog = [
model
- for model in (normalize_catalog_entry(raw) for raw in (catalog or load_model_catalog(install_dir)))
+ for model in (normalize_catalog_entry(raw) for raw in source_catalog)
if model is not None
]
evidence = load_evidence() if evidence is None else evidence
@@ -1651,6 +1708,9 @@ def build_models_payload(gpu_info: Optional[GPUInfo], loaded_model: Optional[str
])
if "turboquant" in normalize_key(runtime_profile_text):
runtime = "turboquant"
+ if normalize_key(runtime) == "vllm":
+ # The installer set a vLLM snapshot up; no GGUF is its selection.
+ recommended_entry = None
data_root = Path(data_dir) if data_dir is not None else Path(install_dir) / "data"
models_dir = model_files_dir(data_root)
if downloaded_files_override is not None:
@@ -1875,6 +1935,19 @@ def build_models_payload(gpu_info: Optional[GPUInfo], loaded_model: Optional[str
}
append_model(fallback, path, "downloaded")
+ if normalize_key(runtime) == "vllm":
+ # A vLLM install serves its LLM_MODEL catalog entry; normalization
+ # (GGUF only) leaves vLLM entries out of the list above.
+ llm_model = read_env_value("LLM_MODEL", install_dir)
+ vllm_entry = next((raw for raw in source_catalog if is_vllm_entry(raw)
+ and llm_model in {raw.get("llm_model_name"), raw.get("id")}), None)
+ if vllm_entry is not None:
+ row = vllm_model_row(vllm_entry, loaded_model, context_length)
+ response_models.append(row)
+ configured_model_id = row["id"]
+ if row["status"] == "loaded":
+ current_model_id = row["id"]
+
if isinstance(loaded_model, str) and loaded_model.strip() and current_model_id is None:
# An external runtime can report a model that is neither in our catalog
# nor an inspectable local GGUF. Show what is actually running without
diff --git a/ods/extensions/services/dashboard-api/routers/models.py b/ods/extensions/services/dashboard-api/routers/models.py
index c0a8e786e..e8aa4d501 100644
--- a/ods/extensions/services/dashboard-api/routers/models.py
+++ b/ods/extensions/services/dashboard-api/routers/models.py
@@ -54,6 +54,7 @@ from host_agent_client import (
request_json as request_agent_json,
)
from models import ModelLibraryGpu, ModelLibraryResponse
+from model_vllm import is_vllm_entry
from pixel_runtime_state import pixel_stream_active
from runtime_projection import active_runtime_projection
from context_policy import HERMES_MIN_CONTEXT
@@ -229,6 +230,16 @@ def _model_activation_mode_denial(
"model server. Change the model in that server, or rerun the "
"installer with an ODS-managed backend to activate downloaded models."
)
+ elif normalized_backend == "vllm":
+ # The vLLM engine serves one installer-set snapshot; activation swaps
+ # llama-server GGUFs (the host agent refuses it as well).
+ code = "vllm_runtime_installer_managed"
+ reason = "vllm_backend_selected"
+ message = (
+ "This install runs its model on the vLLM engine, and models cannot "
+ "be switched here. To use another model, run the ODS installer "
+ "again and choose it."
+ )
elif "unknown" in {effective_mode, configured_mode}:
code = "ods_mode_unknown"
reason = "mode_unknown"
@@ -1517,6 +1528,22 @@ def _hf_requested_projector(details: dict[str, Any], body: dict[str, Any]) -> di
return projector
+def _refuse_vllm_entry(model: dict[str, Any], action: str) -> None:
+ """409 for a vLLM catalog entry: the installer downloads and runs it.
+
+ Its snapshot is a model folder, not a GGUF the model lifecycle can
+ download, switch to or delete (scripts/download-hf-manifest.py and
+ LLM_BACKEND=vllm).
+ """
+ if not is_vllm_entry(model):
+ return
+ raise HTTPException(status_code=409, detail={
+ "error": f"{model.get('name') or model.get('id')} runs only on the vLLM engine. {action}",
+ "code": "vllm_model_installer_managed",
+ "requestedModelId": model.get("id"),
+ })
+
+
def _hf_download_payload(record: dict[str, Any]) -> dict[str, Any]:
"""The host-agent download request for a model record, projector included."""
payload: dict[str, Any] = {
@@ -2705,7 +2732,13 @@ def loaded_model_display_name(loaded_model: str | None) -> str | None:
"""
if not loaded_model:
return None
- catalog = [entry for entry in map(normalize_catalog_entry, _load_library()) if entry is not None]
+ library = _load_library()
+ if LLM_BACKEND == "vllm":
+ # vLLM serves --served-model-name; its entry has no GGUF to match.
+ entry = next((raw for raw in library if is_vllm_entry(raw)
+ and raw.get("served_model_name") == loaded_model), None)
+ return entry.get("name") if entry else None
+ catalog = [entry for entry in map(normalize_catalog_entry, library) if entry is not None]
entry = find_catalog_model(catalog, loaded_model, loaded_model)
return entry["name"] if entry else None
@@ -2901,6 +2934,7 @@ def download_model(model_id: str, api_key: str = Depends(verify_api_key)):
model = _find_model_in_library(model_id)
if model is None:
raise HTTPException(status_code=404, detail=f"Model '{model_id}' not found in library")
+ _refuse_vllm_entry(model, "The ODS installer downloads it when it sets that engine up.")
bootstrap_conflict = _bootstrap_upgrade_download_conflict()
if bootstrap_conflict is not None:
@@ -3099,6 +3133,7 @@ def _activate_model(model_id: str, body: dict[str, Any] | None, *, chat_template
model = _find_loadable_model(model_id)
if model is None:
raise HTTPException(status_code=404, detail=f"Model '{model_id}' not found in library or local GGUF files")
+ _refuse_vllm_entry(model, "Only the ODS installer sets that engine up and runs it.")
requested_context = _requested_activation_context(body)
already_active, loaded_model = _already_active_model(model_id, model)
@@ -3292,6 +3327,7 @@ def delete_model(model_id: str, api_key: str = Depends(verify_api_key)):
model = _find_model_in_library(model_id) or _find_local_gguf_model(model_id)
if model is None:
raise HTTPException(status_code=404, detail=f"Model '{model_id}' not found in library or local GGUF files")
+ _refuse_vllm_entry(model, "The ODS installer manages its files.")
payload = {
"gguf_file": model["gguf_file"],
diff --git a/ods/extensions/services/dashboard-api/setup_chat_route.py b/ods/extensions/services/dashboard-api/setup_chat_route.py
index 747906c09..8f161d067 100644
--- a/ods/extensions/services/dashboard-api/setup_chat_route.py
+++ b/ods/extensions/services/dashboard-api/setup_chat_route.py
@@ -7,8 +7,10 @@ KEYS = (
"LLM_API_URL", "OLLAMA_URL", "LLM_API_BASE_PATH", "LLM_MODEL", "LLM_BACKEND",
"GGUF_FILE", "ODS_MODEL_SWITCHBOARD", "LITELLM_KEY", "LITELLM_MASTER_KEY",
"OPEN_WEBUI_LLM_BASE_URL", "OPEN_WEBUI_LLM_API_KEY", "OPEN_WEBUI_TASK_MODEL",
- "AMD_INFERENCE_LOCATION",
+ "AMD_INFERENCE_LOCATION", "VLLM_SERVED_MODEL",
)
+LLAMA_SERVER_HOSTS = ("llama-server", "ods-llama-server")
+VLLM_HOSTS = ("vllm-server", "ods-vllm-server")
def api_base(raw: str, path: str = "/v1") -> str:
@@ -45,6 +47,10 @@ def resolve_chat_route(default_url: str) -> tuple[str, str, dict[str, str]]:
# A Windows-hosted llama-server requires its key, which only LiteLLM
# holds; setup chat reaches it through the gateway.
raw = "http://litellm:4000"
+ if backend == "vllm" and urlsplit(raw).hostname in LLAMA_SERVER_HOSTS:
+ # LLM_BACKEND=vllm turns the in-stack llama-server off; the vLLM
+ # engine answers through the gateway (LiteLLM, then model-router).
+ raw = "http://litellm:4000"
base = api_base(raw, env["LLM_API_BASE_PATH"] or "/v1")
target = urlsplit(base)
local_gateway = target.scheme == "http" and target.hostname in ("litellm", "ods-litellm") and target.port == 4000
@@ -63,7 +69,10 @@ def resolve_chat_route(default_url: str) -> tuple[str, str, dict[str, str]]:
elif paired_webui:
key = env["OPEN_WEBUI_LLM_API_KEY"]
model = env["OPEN_WEBUI_TASK_MODEL"] or model
- elif target.hostname in ("llama-server", "ods-llama-server", "host.docker.internal") and backend != "external":
+ elif target.hostname in VLLM_HOSTS and backend == "vllm":
+ # vLLM serves exactly --served-model-name and refuses any other id.
+ model = env["VLLM_SERVED_MODEL"] or model
+ elif target.hostname in (*LLAMA_SERVER_HOSTS, "host.docker.internal") and backend != "external":
# Every managed llama-server serves its model as --alias <GGUF_FILE>.
model = env["GGUF_FILE"] or model
headers = {"Content-Type": "application/json"}
diff --git a/ods/extensions/services/dashboard-api/tests/test_vllm_runtime.py b/ods/extensions/services/dashboard-api/tests/test_vllm_runtime.py
new file mode 100644
index 000000000..8f2ea40f2
--- /dev/null
+++ b/ods/extensions/services/dashboard-api/tests/test_vllm_runtime.py
@@ -0,0 +1,313 @@
+"""The dashboard on a vLLM install (LLM_BACKEND=vllm, docker-compose.vllm.yml).
+
+vllm-server replaces the in-stack llama-server: the LLM row probes it, its
+/v1/models row gives the context (no /props), the Models page shows the
+installed vLLM catalog model, readiness names vllm-server (and suggests no
+restart during a cold start), setup chat names the served model, and the
+model lifecycle refuses vLLM entries with a 409 instead of a 500.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from config import _apply_vllm_service_override
+from models import BootstrapStatus, GPUInfo, ServiceStatus
+
+CATALOG = Path(__file__).resolve().parents[4] / "config" / "model-library.json"
+SERVED = "GLM-5.3-Flash-NVFP4"
+
+
+def _vllm_entry(model_id="glm-5.3-flash-nvfp4-vllm"):
+ models = json.loads(CATALOG.read_text(encoding="utf-8"))["models"]
+ return next(model for model in models if model["id"] == model_id)
+
+
+# --- config: the LLM row probes vllm-server ---
+
+
+def test_vllm_route_repoints_the_llm_row_at_vllm_server():
+ services = {"llama-server": {"host": "llama-server", "port": 8080, "health": "/health",
+ "name": "llama-server (LLM Inference)", "container_name": "ods-llama-server"}}
+ _apply_vllm_service_override(services, {"LLM_BACKEND": "vllm"})
+ assert services["llama-server"] == {
+ "host": "vllm-server", "port": 8000, "health": "/health", "name": "vLLM (LLM Inference)",
+ "container_name": "ods-vllm-server", "external_link": False,
+ }
+
+
[email protected]("backend", ["", "llama-server", "external", "lemonade"])
+def test_other_routes_keep_the_llm_row(backend):
+ row = {"host": "llama-server", "port": 8080, "health": "/health"}
+ services = {"llama-server": dict(row)}
+ _apply_vllm_service_override(services, {"LLM_BACKEND": backend})
+ assert services["llama-server"] == row
+
+
+# --- helpers: context from /v1/models, no /props ---
+
+
+def _client(monkeypatch, payload):
+ response = MagicMock()
+ response.json = MagicMock(return_value=payload)
+ client = AsyncMock()
+ client.get = AsyncMock(return_value=response)
+ monkeypatch.setattr("helpers._get_httpx_client", AsyncMock(return_value=client))
+ return client
+
+
+async def test_vllm_context_is_the_served_rows_max_model_len(monkeypatch):
+ from helpers import get_llama_context_size, get_llama_vision_support
+
+ monkeypatch.setattr("helpers.SERVICES", {"llama-server": {"host": "vllm-server", "port": 8000}})
+ monkeypatch.setattr("helpers.LLM_BACKEND", "vllm")
+ monkeypatch.setattr("helpers._host_native_llm", lambda: False)
+ client = _client(monkeypatch, {"object": "list", "data": [
+ {"id": SERVED, "object": "model", "owned_by": "vllm", "max_model_len": 131072}]})
+
+ assert await get_llama_context_size(model_hint=SERVED) == 131072
+ assert await get_llama_context_size(model_hint=None) == 131072
+ assert [call.args[0] for call in client.get.call_args_list] == ["http://vllm-server:8000/v1/models"] * 2
+ # Another id, or a row without a usable length, reports no context.
+ assert await get_llama_context_size(model_hint="Other") is None
+ _client(monkeypatch, {"data": [{"id": SERVED, "max_model_len": "131072"}]})
+ assert await get_llama_context_size(model_hint=SERVED) is None
+ _client(monkeypatch, ["not", "an", "object"])
+ assert await get_llama_context_size(model_hint=SERVED) is None
+ # vLLM reports no modalities: unknown, and nothing asks for /props.
+ client = _client(monkeypatch, {})
+ assert await get_llama_vision_support() is None
+ client.get.assert_not_called()
+
+
+VLLM_METRICS = """# HELP vllm:num_requests_running Number of requests in model execution batches.
+# TYPE vllm:num_requests_running gauge
+vllm:num_requests_running{engine="0",model_name="GLM-5.3-Flash-NVFP4"} 1.0
+# TYPE vllm:generation_tokens_total counter
+vllm:generation_tokens_total{engine="0",model_name="GLM-5.3-Flash-NVFP4"} 4321.0
+vllm:generation_tokens_created{engine="0",model_name="GLM-5.3-Flash-NVFP4"} 1.7e9
+vllm:prompt_tokens_total{engine="0",model_name="GLM-5.3-Flash-NVFP4"} 9876.0
+"""
+
+
+async def test_vllm_metrics_count_tokens_without_claiming_a_rate(monkeypatch, tmp_path):
+ import helpers
+
+ monkeypatch.setattr("helpers.SERVICES", {"llama-server": {"host": "vllm-server", "port": 8000}})
+ monkeypatch.setattr("helpers.LLM_BACKEND", "vllm")
+ monkeypatch.setattr("helpers._host_native_llm", lambda: False)
+ monkeypatch.setattr(helpers, "_TOKEN_FILE", tmp_path / "token_counter.json")
+ helpers._prev_tokens.clear()
+ response = MagicMock()
+ response.text = VLLM_METRICS
+ response.raise_for_status = MagicMock()
+ client = AsyncMock()
+ client.get = AsyncMock(return_value=response)
+ monkeypatch.setattr("helpers._get_httpx_client", AsyncMock(return_value=client))
+
+ result = await helpers._fetch_llama_metrics(model_hint=SERVED)
+
+ assert result["tokens_per_second"] is None and result["_available"] is False
+ assert result["inference_active"] is True
+ assert result["_counters"] == (4321.0, None)
+ # One /metrics read: vLLM has no llama.cpp /slots to ask while it is busy.
+ assert [call.args[0] for call in client.get.call_args_list] == ["http://vllm-server:8000/metrics"]
+
+
+# --- readiness: names vllm-server, no restart while it starts ---
+
+
+def _readiness(monkeypatch, status, *, startup_pending=False, backend="vllm"):
+ import main
+
+ monkeypatch.setattr(main, "LLM_BACKEND", backend)
+ statuses = [
+ ServiceStatus(id="llama-server", name="vLLM (LLM Inference)", port=8000, external_port=8000,
+ status=status, startup_pending=startup_pending),
+ ServiceStatus(id="open-webui", name="Open WebUI", port=3000, external_port=3000, status="healthy"),
+ ]
+ result = main._build_readiness_payload(
+ service_statuses=statuses, loaded_model=None if status != "healthy" else SERVED,
+ context_size=None if status != "healthy" else 131072, bootstrap_info=BootstrapStatus(active=False),
+ host_agent={"available": True}, stt_model_cached=None, stt_model_name="whisper")
+ return next(check for check in result["checks"] if check["id"] == "chat")
+
+
+def test_readiness_waits_calmly_through_a_vllm_cold_start(monkeypatch):
+ chat = _readiness(monkeypatch, "degraded", startup_pending=True)
+ assert chat["detail"].startswith("The vLLM engine is starting")
+ assert chat.get("repair") is None
+
+ chat = _readiness(monkeypatch, "down")
+ assert chat["detail"] == "vllm-server is not healthy"
+ assert chat["repair"] == "ods restart vllm-server"
+
+ chat = _readiness(monkeypatch, "healthy")
+ assert chat["ready"] is True and chat["detail"] == f"{SERVED} loaded with 131072 context"
+
+
+def test_llama_server_readiness_is_unchanged(monkeypatch):
+ chat = _readiness(monkeypatch, "degraded", startup_pending=True, backend="llama-server")
+ assert chat["detail"] == "llama-server is not healthy"
+ assert chat["repair"] == "ods restart llama-server"
+
+
+# --- Models page and model lifecycle ---
+
+
+def _vllm_install(monkeypatch, tmp_path, env_text):
+ import routers.models as models_router
+
+ install_dir = tmp_path / "ods"
+ data_dir = install_dir / "data"
+ (data_dir / "models").mkdir(parents=True)
+ (install_dir / "config").mkdir()
+ (install_dir / "config" / "model-library.json").write_text(CATALOG.read_text(encoding="utf-8"),
+ encoding="utf-8")
+ (install_dir / ".env").write_text(env_text, encoding="utf-8")
+ monkeypatch.delenv("LLM_BACKEND", raising=False)
+ monkeypatch.delenv("LLM_MODEL", raising=False)
+ monkeypatch.setattr(models_router, "INSTALL_DIR", str(install_dir))
+ monkeypatch.setattr(models_router, "DATA_DIR", str(data_dir))
+ monkeypatch.setattr(models_router, "_LIBRARY_PATH", install_dir / "config" / "model-library.json")
+ monkeypatch.setattr(models_router, "_MODELS_DIR", data_dir / "models")
+ monkeypatch.setattr(models_router, "_ENV_PATH", install_dir / ".env")
+ monkeypatch.setattr(models_router, "ODS_MODE_EFFECTIVE", "local")
+ monkeypatch.setattr(models_router, "get_gpu_info", lambda: None)
+ monkeypatch.setattr(models_router, "_request_agent_download",
+ lambda payload: pytest.fail("a vLLM snapshot reached the GGUF download"))
+ monkeypatch.setattr(models_router, "_request_agent_waiting_short_holds",
+ lambda *args, **kwargs: pytest.fail("a vLLM snapshot reached the GGUF delete"))
+ return models_router, install_dir, data_dir
+
+
+VLLM_ENV = ("ODS_MODE=local\nLLM_BACKEND=vllm\nLLM_MODEL=glm-5.3-flash-nvfp4\n"
+ "VLLM_SERVED_MODEL=GLM-5.3-Flash-NVFP4\nGGUF_FILE=\nMAX_CONTEXT=131072\n")
+
+
[email protected]("method,path", [
+ ("post", "/api/models/glm-5.3-flash-nvfp4-vllm/download"),
+ ("delete", "/api/models/glm-5.3-flash-nvfp4-vllm"),
+ ("post", "/api/models/glm-5.3-flash-nvfp4-vllm/load"),
+])
+def test_vllm_entries_are_refused_with_a_clear_409(test_client, monkeypatch, tmp_path, method, path):
+ models_router, _install, _data = _vllm_install(
+ monkeypatch, tmp_path, "ODS_MODE=local\nLLM_BACKEND=llama-server\nGGUF_FILE=a.gguf\n")
+ monkeypatch.setattr(models_router, "LLM_BACKEND", "llama-server")
+ monkeypatch.setattr(models_router, "_configured_ods_mode", lambda: "local")
+ monkeypatch.setattr(models_router, "_windows_hosted_runtime", lambda: False)
+ response = getattr(test_client, method)(path, headers=test_client.auth_headers)
+
+ assert response.status_code == 409
+ detail = response.json()["detail"]
+ assert detail["code"] == "vllm_model_installer_managed"
+ assert detail["requestedModelId"] == "glm-5.3-flash-nvfp4-vllm"
+ assert detail["error"].startswith("GLM 5.3 Flash NVFP4 (vLLM) runs only on the vLLM engine.")
+ assert "ODS installer" in detail["error"]
+
+
+def test_a_vllm_install_refuses_model_switches_before_the_host_agent():
+ import routers.models as models_router
+
+ denial = models_router._model_activation_mode_denial("local", "local", "vllm")
+ assert denial["code"] == "vllm_runtime_installer_managed"
+ assert "vLLM engine" in denial["message"] and "installer" in denial["message"]
+ assert models_router._model_activation_mode_denial("local", "local", "llama-server") is None
+
+
+def _gpu():
+ return GPUInfo(name="NVIDIA GB300", memory_used_mb=200_000, memory_total_mb=288_000, memory_percent=69.0,
+ utilization_percent=10, temperature_c=40, gpu_backend="nvidia")
+
+
[email protected]("loaded,status", [(SERVED, "loaded"), (None, "downloaded")])
+def test_models_page_shows_the_installed_vllm_model(test_client, monkeypatch, tmp_path, loaded, status):
+ models_router, _install, _data = _vllm_install(monkeypatch, tmp_path, VLLM_ENV)
+ monkeypatch.setattr(models_router, "get_gpu_info", _gpu)
+ monkeypatch.setattr(models_router, "get_loaded_model", AsyncMock(return_value=loaded))
+ monkeypatch.setattr(models_router, "_fetch_llama_loaded_model", AsyncMock(return_value=None))
+ monkeypatch.setattr(models_router, "get_llama_metrics", AsyncMock(return_value={"tokens_per_second": 0}))
+ monkeypatch.setattr(models_router, "get_llama_context_size",
+ AsyncMock(return_value=131072 if loaded else None))
+ monkeypatch.setattr(models_router, "LLM_BACKEND", "vllm")
+ monkeypatch.setattr(models_router, "record_model_performance",
+ lambda *args, **kwargs: pytest.fail("no measured throughput on vLLM"))
+ response = test_client.get("/api/models", headers=test_client.auth_headers)
+
+ assert response.status_code == 200
+ payload = response.json()
+ rows = [row for row in payload["models"] if row["id"] == "glm-5.3-flash-nvfp4-vllm"]
+ assert len(rows) == 1
+ row = rows[0]
+ assert row["status"] == status
+ assert row["name"] == "GLM 5.3 Flash NVFP4 (vLLM)"
+ assert row["size"] == "199.7 GB" and row["sizeGb"] == 199.7
+ assert row["contextLength"] == 131072 and row["maxContextLength"] == 131072
+ assert row["quantization"] == "NVFP4"
+ assert row["vramRequired"] == 239.0
+ assert row["metadata"]["source"] == "runtime" and row["metadata"]["runtime"] == "vllm"
+ assert row["metadata"]["servedModelName"] == SERVED
+ assert row["gguf"] is None and row["downloadUrl"] is None
+ assert payload["configuredModel"] == "glm-5.3-flash-nvfp4-vllm"
+ assert payload["llmBackend"] == "vllm"
+ assert payload["currentModel"] == ("glm-5.3-flash-nvfp4-vllm" if loaded else None)
+ # The served model is the catalog row, never an anonymous runtime row,
+ # and no GGUF claims to be the installer's selection.
+ assert not [row for row in payload["models"] if row["id"].startswith("runtime-")]
+ assert not [row for row in payload["models"] if row["recommended"]]
+ # Only the installed vLLM entry is listed; others need the installer.
+ assert "qwen3.8-flash-next-nvfp4-vllm" not in {row["id"] for row in payload["models"]}
+
+
+def test_status_names_the_served_vllm_model_by_its_catalog_name(monkeypatch, tmp_path):
+ models_router, _install, _data = _vllm_install(monkeypatch, tmp_path, VLLM_ENV)
+ monkeypatch.setattr(models_router, "LLM_BACKEND", "vllm")
+ assert models_router.loaded_model_display_name(SERVED) == "GLM 5.3 Flash NVFP4 (vLLM)"
+ assert models_router.loaded_model_display_name("Unknown-Model") is None
+ # A llama-server route never names a vLLM entry.
+ monkeypatch.setattr(models_router, "LLM_BACKEND", "llama-server")
+ assert models_router.loaded_model_display_name(SERVED) is None
+
+
+def test_llama_install_never_lists_vllm_entries(monkeypatch, tmp_path):
+ from performance_oracle import build_models_payload
+
+ install = tmp_path / "ods"
+ install.mkdir()
+ (install / ".env").write_text("LLM_BACKEND=llama-server\nLLM_MODEL=glm-5.3-flash-nvfp4\n", encoding="utf-8")
+ monkeypatch.delenv("LLM_BACKEND", raising=False)
+ monkeypatch.delenv("LLM_MODEL", raising=False)
+ payload = build_models_payload(None, None, 0.0, install, install / "data", None,
+ catalog=[_vllm_entry()], downloaded_files_override={})
+ assert payload["models"] == []
+
+
+# --- setup chat ---
+
+
[email protected]("values,url,model", [
+ # The installer's gateway route: LiteLLM, then model-router, to vLLM.
+ ({"LLM_API_URL": "http://litellm:4000", "ODS_MODEL_SWITCHBOARD": "enabled"},
+ "http://litellm:4000/v1", "ods/current"),
+ # A stale llama-server URL goes through the gateway, never the off container.
+ ({"LLM_API_URL": "http://llama-server:8080", "ODS_MODEL_SWITCHBOARD": "enabled"},
+ "http://litellm:4000/v1", "ods/current"),
+ # Straight to the engine: the exact served id.
+ ({"LLM_API_URL": "http://vllm-server:8000"}, "http://vllm-server:8000/v1", SERVED),
+])
+def test_setup_chat_reaches_the_vllm_engine(monkeypatch, tmp_path, values, url, model):
+ import config
+ from setup_chat_route import resolve_chat_route
+
+ monkeypatch.setattr(config, "INSTALL_DIR", str(tmp_path))
+ values = {"LLM_BACKEND": "vllm", "LLM_MODEL": "glm-5.3-flash-nvfp4", "VLLM_SERVED_MODEL": SERVED,
+ "LITELLM_KEY": "gateway-key", **values}
+ (tmp_path / ".env").write_text("".join(f"{key}={value}\n" for key, value in values.items()))
+ actual_url, actual_model, _headers = resolve_chat_route("http://vllm-server:8000")
+ assert (actual_url, actual_model) == (url + "/chat/completions", model)
diff --git a/ods/extensions/services/model-router/app/main.py b/ods/extensions/services/model-router/app/main.py
index 1ce966107..8aeb02c0c 100644
--- a/ods/extensions/services/model-router/app/main.py
+++ b/ods/extensions/services/model-router/app/main.py
@@ -879,10 +879,36 @@ class RouterError(Exception):
self.message = message
+# ODS thinking levels. A vllm endpoint maps each to the served model's own
+# chat-template arguments (config/model-library.json thinking_template_kwargs,
+# rendered into endpoints.json as "thinking").
+THINKING_LEVELS = ("off", "low", "medium", "high")
+# OpenAI reasoning_effort values, read as ODS levels.
+_REASONING_EFFORT_LEVELS = {
+ "none": "off", "off": "off", "minimal": "low", "low": "low",
+ "medium": "medium", "high": "high", "xhigh": "high",
+}
+
+
+def _valid_thinking(value: Any) -> bool:
+ """An endpoint ``thinking`` map: {"default": level, "templateKwargs": {level: {...}}}."""
+ if not _has_keys(value, {"default", "templateKwargs"}, {"default", "templateKwargs"}):
+ return False
+ levels = value["templateKwargs"]
+ return (
+ value["default"] in THINKING_LEVELS
+ and isinstance(levels, dict)
+ and set(levels) == set(THINKING_LEVELS)
+ and all(isinstance(kwargs, dict) and all(isinstance(key, str) for key in kwargs)
+ for kwargs in levels.values())
+ )
+
+
def _load_endpoints() -> dict[str, dict[str, Any]]:
- """Allowlist endpointId -> {baseUrl, apiKeyEnv?}, re-read when the file
- changes on disk (activation re-renders it). Retains the last good
- allowlist while the file is missing or invalid."""
+ """Allowlist endpointId -> {baseUrl, apiKeyEnv?, thinking?}, re-read when
+ the file changes on disk (activation re-renders it). Retains the last good
+ allowlist while the file is missing or invalid. An endpoint whose thinking
+ map is malformed is left out, so its route fails closed."""
try:
stat = ENDPOINTS_PATH.stat()
except OSError as exc:
@@ -900,9 +926,14 @@ def _load_endpoints() -> dict[str, dict[str, Any]]:
base_url = str(entry.get("baseUrl") or "")
if not endpoint_id or not base_url.startswith(("http://", "https://")):
continue
+ thinking = entry.get("thinking")
+ if thinking is not None and not _valid_thinking(thinking):
+ logger.error("endpoint %s has a malformed thinking map; it is not allowlisted", endpoint_id)
+ continue
endpoints[endpoint_id] = {
"baseUrl": base_url.rstrip("/"),
"apiKeyEnv": str(entry.get("apiKeyEnv") or ""),
+ "thinking": thinking,
}
except (OSError, ValueError) as exc:
logger.error("endpoints allowlist unreadable; retaining previous: %s", exc)
@@ -1129,6 +1160,7 @@ def _active_route() -> dict[str, Any]:
"endpointId": endpoint_id,
"baseUrl": endpoint["baseUrl"],
"apiKeyEnv": endpoint["apiKeyEnv"],
+ "thinking": endpoint.get("thinking"),
"queueMode": str(availability.get("mode") or "serve_active") == "queue",
}
@@ -1146,6 +1178,60 @@ def _llama_tool_compat(route: dict[str, Any]) -> bool:
return route["backendKind"] == "llama-server"
+def _thinking_level(payload: dict[str, Any], default: str) -> str:
+ """The ODS thinking level a chat request asks for.
+
+ A ``reasoning_effort`` names it, top-level (OpenAI's field) or in
+ ``chat_template_kwargs`` (vLLM passes both to the template alike);
+ otherwise the ODS consumers' on/off switch,
+ ``chat_template_kwargs.enable_thinking`` (false is off, true the route's
+ default level, at least medium); otherwise the default.
+ """
+ options = payload.get("chat_template_kwargs")
+ options = options if isinstance(options, dict) else {}
+ effort = payload.get("reasoning_effort")
+ if effort is None:
+ effort = options.get("reasoning_effort")
+ if effort is not None:
+ level = _REASONING_EFFORT_LEVELS.get(effort.strip().lower()) if isinstance(effort, str) else None
+ if level is None:
+ raise RouterError(400, "invalid_request_error",
+ "reasoning_effort must be one of " + ", ".join(_REASONING_EFFORT_LEVELS))
+ return level
+ enabled = options.get("enable_thinking")
+ if enabled is False:
+ return "off"
+ if enabled is True and default == "off":
+ return "medium"
+ return default
+
+
+def _vllm_thinking_payload(payload: dict[str, Any], thinking: dict[str, Any]) -> tuple[dict[str, Any], str]:
+ """The request with its thinking level in the served model's own template terms.
+
+ Chat templates disagree: GLM-5.3 reads only reasoning_effort low or high
+ (any other value thinks at "max", about twice as slow) and Qwen3.8 fails
+ with HTTP 400 on any reasoning_effort but xhigh, medium and low, so no
+ level, reasoning_effort or enable_thinking is forwarded as sent. The
+ level's arguments replace them; the other chat_template_kwargs a client
+ sends are the model's own and stay. Returns the new payload and the level.
+ """
+ options = payload.get("chat_template_kwargs")
+ if options is not None and not isinstance(options, dict):
+ raise RouterError(400, "invalid_request_error", "chat_template_kwargs must be an object")
+ level = _thinking_level(payload, thinking["default"])
+ kwargs = {
+ **thinking["templateKwargs"][level],
+ **{key: value for key, value in (options or {}).items()
+ if key not in {"enable_thinking", "reasoning_effort"}},
+ }
+ translated = {key: value for key, value in payload.items()
+ if key not in {"reasoning_effort", "chat_template_kwargs"}}
+ if kwargs:
+ translated["chat_template_kwargs"] = kwargs
+ return translated, level
+
+
def _offered_tool_evidence(payload: dict[str, Any]) -> dict[str, Any]:
"""Fingerprint the outgoing tools, never retain their names or schema text.
@@ -2039,6 +2125,15 @@ async def _forward_inner(request: Request, path: str, payload: dict[str, Any],
"type": "route_changed", "code": "409"}}, status_code=409), False
payload["model"] = route["runtimeModelId"]
+ thinking_level = None
+ if path == "/v1/chat/completions" and route["backendKind"] == "vllm" and route["thinking"]:
+ try:
+ payload, thinking_level = _vllm_thinking_payload(payload, route["thinking"])
+ except RouterError as exc:
+ return JSONResponse(
+ {"error": {"message": exc.message, "type": exc.code, "code": str(exc.status)}},
+ status_code=exc.status,
+ ), False
# The stable client alias hides the selected template from agents. Adapt
# only outgoing Nemo history after resolving the current route; audit and
# response IDs remain unchanged, including across model switches.
@@ -2094,6 +2189,8 @@ async def _forward_inner(request: Request, path: str, payload: dict[str, Any],
}
if thinking_tool_compat:
ods_headers["X-ODS-Tool-Stream-Compatibility"] = "thinking-nonstream"
+ if thinking_level is not None:
+ ods_headers["X-ODS-Thinking-Level"] = thinking_level
url = route["baseUrl"] + path
client: httpx.AsyncClient = app.state.http
diff --git a/ods/extensions/services/model-router/tests/test_router.py b/ods/extensions/services/model-router/tests/test_router.py
index 94574f5d8..5144d3950 100644
--- a/ods/extensions/services/model-router/tests/test_router.py
+++ b/ods/extensions/services/model-router/tests/test_router.py
@@ -1502,6 +1502,114 @@ class TestVllmRoute:
"key": "ok"}
+# The two catalog maps (config/model-library.json thinking_template_kwargs),
+# as render-runtime-configs.py writes them into the vllm endpoint.
+_GLM_THINKING = {"default": "medium", "templateKwargs": {
+ "off": {"reasoning_effort": "low"}, "low": {"reasoning_effort": "low"},
+ "medium": {"reasoning_effort": "high"}, "high": {}}}
+_QWEN_THINKING = {"default": "medium", "templateKwargs": {
+ "off": {"enable_thinking": False}, "low": {"reasoning_effort": "low"},
+ "medium": {"reasoning_effort": "medium"}, "high": {"reasoning_effort": "xhigh"}}}
+
+
+def _write_vllm_thinking_route(tmp_path, write_state, thinking):
+ (tmp_path / "endpoints.json").write_text(json.dumps({"endpoints": [
+ {"id": "vllm-default", "baseUrl": "http://vllm-server:8000", "thinking": thinking},
+ ]}), encoding="utf-8")
+ write_state(runtime=_VLLM_MODEL, endpoint="vllm-default",
+ mutate=lambda doc: doc["active"]["backend"].update(kind="vllm"))
+
+
+class TestVllmThinking:
+ """ODS thinking levels reach vLLM in the served model's own template terms."""
+
+ def _send(self, router, tmp_path, thinking, extra, kind="vllm"):
+ mod, client, write_state, _calls = router
+ if kind == "vllm":
+ _write_vllm_thinking_route(tmp_path, write_state, thinking)
+ else:
+ write_state()
+ sent = []
+
+ def handler(request):
+ sent.append(json.loads(request.content))
+ return httpx.Response(200, json={"model": sent[-1]["model"], "choices": [
+ {"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}]})
+
+ _use_upstream(mod, handler)
+ response = client.post("/v1/chat/completions", json={
+ "model": "ods/current", "messages": [{"role": "user", "content": "hi"}], **extra})
+ return response, sent
+
+ @pytest.mark.parametrize("thinking,extra,expected,level", [
+ # No hint: the catalog default, medium, which GLM spells "high".
+ (_GLM_THINKING, {}, {"reasoning_effort": "high"}, "medium"),
+ (_QWEN_THINKING, {}, {"reasoning_effort": "medium"}, "medium"),
+ # The ODS consumers' switch: false is off; GLM cannot stop thinking.
+ (_GLM_THINKING, {"chat_template_kwargs": {"enable_thinking": False}},
+ {"reasoning_effort": "low"}, "off"),
+ (_QWEN_THINKING, {"chat_template_kwargs": {"enable_thinking": False}},
+ {"enable_thinking": False}, "off"),
+ (_QWEN_THINKING, {"chat_template_kwargs": {"enable_thinking": True}},
+ {"reasoning_effort": "medium"}, "medium"),
+ # OpenAI's reasoning_effort, never forwarded literally: Qwen's
+ # template fails on "high" and GLM's thinks at max on "medium".
+ (_QWEN_THINKING, {"reasoning_effort": "high"}, {"reasoning_effort": "xhigh"}, "high"),
+ (_GLM_THINKING, {"reasoning_effort": "high"}, None, "high"),
+ (_GLM_THINKING, {"reasoning_effort": "low"}, {"reasoning_effort": "low"}, "low"),
+ (_QWEN_THINKING, {"reasoning_effort": "none"}, {"enable_thinking": False}, "off"),
+ (_QWEN_THINKING, {"reasoning_effort": "minimal"}, {"reasoning_effort": "low"}, "low"),
+ # vLLM hands chat_template_kwargs.reasoning_effort to the template
+ # like the top-level field, so it is translated the same way.
+ (_QWEN_THINKING, {"chat_template_kwargs": {"reasoning_effort": "high"}},
+ {"reasoning_effort": "xhigh"}, "high"),
+ (_GLM_THINKING, {"chat_template_kwargs": {"reasoning_effort": "medium"}},
+ {"reasoning_effort": "high"}, "medium"),
+ # A client's other template arguments are the model's own and stay.
+ (_QWEN_THINKING, {"chat_template_kwargs": {"enable_thinking": True, "preserve_thinking": True}},
+ {"reasoning_effort": "medium", "preserve_thinking": True}, "medium"),
+ ])
+ def test_level_becomes_the_models_template_arguments(
+ self, router, tmp_path, thinking, extra, expected, level):
+ response, sent = self._send(router, tmp_path, thinking, extra)
+
+ assert response.status_code == 200
+ assert len(sent) == 1
+ assert sent[0].get("chat_template_kwargs") == expected
+ assert "reasoning_effort" not in sent[0]
+ assert response.headers["X-ODS-Thinking-Level"] == level
+
+ def test_unknown_reasoning_effort_is_refused_before_vllm(self, router, tmp_path):
+ response, sent = self._send(router, tmp_path, _QWEN_THINKING, {"reasoning_effort": "max"})
+
+ assert response.status_code == 400
+ assert response.json()["error"]["type"] == "invalid_request_error"
+ assert sent == []
+
+ def test_non_object_template_arguments_are_refused(self, router, tmp_path):
+ response, sent = self._send(router, tmp_path, _GLM_THINKING, {"chat_template_kwargs": ["x"]})
+
+ assert response.status_code == 400
+ assert sent == []
+
+ def test_llama_server_route_forwards_thinking_unchanged(self, router, tmp_path):
+ extra = {"chat_template_kwargs": {"enable_thinking": False}, "reasoning_effort": "high"}
+ response, sent = self._send(router, tmp_path, None, extra, kind="llama-server")
+
+ assert response.status_code == 200
+ assert sent[0]["chat_template_kwargs"] == {"enable_thinking": False}
+ assert sent[0]["reasoning_effort"] == "high"
+ assert "X-ODS-Thinking-Level" not in response.headers
+
+ def test_malformed_thinking_map_fails_the_route_closed(self, router, tmp_path):
+ broken = {"default": "medium", "templateKwargs": {"off": {}, "low": {}, "medium": {}}}
+ response, sent = self._send(router, tmp_path, broken, {})
+
+ assert response.status_code == 503
+ assert response.json()["error"]["type"] == "endpoint_not_allowlisted"
+ assert sent == []
+
+
class TestStateTrust:
def test_schema_invalid_state_is_not_routable(self, router):
mod, client, write_state, calls = router
diff --git a/ods/extensions/services/pixel-agent/host/pixel_ingress.mjs b/ods/extensions/services/pixel-agent/host/pixel_ingress.mjs
index 9ad17c4e7..e6e3ce374 100644
--- a/ods/extensions/services/pixel-agent/host/pixel_ingress.mjs
+++ b/ods/extensions/services/pixel-agent/host/pixel_ingress.mjs
@@ -169,6 +169,8 @@ const ALLOWED_SERVICES = new Set([
"dashboard-api",
"ods-llama-server",
"llama-server",
+ "ods-vllm-server",
+ "vllm-server",
"ods-searxng",
"searxng",
"ods-langfuse",
@@ -1786,6 +1788,46 @@ export async function dockerRuntime(deps = defaultDeps) {
return { model, context_length: contextLength };
}
+function commandFlag(command, flag) {
+ const index = command.indexOf(flag);
+ if (index >= 0) return command[index + 1];
+ const joined = command.find((item) => item.startsWith(`${flag}=`));
+ return joined === undefined ? undefined : joined.slice(flag.length + 1);
+}
+
+// LLM_BACKEND=vllm: ods-vllm-server serves --served-model-name with the
+// context its --max-model-len (catalog VLLM_ARGS) fixes; there is no
+// ods-llama-server, or only one an earlier install left stopped.
+export async function dockerVllmRuntime(deps = defaultDeps) {
+ const { stdout } = await execFilePromise(
+ deps.execFile,
+ "docker",
+ ["inspect", "ods-vllm-server", "--format", "{{json .Config.Cmd}}"],
+ { timeout: DOCKER_TIMEOUT_MS, maxBuffer: 64 * 1024, windowsHide: true }
+ );
+ let command;
+ try {
+ command = JSON.parse(String(stdout || ""));
+ } catch {
+ return null;
+ }
+ if (!Array.isArray(command) || !command.every((item) => typeof item === "string")) {
+ return null;
+ }
+ const model = commandFlag(command, "--served-model-name");
+ const contextLength = Number(commandFlag(command, "--max-model-len"));
+ if (
+ typeof model !== "string" ||
+ !MODEL_NAME_RE.test(model) ||
+ !Number.isInteger(contextLength) ||
+ contextLength < 4096 ||
+ contextLength > 10_000_000
+ ) {
+ return null;
+ }
+ return { model, context_length: contextLength };
+}
+
export async function dockerRouterRuntime(deps = defaultDeps) {
const { stdout } = await execFilePromise(
deps.execFile,
@@ -1884,7 +1926,9 @@ export async function writeStatus(
// to label that provider with an unrelated local llama model.
if (configuredRuntime === undefined && runtime === null && docker === "ok") {
try {
- runtime = await dockerRuntime(deps);
+ runtime = apps.some(({ name }) => name === "ods-vllm-server")
+ ? await dockerVllmRuntime(deps)
+ : await dockerRuntime(deps);
} catch {
runtime = null;
}
diff --git a/ods/extensions/services/pixel-agent/plugin/projection.mjs b/ods/extensions/services/pixel-agent/plugin/projection.mjs
index 4238c623d..ea04c6633 100644
--- a/ods/extensions/services/pixel-agent/plugin/projection.mjs
+++ b/ods/extensions/services/pixel-agent/plugin/projection.mjs
@@ -50,6 +50,9 @@ registerApp(["ods-whisper", "whisper"], "Whisper", "speech-to-text API", "whispe
registerApp(["ods-embeddings", "embeddings"], "TEI", "embeddings API", "embeddings", "/info");
registerApp(["ods-litellm", "litellm"], "LiteLLM", "model gateway", "litellm");
registerApp(["ods-llama-server", "llama-server"], "llama-server", "local model API", "llama");
+// LLM_BACKEND=vllm: the vLLM engine publishes no host port; model-router
+// reaches it on the ODS network, so it has no local URL.
+registerApp(["ods-vllm-server", "vllm-server"], "vLLM", "local model engine behind the model router", null);
registerApp(["ods-privacy-shield"], "Privacy Shield", "PII protection API", "privacy_shield", "/docs");
registerApp(["ods-token-spy"], "Token Spy", "usage monitor", "token_spy", "/dashboard");
registerApp(["ods-ape"], "APE", "agent policy engine", "ape");
@@ -77,6 +80,8 @@ const ALLOWED_SERVICES = new Set([
"dashboard-api",
"ods-llama-server",
"llama-server",
+ "ods-vllm-server",
+ "vllm-server",
"ods-searxng",
"searxng",
"ods-langfuse",
@@ -305,13 +310,16 @@ function validAppPorts(value) {
function enrichApp(app, appPorts) {
const metadata = APP_DIRECTORY.get(app.name);
if (!metadata) return { name: app.name, status: app.status };
- return {
+ const enriched = {
name: app.name,
status: app.status,
display_name: metadata.display_name,
purpose: metadata.purpose,
- url: `http://localhost:${appPorts[metadata.port_key]}${metadata.url_path}`,
};
+ if (metadata.port_key !== null) {
+ enriched.url = `http://localhost:${appPorts[metadata.port_key]}${metadata.url_path}`;
+ }
+ return enriched;
}
// Validate a fully parsed projection object against the strict schema.
diff --git a/ods/extensions/services/pixel-agent/plugin/tool-content.mjs b/ods/extensions/services/pixel-agent/plugin/tool-content.mjs
index 842426aa9..b30fa8fd5 100644
--- a/ods/extensions/services/pixel-agent/plugin/tool-content.mjs
+++ b/ods/extensions/services/pixel-agent/plugin/tool-content.mjs
@@ -22,11 +22,16 @@ function freshness(value) {
return value ? "stale" : "current";
}
+function appDescription({ name, status, display_name: displayName, purpose, url }) {
+ if (!displayName || !purpose) return `${name} (${status})`;
+ // An app without a host port (the vLLM engine) has no local URL.
+ return url
+ ? `${displayName} [${name}] (${status}; ${purpose}; ${url})`
+ : `${displayName} [${name}] (${status}; ${purpose})`;
+}
+
function appList(apps) {
- return apps.map(({ name, status, display_name: displayName, purpose, url }) => {
- if (!displayName || !purpose || !url) return `${name} (${status})`;
- return `${displayName} [${name}] (${status}; ${purpose}; ${url})`;
- }).join(", ");
+ return apps.map(appDescription).join(", ");
}
export function statusToolText(projection) {
@@ -54,10 +59,7 @@ export function appsToolText(projection) {
if (projection.app_count === 0) {
return `ODS reports 0 of 0 applications online in its ${freshness(projection.stale)} projection at ${projection.timestamp}. ${EVIDENCE_BOUNDARY}`;
}
- const first = projection.apps[0];
- const firstDescription = first.display_name
- ? `${first.display_name} [${first.name}] (${first.status}; ${first.purpose}; ${first.url})`
- : `${first.name} (${first.status})`;
+ const firstDescription = appDescription(projection.apps[0]);
return [
`ODS reports ${projection.online_app_count} of ${projection.app_count} applications online in its ${freshness(projection.stale)} projection at ${projection.timestamp}.`,
`The first is ${firstDescription}.`,
diff --git a/ods/extensions/services/pixel-agent/tests/pixel_ingress.test.mjs b/ods/extensions/services/pixel-agent/tests/pixel_ingress.test.mjs
index 400a408de..e2b229a81 100644
--- a/ods/extensions/services/pixel-agent/tests/pixel_ingress.test.mjs
+++ b/ods/extensions/services/pixel-agent/tests/pixel_ingress.test.mjs
@@ -35,6 +35,7 @@ const {
validateConfig,
dockerApps,
dockerRuntime,
+ dockerVllmRuntime,
dockerRouterRuntime,
writeStatus,
start,
@@ -1957,6 +1958,77 @@ test("runtime projection rejects paths, malformed contexts, and non-JSON output"
}
});
+// LLM_BACKEND=vllm (docker-compose.vllm.yml): the command Compose builds
+// for ods-vllm-server, catalog VLLM_ARGS included.
+const VLLM_COMMAND = [
+ "--model", "/model", "--served-model-name", "GLM-5.3-Flash-NVFP4", "--host", "0.0.0.0",
+ "--port", "8000", "--gpu-memory-utilization", "0.90", "--max-model-len", "131072",
+ "--kv-cache-dtype", "fp8", "--speculative-config.method=mtp",
+];
+
+test("vLLM runtime projection reads the served name and context from ods-vllm-server", async () => {
+ const fakeDocker = (cmd, args, opts, callback) => {
+ assert.equal(cmd, "docker");
+ assert.deepEqual(args, ["inspect", "ods-vllm-server", "--format", "{{json .Config.Cmd}}"]);
+ assert.ok(opts.timeout > 0, "must have explicit timeout");
+ callback(null, JSON.stringify(VLLM_COMMAND), "");
+ };
+ assert.deepEqual(await dockerVllmRuntime({ execFile: fakeDocker }), {
+ model: "GLM-5.3-Flash-NVFP4",
+ context_length: 131072,
+ });
+ for (const command of [
+ VLLM_COMMAND.filter((item) => item !== "--max-model-len" && item !== "131072"),
+ VLLM_COMMAND.map((item) => (item === "131072" ? "2048" : item)),
+ VLLM_COMMAND.map((item) => (item === "GLM-5.3-Flash-NVFP4" ? "../secret" : item)),
+ ["--model", "/model", "--served-model-name=Joined-Name", "--max-model-len=65536"],
+ "not-json",
+ ]) {
+ const reply = (_cmd, _args, _opts, callback) => {
+ callback(null, command === "not-json" ? command : JSON.stringify(command), "");
+ };
+ const expected = Array.isArray(command) && command.includes("--max-model-len=65536")
+ ? { model: "Joined-Name", context_length: 65536 }
+ : null;
+ assert.deepEqual(await dockerVllmRuntime({ execFile: reply }), expected);
+ }
+});
+
+test("status on a vLLM install lists the engine and reads its runtime, not a stale llama-server", async () => {
+ const statusFile = path.join(DIR, "vllm-runtime-status.json");
+ const inspected = [];
+ const fakeDocker = (_cmd, args, _opts, callback) => {
+ if (args[0] === "ps") {
+ callback(null, [
+ JSON.stringify({ Names: "ods-vllm-server", Status: "Up 4 minutes (health: starting)" }),
+ // An earlier llama-server install left its container behind, stopped.
+ JSON.stringify({ Names: "ods-llama-server", State: "exited", Status: "Exited (0) 2 days ago" }),
+ "",
+ ].join("\n"), "");
+ return;
+ }
+ if (args[0] === "exec") {
+ callback(new Error("model-router state unavailable"));
+ return;
+ }
+ inspected.push(args[1]);
+ callback(null, JSON.stringify(args[1] === "ods-vllm-server" ? VLLM_COMMAND
+ : ["--model", "/models/Stale.gguf", "--ctx-size", "32768"]), "");
+ };
+ const projection = await writeStatus(true, 18999, statusFile, "2.6.0", {
+ fetch: async () => ({ status: 200, body: { cancel: async () => {} } }),
+ execFile: fakeDocker,
+ setTimeout,
+ clearTimeout,
+ });
+ assert.deepEqual(projection.apps, [
+ { name: "ods-llama-server", status: "stopped" },
+ { name: "ods-vllm-server", status: "starting" },
+ ]);
+ assert.deepEqual(projection.runtime, { model: "GLM-5.3-Flash-NVFP4", context_length: 131072 });
+ assert.deepEqual(inspected, ["ods-vllm-server"]);
+});
+
test("router runtime projection returns the exact active stable-alias target", async () => {
const fakeDocker = (cmd, args, opts, callback) => {
assert.equal(cmd, "docker");
diff --git a/ods/extensions/services/pixel-agent/tests/projection.test.mjs b/ods/extensions/services/pixel-agent/tests/projection.test.mjs
index 142c89ccd..c436c60a7 100644
--- a/ods/extensions/services/pixel-agent/tests/projection.test.mjs
+++ b/ods/extensions/services/pixel-agent/tests/projection.test.mjs
@@ -196,6 +196,24 @@ test("application payloads expose allowlisted purposes and configured local URLs
]);
});
+test("the vLLM engine is an allowlisted app with no local URL", async () => {
+ const raw = JSON.stringify(goodProjection({
+ apps: [
+ { name: "ods-vllm-server", status: "starting" },
+ { name: "vllm-server", status: "healthy" },
+ ],
+ }));
+ const fsImpl = memoryFs({ [FIXED]: makeEntry(raw) });
+ const projection = await readProjection(FIXED, fsImpl, Date.now());
+
+ assert.deepEqual(appsPayload(projection).apps, [
+ { name: "ods-vllm-server", status: "starting", display_name: "vLLM",
+ purpose: "local model engine behind the model router" },
+ { name: "vllm-server", status: "healthy", display_name: "vLLM",
+ purpose: "local model engine behind the model router" },
+ ]);
+});
+
test("starting application status is accepted", async () => {
const raw = JSON.stringify(
goodProjection({
diff --git a/ods/extensions/services/pixel-agent/tests/tool_content.test.mjs b/ods/extensions/services/pixel-agent/tests/tool_content.test.mjs
index 0964a9d72..716a0af79 100644
--- a/ods/extensions/services/pixel-agent/tests/tool_content.test.mjs
+++ b/ods/extensions/services/pixel-agent/tests/tool_content.test.mjs
@@ -43,6 +43,22 @@ test("apps result states the count and first application without JSON parsing",
assert.ok(!text.startsWith("{"));
});
+test("an application without a host port is described without a URL", () => {
+ // LLM_BACKEND=vllm: model-router reaches the vLLM engine on the ODS network.
+ const vllm = {
+ name: "ods-vllm-server",
+ status: "starting",
+ display_name: "vLLM",
+ purpose: "local model engine behind the model router",
+ };
+ for (const apps of [[vllm, projection.apps[0]], [projection.apps[0], vllm]]) {
+ const text = appsToolText({ ...projection, apps });
+ assert.match(text, /vLLM \[ods-vllm-server\] \(starting; local model engine behind the model router\)/);
+ assert.doesNotMatch(text, /undefined/);
+ assert.match(text, /Dashboard \[ods-dashboard\] \(healthy; ODS control center; http:\/\/localhost:3001\/\)/);
+ }
+});
+
test("status result states each bounded host fact in natural language", () => {
const text = statusToolText(projection);
assert.match(text, /Pixel availability: available/);
diff --git a/ods/lib/preflight-llm-route.sh b/ods/lib/preflight-llm-route.sh
index ace9526e3..65484e289 100644
--- a/ods/lib/preflight-llm-route.sh
+++ b/ods/lib/preflight-llm-route.sh
@@ -7,19 +7,30 @@ ods_preflight_host_native_llm() {
[[ -n "${NATIVE_LLM_BASE_URL:-}" ]]
}
+# True when .env selects the vLLM engine (LLM_BACKEND=vllm,
+# docker-compose.vllm.yml): the vllm-server container serves the model on the
+# ODS network without a host port, and the in-stack llama-server is off.
+# Clients reach it through model-router and LiteLLM.
+ods_preflight_vllm_engine() {
+ local backend
+ backend=$(printf '%s' "${LLM_BACKEND:-}" | tr '[:upper:]' '[:lower:]')
+ [[ "$backend" == "vllm" ]]
+}
+
ods_preflight_uses_litellm() {
# ods-preflight.sh runs under macOS /bin/bash 3.2, which has no Bash 4 case conversion.
local mode
mode=$(printf '%s' "${ODS_MODE:-local}" | tr '[:upper:]' '[:lower:]')
- ods_preflight_host_native_llm || [[ -n "${EXTERNAL_LLM_URL:-}" ]] || [[ "$mode" == "cloud" ]]
+ ods_preflight_host_native_llm || ods_preflight_vllm_engine \
+ || [[ -n "${EXTERNAL_LLM_URL:-}" ]] || [[ "$mode" == "cloud" ]]
}
# Status leaves out the in-stack inference services a LiteLLM route does not
# start: llama-server always, model-router unless a host-native llama-server
-# keeps it for model switching.
+# keeps it for model switching or the vLLM engine is served through it.
ods_status_skips_managed_inference() {
local sid="$1"
ods_preflight_uses_litellm || return 1
[[ "$sid" == "llama-server" ]] && return 0
- [[ "$sid" == "model-router" ]] && ! ods_preflight_host_native_llm
+ [[ "$sid" == "model-router" ]] && ! ods_preflight_host_native_llm && ! ods_preflight_vllm_engine
}
diff --git a/ods/ods-cli b/ods/ods-cli
index 88ad79bdf..8c913d6f5 100755
--- a/ods/ods-cli
+++ b/ods/ods-cli
@@ -807,10 +807,17 @@ load_env() {
#=============================================================================
. "$SCRIPT_DIR/lib/service-registry.sh"
-# Resolve a user-provided service name to a compose service ID
+# Resolve a user-provided service name to the compose service ID that serves
+# it on this install. With LLM_BACKEND=vllm the vLLM engine serves the model
+# and the in-stack llama-server is profiled out (docker-compose.vllm.yml).
+# Naming llama-server to Compose would enable its profile and start a second
+# engine on the GPU, so "llm" and "llama-server" mean vllm-server there.
resolve_service() {
- local resolved
+ local resolved backend="${LLM_BACKEND:-}"
resolved=$(sr_resolve "$1")
+ if [[ "$resolved" == "llama-server" && "${backend,,}" == "vllm" ]]; then
+ resolved="vllm-server"
+ fi
echo "$resolved"
}
@@ -957,6 +964,18 @@ get_compose_flags() {
if [[ "$_ods_mode" != "cloud" ]] && [[ "$cached" == *"docker-compose.cloud.yml"* ]]; then
stale=1
fi
+ # LLM_BACKEND=vllm swaps the in-stack llama-server for the vLLM
+ # engine overlay, which (like a host-native route) leaves out the
+ # compose.local.yaml waits on llama-server. A cache from the other
+ # route starts the wrong engine.
+ local _llm_backend="${LLM_BACKEND:-}"
+ if [[ "${_llm_backend,,}" == "vllm" ]]; then
+ if [[ "$cached" != *"docker-compose.vllm.yml"* || "$cached" == *"compose.local.yaml"* ]]; then
+ stale=1
+ fi
+ elif [[ "$cached" == *"docker-compose.vllm.yml"* ]]; then
+ stale=1
+ fi
# The retired Lemonade overlay can outlive an update that kept
# the old file on disk; the stack must be resolved again.
if [[ "$cached" == *"docker-compose.lemonade-external.yml"* ]]; then
@@ -1452,6 +1471,29 @@ _ods_cli_service_health() {
fi
}
+# The vLLM engine (LLM_BACKEND=vllm) publishes no host port. Its Docker
+# healthcheck asks /health, which answers only once the model is loaded, so
+# Docker's state is its status: healthy, starting (a cold start loads the
+# weights and compiles kernels for up to 15 minutes), unhealthy, or stopped.
+_ODS_VLLM_NAME="vLLM (LLM Inference)"
+_ods_cli_vllm_health() {
+ local inspect state="" health=""
+ if ! inspect=$(docker inspect --format \
+ '{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' \
+ ods-vllm-server 2>&1); then
+ printf 'stopped\n'
+ return 0
+ fi
+ read -r state health <<<"$inspect"
+ if [[ "$state" != "running" ]]; then
+ printf 'stopped\n'
+ elif [[ "$health" == "healthy" || "$health" == "starting" ]]; then
+ printf '%s\n' "$health"
+ else
+ printf 'unhealthy\n'
+ fi
+}
+
cmd_status() {
local json_mode="false"
while [[ $# -gt 0 ]]; do
@@ -1568,6 +1610,13 @@ cmd_status() {
fi
fi
done
+ if ods_preflight_vllm_engine; then
+ case "$(_ods_cli_vllm_health)" in
+ healthy) success "$_ODS_VLLM_NAME: healthy" ;;
+ starting) log "$_ODS_VLLM_NAME: starting. A first start loads the model and compiles kernels, which can take 15 minutes. Follow it with: ods logs llm" ;;
+ *) warn "$_ODS_VLLM_NAME: not responding. See why with: ods logs llm" ;;
+ esac
+ fi
echo ""
@@ -1675,6 +1724,17 @@ cmd_status_json() {
--arg containerStatus "$container_status" \
'{id: $id, name: $name, category: $category, port: $port, status: $status, containerStatus: $containerStatus}' >> "$tmp"
done
+ if ods_preflight_vllm_engine; then
+ local vllm_status vllm_container="unknown"
+ vllm_status="$(_ods_cli_vllm_health)"
+ if [[ "$container_state_known" == "true" ]]; then
+ vllm_container="stopped"
+ grep -Fxq -- vllm-server <<<"$running_services" && vllm_container="running"
+ fi
+ # No host port: model-router and LiteLLM reach it on the ODS network.
+ jq -n --arg name "$_ODS_VLLM_NAME" --arg status "$vllm_status" --arg containerStatus "$vllm_container" \
+ '{id: "vllm-server", name: $name, category: "core", port: "", status: $status, containerStatus: $containerStatus}' >> "$tmp"
+ fi
# GPU summary (optional)
local gpu_summary_json="null"
@@ -3008,8 +3068,9 @@ if llm_backend:
local_warn = llm_backend.get('local_running_warning', False)
recovery = llm_backend.get('recovery_hint', '')
- is_external = llm_provider != "llama-server"
- backend_label = f"{llm_provider} (external)" if is_external else "llama-server (local)"
+ # The vLLM engine (vllm-server) is ODS's own local model server, like llama-server.
+ is_external = llm_provider not in ("llama-server", "vLLM")
+ backend_label = f"{llm_provider} (external)" if is_external else f"{llm_provider} (local)"
if llm_status == 'ok':
print(f" {GREEN}✓{NC} LLM Backend: {backend_label} — responding")
@@ -4155,6 +4216,8 @@ cmd_mode() {
check_install; cd "$INSTALL_DIR"
load_env || true
local mode="${1:-}"
+ local configured_backend
+ configured_backend=$(grep -m1 "^LLM_BACKEND=" .env 2>/dev/null | cut -d= -f2- | tr -d '"\047\r' || true)
if [[ -z "$mode" ]]; then
# Show current mode
@@ -4163,6 +4226,8 @@ cmd_mode() {
local api_url=$(grep "^LLM_API_URL=" .env 2>/dev/null | cut -d= -f2 || true)
local model=$(grep "^LLM_MODEL=" .env 2>/dev/null | cut -d= -f2 || true)
local tier=$(grep "^TIER=" .env 2>/dev/null | cut -d= -f2 || true)
+ local engine="llama-server"
+ [[ "${configured_backend,,}" != "vllm" ]] || engine="vLLM engine (vllm-server)"
echo -e "${BLUE}━━━ ODS Mode ━━━${NC}"
echo ""
echo -e "Current mode: ${GREEN}${current}${NC}"
@@ -4177,11 +4242,11 @@ cmd_mode() {
echo " Cost: ~\$0.003-0.06/1K tokens"
;;
local)
- echo " LLM: Local llama-server"
+ echo " LLM: Local $engine"
echo " Cost: \$0 (electricity only)"
;;
hybrid)
- echo " LLM: Local llama-server → Cloud fallback on failure"
+ echo " LLM: Local $engine → Cloud fallback on failure"
echo " Cost: \$0 when local works, cloud rates on fallback"
;;
esac
@@ -4201,8 +4266,6 @@ cmd_mode() {
*) error "Unknown mode: $mode. Use: local, cloud, hybrid" ;;
esac
- local configured_backend
- configured_backend=$(grep -m1 "^LLM_BACKEND=" .env 2>/dev/null | cut -d= -f2- | tr -d '"\047\r' || true)
if [[ "${configured_backend,,}" == "external" ]]; then
# Fleet row 4: "installer-managed" did not say what to do or where.
echo -e "${RED}✗${NC} ODS is in API mode, which only the installer turns on and off. Nothing was changed." >&2
@@ -4210,6 +4273,14 @@ cmd_mode() {
echo " Need help? Ask on the ODS Discord: https://discord.gg/4ntNp9MAwC" >&2
exit 1
fi
+ if [[ "${configured_backend,,}" == "vllm" && "$mode" != "local" ]]; then
+ # Cloud mode has no local model server, and the resolver refuses the
+ # vLLM engine with it; hybrid mode was never proven with vLLM.
+ echo -e "${RED}✗${NC} ODS runs this model on the vLLM engine, which works only in local mode. Nothing was changed." >&2
+ echo " To use $mode mode, run the ODS installer again without the vLLM engine." >&2
+ echo " Need help? Ask on the ODS Discord: https://discord.gg/4ntNp9MAwC" >&2
+ exit 1
+ fi
# Update .env
_env_set "ODS_MODE" "$mode"
@@ -4219,9 +4290,12 @@ cmd_mode() {
local switchboard_mode native_llm
switchboard_mode=$(grep -m1 "^ODS_MODEL_SWITCHBOARD=" .env 2>/dev/null | cut -d= -f2- | tr -d '"\047\r' || true)
native_llm=$(grep -m1 "^NATIVE_LLM_BASE_URL=" .env 2>/dev/null | cut -d= -f2- | tr -d '"\047\r' || true)
- # A host-native llama-server install keeps the in-stack llama-server
- # off; LiteLLM is what reaches the native server (06-directories.sh).
- if [[ -n "$native_llm" ]]; then
+ # A host-native llama-server install and a vLLM install keep the
+ # in-stack llama-server off; LiteLLM is what reaches the model
+ # (06-directories.sh), through model-router for vLLM.
+ local llama_off="false"
+ [[ -n "$native_llm" || "${configured_backend,,}" == "vllm" ]] && llama_off="true"
+ if [[ "$llama_off" == "true" ]]; then
api_url="http://litellm:4000"
else
api_url="http://llama-server:8080"
@@ -4230,7 +4304,7 @@ cmd_mode() {
if [[ "${switchboard_mode,,}" == "enabled" ]]; then
_env_set "HERMES_LLM_BASE_URL" "http://model-router:9099/v1"
_env_set "HERMES_LLM_API_KEY" "no-key"
- elif [[ -n "$native_llm" ]]; then
+ elif [[ "$llama_off" == "true" ]]; then
_env_set "HERMES_LLM_BASE_URL" "http://litellm:4000/v1"
if [[ -n "${LITELLM_KEY:-}" ]]; then
_env_set "HERMES_LLM_API_KEY" "$LITELLM_KEY"
diff --git a/ods/scripts/mode-switch.sh b/ods/scripts/mode-switch.sh
index c94e62a30..ecbb15f65 100755
--- a/ods/scripts/mode-switch.sh
+++ b/ods/scripts/mode-switch.sh
@@ -80,17 +80,26 @@ switch_mode() {
if [[ "$configured_backend" == "external" ]]; then
error "External LLM routing is installer-managed. Run './install.sh --no-external-llm' first, then retry the mode switch."
fi
+ if [[ "$configured_backend" == "vllm" && "$mode" != "local" ]]; then
+ # Cloud mode has no local model server, and the resolver refuses the
+ # vLLM engine with it; hybrid mode was never proven with vLLM.
+ error "ODS runs this model on the vLLM engine, which works only in local mode. Nothing was changed. To use $mode mode, run the ODS installer again without the vLLM engine. Need help? Ask on the ODS Discord: https://discord.gg/4ntNp9MAwC"
+ fi
# Update .env
env_set "ODS_MODE" "$mode"
if [[ "$mode" == "local" ]]; then
- local switchboard_mode native_llm
+ local switchboard_mode native_llm llama_off="false"
switchboard_mode=$(grep -m1 "^ODS_MODEL_SWITCHBOARD=" "$ENV_FILE" 2>/dev/null | cut -d= -f2- | tr -d '"\047\r' || true)
native_llm=$(grep -m1 "^NATIVE_LLM_BASE_URL=" "$ENV_FILE" 2>/dev/null | cut -d= -f2- | tr -d '"\047\r' || true)
- # A host-native llama-server install keeps the in-stack llama-server
- # off; LiteLLM is what reaches the native server (06-directories.sh).
- if [[ -n "$native_llm" ]]; then
+ # A host-native llama-server install and a vLLM install keep the
+ # in-stack llama-server off; LiteLLM is what reaches the model
+ # (06-directories.sh), through model-router for vLLM.
+ if [[ -n "$native_llm" || "$configured_backend" == "vllm" ]]; then
+ llama_off="true"
+ fi
+ if [[ "$llama_off" == "true" ]]; then
env_set "LLM_API_URL" "http://litellm:4000"
else
env_set "LLM_API_URL" "http://llama-server:8080"
@@ -99,7 +108,7 @@ switch_mode() {
if [[ "$switchboard_mode" == "enabled" ]]; then
env_set "HERMES_LLM_BASE_URL" "http://model-router:9099/v1"
env_set "HERMES_LLM_API_KEY" "no-key"
- elif [[ -n "$native_llm" ]]; then
+ elif [[ "$llama_off" == "true" ]]; then
env_set "HERMES_LLM_BASE_URL" "http://litellm:4000/v1"
local litellm_key
litellm_key=$(grep -m1 "^LITELLM_KEY=" "$ENV_FILE" 2>/dev/null | cut -d= -f2- | tr -d '"\047\r' || true)
diff --git a/ods/scripts/ods-doctor.sh b/ods/scripts/ods-doctor.sh
index 6930596be..268ca815a 100755
--- a/ods/scripts/ods-doctor.sh
+++ b/ods/scripts/ods-doctor.sh
@@ -445,11 +445,57 @@ _doctor_check_llama_server() {
fi
}
+# LLM_BACKEND=vllm (docker-compose.vllm.yml): vllm-server serves the model on
+# the ODS network without a host port, and the in-stack llama-server is off.
+# Docker's healthcheck asks vLLM's /health, which answers only once the model
+# is loaded, so its state is the engine's status. A cold start loads the
+# weights and compiles kernels for up to 15 minutes: that is starting, not a
+# fault. A llama-server container left by an earlier install still running
+# would compete for the GPU.
+_doctor_check_vllm_engine() {
+ local inspect="" state="" health=""
+ LLM_PROVIDER="vLLM"
+ LLM_URL="http://vllm-server:8000 (ODS network only)"
+ LLM_MODEL="${VLLM_SERVED_MODEL:-}"
+ LLM_RECOVERY=""
+ LLM_LOCAL_WARNING="false"
+ if [[ "$DOCKER_DAEMON" == "true" ]] && inspect=$(docker inspect --format \
+ '{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' \
+ ods-vllm-server 2>&1); then
+ read -r state health <<<"$inspect"
+ fi
+ if [[ "$state" == "running" && "$health" == "healthy" ]]; then
+ LLM_STATUS="ok"
+ log_ok "LLM backend: vLLM (local) — responding"
+ [[ -z "$LLM_MODEL" ]] || log_ok " Model : $LLM_MODEL"
+ elif [[ "$state" == "running" && "$health" == "starting" ]]; then
+ LLM_STATUS="starting"
+ log_info "LLM backend: vLLM (local) — starting: a first start loads the model and compiles kernels, which can take 15 minutes"
+ log_info " Follow : ods logs llm"
+ else
+ LLM_STATUS="fail"
+ if [[ "$state" == "running" ]]; then
+ LLM_RECOVERY="ods restart vllm-server"
+ log_fail "LLM backend: vLLM (local) — container running but not healthy"
+ else
+ LLM_RECOVERY="ods start vllm-server"
+ log_fail "LLM backend: vLLM (local) — container not running"
+ fi
+ log_info " Recovery : $LLM_RECOVERY (see why with: ods logs llm)"
+ fi
+ if [[ "$DOCKER_DAEMON" == "true" ]] && docker ps --format '{{.Names}}' 2>/dev/null | grep -qx 'ods-llama-server'; then
+ LLM_LOCAL_WARNING="true"
+ log_warn " Note : A llama-server container from an earlier install is running beside vLLM; stop it with: docker stop ods-llama-server"
+ fi
+}
+
_doctor_check_llm_backend() {
local ext_url="${EXTERNAL_LLM_URL:-}"
local ext_provider="${EXTERNAL_LLM_PROVIDER:-}"
local ext_model="${EXTERNAL_LLM_MODEL:-}"
local mode="${ODS_MODE:-local}"
+ local backend
+ backend=$(printf '%s' "${LLM_BACKEND:-}" | tr '[:upper:]' '[:lower:]')
if [ -n "$ext_url" ]; then
# External LLM mode — skip llama-server check
@@ -457,6 +503,8 @@ _doctor_check_llm_backend() {
"$ROOT_DIR/config/litellm/external-upstream.key"
elif [[ -n "${NATIVE_LLM_BASE_URL:-}" ]]; then
_doctor_check_host_native_llm
+ elif [[ "$backend" == "vllm" && "$mode" != "cloud" ]]; then
+ _doctor_check_vllm_engine
elif [[ "$mode" == "cloud" ]]; then
local cloud_url="${LLM_API_URL:-}"
if [ -n "$cloud_url" ]; then
@@ -1114,6 +1162,14 @@ def _looks_like_local_llama_route(value):
}
+def _url_hostname(value):
+ """The lower-case host of a URL, or "" when it names none."""
+ try:
+ return (parse.urlsplit(value.strip()).hostname or "").lower() if "://" in value else ""
+ except ValueError:
+ return ""
+
+
def _source(path):
try:
return path.relative_to(root_dir).as_posix()
@@ -1194,8 +1250,12 @@ def _collect_inference_contract():
generic_external = bool(external_llm_url.strip()) or llm_backend.strip().lower() == "external"
external_inference = ods_mode == "cloud" or generic_external
+ # LLM_BACKEND=vllm: vllm-server serves the model behind model-router;
+ # llama-server is off and clients use the LiteLLM gateway.
+ vllm_engine = llm_backend.strip().lower() == "vllm" and not external_inference
+ vllm_overlay = _has_compose_file(compose_files, "docker-compose.vllm.yml")
expected_owner = "external" if external_inference else "ods"
- expected_gateway = "litellm" if external_inference or host_native else "llama-server"
+ expected_gateway = "litellm" if external_inference or host_native or vllm_engine else "llama-server"
issues = []
if lemonade_era:
@@ -1293,7 +1353,27 @@ def _collect_inference_contract():
)
)
- if ods_mode == "local" and not host_native and not generic_external and not lemonade_era:
+ if vllm_engine:
+ if compose_flags_exists and not vllm_overlay:
+ issues.append(
+ _inference_issue(
+ "ODS-RUNTIME-VLLM-OVERLAY-MISSING",
+ "blocker",
+ ".compose-flags",
+ "LLM_BACKEND=vllm but docker-compose.vllm.yml is not in the resolved compose stack.",
+ )
+ )
+ if _url_hostname(llm_api_url) in {"llama-server", "ods-llama-server"}:
+ issues.append(
+ _inference_issue(
+ "ODS-RUNTIME-VLLM-LLAMA-ROUTE",
+ "blocker",
+ ".env",
+ f"The vLLM engine serves the model, but LLM_API_URL points at the in-stack llama-server ({llm_api_url}), which is off.",
+ )
+ )
+
+ if ods_mode == "local" and not host_native and not generic_external and not lemonade_era and not vllm_engine:
if compose_flags_exists and cloud_overlay:
issues.append(
_inference_issue(
@@ -1327,6 +1407,8 @@ def _collect_inference_contract():
"ODS-RUNTIME-HOST-NATIVE-KEY-MISSING": "The Windows llama-server route has no API key",
"ODS-RUNTIME-LOCAL-CLOUD-OVERLAY": "Local mode still has the cloud compose overlay",
"ODS-RUNTIME-LOCAL-LITELLM-ROUTE": "Local mode routes through LiteLLM unexpectedly",
+ "ODS-RUNTIME-VLLM-OVERLAY-MISSING": "The vLLM route is missing its compose overlay",
+ "ODS-RUNTIME-VLLM-LLAMA-ROUTE": "The vLLM route still sends clients to the in-stack llama-server",
}
next_steps = {
"ODS-RUNTIME-CLOUD-OVERLAY-MISSING": [
@@ -1368,6 +1450,12 @@ def _collect_inference_contract():
"ODS-RUNTIME-MODE-UNKNOWN": [
"Set ODS_MODE to local, cloud, or hybrid.",
],
+ "ODS-RUNTIME-VLLM-OVERLAY-MISSING": [
+ "Run `ods restart`: the CLI resolves the compose stack again from LLM_BACKEND=vllm.",
+ ],
+ "ODS-RUNTIME-VLLM-LLAMA-ROUTE": [
+ "Run `ods mode local` to point LLM_API_URL at the LiteLLM gateway, then `ods restart`.",
+ ],
}
for issue in issues:
diagnoses.append(
@@ -1398,6 +1486,8 @@ def _collect_inference_contract():
"host_native_overlay": host_native_overlay,
"local_inference_overlay": local_inference_overlay,
"native_key_configured": native_key_configured,
+ "vllm_engine": vllm_engine,
+ "vllm_overlay": vllm_overlay,
},
"issues": issues,
"issue_counts": {
@@ -1778,14 +1868,20 @@ llm_local_warn = os.environ.get("LLM_LOCAL_WARNING", "false") == "true"
llm_failure = os.environ.get("LLM_FAILURE", "")
if llm_status == "fail" and llm_recovery:
- if llm_provider != "llama-server" and llm_failure in ("key-refused", "key-required"):
+ if llm_provider == "vLLM":
+ fix_hints.append(f"The vLLM engine (vllm-server) is not serving the model. Hint: {llm_recovery}")
+ elif llm_provider != "llama-server" and llm_failure in ("key-refused", "key-required"):
fix_hints.append(f"External LLM backend ({llm_provider}) answered but did not accept a stored API key. Hint: {llm_recovery}")
elif llm_provider != "llama-server":
fix_hints.append(f"External LLM backend ({llm_provider}) is unreachable. Hint: {llm_recovery}")
else:
fix_hints.append(f"Local llama-server is unreachable. Hint: {llm_recovery}")
-if llm_local_warn:
+if llm_local_warn and llm_provider == "vLLM":
+ # `ods stop llama-server` names vllm-server on this route; stop the container itself.
+ fix_hints.append("A llama-server container from an earlier install is running beside the vLLM engine "
+ "and competes for its GPU. Stop it with: docker stop ods-llama-server")
+elif llm_local_warn:
fix_hints.append("External LLM is configured but the local llama-server container is still running. Stop it with: ods stop llama-server")
# Rootless daemon without (enough) subordinate IDs: images fail to extract
# with "uid/gid map out of range" long after doctor reports healthy Docker.
diff --git a/ods/scripts/ods-test.sh b/ods/scripts/ods-test.sh
index 0a31767df..71599c4d4 100755
--- a/ods/scripts/ods-test.sh
+++ b/ods/scripts/ods-test.sh
@@ -46,6 +46,20 @@ fi
LLM_HOST="${LLM_HOST:-localhost}"
LLM_PORT="${LLM_PORT:-${SERVICE_PORTS[llama-server]:-11434}}"
LLM_URL="http://${LLM_HOST}:${LLM_PORT}"
+LLM_HEALTH_PATH="/health"
+LLM_ENGINE="llama-server"
+LLM_GATEWAY_KEY=""
+# LLM_BACKEND=vllm: vllm-server publishes no host port. ODS clients reach it
+# through LiteLLM as ods/current (model-router adapts the request to vLLM), so
+# the LLM checks go there too, with the gateway key (_llm_chat).
+_ods_test_backend="$(printf '%s' "${LLM_BACKEND:-}" | tr '[:upper:]' '[:lower:]')"
+if [[ "$_ods_test_backend" == "vllm" ]]; then
+ LLM_URL="http://${LLM_HOST}:${LITELLM_PORT:-4000}"
+ LLM_HEALTH_PATH="/health/readiness"
+ LLM_ENGINE="vllm-server"
+ LLM_GATEWAY_KEY="${LITELLM_KEY:-${LITELLM_MASTER_KEY:-}}"
+fi
+unset _ods_test_backend
WHISPER_HOST="${WHISPER_HOST:-localhost}"
WHISPER_PORT="${WHISPER_PORT:-${SERVICE_PORTS[whisper]:-9000}}"
TTS_HOST="${TTS_HOST:-localhost}"
@@ -108,11 +122,32 @@ _now_ms() {
# /v1/models as test_llm always has; "local" when the listing fails, and the
# request that follows then reports the fault.
_llm_model_id() {
+ # The gateway serves the active model under its stable alias.
+ if [[ -n "$LLM_GATEWAY_KEY" ]]; then
+ echo "ods/current"
+ return 0
+ fi
local model_id
model_id=$(curl -s --max-time 10 "$LLM_URL/v1/models" 2>/dev/null | grep -o '"id":"[^"]*"' | head -1 | cut -d'"' -f4) || model_id=""
echo "${model_id:-local}"
}
+# POST one chat completion ($2) to the LLM route within $1 seconds. A gateway
+# key travels on curl's stdin (-H @-), never in argv.
+_llm_chat() {
+ local seconds="$1" payload="$2"
+ if [[ -n "$LLM_GATEWAY_KEY" ]]; then
+ printf 'Authorization: Bearer %s\n' "$LLM_GATEWAY_KEY" \
+ | curl -s --max-time "$seconds" -X POST "$LLM_URL/v1/chat/completions" \
+ -H "Content-Type: application/json" -H @- \
+ -d "$payload" 2>/dev/null
+ else
+ curl -s --max-time "$seconds" -X POST "$LLM_URL/v1/chat/completions" \
+ -H "Content-Type: application/json" \
+ -d "$payload" 2>/dev/null
+ fi
+}
+
# Judge a chat completion (read on stdin) by its JSON structure, not by
# grepping for a word: a reply with "tool_calls": null and no call contains the
# word too. $1 is "text" (an assistant answer; a thinking model may spend a
@@ -340,13 +375,16 @@ test_gpu() {
}
test_llm() {
- print_section "LLM Inference (llama-server)"
+ print_section "LLM Inference ($LLM_ENGINE)"
# A failed check is already recorded; return 0 so the short-circuit only
# skips this section's dependent checks instead of aborting the whole run
# under set -e (which lost the summary and the --json document).
- test_http "LLM Health" "$LLM_URL/health" "200" || return 0
- test_http "LLM Models API" "$LLM_URL/v1/models" "200" || log "LLM models API check failed"
+ test_http "LLM Health" "$LLM_URL$LLM_HEALTH_PATH" "200" || return 0
+ # The gateway lists only its aliases, and only with its key.
+ if [[ -z "$LLM_GATEWAY_KEY" ]]; then
+ test_http "LLM Models API" "$LLM_URL/v1/models" "200" || log "LLM models API check failed"
+ fi
if [[ "$QUICK_MODE" == "true" ]]; then
record_result "LLM Inference" "skip" "quick mode"
@@ -360,10 +398,7 @@ test_llm() {
local payload="{\"model\": \"$model_id\", \"messages\": [{\"role\": \"user\", \"content\": \"Say hello\"}], \"max_tokens\": 10}"
local response
- response=$(curl -s --max-time 30 \
- -X POST "$LLM_URL/v1/chat/completions" \
- -H "Content-Type: application/json" \
- -d "$payload" 2>/dev/null) || response=""
+ response=$(_llm_chat 30 "$payload") || response=""
if printf '%s' "$response" | _completion_ok text; then
local tokens_used
@@ -393,11 +428,8 @@ test_tool_calling() {
local payload="{\"model\": \"$model_id\", \"messages\": [{\"role\": \"user\", \"content\": \"What is the weather in Tokyo?\"}], \"tools\": $tools, \"tool_choice\": \"auto\", \"max_tokens\": 100}"
local response
- response=$(curl -s --max-time 30 \
- -X POST "$LLM_URL/v1/chat/completions" \
- -H "Content-Type: application/json" \
- -d "$payload" 2>/dev/null) || response=""
-
+ response=$(_llm_chat 30 "$payload") || response=""
+
if printf '%s' "$response" | _completion_ok tool; then
record_result "Tool Calling" "pass" "function called"
print_test "Tool Calling" "pass" "function called"
@@ -495,7 +527,7 @@ test_voice_roundtrip() {
if curl -s --max-time 5 "http://${TTS_HOST}:${TTS_PORT}/v1/audio/voices" &>/dev/null; then
tts_ready=true
fi
- if curl -s --max-time 5 "$LLM_URL/health" &>/dev/null; then
+ if curl -s --max-time 5 "$LLM_URL$LLM_HEALTH_PATH" &>/dev/null; then
llm_ready=true
fi
@@ -522,10 +554,7 @@ test_voice_roundtrip() {
model_id=$(_llm_model_id)
local llm_payload="{\"model\": \"$model_id\", \"messages\": [{\"role\": \"user\", \"content\": \"What is the weather today?\"}], \"max_tokens\": 50}"
local llm_response
- llm_response=$(curl -s --max-time 15 \
- -X POST "$LLM_URL/v1/chat/completions" \
- -H "Content-Type: application/json" \
- -d "$llm_payload" 2>/dev/null) || llm_response=""
+ llm_response=$(_llm_chat 15 "$llm_payload") || llm_response=""
if ! printf '%s' "$llm_response" | _completion_ok text; then
record_result "Voice Round-Trip" "fail" "LLM step failed"
@@ -669,14 +698,14 @@ _print_text_summary() {
echo "Actionable fixes:"
if [[ "${RESULTS_STATUS[0]:-}" == "fail" ]] && [[ "${RESULTS_NAMES[0]:-}" == *"LLM"* ]]; then
- echo " - LLM not responding - check: docker logs ods-llama-server"
+ echo " - LLM not responding - check: docker logs ods-${LLM_ENGINE}"
fi
local i
for i in "${!RESULTS_NAMES[@]}"; do
if [[ "${RESULTS_STATUS[$i]}" == "fail" ]]; then
case "${RESULTS_NAMES[$i]}" in
- "Tool Calling") echo " - Tool calling failed - check llama-server tool support" ;;
+ "Tool Calling") echo " - Tool calling failed - check $LLM_ENGINE tool support" ;;
"Whisper Port") echo " - Whisper not running - start: docker compose up whisper" ;;
"TTS Port") echo " - TTS not running - start: docker compose up kokoro-tts" ;;
esac
diff --git a/ods/scripts/preserve-active-model.py b/ods/scripts/preserve-active-model.py
index 202404195..2170ce60b 100644
--- a/ods/scripts/preserve-active-model.py
+++ b/ods/scripts/preserve-active-model.py
@@ -13,6 +13,9 @@ installs wrote. Every mode emits a small allowlisted dotenv fragment without
``eval``. The default mode also keeps the settings model activation owns for
the same model on this host's llama-server container: an import's vision
projector and a fixed chat template, each only while it still checks out.
+On a vLLM install (``LLM_BACKEND=vllm``) the default mode keeps the vLLM
+route, its catalog snapshot and GPU instead (``preserved_vllm_contract``),
+and exits 2 rather than let the rerun fall back to a llama.cpp model.
Round F serves every model as ``--alias <GGUF_FILE>``, so a served model id is
the GGUF filename. For one release the retired Lemonade forms of the same id
@@ -548,6 +551,100 @@ def repaired_served_model_contract(
return env, served_id, contract
+def is_vllm_runtime(env: dict[str, str]) -> bool:
+ """A ``.env`` whose model the vLLM engine serves (LLM_BACKEND=vllm)."""
+ return env.get("LLM_BACKEND", "").strip().lower() == "vllm"
+
+
+_VLLM_GPU_UUID = re.compile(r"[A-Za-z0-9][A-Za-z0-9-]*")
+_VLLM_MEMORY_SHARE = re.compile(r"(?:0(?:\.[0-9]+)?|1(?:\.0+)?)")
+
+
+def vllm_snapshot_installed(models_dir: Path, folder: str, files: list[dict[str, Any]]) -> bool:
+ """Every manifest file is in ``models_dir/folder`` under its final name, at its size.
+
+ scripts/download-hf-manifest.py renames a file into place only after its
+ digest matched, so a complete set of final names at the catalog sizes is a
+ complete verified download. Links are refused, as the GGUF check does.
+ """
+ try:
+ root = models_dir.resolve()
+ snapshot = models_dir / folder
+ if snapshot.is_symlink() or not snapshot.is_dir() or snapshot.resolve().parent != root:
+ return False
+ for item in files:
+ info = (snapshot / item["path"]).lstat()
+ if not stat.S_ISREG(info.st_mode) or info.st_size != item["size"]:
+ return False
+ if not (snapshot / item["path"]).resolve().is_relative_to(snapshot.resolve()):
+ return False
+ except (OSError, RuntimeError):
+ return False
+ return True
+
+
+def preserved_vllm_contract(args: argparse.Namespace) -> dict[str, str] | None:
+ """The retained vLLM route, while its catalog snapshot is still installed.
+
+ The route stays on vLLM (LLM_BACKEND=vllm) with the same model and GPU, and
+ clears the GGUF keys so no llama.cpp model is downloaded or started. The
+ model folder and served id must be the catalog entry's (a new revision is
+ a new download, which this never starts). VLLM_ARGS comes from this
+ release's catalog entry for that snapshot: its flags are measured with the
+ release's vLLM image, which the installer pins separately (VLLM_IMAGE).
+ The GPU and its memory share are the host's and stay as installed.
+ """
+ env = parse_dotenv(args.env)
+ if (
+ not is_vllm_runtime(env)
+ or env.get("ODS_MODE", "local").strip().lower() != "local"
+ or env.get("EXTERNAL_LLM_URL")
+ or env.get("NATIVE_LLM_BASE_URL")
+ ):
+ return None
+ sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "extensions/services/dashboard-api"))
+ import model_vllm
+
+ llm_model = env.get("LLM_MODEL", "").strip()
+ matches = [
+ item for item in load_records(args.catalog, None)
+ if model_vllm.is_vllm_entry(item) and item.get("llm_model_name") == llm_model
+ ]
+ if len(matches) != 1 or model_vllm.entry_problems(matches[0]):
+ return None
+ model = matches[0]
+ folder = model_vllm.model_dir_name(model)
+ gpu_uuid = env.get("VLLM_GPU_UUID", "")
+ share = env.get("VLLM_GPU_MEMORY_UTILIZATION", "")
+ if (
+ env.get("VLLM_MODEL_DIR") != folder
+ or env.get("VLLM_SERVED_MODEL") != model["served_model_name"]
+ or not _VLLM_GPU_UUID.fullmatch(gpu_uuid)
+ or (share and (not _VLLM_MEMORY_SHARE.fullmatch(share) or float(share) <= 0))
+ or not vllm_snapshot_installed(args.models_dir, folder, model["files"])
+ ):
+ return None
+ source = env.get("MODEL_SELECTION_SOURCE", "")
+ contract = {
+ "LLM_BACKEND": "vllm",
+ "LLM_MODEL": str(model["llm_model_name"]),
+ # A vLLM route has no GGUF; empty values override the tier map's.
+ "GGUF_FILE": "",
+ "GGUF_URL": "",
+ "GGUF_SHA256": "",
+ "MAX_CONTEXT": str(model["context_length"]),
+ "LLM_MODEL_SIZE_MB": str(round(model_vllm.manifest_bytes(model) / 1_000_000)),
+ "MODEL_SELECTION_SOURCE": source if source in {"installer", "dashboard", "operator"} else "preserved-local",
+ "VLLM_MODEL_DIR": folder,
+ "VLLM_SERVED_MODEL": str(model["served_model_name"]),
+ "VLLM_ARGS": model_vllm.vllm_args_text(model),
+ "VLLM_GPU_UUID": gpu_uuid,
+ }
+ if share:
+ contract["VLLM_GPU_MEMORY_UTILIZATION"] = share
+ return contract
+
+
def is_local_runtime(env: dict[str, str]) -> bool:
"""A ``.env`` whose model this host's managed llama-server serves."""
if env.get("EXTERNAL_LLM_URL", "") or env.get("LEMONADE_EXTERNAL", "false").lower() == "true":
@@ -981,6 +1078,16 @@ def main() -> int:
)
print(f"Repairing the saved description of the served model ({served_id}): "
f"{changes}. The served model is unchanged.", file=sys.stderr)
+ elif not (args.native_llm or args.external_lemonade) and is_vllm_runtime(parse_dotenv(args.env)):
+ contract = preserved_vllm_contract(args)
+ if contract is None:
+ # Falling through to the recommendation would turn the vLLM
+ # install into a llama.cpp one and download a GGUF.
+ print("This installation runs its model on the vLLM engine, but its saved model is not "
+ "a complete vLLM catalog snapshot for the GPU it names; refusing to replace it. "
+ "Run the installer again and choose the vLLM model, or use --reselect-model.",
+ file=sys.stderr)
+ return 2
else:
contract = preserved_contract(args)
if contract is None:
diff --git a/ods/scripts/render-runtime-configs.py b/ods/scripts/render-runtime-configs.py
index 397e3edc2..38ee8233d 100644
--- a/ods/scripts/render-runtime-configs.py
+++ b/ods/scripts/render-runtime-configs.py
@@ -16,9 +16,9 @@ import stat
import sys
import tempfile
import time
-from dataclasses import asdict, dataclass
+from dataclasses import asdict, dataclass, replace
from pathlib import Path
-from typing import Callable
+from typing import Any, Callable
from urllib.parse import urlsplit
@@ -136,6 +136,9 @@ class RenderInputs:
switchboard_mode: str = "enabled"
# Model server behind model-router: llama-server | vllm (model_server()).
llm_backend: str = "llama-server"
+ # The vllm route's thinking map for model-router (vllm_thinking()):
+ # {"default": <level>, "templateKwargs": {<level>: chat_template_kwargs}}.
+ vllm_thinking: dict[str, Any] | None = None
@dataclass(frozen=True)
@@ -172,6 +175,30 @@ def model_server(llm_backend: str) -> str:
return "vllm" if llm_backend == "vllm" else "llama-server"
+def vllm_thinking(model: str, catalog_path: Path = ROOT / "config" / "model-library.json") -> dict[str, Any]:
+ """The thinking map model-router applies on the vllm route for ``model``.
+
+ ``model`` is .env LLM_MODEL: the vLLM catalog entry's llm_model_name (or
+ its id). vLLM serves one model, so its endpoint carries that model's
+ default level (``reasoning``) and the chat-template arguments of each
+ level (``thinking_template_kwargs``); see model_vllm.py.
+ """
+ sys.path.insert(0, str(ROOT / "extensions" / "services" / "dashboard-api"))
+ import model_vllm
+
+ catalog = json.loads(catalog_path.read_text(encoding="utf-8"))
+ matches = [
+ entry for entry in catalog.get("models", [])
+ if model_vllm.is_vllm_entry(entry) and model in {entry.get("llm_model_name"), entry.get("id")}
+ ]
+ if len(matches) != 1:
+ raise ValueError(
+ f"LLM_BACKEND=vllm needs --model to name one vLLM catalog entry (its llm_model_name); got {model!r}"
+ )
+ entry = model_vllm.require_valid(matches[0])
+ return {"default": entry["reasoning"], "templateKwargs": entry["thinking_template_kwargs"]}
+
+
def hermes_model_id(inputs: RenderInputs) -> str:
if inputs.switchboard_mode == "enabled":
return "ods/current"
@@ -669,8 +696,12 @@ def render_model_router_endpoints(inputs: RenderInputs) -> RenderedFile:
if inputs.llm_backend == "vllm":
# The in-stack vllm-server container on its internal port. It takes no
- # key; the llama-server origin and key inputs do not describe it.
- endpoint = {"id": VLLM_ENDPOINT_ID, "baseUrl": VLLM_SERVER_ORIGIN}
+ # key; the llama-server origin and key inputs do not describe it. The
+ # router translates a request's thinking level with the served
+ # model's own map, which only this endpoint knows.
+ if inputs.vllm_thinking is None:
+ raise ValueError("the vllm-server endpoint needs its model's thinking map (vllm_thinking)")
+ endpoint = {"id": VLLM_ENDPOINT_ID, "baseUrl": VLLM_SERVER_ORIGIN, "thinking": inputs.vllm_thinking}
else:
# Otherwise the model server is upstream llama-server: the in-stack
# container, or a host-native one reached at
@@ -921,15 +952,15 @@ def render(args: argparse.Namespace) -> dict[str, object]:
"litellm-switchboard is local-runtime-only and cannot be rendered "
"for ODS_MODE=cloud"
)
- files = [
- RENDERERS[name](inputs)
- for name in select_surfaces(
- args.surface,
- inputs.ods_mode,
- inputs.switchboard_mode,
- inputs.remote_llm_enabled,
- )
- ]
+ surfaces = select_surfaces(
+ args.surface,
+ inputs.ods_mode,
+ inputs.switchboard_mode,
+ inputs.remote_llm_enabled,
+ )
+ if inputs.llm_backend == "vllm" and "model-router-endpoints" in surfaces:
+ inputs = replace(inputs, vllm_thinking=vllm_thinking(inputs.model))
+ files = [RENDERERS[name](inputs) for name in surfaces]
written: list[str] = []
if args.write:
output_root = Path(args.output_root)
diff --git a/ods/tests/ci-suite.txt b/ods/tests/ci-suite.txt
index f4a9e000f..9c8c9f3a6 100644
--- a/ods/tests/ci-suite.txt
+++ b/ods/tests/ci-suite.txt
@@ -63,6 +63,8 @@ tests/test-comfyui-user-dir-persisted.sh
tests/test-comfyui-cuda-cache-persisted.sh
tests/test-llama-cuda-cache-persisted.sh
tests/test-vllm-engine-service.sh
+tests/test-cli-vllm-engine.sh
+tests/test-doctor-vllm-engine.sh
tests/test_comfyui_extension_contract.py
tests/test_extension_assets.py
tests/test-compose-failure-evidence.sh
@@ -145,6 +147,9 @@ tests/test-model-library-coverage.py
tests/test-model-library-verdicts.py
tests/test-model-library-vllm.py
tests/test_select_model_vllm.py
+tests/test_host_agent_vllm.py
+tests/test_preserve_active_model_vllm.py
+tests/test_doctor_vllm_route.py
tests/test-native-llm-phase02.sh
tests/test-network-security.sh
tests/test-network-timeouts.sh
diff --git a/ods/tests/test-cli-vllm-engine.sh b/ods/tests/test-cli-vllm-engine.sh
new file mode 100644
index 000000000..6dc313c33
--- /dev/null
+++ b/ods/tests/test-cli-vllm-engine.sh
@@ -0,0 +1,198 @@
+#!/usr/bin/env bash
+# The CLI on a vLLM install (LLM_BACKEND=vllm, docker-compose.vllm.yml): the
+# vllm-server container serves the model and the in-stack llama-server is
+# profiled out. A saved Compose file list from the other route is stale; "llm"
+# and "llama-server" name vllm-server for logs and lifecycle commands (naming
+# llama-server to Compose would enable its profile and start a second engine);
+# status reports vllm-server from Docker's health (it has no host port) and no
+# llama-server fault; ods mode keeps the route local and never points clients
+# back at llama-server. Docker, curl and the resolver are stubs.
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+FIXTURE="$(mktemp -d)"
+trap 'rm -rf "$FIXTURE"' EXIT
+install_dir="$FIXTURE/install"
+mkdir -p "$install_dir/lib" "$install_dir/extensions/services" "$install_dir/scripts" "$FIXTURE/bin"
+cp "$ROOT_DIR/ods-cli" "$install_dir/ods-cli"
+cp "$ROOT_DIR/scripts/mode-switch.sh" "$install_dir/scripts/mode-switch.sh"
+for library in safe-env.sh service-registry.sh python-cmd.sh preflight-llm-route.sh; do
+ cp "$ROOT_DIR/lib/$library" "$install_dir/lib/"
+done
+for compose in docker-compose.base.yml docker-compose.nvidia.yml docker-compose.vllm.yml; do
+ printf 'services: {}\n' > "$install_dir/$compose"
+done
+
+fail() { printf 'FAIL: %s\n' "$*" >&2; exit 1; }
+pass() { printf 'PASS: %s\n' "$*"; }
+
+for entry in 'llama-server core 8080 ods-llama-server [llm]' 'model-router core 9099 ods-model-router []' \
+ 'dashboard core 3001 ods-dashboard []'; do
+ read -r service category port container aliases <<< "$entry"
+ mkdir -p "$install_dir/extensions/services/$service"
+ cat > "$install_dir/extensions/services/$service/manifest.yaml" <<MANIFEST
+schema_version: ods.services.v1
+service:
+ id: $service
+ name: $service
+ aliases: $aliases
+ container_name: $container
+ category: $category
+ port: $port
+ health: /health
+MANIFEST
+done
+
+VLLM_FLAGS='-f docker-compose.base.yml -f docker-compose.nvidia.yml -f docker-compose.vllm.yml'
+# The resolver selects the vLLM overlay from the install's LLM_BACKEND.
+cat > "$install_dir/scripts/resolve-compose-stack.sh" <<'STUB'
+#!/usr/bin/env bash
+set -euo pipefail
+printf 'resolve\n' >> "$CLI_CALLS"
+if grep -qx 'LLM_BACKEND=vllm' "$INSTALL_DIR/.env"; then
+ printf '%s\n' '-f docker-compose.base.yml -f docker-compose.nvidia.yml -f docker-compose.vllm.yml'
+else
+ printf '%s\n' '-f docker-compose.base.yml -f docker-compose.nvidia.yml'
+fi
+STUB
+chmod +x "$install_dir/scripts/resolve-compose-stack.sh"
+cat > "$FIXTURE/bin/docker" <<'STUB'
+#!/usr/bin/env bash
+set -euo pipefail
+printf 'docker %s\n' "$*" >> "$CLI_CALLS"
+case "$*" in
+ info*) printf '16\n' ;;
+ 'inspect --format '*' ods-vllm-server')
+ [[ "$VLLM_STATE" == missing ]] && { printf 'Error: No such object: ods-vllm-server\n' >&2; exit 1; }
+ printf '%s\n' "$VLLM_STATE" ;;
+ compose*'ps --format {{.Service}}') printf 'vllm-server\nmodel-router\ndashboard\n' ;;
+ compose*) ;;
+ *) exit 64 ;;
+esac
+STUB
+cat > "$FIXTURE/bin/curl" <<'STUB'
+#!/usr/bin/env bash
+printf 'curl %s\n' "$*" >> "$CLI_CALLS"
+exit 0
+STUB
+chmod +x "$FIXTURE/bin/"*
+
+export CLI_CALLS="$FIXTURE/calls" VLLM_STATE="running healthy"
+write_env() {
+ printf '%s\n' 'GPU_BACKEND=nvidia' 'ODS_MODE=local' 'LLM_BACKEND=vllm' \
+ 'LLM_MODEL=glm-5.3-flash-nvfp4' 'VLLM_SERVED_MODEL=GLM-5.3-Flash-NVFP4' \
+ 'ODS_MODEL_SWITCHBOARD=enabled' 'LLM_API_URL=http://litellm:4000' \
+ 'HERMES_DASHBOARD_SESSION_TOKEN=fixture-session' "$@" > "$install_dir/.env"
+}
+cli() {
+ : > "$CLI_CALLS"
+ INSTALL_DIR="$install_dir" ODS_HOME="$install_dir" HOME="$FIXTURE" PATH="$FIXTURE/bin:$PATH" \
+ bash "$install_dir/ods-cli" "$@"
+}
+
+# 1. A Compose file list saved for llama-server is stale on the vLLM route,
+# and the model service of logs is vllm-server.
+write_env
+printf '%s\n' '-f docker-compose.base.yml -f docker-compose.nvidia.yml' > "$install_dir/.compose-flags"
+cli logs llm 5 > /dev/null
+grep -qx 'resolve' "$CLI_CALLS" || fail 'a llama-server stack cache was reused on the vLLM route'
+grep -qx "docker compose $VLLM_FLAGS logs -f --tail 5 vllm-server" "$CLI_CALLS" \
+ || fail "ods logs llm did not follow vllm-server: $(cat "$CLI_CALLS")"
+pass 'a llama-server stack cache is re-resolved and ods logs llm shows vllm-server'
+
+printf '%s\n' "$VLLM_FLAGS -f extensions/services/hermes/compose.local.yaml" > "$install_dir/.compose-flags"
+cli logs llama-server 5 > /dev/null
+grep -qx 'resolve' "$CLI_CALLS" || fail 'a cache with llama-server waits was reused on the vLLM route'
+grep -q ' logs -f --tail 5 vllm-server$' "$CLI_CALLS" || fail 'ods logs llama-server reached llama-server'
+pass 'compose.local.yaml waits on llama-server make a vLLM cache stale'
+
+printf '%s\n' "$VLLM_FLAGS" > "$install_dir/.compose-flags"
+cli logs llm 5 > /dev/null
+grep -q '^resolve$' "$CLI_CALLS" && fail 'a current vLLM cache was resolved again'
+pass 'a current vLLM cache is reused'
+
+write_env 'LLM_BACKEND=llama-server'
+sed -i '/^LLM_BACKEND=vllm$/d' "$install_dir/.env"
+printf '%s\n' "$VLLM_FLAGS" > "$install_dir/.compose-flags"
+cli logs llm 5 > /dev/null
+grep -qx 'resolve' "$CLI_CALLS" || fail 'a vLLM cache was reused on the llama-server route'
+grep -q ' logs -f --tail 5 llama-server$' "$CLI_CALLS" || fail 'a llama-server install lost its llm alias'
+pass 'a llama-server install re-resolves a vLLM cache and keeps llm on llama-server'
+
+# 2. Lifecycle commands never name llama-server to Compose.
+write_env
+printf '%s\n' "$VLLM_FLAGS" > "$install_dir/.compose-flags"
+cli restart llm > /dev/null
+grep -q 'up -d --force-recreate --no-build --pull never vllm-server$' "$CLI_CALLS" \
+ || fail "ods restart llm did not recreate vllm-server: $(cat "$CLI_CALLS")"
+cli stop llm > /dev/null
+grep -q ' stop vllm-server$' "$CLI_CALLS" || fail 'ods stop llm did not stop vllm-server'
+cli start llama-server > /dev/null
+grep -q ' up -d vllm-server$' "$CLI_CALLS" || fail 'ods start llama-server did not start vllm-server'
+for command in restart stop start; do
+ cli "$command" llm > /dev/null
+ if grep -qE '^docker compose .* llama-server$' "$CLI_CALLS"; then
+ fail "ods $command named llama-server to Compose"
+ fi
+done
+pass 'restart, stop and start of llm or llama-server reach vllm-server only'
+
+# 3. Status: vllm-server from Docker's health, no llama-server fault.
+for state in 'running healthy:healthy' 'running starting:starting' 'running unhealthy:unhealthy' \
+ 'exited none:stopped' 'missing:stopped'; do
+ VLLM_STATE="${state%%:*}"
+ cli status --json > "$FIXTURE/status.json"
+ jq -e --arg status "${state##*:}" '.services[] | select(.id == "vllm-server")
+ | .status == $status and .containerStatus == "running" and .port == ""' "$FIXTURE/status.json" >/dev/null \
+ || fail "vllm-server should be ${state##*:} for '$VLLM_STATE': $(cat "$FIXTURE/status.json")"
+ jq -e 'all(.services[]; .id != "llama-server") and any(.services[]; .id == "model-router")' \
+ "$FIXTURE/status.json" >/dev/null || fail 'status JSON reported llama-server or dropped model-router'
+done
+VLLM_STATE="running starting"
+cli status > "$FIXTURE/status.txt" 2>&1
+grep -q 'vLLM (LLM Inference): starting. A first start loads the model' "$FIXTURE/status.txt" \
+ || fail "a cold start did not read as starting: $(cat "$FIXTURE/status.txt")"
+grep -q 'llama-server' "$FIXTURE/status.txt" && fail "status mentioned llama-server: $(cat "$FIXTURE/status.txt")"
+grep -q '127.0.0.1:8080/health' "$CLI_CALLS" && fail 'status probed the profiled-out llama-server port'
+VLLM_STATE="running healthy"
+cli status > "$FIXTURE/status.txt" 2>&1
+grep -q 'vLLM (LLM Inference): healthy' "$FIXTURE/status.txt" || fail 'a ready vLLM engine was not healthy'
+pass 'status reports vllm-server from Docker health, starting during a cold start, and no llama-server'
+
+# 4. ods mode and mode-switch.sh keep the vLLM route local.
+for mode in cloud hybrid; do
+ write_env
+ cp "$install_dir/.env" "$FIXTURE/before.env"
+ if cli mode "$mode" > "$FIXTURE/mode.out" 2>&1; then fail "ods mode $mode was accepted on vLLM"; fi
+ grep -q 'vLLM engine, which works only in local mode. Nothing was changed.' "$FIXTURE/mode.out" \
+ || fail "ods mode $mode refusal: $(cat "$FIXTURE/mode.out")"
+ grep -q 'discord.gg' "$FIXTURE/mode.out" || fail 'the refusal names no help'
+ cmp -s "$FIXTURE/before.env" "$install_dir/.env" || fail "ods mode $mode changed .env"
+ if (cd "$install_dir/scripts" && bash ./mode-switch.sh "$mode") > "$FIXTURE/mode.out" 2>&1; then
+ fail "mode-switch.sh $mode was accepted on vLLM"
+ fi
+ grep -q 'works only in local mode' "$FIXTURE/mode.out" || fail "mode-switch.sh $mode refusal"
+ cmp -s "$FIXTURE/before.env" "$install_dir/.env" || fail "mode-switch.sh $mode changed .env"
+done
+write_env 'LLM_API_URL=http://llama-server:8080' 'ODS_MODE=hybrid'
+cli mode local > /dev/null
+grep -qx 'LLM_API_URL=http://litellm:4000' "$install_dir/.env" || fail 'ods mode local pointed back at llama-server'
+grep -qx 'HERMES_LLM_BASE_URL=http://model-router:9099/v1' "$install_dir/.env" || fail 'Hermes left model-router'
+write_env 'LLM_API_URL=http://llama-server:8080'
+(cd "$install_dir/scripts" && bash ./mode-switch.sh local) > /dev/null
+grep -qx 'LLM_API_URL=http://litellm:4000' "$install_dir/.env" || fail 'mode-switch.sh local pointed back at llama-server'
+cli mode > "$FIXTURE/mode.out" 2>&1
+grep -q 'LLM: Local vLLM engine (vllm-server)' "$FIXTURE/mode.out" || fail 'ods mode did not name the vLLM engine'
+pass 'ods mode refuses cloud and hybrid on vLLM and local keeps clients off llama-server'
+
+# 5. ods doctor's summary names the vLLM engine as ODS's own local model server.
+awk '/python3 - "\$report_file" <<'"'"'PY'"'"'/ {take=1; next} take && /^PY$/ {exit} take {print}' "$ROOT_DIR/ods-cli" > "$FIXTURE/doctor-view.py"
+[[ -s "$FIXTURE/doctor-view.py" ]] || fail 'the doctor report renderer was not found in ods-cli'
+printf '%s\n' '{"runtime": {"llm_backend": {"status": "ok", "provider": "vLLM", "url": "http://vllm-server:8000 (ODS network only)"}}}' > "$FIXTURE/doctor.json"
+# The renderer exits 1 when this partial report lacks the Docker checks; only its LLM line matters here.
+view_status=0
+PYTHONIOENCODING=utf-8 python3 "$FIXTURE/doctor-view.py" "$FIXTURE/doctor.json" > "$FIXTURE/doctor.out" 2>&1 || view_status=$?
+[[ "$view_status" -le 1 ]] || fail "the doctor report renderer crashed ($view_status): $(cat "$FIXTURE/doctor.out")"
+grep -q 'LLM Backend: vLLM (local) — responding' "$FIXTURE/doctor.out" \
+ || fail "ods doctor must call the vLLM engine local: $(cat "$FIXTURE/doctor.out")"
+pass 'ods doctor names the vLLM engine as a local model server'
diff --git a/ods/tests/test-doctor-vllm-engine.sh b/ods/tests/test-doctor-vllm-engine.sh
new file mode 100644
index 000000000..dee990766
--- /dev/null
+++ b/ods/tests/test-doctor-vllm-engine.sh
@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+# ods doctor on a vLLM install (LLM_BACKEND=vllm, docker-compose.vllm.yml):
+# vllm-server has no host port and llama-server is off, so the LLM check reads
+# Docker's health of ods-vllm-server. A cold start (weights load plus kernel
+# compiles, up to 15 minutes) reads as starting, not a fault; a missing
+# llama-server is no fault; a llama-server container an earlier install left
+# running is named, with a stop command that does not stop vLLM.
+# Extracted production functions consume the fixture globals below.
+# shellcheck disable=SC2034
+set -euo pipefail
+
+root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+if (( BASH_VERSINFO[0] < 4 )); then
+ for candidate in /opt/homebrew/bin/bash /usr/local/bin/bash; do
+ if [[ -x "$candidate" ]]; then exec "$candidate" "$0" "$@"; fi
+ done
+ echo 'Bash 4+ is required' >&2
+ exit 1
+fi
+extract() {
+ awk -v name="$1" '$0 ~ "^" name "[(][)]" { active=1 }
+ active {print} active && /^}/ {exit}' "$root/scripts/ods-doctor.sh"
+}
+eval "$(extract _doctor_check_vllm_engine)"
+eval "$(extract _doctor_check_llm_backend)"
+fail() { printf '[FAIL] %s\n' "$*" >&2; exit 1; }
+pass() { printf '[PASS] %s\n' "$*"; }
+messages=()
+log_ok() { messages+=("ok:$*"); }
+log_fail() { messages+=("fail:$*"); }
+log_info() { messages+=("info:$*"); }
+log_warn() { messages+=("warn:$*"); }
+_doctor_check_llama_server() { LLM_PROVIDER="llama-server"; LLM_STATUS="llama-checked"; }
+ROOT_DIR="$root"
+vllm_state="running healthy"
+llama_running=false
+docker() {
+ case "$1" in
+ inspect)
+ [[ "${*: -1}" == ods-vllm-server ]] || fail "inspected ${*: -1}"
+ [[ "$vllm_state" != missing ]] || { printf 'Error: No such object: ods-vllm-server\n'; return 1; }
+ printf '%s\n' "$vllm_state" ;;
+ ps) [[ "$llama_running" == false ]] || printf 'ods-llama-server\n' ;;
+ *) fail "unexpected docker $*" ;;
+ esac
+}
+DOCKER_DAEMON=true
+ODS_MODE=local
+VLLM_SERVED_MODEL=GLM-5.3-Flash-NVFP4
+unset EXTERNAL_LLM_URL NATIVE_LLM_BASE_URL
+
+for backend in vllm VLLM; do
+ LLM_BACKEND="$backend"
+ _doctor_check_llm_backend
+ [[ "$LLM_PROVIDER" == vLLM && "$LLM_STATUS" == ok && "$LLM_MODEL" == GLM-5.3-Flash-NVFP4 ]] \
+ || fail "LLM_BACKEND=$backend did not check the vLLM engine: $LLM_PROVIDER $LLM_STATUS"
+done
+pass 'LLM_BACKEND=vllm checks vllm-server, never llama-server'
+
+messages=()
+vllm_state="running starting"
+_doctor_check_llm_backend
+[[ "$LLM_STATUS" == starting && -z "$LLM_RECOVERY" ]] || fail "a cold start was not starting: $LLM_STATUS"
+said="$(printf '%s\n' "${messages[@]}")"
+grep -q '^info:LLM backend: vLLM (local) — starting: a first start loads the model' <<<"$said" \
+ || fail "cold start message: ${messages[*]}"
+if grep -q '^fail:' <<<"$said"; then fail "a cold start reported a fault: ${messages[*]}"; fi
+pass 'a cold start reads as starting, with no fault and no restart advice'
+
+for state in 'running unhealthy:ods restart vllm-server' 'exited none:ods start vllm-server' \
+ 'missing:ods start vllm-server'; do
+ vllm_state="${state%%:*}"
+ _doctor_check_llm_backend
+ [[ "$LLM_STATUS" == fail && "$LLM_RECOVERY" == "${state##*:}" ]] \
+ || fail "'$vllm_state' should fail with ${state##*:}: $LLM_STATUS $LLM_RECOVERY"
+done
+pass 'an unhealthy or stopped engine fails with a vllm-server recovery'
+
+messages=()
+vllm_state="running healthy"
+llama_running=true
+_doctor_check_llm_backend
+[[ "$LLM_STATUS" == ok && "$LLM_LOCAL_WARNING" == true ]] || fail 'a running stale llama-server was not noted'
+said="$(printf '%s\n' "${messages[@]}")"
+grep -q 'docker stop ods-llama-server' <<<"$said" || fail "stale llama-server note: ${messages[*]}"
+grep -q 'Stop it with: docker stop ods-llama-server' "$root/scripts/ods-doctor.sh" \
+ || fail 'the fix hint must stop the container, since ods stop llama-server names vllm-server on this route'
+pass 'a llama-server left running beside vLLM is named with a stop that keeps vLLM'
+
+LLM_BACKEND=llama-server
+llama_running=false
+_doctor_check_llm_backend
+[[ "$LLM_STATUS" == llama-checked ]] || fail 'a llama-server install left the llama-server check'
+LLM_BACKEND=vllm
+ODS_MODE=cloud
+_doctor_check_llm_backend
+[[ "$LLM_PROVIDER" != vLLM ]] || fail 'cloud mode checked the vLLM engine'
+pass 'llama-server and cloud installs keep their checks'
diff --git a/ods/tests/test-model-library-vllm.py b/ods/tests/test-model-library-vllm.py
index d29d638c4..1058fedb7 100644
--- a/ods/tests/test-model-library-vllm.py
+++ b/ods/tests/test-model-library-vllm.py
@@ -159,6 +159,11 @@ def _mutate(path: str, value):
("selection.discrete", 50, "selection.discrete must be 0"),
("selection.vllm", KeyError, "selection.vllm must be a whole number"),
("served_model_name", "GLM 5.3", "served_model_name must be letters, digits and ._-"),
+ ("thinking_template_kwargs", KeyError, "thinking_template_kwargs must map each of off, low, medium, high"),
+ ("thinking_template_kwargs.high", KeyError, "thinking_template_kwargs must map each of"),
+ ("thinking_template_kwargs.medium", "high", "thinking_template_kwargs.medium must map"),
+ ("thinking_template_kwargs.off", {"reasoning_effort": "lo w"}, "thinking_template_kwargs.off must map"),
+ ("thinking_template_kwargs.low", {"Reasoning-Effort": "low"}, "thinking_template_kwargs.low must map"),
])
def test_the_schema_catches_broken_entries(path, value, expected):
problems = entry_problems(_mutate(path, value))
@@ -213,6 +218,55 @@ def test_the_entries_keep_the_flags_measured_on_a_dgx_station(model_id):
assert switches == MEASURED_SWITCHES[model_id], model_id
+# Each ODS level in the model's own chat-template arguments, read from the
+# templates on the DGX Station (10-27). GLM-5.3's template has no
+# enable_thinking and accepts only reasoning_effort low or high: any other
+# value (none, medium, xhigh) thinks at "max", so "low" is the nearest to off
+# and high omits the field. Qwen3.8's turns thinking off with enable_thinking
+# false and answers HTTP 400 for any reasoning_effort but xhigh (its
+# default), medium and low. Measured (GB300, temperature 0.6): GLM max 7.5 s
+# vs high 3.4 s and low 2.8 s; Qwen xhigh 5.0 s, medium 3.7 s, low 3.4 s.
+THINKING = {
+ "glm-5.3-flash-nvfp4-vllm": {
+ "off": {"reasoning_effort": "low"}, "low": {"reasoning_effort": "low"},
+ "medium": {"reasoning_effort": "high"}, "high": {},
+ },
+ "qwen3.8-flash-next-nvfp4-vllm": {
+ "off": {"enable_thinking": False}, "low": {"reasoning_effort": "low"},
+ "medium": {"reasoning_effort": "medium"}, "high": {"reasoning_effort": "xhigh"},
+ },
+}
+TEMPLATE_EFFORTS = {
+ "glm-5.3-flash-nvfp4-vllm": {"low", "high"},
+ "qwen3.8-flash-next-nvfp4-vllm": {"low", "medium", "xhigh"},
+}
+
+
[email protected]("model_id", sorted(THINKING))
+def test_each_level_uses_the_models_own_template_arguments(model_id):
+ model = {item["id"]: item for item in _vllm_models()}[model_id]
+ assert model["thinking_template_kwargs"] == THINKING[model_id]
+ # No level hands the template a reasoning_effort it does not accept.
+ for level, kwargs in model["thinking_template_kwargs"].items():
+ assert kwargs.get("reasoning_effort", "low") in TEMPLATE_EFFORTS[model_id], (model_id, level)
+
+
+def test_every_vllm_entry_has_a_thinking_map():
+ assert {model["id"] for model in _vllm_models()} == set(THINKING)
+
+
[email protected]("model_id", sorted(MEASURED_FLAGS))
+def test_the_entries_declare_tools_and_vision(model_id):
+ # The host agent's route proof publishes these catalog booleans as the
+ # route's capabilities (model-state, Pixel connection metadata). Both
+ # checkpoints are image-text-to-text and run vLLM's tool parser; on the
+ # DGX Station GB300 GLM made tool calls (streamed and not) and read an
+ # image through model-router. Without them the route claimed neither.
+ model = {item["id"]: item for item in _vllm_models()}[model_id]
+ assert model.get("tools") is True and model.get("vision") is True, model_id
+ assert "--enable-auto-tool-choice" in model["vllm_args"], model_id
+
+
def test_the_broken_entry_cases_edit_what_they_name():
# "repeats README.md" needs README.md after files[0]; "sha256 must be
# lowercase hex" needs files[8] to be an LFS file.
diff --git a/ods/tests/test-ods-preflight-llm-route.sh b/ods/tests/test-ods-preflight-llm-route.sh
index c6438f816..415e60e1f 100644
--- a/ods/tests/test-ods-preflight-llm-route.sh
+++ b/ods/tests/test-ods-preflight-llm-route.sh
@@ -45,6 +45,32 @@ assert_skipped skipped model-router cloud ''
assert_skipped skipped llama-server local http://localhost:8080
assert_skipped shown model-router local http://localhost:8080
+# LLM_BACKEND=vllm: vllm-server (no host port) serves the model through
+# model-router and LiteLLM; the in-stack llama-server is off.
+for backend in vllm VLLM; do
+ if ! (export LLM_BACKEND="$backend" ODS_MODE=local EXTERNAL_LLM_URL='' NATIVE_LLM_BASE_URL=''
+ ods_preflight_vllm_engine && ods_preflight_uses_litellm); then
+ printf 'FAIL LLM_BACKEND=%s should select the gateway route\n' "$backend" >&2
+ exit 1
+ fi
+ if ! (export LLM_BACKEND="$backend" ODS_MODE=local EXTERNAL_LLM_URL='' NATIVE_LLM_BASE_URL=''
+ ods_status_skips_managed_inference llama-server); then
+ printf 'FAIL LLM_BACKEND=%s status should leave out llama-server\n' "$backend" >&2
+ exit 1
+ fi
+ if (export LLM_BACKEND="$backend" ODS_MODE=local EXTERNAL_LLM_URL='' NATIVE_LLM_BASE_URL=''
+ ods_status_skips_managed_inference model-router); then
+ printf 'FAIL LLM_BACKEND=%s status must keep model-router, which reaches vllm-server\n' "$backend" >&2
+ exit 1
+ fi
+done
+for backend in '' llama-server external; do
+ if (export LLM_BACKEND="$backend"; ods_preflight_vllm_engine); then
+ printf 'FAIL LLM_BACKEND=%s is not the vLLM engine\n' "$backend" >&2
+ exit 1
+ fi
+done
+
# ods-preflight.sh is #!/bin/bash, which is Bash 3.2 on macOS: the route
# selector must not use Bash 4 case conversion (${var,,} aborts the preflight).
if grep -nE '\$\{[A-Za-z_][A-Za-z_0-9]*(,,?|\^\^?)\}' "$ROOT_DIR/lib/preflight-llm-route.sh"; then
diff --git a/ods/tests/test-ods-test-tool-calling.sh b/ods/tests/test-ods-test-tool-calling.sh
index 9c17b0f8f..dccb55773 100755
--- a/ods/tests/test-ods-test-tool-calling.sh
+++ b/ods/tests/test-ods-test-tool-calling.sh
@@ -105,7 +105,8 @@ class Handler(BaseHTTPRequestHandler):
self.wfile.write(data)
def do_GET(self):
- if self.path == "/health":
+ # /health/readiness: LiteLLM's, which a vLLM install's checks ask.
+ if self.path in ("/health", "/health/readiness"):
self.reply(200, {"status": "ok"})
elif self.path == "/v1/models":
self.reply(200, {"object": "list", "data": [{"id": MODEL, "object": "model"}]})
@@ -122,7 +123,7 @@ class Handler(BaseHTTPRequestHandler):
request = None
if request is not None:
with open(os.path.join(WORK, "requests.jsonl"), "a") as log:
- log.write(json.dumps(request) + "\n")
+ log.write(json.dumps({**request, "_authorization": self.headers.get("Authorization")}) + "\n")
elif mode in WORKING:
self.reply(400, {"error": {"code": 400, "type": "invalid_request_error",
"message": "Failed to parse the request body as JSON"}})
@@ -205,3 +206,19 @@ expect "Tool Calling reads the install's .env by default (empty HOME)" call tool
expect "LLM Inference passes on an assistant answer" text llm "LLM Inference" pass
expect "LLM Inference passes when a thinking model only reasoned" reasoning llm "LLM Inference" pass
expect "LLM Inference fails on an error body" error llm "LLM Inference" fail
+
+# LLM_BACKEND=vllm: vllm-server publishes no host port, so the LLM checks go
+# through the LiteLLM gateway (its readiness, then ods/current with its key),
+# as ODS clients reach the engine.
+printf 'LLM_BACKEND=vllm\nLITELLM_PORT=%s\nLITELLM_KEY=sk-gateway-fixture\n' "$(cat "$TMP_DIR/port")" > "$INSTALL/.env"
+expect "Tool Calling on a vLLM install goes through the gateway" call tool-calling "Tool Calling" pass
+python3 - "$TMP_DIR/requests.jsonl" <<'PY' || fail "the vLLM check did not ask the gateway for ods/current with its key"
+import json, sys
+with open(sys.argv[1]) as stream:
+ requests = [json.loads(line) for line in stream if line.strip()]
+assert requests, "no parseable chat request reached the gateway"
+assert requests[-1]["model"] == "ods/current", requests[-1]["model"]
+assert requests[-1]["_authorization"] == "Bearer sk-gateway-fixture", requests[-1]["_authorization"]
+PY
+pass "the vLLM check names ods/current and sends the gateway key"
+expect "LLM Inference on a vLLM install passes through the gateway" text llm "LLM Inference" pass
diff --git a/ods/tests/test-render-runtime-configs.py b/ods/tests/test-render-runtime-configs.py
index 60a864897..d1d47bca8 100644
--- a/ods/tests/test-render-runtime-configs.py
+++ b/ods/tests/test-render-runtime-configs.py
@@ -528,8 +528,20 @@ def test_llama_server_backends_render_todays_bytes() -> None:
assert flagged["files"] == baseline["files"], value
+# The vllm endpoint carries the served model's thinking map (catalog
+# reasoning + thinking_template_kwargs): model-router translates a request's
+# level with it, since GLM-5.3 and Qwen3.8 read different template arguments.
+VLLM_MODEL = ("--model", "glm-5.3-flash-nvfp4")
+GLM_THINKING = {"default": "medium", "templateKwargs": {
+ "off": {"reasoning_effort": "low"}, "low": {"reasoning_effort": "low"},
+ "medium": {"reasoning_effort": "high"}, "high": {}}}
+QWEN_THINKING = {"default": "medium", "templateKwargs": {
+ "off": {"enable_thinking": False}, "low": {"reasoning_effort": "low"},
+ "medium": {"reasoning_effort": "medium"}, "high": {"reasoning_effort": "xhigh"}}}
+
+
def test_vllm_backend_routes_model_router_to_vllm_server() -> None:
- surface = ("--surface", "model-router-endpoints")
+ surface = ("--surface", "model-router-endpoints", *VLLM_MODEL)
for payload in (
run_renderer(*surface, "--llm-backend", "vllm", env=llm_backend_env()),
run_renderer(*surface, env=llm_backend_env("vllm")),
@@ -541,14 +553,42 @@ def test_vllm_backend_routes_model_router_to_vllm_server() -> None:
assert payload["inputs"]["llm_backend"] == "vllm"
content = file_by_surface(payload, "model-router-endpoints")["content"]
assert json.loads(content) == {
- "endpoints": [{"id": "vllm-default", "baseUrl": "http://vllm-server:8000"}],
+ "endpoints": [{"id": "vllm-default", "baseUrl": "http://vllm-server:8000",
+ "thinking": GLM_THINKING}],
}
+def test_vllm_endpoint_carries_each_models_own_thinking_map() -> None:
+ surface = ("--surface", "model-router-endpoints", "--llm-backend", "vllm")
+ for model, thinking in (("glm-5.3-flash-nvfp4", GLM_THINKING),
+ ("qwen3.8-flash-next-nvfp4", QWEN_THINKING),
+ # The catalog id names the same entry.
+ ("qwen3.8-flash-next-nvfp4-vllm", QWEN_THINKING)):
+ payload = run_renderer(*surface, "--model", model, env=llm_backend_env())
+ endpoint = json.loads(file_by_surface(payload, "model-router-endpoints")["content"])["endpoints"][0]
+ assert endpoint["thinking"] == thinking, model
+
+
+def test_vllm_endpoint_refuses_a_model_without_a_vllm_catalog_entry() -> None:
+ for model in ("qwen3.5-9b", "no-such-model"):
+ proc = subprocess.run(
+ [sys.executable, str(SCRIPT), "--surface", "model-router-endpoints",
+ "--llm-backend", "vllm", "--model", model],
+ cwd=ROOT, env=llm_backend_env(), text=True, capture_output=True,
+ )
+ assert proc.returncode == 2, model
+ assert "needs --model to name one vLLM catalog entry" in proc.stderr, proc.stderr
+ # Only the router endpoint needs the entry; other surfaces render as before.
+ switchboard = run_renderer("--surface", "litellm-switchboard", "--llm-backend", "vllm",
+ "--litellm-key", "sk-test-litellm", env=llm_backend_env())
+ assert switchboard["inputs"]["vllm_thinking"] is None
+
+
def test_vllm_backend_changes_only_the_router_endpoint_and_env_backend() -> None:
- baseline = run_renderer(*ALL_ENABLED, env=llm_backend_env())
- vllm = run_renderer(*ALL_ENABLED, "--llm-backend", "vllm", env=llm_backend_env())
- assert file_by_surface(vllm, "env")["content"] == LLAMA_SERVER_ENV.replace(
+ baseline = run_renderer(*ALL_ENABLED, *VLLM_MODEL, env=llm_backend_env())
+ vllm = run_renderer(*ALL_ENABLED, *VLLM_MODEL, "--llm-backend", "vllm", env=llm_backend_env())
+ assert "LLM_BACKEND=vllm\n" in file_by_surface(vllm, "env")["content"]
+ assert file_by_surface(vllm, "env")["content"] == file_by_surface(baseline, "env")["content"].replace(
"LLM_BACKEND=llama-server\n", "LLM_BACKEND=vllm\n")
changed = {after["surface"] for after, before in zip(vllm["files"], baseline["files"])
if after != before}
@@ -934,6 +974,8 @@ def main() -> int:
test_router_endpoints_strip_trailing_v1,
test_llama_server_backends_render_todays_bytes,
test_vllm_backend_routes_model_router_to_vllm_server,
+ test_vllm_endpoint_carries_each_models_own_thinking_map,
+ test_vllm_endpoint_refuses_a_model_without_a_vllm_catalog_entry,
test_vllm_backend_changes_only_the_router_endpoint_and_env_backend,
test_retired_lemonade_flags_are_accepted_and_ignored,
test_litellm_lemonade_surface_is_retired,
diff --git a/ods/tests/test-router-transport.py b/ods/tests/test-router-transport.py
index ccc9e5112..f6821d584 100644
--- a/ods/tests/test-router-transport.py
+++ b/ods/tests/test-router-transport.py
@@ -64,6 +64,20 @@ class OwnershipTests(unittest.TestCase):
self.assertEqual(calls[2].kwargs["timeout"], 15)
self.assertNotIn(".Config.Env", calls[1].args[0][5])
+ def test_the_vllm_endpoint_is_named_and_no_other_is_accepted(self):
+ # LLM_BACKEND=vllm: vllm-server has no host port; the router container
+ # reaches it at the vllm-default endpoint's origin.
+ result, calls = self.run_request(endpoint_id="vllm-default", response=b"")
+ self.assertEqual(result, "")
+ self.assertEqual(json.loads(calls[2].kwargs["input"])["endpoint_id"], "vllm-default")
+ for endpoint_id in ("lemonade-default", "keyed", ""):
+ with self.subTest(endpoint_id=endpoint_id), \
+ patch.object(transport.subprocess, "run", side_effect=AssertionError("docker ran")):
+ with self.assertRaises(ValueError):
+ transport.request(self.root, ORIGIN, "/health", endpoint_id=endpoint_id)
+ with self.assertRaises(ValueError):
+ transport.exchange(self.root, ORIGIN, "/health", endpoint_id=endpoint_id)
+
def test_exchange_asks_for_an_envelope_and_returns_status_and_text(self):
envelope = json.dumps({"status": 400, "body": "Unable to generate parser for this template."}).encode()
results = [self.process((CONTAINER_ID + "\n").encode()), self.process(json.dumps(self.info).encode()),
diff --git a/ods/tests/test_doctor_vllm_route.py b/ods/tests/test_doctor_vllm_route.py
new file mode 100644
index 000000000..542fc546f
--- /dev/null
+++ b/ods/tests/test_doctor_vllm_route.py
@@ -0,0 +1,93 @@
+"""ods doctor's inference contract on a vLLM install (LLM_BACKEND=vllm).
+
+The route keeps llama-server off and sends clients through LiteLLM to
+model-router and vllm-server, so the doctor expects the LiteLLM gateway and
+the docker-compose.vllm.yml overlay, flags an LLM_API_URL that still names
+the in-stack llama-server, and no longer warns that local mode routes through
+LiteLLM. The contract functions are compiled from scripts/ods-doctor.sh's
+embedded Python without running the doctor.
+"""
+
+from __future__ import annotations
+
+import ast
+import json
+import os
+import pathlib
+import re
+import shlex
+import sys
+from urllib import error, parse, request
+
+import pytest
+
+DOCTOR = pathlib.Path(__file__).resolve().parents[1] / "scripts" / "ods-doctor.sh"
+NAMES = {
+ "_clean_env", "_read_text", "_parse_kv_lines", "_truthy", "_env_file_values", "_compose_files_from_flags",
+ "_has_compose_file", "_looks_like_litellm_route", "_looks_like_local_llama_route",
+ "_url_hostname", "_source", "_evidence", "_diagnosis", "_inference_issue",
+ "_collect_inference_contract",
+}
+
+
+def _contract(root: pathlib.Path):
+ text = DOCTOR.read_text(encoding="utf-8")
+ start = text.index("<<'PY'\n") + len("<<'PY'\n")
+ source = text[start:text.index("\nPY\n", start)]
+ tree = ast.parse(source)
+ module = ast.Module(body=[node for node in tree.body
+ if isinstance(node, ast.FunctionDef) and node.name in NAMES], type_ignores=[])
+ namespace = {"root_dir": root, "os": os, "re": re, "json": json, "shlex": shlex, "sys": sys,
+ "parse": parse, "error": error, "request": request, "pathlib": pathlib}
+ exec(compile(module, str(DOCTOR), "exec"), namespace)
+ assert NAMES <= set(namespace), NAMES - set(namespace)
+ return namespace["_collect_inference_contract"]
+
+
+VLLM_FLAGS = "-f docker-compose.base.yml -f docker-compose.nvidia.yml -f docker-compose.vllm.yml"
+
+
+def doctor(tmp_path, monkeypatch):
+ for key in ("ODS_MODE", "LLM_BACKEND", "LLM_API_URL", "EXTERNAL_LLM_URL", "NATIVE_LLM_BASE_URL",
+ "HERMES_LLM_BASE_URL", "GPU_BACKEND"):
+ monkeypatch.delenv(key, raising=False)
+
+ def run(env: dict[str, str], flags: str | None = VLLM_FLAGS):
+ (tmp_path / ".env").write_text("".join(f"{k}={v}\n" for k, v in env.items()), encoding="utf-8")
+ if flags is None:
+ (tmp_path / ".compose-flags").unlink(missing_ok=True)
+ else:
+ (tmp_path / ".compose-flags").write_text(flags + "\n", encoding="utf-8")
+ return _contract(tmp_path)()
+ return run
+
+
+VLLM_ENV = {"ODS_MODE": "local", "GPU_BACKEND": "nvidia", "LLM_BACKEND": "vllm",
+ "LLM_API_URL": "http://litellm:4000", "HERMES_LLM_BASE_URL": "http://model-router:9099/v1"}
+
+
+def test_a_vllm_install_expects_the_gateway_and_has_no_issue(doctor):
+ report = doctor(VLLM_ENV)
+ assert report["expected_gateway"] == "litellm"
+ assert report["expected_inference_owner"] == "ods"
+ assert report["signals"]["vllm_engine"] is True and report["signals"]["vllm_overlay"] is True
+ assert report["issues"] == []
+
+
+def test_a_missing_overlay_or_a_llama_server_url_is_a_blocker(doctor):
+ report = doctor(VLLM_ENV, flags="-f docker-compose.base.yml -f docker-compose.nvidia.yml")
+ assert [issue["id"] for issue in report["issues"]] == ["ODS-RUNTIME-VLLM-OVERLAY-MISSING"]
+ for url in ("http://llama-server:8080", "http://ods-llama-server:8080/v1"):
+ report = doctor({**VLLM_ENV, "LLM_API_URL": url})
+ assert [issue["id"] for issue in report["issues"]] == ["ODS-RUNTIME-VLLM-LLAMA-ROUTE"], url
+ assert report["diagnoses"][0]["severity"] == "blocker"
+ assert "ods mode local" in report["diagnoses"][0]["next_steps"][0]
+
+
+def test_llama_server_installs_keep_their_contract(doctor):
+ llama = {**VLLM_ENV, "LLM_BACKEND": "llama-server"}
+ report = doctor(llama, flags="-f docker-compose.base.yml -f docker-compose.nvidia.yml")
+ assert report["expected_gateway"] == "llama-server"
+ assert [issue["id"] for issue in report["issues"]] == ["ODS-RUNTIME-LOCAL-LITELLM-ROUTE"]
+ assert report["signals"]["vllm_engine"] is False
diff --git a/ods/tests/test_hf_manifest_download.py b/ods/tests/test_hf_manifest_download.py
index 5cf8b0205..8a773c48a 100644
--- a/ods/tests/test_hf_manifest_download.py
+++ b/ods/tests/test_hf_manifest_download.py
@@ -68,6 +68,7 @@ def write_catalog(path: Path, files: dict[str, bytes], **overrides) -> Path:
"served_model_name": "Test-Model-NVFP4", "llm_model_name": "test-model-nvfp4",
"context_length": 8192, "min_gpu_mib": 1024, "compute_capability": ["10.0"],
"reasoning": "off", "tool_call_parser": "hermes", "reasoning_parser": "qwen3",
+ "thinking_template_kwargs": {"off": {"enable_thinking": False}, "low": {}, "medium": {}, "high": {}},
"vllm_args": ["--max-model-len", "8192", "--enable-auto-tool-choice",
"--tool-call-parser", "hermes", "--reasoning-parser", "qwen3"],
"selection": {"discrete": 0, "unified": 0, "cpu": 0, "vllm": 1},
diff --git a/ods/tests/test_host_agent_vllm.py b/ods/tests/test_host_agent_vllm.py
new file mode 100644
index 000000000..06236f4dc
--- /dev/null
+++ b/ods/tests/test_host_agent_vllm.py
@@ -0,0 +1,278 @@
+"""The host agent on a vLLM install (LLM_BACKEND=vllm, docker-compose.vllm.yml).
+
+vllm-server serves the one model on the ODS network without a host port, so
+the agent proves it through the owned model-router container against the
+router's vllm-default endpoint: /health is an empty 200, /v1/models lists the
+served id with max_model_len (vLLM has no /props), and the completion names
+that exact id. A cold start reads as loading while Docker reports the
+container starting. Model switching, profile checks and every path that
+would name llama-server to Compose refuse with a clear message instead.
+No Docker, network or real install is used.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import io
+import json
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[1]
+_spec = importlib.util.spec_from_file_location("ods_host_agent_vllm_test", ROOT / "bin" / "ods-host-agent.py")
+agent = importlib.util.module_from_spec(_spec)
+sys.modules["ods_host_agent_vllm_test"] = agent
+_spec.loader.exec_module(agent)
+
+SERVED = "GLM-5.3-Flash-NVFP4"
+ENV = {
+ "ODS_MODE": "local", "GPU_BACKEND": "nvidia", "LLM_BACKEND": "vllm",
+ "LLM_MODEL": "glm-5.3-flash-nvfp4", "VLLM_SERVED_MODEL": SERVED,
+ "VLLM_MODEL_DIR": "nvidia--GLM-5.3-Flash-NVFP4@da920bb0",
+ "GGUF_FILE": "", "CTX_SIZE": "131072", "MAX_CONTEXT": "131072",
+}
+# GLM-5.3's template has no enable_thinking; its catalog "off" is this.
+GLM_OFF = {"reasoning_effort": "low"}
+
+
+def models(model=SERVED, context=131072):
+ return json.dumps({"object": "list", "data": [{
+ "id": model, "object": "model", "owned_by": "vllm", "root": "/model", "max_model_len": context}]})
+
+
+def completion(model=SERVED, content="READY"):
+ return json.dumps({"model": model, "choices": [{"message": {"role": "assistant", "content": content},
+ "finish_reason": "stop"}]})
+
+
+class FakeVllm:
+ """vllm-server as the router transport reaches it."""
+
+ def __init__(self, *, health=200, listed=None, answer=None, container="healthy"):
+ self.health, self.listed, self.answer, self.container = health, listed, answer, container
+ self.calls = []
+
+ def request(self, install_dir, origin, path, **kwargs):
+ self.calls.append((origin, path, kwargs))
+ if path == "/v1/models":
+ return self.listed or models()
+ if path == "/v1/chat/completions":
+ return self.answer or completion()
+ # vLLM has no /props: the transport fails an HTTP 404.
+ raise OSError(f"Docker runtime transport failed (exit 1): runtime returned HTTP 404 for {path}")
+
+ def exchange(self, install_dir, origin, path, **kwargs):
+ self.calls.append((origin, path, kwargs))
+ if isinstance(self.health, Exception):
+ raise self.health
+ return self.health, ""
+
+
+def vllm(monkeypatch, tmp_path):
+ fake = FakeVllm()
+ monkeypatch.setattr(agent, "_router_transport_request", fake.request)
+ monkeypatch.setattr(agent, "_router_transport_exchange", fake.exchange)
+ monkeypatch.setattr(agent, "_container_health", lambda container: fake.container)
+ monkeypatch.setattr(agent, "INSTALL_DIR", tmp_path)
+ monkeypatch.delenv("ODS_HOST_INSTALL_DIR", raising=False)
+ (tmp_path / "config").mkdir()
+ shutil.copyfile(ROOT / "config" / "model-library.json", tmp_path / "config" / "model-library.json")
+ return fake
+
+
+def prove(env=ENV, **kwargs):
+ return agent._wait_for_model_readiness(
+ env, model_id="glm-5.3-flash-nvfp4-vllm", gguf_file=SERVED, llm_model_name=env["LLM_MODEL"],
+ attempts=kwargs.pop("attempts", 1), initial_delay=0, interval=0, return_proof=True, **kwargs)
+
+
+def test_route_backend_and_endpoint_follow_llm_backend():
+ assert agent._initial_switchboard_backend(ENV) == ("vllm", "vllm-default", None)
+ assert agent._switchboard_state.default_endpoint_id("vllm") == "vllm-default"
+ for backend in ("", "llama-server", "external", "lemonade"):
+ assert agent._initial_switchboard_backend({**ENV, "LLM_BACKEND": backend}) == (
+ "llama-server", "llama-server-default", None), backend
+
+
+def test_vllm_is_proven_through_the_routers_vllm_endpoint(vllm, monkeypatch):
+ monkeypatch.setattr(agent.subprocess, "run", lambda *a, **k: pytest.fail("no host port to probe"))
+ proof = prove()
+
+ assert proof["identity"] == SERVED
+ assert proof["contextLength"] == 131072 and proof["contextVerified"] is True
+ assert "vision" not in proof
+ assert [call[1] for call in vllm.calls] == ["/health", "/v1/models", "/v1/chat/completions"]
+ assert all(call[0] == "http://vllm-server:8000" for call in vllm.calls)
+ assert all(call[2]["endpoint_id"] == "vllm-default" and call[2]["api_key"] == "" for call in vllm.calls)
+ payload = vllm.calls[-1][2]["payload"]
+ # vLLM answers 404 for any name but the served one.
+ assert payload["model"] == SERVED
+ assert payload["chat_template_kwargs"] == GLM_OFF
+
+
+def test_qwen_turns_thinking_off_with_its_own_template_argument(vllm):
+ env = {**ENV, "LLM_MODEL": "qwen3.8-flash-next-nvfp4", "VLLM_SERVED_MODEL": "Qwen3.8-Flash-Next-NVFP4"}
+ vllm.listed = models("Qwen3.8-Flash-Next-NVFP4")
+ vllm.answer = completion("Qwen3.8-Flash-Next-NVFP4")
+ assert prove(env)["identity"] == "Qwen3.8-Flash-Next-NVFP4"
+ assert vllm.calls[-1][2]["payload"]["chat_template_kwargs"] == {"enable_thinking": False}
+
+
+def test_a_short_or_missing_served_context_never_proves_the_route(vllm):
+ for listed in (models(context=32768), json.dumps({"data": [{"id": SERVED}]}), models("Other-Model")):
+ vllm.listed, vllm.calls = listed, []
+ diagnosis = {}
+ assert prove(diagnosis=diagnosis) == {}
+ assert "/v1/chat/completions" not in [call[1] for call in vllm.calls]
+
+
+def test_cold_start_reads_as_loading_while_docker_reports_starting(vllm):
+ vllm.health = OSError("Docker runtime transport failed (exit 1): runtime transport failed: URLError")
+ vllm.container = "starting"
+ assert agent._runtime_health(ENV) == "loading"
+ diagnosis = {}
+ assert prove(diagnosis=diagnosis) == {}
+ assert diagnosis["reason"].startswith("vLLM is still starting")
+ assert "final" not in diagnosis
+
+ vllm.health = 503
+ assert agent._runtime_health(ENV) == "loading"
+ # Past its start period Docker says unhealthy: an error, not loading.
+ vllm.container = "unhealthy"
+ assert agent._runtime_health(ENV) == "error"
+ vllm.health = OSError("connection refused")
+ with pytest.raises(OSError):
+ agent._runtime_health(ENV)
+ vllm.health, vllm.container = 200, "healthy"
+ assert agent._runtime_health(ENV) == "ok"
+
+
+def test_llama_server_health_is_unchanged(monkeypatch):
+ monkeypatch.setattr(agent, "_runtime_http", lambda env, path, **kwargs: "")
+ monkeypatch.setattr(agent, "_router_transport_exchange", lambda *a, **k: pytest.fail("vLLM probe"))
+ assert agent._runtime_health({"LLM_BACKEND": "llama-server"}) == "error"
+
+
+def test_catalog_entry_comes_from_llm_model_not_a_leftover_gguf(vllm):
+ leftover = {**ENV, "GGUF_FILE": "Qwen3.5-9B-Q4_K_M.gguf"}
+ model_id, model = agent._catalog_model_for_current_env(leftover)
+ assert model_id == "glm-5.3-flash-nvfp4-vllm" and model["runtime"] == "vllm"
+ # A llama-server route never resolves to a vLLM entry.
+ llama = {"LLM_BACKEND": "llama-server", "LLM_MODEL": "glm-5.3-flash-nvfp4", "GGUF_FILE": ""}
+ assert agent._catalog_model_for_current_env(llama) == ("glm-5.3-flash-nvfp4", {})
+ assert agent._vllm_probe_thinking_off(ENV) == GLM_OFF
+ assert agent._vllm_probe_thinking_off(llama) is None
+
+
+def test_compose_resolution_reads_llm_backend_from_the_install(monkeypatch, tmp_path):
+ assert "LLM_BACKEND" in agent._HOST_LLM_COMPOSE_SELECTORS
+ assert "LLM_BACKEND" in agent._PREPARE_COMPOSE_SELECTORS
+ (tmp_path / "scripts").mkdir()
+ (tmp_path / "scripts" / "resolve-compose-stack.sh").write_text("", encoding="utf-8")
+ (tmp_path / ".env").write_text("LLM_BACKEND=vllm\nODS_MODE=local\n", encoding="utf-8")
+ monkeypatch.setattr(agent, "INSTALL_DIR", tmp_path)
+ monkeypatch.setattr(agent, "_find_usable_bash", lambda: "bash")
+ monkeypatch.setattr(agent.platform, "system", lambda: "Linux")
+ # A stale value exported when the agent started must not win.
+ monkeypatch.setenv("LLM_BACKEND", "llama-server")
+ seen = {}
+
+ def run(cmd, **kwargs):
+ seen.update(kwargs["env"])
+ return subprocess.CompletedProcess(cmd, 0, stdout="-f docker-compose.base.yml", stderr="")
+
+ monkeypatch.setattr(agent.subprocess, "run", run)
+ agent._run_compose_resolver()
+ assert seen["LLM_BACKEND"] == "vllm"
+
+
+def test_router_configs_render_with_the_installed_backend(monkeypatch, tmp_path):
+ (tmp_path / "scripts").mkdir()
+ (tmp_path / "scripts" / "render-runtime-configs.py").write_text("", encoding="utf-8")
+ commands = []
+
+ def run(cmd, **kwargs):
+ commands.append(cmd)
+ return subprocess.CompletedProcess(cmd, 0, stdout="", stderr="")
+
+ monkeypatch.setattr(agent.subprocess, "run", run)
+ monkeypatch.setenv("LLM_BACKEND", "llama-server")
+ agent._render_model_router_runtime_configs(
+ tmp_path, {**ENV, "ODS_MODEL_SWITCHBOARD": "enabled"},
+ model=ENV["LLM_MODEL"], gguf_file="", context_length=131072)
+ assert commands and all(cmd[cmd.index("--llm-backend") + 1] == "vllm" for cmd in commands)
+ assert all(cmd[cmd.index("--model") + 1] == "glm-5.3-flash-nvfp4" for cmd in commands)
+ commands.clear()
+ agent._render_model_router_runtime_configs(
+ tmp_path, {"ODS_MODEL_SWITCHBOARD": "enabled", "GGUF_FILE": "m.gguf"},
+ model="m", gguf_file="m.gguf", context_length=8192)
+ assert all(cmd[cmd.index("--llm-backend") + 1] == "llama-server" for cmd in commands)
+
+
+class Handler:
+ def __init__(self):
+ self.wfile = io.BytesIO()
+ self.response_code = None
+
+ def send_response(self, code):
+ self.response_code = code
+
+ def send_header(self, name, value):
+ pass
+
+ def end_headers(self):
+ pass
+
+ def body(self):
+ return json.loads(self.wfile.getvalue().decode("utf-8"))
+
+
+def _install(tmp_path, monkeypatch, env_text):
+ (tmp_path / ".env").write_text(env_text, encoding="utf-8")
+ (tmp_path / "config").mkdir(exist_ok=True)
+ shutil.copyfile(ROOT / "config" / "model-library.json", tmp_path / "config" / "model-library.json")
+ monkeypatch.setattr(agent, "INSTALL_DIR", tmp_path)
+ monkeypatch.setattr(agent, "STARTUP_ODS_MODE", "local")
+ monkeypatch.setattr(agent, "_managed_wsl_runtime", lambda env: {"managed": False})
+ monkeypatch.setattr(agent, "_recreate_llama_server", lambda *a, **k: pytest.fail("llama-server recreated"))
+ monkeypatch.setattr(agent, "_compose_restart_llama_server", lambda *a, **k: pytest.fail("llama-server started"))
+
+
+def test_a_vllm_install_refuses_model_switches_before_any_change(monkeypatch, tmp_path):
+ _install(tmp_path, monkeypatch, "ODS_MODE=local\nLLM_BACKEND=vllm\nLLM_MODEL=glm-5.3-flash-nvfp4\n")
+ monkeypatch.setattr(agent, "_load_model_library_records", lambda: pytest.fail("refused before lookup"))
+ before = (tmp_path / ".env").read_bytes()
+ handler = Handler()
+ agent.AgentHandler._do_model_activate(handler, "qwen3.5-9b-q4")
+ assert handler.response_code == 409
+ body = handler.body()
+ assert body["code"] == "vllm_runtime_installer_managed"
+ assert "vLLM engine" in body["error"] and "installer" in body["hint"]
+ assert "discord.gg" in body["hint"]
+ assert (tmp_path / ".env").read_bytes() == before
+
+
+def test_a_llama_install_refuses_a_vllm_catalog_model(monkeypatch, tmp_path):
+ _install(tmp_path, monkeypatch, "ODS_MODE=local\nLLM_BACKEND=llama-server\nGGUF_FILE=a.gguf\n")
+ handler = Handler()
+ agent.AgentHandler._do_model_activate(handler, "glm-5.3-flash-nvfp4-vllm")
+ assert handler.response_code == 409
+ body = handler.body()
+ assert body["code"] == "vllm_model_installer_only"
+ assert body["requestedModelId"] == "glm-5.3-flash-nvfp4-vllm"
+ assert body["error"].startswith("GLM 5.3 Flash NVFP4 (vLLM) runs only on the vLLM engine")
+
+
+def test_llama_server_is_never_named_to_compose_on_a_vllm_install(monkeypatch, tmp_path):
+ (tmp_path / ".env").write_text("LLM_BACKEND=vllm\n", encoding="utf-8")
+ monkeypatch.setattr(agent, "INSTALL_DIR", tmp_path)
+ monkeypatch.setattr(agent, "CORE_SERVICE_IDS", {"llama-server", "litellm", "vllm-server"})
+ monkeypatch.setattr(agent.subprocess, "run", lambda *a, **k: pytest.fail("compose ran"))
+ ok, error = agent.docker_compose_recreate(["litellm", "llama-server"])
+ assert not ok and "vllm-server" in error
+ assert "vllm-server" in agent.ALWAYS_ON_SERVICES
diff --git a/ods/tests/test_preserve_active_model_vllm.py b/ods/tests/test_preserve_active_model_vllm.py
new file mode 100644
index 000000000..96abfdfe7
--- /dev/null
+++ b/ods/tests/test_preserve_active_model_vllm.py
@@ -0,0 +1,164 @@
+"""An installer rerun (ods update) keeps a vLLM install on vLLM.
+
+scripts/preserve-active-model.py's default mode used to keep only a
+llama-server GGUF route, so a .env with LLM_BACKEND=vllm got no contract and
+the rerun fell back to the tier's GGUF. It now keeps the vLLM route while the
+catalog snapshot is installed, clears the GGUF keys, and exits 2 rather than
+fall back when it cannot. Runs the helper with this Python; no installer,
+Docker or network.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import shlex
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[1]
+HELPER = ROOT / "scripts" / "preserve-active-model.py"
+sys.path.insert(0, str(ROOT / "extensions" / "services" / "dashboard-api"))
+import model_vllm # noqa: E402
+
+REVISION = "0123456789abcdef0123456789abcdef01234567"
+FILES = {"config.json": b'{"model_type": "test"}', "model-00001-of-00001.safetensors": b"w" * 4096,
+ "model.safetensors.index.json": b"{}", "tokenizer_config.json": b"{}"}
+LFS = {"model-00001-of-00001.safetensors", "model.safetensors.index.json"}
+ARGS = ["--max-model-len", "65536", "--enable-auto-tool-choice", "--tool-call-parser", "glm47",
+ "--reasoning-parser", "glm47", "--speculative-config.method=mtp"]
+
+
+def blob_sha1(content: bytes) -> str:
+ return hashlib.sha1(b"blob %d\0" % len(content) + content).hexdigest()
+
+
+def entry() -> dict:
+ files = [{"path": path, "size": len(data), "sha256": hashlib.sha256(data).hexdigest()} if path in LFS
+ else {"path": path, "size": len(data), "git_blob_sha1": blob_sha1(data)}
+ for path, data in FILES.items()]
+ return model_vllm.require_valid({
+ "id": "test-model-nvfp4-vllm", "name": "Test Model NVFP4 (vLLM)", "runtime": "vllm",
+ "hf_repo": "nvidia/Test-Model-NVFP4", "revision": REVISION, "files": files,
+ "served_model_name": "Test-Model-NVFP4", "llm_model_name": "test-model-nvfp4",
+ "context_length": 65536, "min_gpu_mib": 1024, "compute_capability": ["10.0"],
+ "reasoning": "medium", "tool_call_parser": "glm47", "reasoning_parser": "glm47",
+ "thinking_template_kwargs": {"off": {"reasoning_effort": "low"}, "low": {"reasoning_effort": "low"},
+ "medium": {"reasoning_effort": "high"}, "high": {}},
+ "vllm_args": ARGS, "selection": {"discrete": 0, "unified": 0, "cpu": 0, "vllm": 1},
+ })
+
+
+FOLDER = "nvidia--Test-Model-NVFP4@01234567"
+VLLM_ENV = {
+ "ODS_MODE": "local", "LLM_BACKEND": "vllm", "LLM_MODEL": "test-model-nvfp4",
+ "GGUF_FILE": "", "MAX_CONTEXT": "65536", "CTX_SIZE": "65536",
+ "VLLM_MODEL_DIR": FOLDER, "VLLM_SERVED_MODEL": "Test-Model-NVFP4",
+ # The args an older release measured; a rerun takes this release's.
+ "VLLM_ARGS": "--max-model-len 65536 --old-flag", "VLLM_GPU_UUID": "GPU-0f1e2d3c-aaaa-bbbb-cccc-0123456789ab",
+ "VLLM_GPU_MEMORY_UTILIZATION": "0.85", "MODEL_SELECTION_SOURCE": "installer",
+}
+
+
+def install(tmp_path):
+ gguf = {"id": "tiny-q4", "name": "Tiny", "llm_model_name": "tiny", "gguf_file": "tiny.gguf",
+ "gguf_url": "https://huggingface.co/a/b/resolve/main/tiny.gguf"}
+ (tmp_path / "model-library.json").write_text(json.dumps({"models": [gguf, entry()]}), encoding="utf-8")
+ snapshot = tmp_path / "data" / "models" / FOLDER
+ snapshot.mkdir(parents=True)
+ for path, data in FILES.items():
+ (snapshot / path).write_bytes(data)
+ return tmp_path
+
+
+def run(install: Path, env: dict[str, str]) -> subprocess.CompletedProcess:
+ (install / ".env").write_text("".join(f"{key}={value}\n" for key, value in env.items()), encoding="utf-8")
+ return subprocess.run(
+ [sys.executable, str(HELPER), "--env", str(install / ".env"),
+ "--catalog", str(install / "model-library.json"), "--models-dir", str(install / "data" / "models"),
+ "--state", str(install / "data" / "model-state.json"), "--backend", "nvidia", "--local-model"],
+ capture_output=True, text=True, timeout=60)
+
+
+def contract(stdout: str) -> dict[str, str]:
+ return {key: shlex.split(value)[0] if value != '""' else ""
+ for key, _, value in (line.partition("=") for line in stdout.splitlines())}
+
+
+def test_an_installed_vllm_route_is_kept_and_no_gguf_is_named(install):
+ result = run(install, VLLM_ENV)
+
+ assert result.returncode == 0, result.stderr
+ assert contract(result.stdout) == {
+ "LLM_BACKEND": "vllm",
+ "LLM_MODEL": "test-model-nvfp4",
+ "GGUF_FILE": "", "GGUF_URL": "", "GGUF_SHA256": "",
+ "MAX_CONTEXT": "65536",
+ "LLM_MODEL_SIZE_MB": "0",
+ "MODEL_SELECTION_SOURCE": "installer",
+ "VLLM_MODEL_DIR": FOLDER,
+ "VLLM_SERVED_MODEL": "Test-Model-NVFP4",
+ "VLLM_ARGS": " ".join(ARGS),
+ "VLLM_GPU_UUID": VLLM_ENV["VLLM_GPU_UUID"],
+ "VLLM_GPU_MEMORY_UTILIZATION": "0.85",
+ }
+ # Empty GGUF keys are written (they replace the tier map's), unlike the
+ # optional llama.cpp settings the helper omits.
+ assert 'GGUF_FILE=""' in result.stdout.splitlines()
+
+
+def test_backend_value_is_read_case_insensitively(install):
+ result = run(install, {**VLLM_ENV, "LLM_BACKEND": "VLLM"})
+ assert result.returncode == 0, result.stderr
+ assert contract(result.stdout)["LLM_BACKEND"] == "vllm"
+
+
[email protected]("damage", ["missing", "short", "partial-only", "linked-folder"])
+def test_an_incomplete_snapshot_stops_the_rerun_instead_of_falling_back(install, damage):
+ snapshot = install / "data" / "models" / FOLDER
+ weights = snapshot / "model-00001-of-00001.safetensors"
+ if damage == "missing":
+ weights.unlink()
+ elif damage == "short":
+ weights.write_bytes(b"w" * 10)
+ elif damage == "partial-only":
+ weights.rename(weights.with_name(weights.name + ".partial"))
+ else:
+ real = install / "elsewhere"
+ snapshot.rename(real)
+ try:
+ snapshot.symlink_to(real, target_is_directory=True)
+ except OSError:
+ pytest.skip("this host cannot create directory symlinks")
+ result = run(install, VLLM_ENV)
+
+ assert result.returncode == 2
+ assert result.stdout == ""
+ assert "vLLM engine" in result.stderr and "refusing to replace it" in result.stderr
+
+
[email protected]("change", [
+ {"VLLM_MODEL_DIR": "nvidia--Test-Model-NVFP4@fedcba98"},
+ {"VLLM_SERVED_MODEL": "Other-Model"},
+ {"LLM_MODEL": "tiny"},
+ {"VLLM_GPU_UUID": ""},
+ {"VLLM_GPU_UUID": "GPU 0; rm -rf /"},
+ {"VLLM_GPU_MEMORY_UTILIZATION": "1.5"},
+ {"VLLM_GPU_MEMORY_UTILIZATION": "0"},
+ {"ODS_MODE": "cloud"},
+])
+def test_a_route_that_is_not_the_catalog_snapshot_is_never_rewritten(install, change):
+ result = run(install, {**VLLM_ENV, **change})
+ assert result.returncode == 2
+ assert result.stdout == ""
+
+
+def test_llama_server_routes_keep_their_behaviour(install):
+ # No vLLM route and no installed GGUF: nothing is preserved, as before.
+ result = run(install, {"ODS_MODE": "local", "LLM_BACKEND": "llama-server",
+ "LLM_MODEL": "tiny", "GGUF_FILE": "tiny.gguf"})
+ assert (result.returncode, result.stdout) == (0, "")
diff --git a/ods/tests/test_select_model_vllm.py b/ods/tests/test_select_model_vllm.py
index d309386c9..6d8200f69 100644
--- a/ods/tests/test_select_model_vllm.py
+++ b/ods/tests/test_select_model_vllm.py
@@ -115,7 +115,8 @@ def test_dgx_station_gets_the_top_priority_vllm_model_on_its_gb300():
assert env["MAX_CONTEXT"] == str(best["context_length"]) == "131072"
assert env["LLM_MODEL_SIZE_MB"] == str(best["size_mb"])
assert env["GGUF_FILE"] == env["GGUF_URL"] == env["GGUF_SHA256"] == ""
- assert env["PIXEL_AGENT_MODEL_READY"] == "false"
+ pixel = ((best.get("app_compatibility") or {}).get("pixel_agent") or {}).get("status")
+ assert env["PIXEL_AGENT_MODEL_READY"] == ("true" if pixel == "verified" else "false")
assert env["MODEL_RECOMMENDATION_POLICY"] == "vllm-gpu-fit-v1"
assert env["MODEL_RECOMMENDATION_SOURCE"] == "catalog_vllm_pre_download"
assert "GPU-test-gb300 (compute capability 10.3, 256703 MiB)" in env["MODEL_RECOMMENDATION_REASON"]