ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.
Python ai-agentsamdcomfyuidockerllama-cppllmlocal-ain8nnvidiaopen-webuiragself-hostedspeech-to-textstrix-halotext-to-speechworkflow-automation
feat(vllm): run an LLM_BACKEND=vllm install end to end (#8298)
Mike Bradleyd06bb8431e3e
diff --git a/ods/bin/model_switchboard/router_transport.py b/ods/bin/model_switchboard/router_transport.pyindex 8d8deb16e..888b839b9 100644--- a/ods/bin/model_switchboard/router_transport.py+++ b/ods/bin/model_switchboard/router_transport.py@@ -1,10 +1,13 @@-"""Probe a host-native llama-server through this installation's router container.+"""Probe a model server through this installation's router container.A WSL host agent cannot reach a Windows loopback-only listener: WSL localhostis not Windows localhost. Docker Desktop containers can, through-``host.docker.internal``. This module runs one bounded stdlib request inside-the owned, running model-router container, against exactly the-``llama-server-default`` origin that the router itself serves inference from.+``host.docker.internal``. The vLLM engine (``LLM_BACKEND=vllm``) publishes no+host port at all; only containers on the ODS network reach it. This module+runs one bounded stdlib request inside the owned, running model-router+container, against exactly the origin of the router endpoint the caller names+(``llama-server-default`` by default, ``vllm-default`` for vLLM), which is the+origin the router itself serves inference from."""from __future__ import annotations@@ -18,6 +21,7 @@ from urllib.parse import urlsplitENDPOINT_ID = "llama-server-default"+_ENDPOINT_IDS = frozenset({ENDPOINT_ID, "vllm-default"})_LIMIT = 65536_ANSWER_LIMIT = 2097152 # the worker's ANSWER_LIMIT for a probe exchange_PATHS = frozenset({"/health", "/props", "/v1/models", "/v1/chat/completions", "/metrics"})@@ -159,19 +163,20 @@ def _owned_router(install_dir: Path, project: str) -> str:def request(install_dir: Path, origin: str, path: str,payload: dict | None = None, api_key: str = "", timeout: float = 5,- *, project: str = "ods") -> str:+ *, project: str = "ods", endpoint_id: str = ENDPOINT_ID) -> str:"""Return bounded HTTP text; never grant readiness or publish model state.- ``origin`` is the router's ``llama-server-default`` base URL without a- path; ``path`` is one of the proof and telemetry routes. An HTTP error- status fails the request.+ ``origin`` is the base URL, without a path, of the router endpoint+ ``endpoint_id``; ``path`` is one of the proof and telemetry routes. An HTTP+ error status fails the request."""- return _send(install_dir, origin, path, payload, api_key, timeout, project, envelope=False)+ return _send(install_dir, origin, path, payload, api_key, timeout, project,+ envelope=False, endpoint_id=endpoint_id)def exchange(install_dir: Path, origin: str, path: str,payload: dict | None = None, api_key: str = "", timeout: float = 5,- *, project: str = "ods") -> tuple[int, str]:+ *, project: str = "ods", endpoint_id: str = ENDPOINT_ID) -> tuple[int, str]:"""Return ``(http_status, text)``, error statuses included, for capability probes.Same ownership, route and size rules as ``request``. A model's own error@@ -179,7 +184,8 @@ def exchange(install_dir: Path, origin: str, path: str,is evidence a probe classifies, so it comes back instead of failing.Redirects are still refused."""- raw = _send(install_dir, origin, path, payload, api_key, timeout, project, envelope=True)+ raw = _send(install_dir, origin, path, payload, api_key, timeout, project,+ envelope=True, endpoint_id=endpoint_id)try:envelope = json.loads(raw)except ValueError as exc:@@ -192,7 +198,9 @@ def exchange(install_dir: Path, origin: str, path: str,def _send(install_dir: Path, origin: str, path: str, payload: dict | None, api_key: str,- timeout: float, project: str, *, envelope: bool) -> str:+ timeout: float, project: str, *, envelope: bool, endpoint_id: str) -> str:+ if endpoint_id not in _ENDPOINT_IDS:+ raise ValueError("Unsupported runtime endpoint")parsed = urlsplit(origin)if (parsed.scheme not in {"http", "https"} or not parsed.hostnameor parsed.username is not None or parsed.password is not None@@ -210,7 +218,7 @@ def _send(install_dir: Path, origin: str, path: str, payload: dict | None, api_kraise ValueError("Runtime proof timeout must be between 0 and 900 seconds")if not re.fullmatch(r"[a-z0-9][a-z0-9_-]*", project):raise ValueError("Invalid Compose project")- message = dict(endpoint_id=ENDPOINT_ID, origin=origin, path=path, payload=payload,+ message = dict(endpoint_id=endpoint_id, origin=origin, path=path, payload=payload,api_key=api_key, timeout=timeout)if envelope:message["envelope"] = Truediff --git a/ods/bin/ods-host-agent.py b/ods/bin/ods-host-agent.pyindex df73eec67..131dd8f8f 100755--- a/ods/bin/ods-host-agent.py+++ b/ods/bin/ods-host-agent.py@@ -148,6 +148,15 @@ if _gguf_inspector_spec is None or _gguf_inspector_spec.loader is None:_gguf_inspector = importlib.util.module_from_spec(_gguf_inspector_spec)_gguf_inspector_spec.loader.exec_module(_gguf_inspector)+# vLLM catalog entries ("runtime": "vllm"): the installer sets them up with+# LLM_BACKEND=vllm; local model switching manages GGUF models only.+_model_vllm_spec = importlib.util.spec_from_file_location(+ "_ods_model_vllm", _MODEL_MEMORY_PATH.with_name("model_vllm.py"))+if _model_vllm_spec is None or _model_vllm_spec.loader is None:+ raise ImportError("Cannot load shared vLLM catalog policy")+_model_vllm = importlib.util.module_from_spec(_model_vllm_spec)+_model_vllm_spec.loader.exec_module(_model_vllm)+def _activation_architecture_refusal(target: Path, env: dict, *, windows_hosted: bool,registered_runtime: dict | None) -> dict | None:@@ -312,7 +321,7 @@ WINDOWS_WHISPER_CUDA_MIN_DRIVER_MAJOR = 575# Distinct from CORE_SERVICE_IDS (which is the allowlist of known service IDs).ALWAYS_ON_SERVICES: frozenset = frozenset({"llama-server", "model-router", "remote-provider-egress", "remote-provider-ssh-tunnel",- "open-webui", "dashboard", "dashboard-api",+ "open-webui", "dashboard", "dashboard-api", "vllm-server",})USER_EXTENSIONS_DIR: Path = Path()EXTENSIONS_DIR: Path = Path()@@ -758,13 +767,57 @@ def _model_activation_public_failure(diagnosis: dict, healthy: bool) -> str:# .env keys that decide whether inference is a container or a host-native-# llama-server. Compose resolution must read them from installed state. The-# LEMONADE_EXTERNAL selector stays one release for un-migrated installs.+# llama-server, and LLM_BACKEND, which selects the vLLM engine overlay+# (LLM_BACKEND=vllm). Compose resolution must read them from installed state.+# The LEMONADE_EXTERNAL selector stays one release for un-migrated installs._HOST_LLM_COMPOSE_SELECTORS = ("AMD_INFERENCE_RUNTIME", "AMD_INFERENCE_RUNTIME_MODE", "AMD_INFERENCE_LOCATION","AMD_INFERENCE_MANAGED", "ODS_HOST_LLM_TRANSPORT", "LEMONADE_EXTERNAL",+ "LLM_BACKEND",)+# LLM_BACKEND=vllm (docker-compose.vllm.yml): vllm-server serves the one model+# in its own container on the ODS network, without a host port, and the+# in-stack llama-server is profiled out. Naming llama-server in a Compose+# command would enable its profile again, so nothing here may do that.+_VLLM_CONTAINER = "ods-vllm-server"+_VLLM_ORIGIN = "http://vllm-server:8000"+_VLLM_ENDPOINT_ID = "vllm-default"+++def _vllm_engine(env: dict) -> bool:+ """Whether .env selects the vLLM engine (LLM_BACKEND=vllm)."""+ return str(env.get("LLM_BACKEND") or "").strip().casefold() == "vllm"+++_ODS_HELP = "Need help? Ask on the ODS Discord: https://discord.gg/4ntNp9MAwC"+_VLLM_LLAMA_SERVER_OFF = (+ "llama-server is off on this install: the vLLM engine (vllm-server) serves the model. "+ "Restart vllm-server instead, or run the ODS installer again to change the engine."+)+++def _vllm_activation_refusal(model_id: str) -> dict:+ """409 body for a model switch on an install the vLLM engine serves."""+ return {+ "error": "This install runs its model on the vLLM engine, and models cannot be switched here.",+ "code": "vllm_runtime_installer_managed",+ "hint": f"To use another model, run the ODS installer again and choose it. {_ODS_HELP}",+ "requestedModelId": model_id,+ }+++def _vllm_model_refusal(model: dict) -> dict:+ """409 body for a vLLM catalog entry asked of llama-server's model lifecycle."""+ return {+ "error": f"{model.get('name') or model.get('id')} runs only on the vLLM engine, "+ "which only the ODS installer sets up.",+ "code": "vllm_model_installer_only",+ "hint": ("It needs one NVIDIA datacenter Blackwell GPU. Run the ODS installer and "+ f"choose the vLLM engine to use it. {_ODS_HELP}"),+ "requestedModelId": model.get("id"),+ }+_SWITCHBOARD_ROUTE_ENV_KEYS = ("GPU_BACKEND",@@ -2969,14 +3022,23 @@ def _switchboard_state_needs_current_env_verification(def _catalog_model_for_current_env(env: dict) -> tuple[str, dict]:- gguf_file = str(env.get("GGUF_FILE") or "").strip()+ """The catalog id and entry the configured route serves.++ Without a catalog entry: the configured model name and ``{}``.++ A vLLM route (LLM_BACKEND=vllm) serves a vLLM entry, found by LLM_MODEL; a+ GGUF_FILE left from an earlier llama-server install never names it. Every+ other route serves a GGUF entry.+ """+ vllm = _vllm_engine(env)+ gguf_file = "" if vllm else str(env.get("GGUF_FILE") or "").strip()llm_model_name = str(env.get("LLM_MODEL") or "").strip()try:library = _load_model_library_records()except RuntimeError:library = []for entry in library:- if not isinstance(entry, dict):+ if not isinstance(entry, dict) or _model_vllm.is_vllm_entry(entry) != vllm:continueentry_id = str(entry.get("id") or "")entry_gguf = str(entry.get("gguf_file") or "")@@ -2992,7 +3054,13 @@ def _catalog_model_for_current_env(env: dict) -> tuple[str, dict]:def _initial_switchboard_backend(env: dict) -> tuple[str, str, str | None]:- """Every managed runtime is upstream llama-server behind one endpoint."""+ """The route's backend kind and router endpoint for the configured runtime.++ LLM_BACKEND=vllm routes to vllm-server; every other managed runtime is+ upstream llama-server behind the llama-server-default endpoint.+ """+ if _vllm_engine(env):+ return "vllm", _VLLM_ENDPOINT_ID, Nonereturn "llama-server", "llama-server-default", None@@ -7602,6 +7670,9 @@ def docker_compose_recreate(service_ids: list[str], *, force_recreate: bool = Trok, error = validate_core_recreate_ids(service_ids)if not ok:return False, error+ if "llama-server" in service_ids and _vllm_engine(load_env(INSTALL_DIR / ".env")):+ # Naming the service would enable its profile and start a second engine.+ return False, _VLLM_LLAMA_SERVER_OFFtry:flags = _core_recreate_compose_flags(resolve_compose_flags(), service_ids)@@ -13103,6 +13174,11 @@ class AgentHandler(BaseHTTPRequestHandler):if not isinstance(sid, str) or not SERVICE_ID_RE.fullmatch(sid):json_response(self, 400, {"error": "Invalid service_id"})return+ if sid == "llama-server" and _vllm_engine(load_env(INSTALL_DIR / ".env")):+ # A container left by an earlier llama-server install would start+ # beside vLLM on the same GPU.+ json_response(self, 409, {"error": _VLLM_LLAMA_SERVER_OFF, "code": "llama_server_off"})+ returnhas_container, restart_error = _service_has_docker_container(sid)if not has_container:@@ -13752,6 +13828,12 @@ class AgentHandler(BaseHTTPRequestHandler):if model is None:json_response(self, 404, {"error": "Unknown model"})return+ if _model_vllm.is_vllm_entry(model) or _vllm_engine(env):+ # The probes read llama.cpp's /props and chat-template errors.+ json_response(self, 409, {"error": "Model checks measure llama.cpp models; "+ "a model on the vLLM engine is not checked",+ "code": "vllm_not_profiled"})+ returnif str(model.get("gguf_file") or "") != str(env.get("GGUF_FILE") or ""):json_response(self, 409, {"error": "Only the running model can be checked; run it first", "code": "not_running"})return@@ -14684,6 +14766,12 @@ class AgentHandler(BaseHTTPRequestHandler):"requestedModelId": model_id,})return+ if _vllm_engine(persisted_env):+ # Activation swaps llama-server's GGUF and rolls back by restoring+ # it; the vLLM engine serves one installed snapshot instead, and+ # starting llama-server here would run a second engine.+ json_response(self, 409, _vllm_activation_refusal(model_id))+ returndef local_gguf_model_from_id(raw_model_id: str) -> dict | None:matching = []@@ -14746,6 +14834,9 @@ class AgentHandler(BaseHTTPRequestHandler):if model is None:json_response(self, 404, {"error": f"Model '{model_id}' not found in library or local GGUF files"})return+ if _model_vllm.is_vllm_entry(model):+ json_response(self, 409, _vllm_model_refusal(model))+ returngguf_file = model.get("gguf_file", "")llm_model_name = model.get("llm_model_name", model_id)@@ -16495,15 +16586,21 @@ def _runtime_uses_router_transport(env: dict) -> bool:def _runtime_endpoint(env: dict) -> tuple[str, str]:- """Return ``(origin, transport)`` of the configured llama-server runtime.-- Every managed runtime is upstream llama-server. ``transport`` is- ``router`` when this agent runs in WSL and the server is the owned- Windows task: WSL localhost is not Windows localhost, so proofs run in- the owned model-router container against its ``llama-server-default``- origin. Every other runtime is reached ``direct``: the container by- Docker DNS or its published port, a host-native server on loopback.+ """Return ``(origin, transport)`` of the configured model server.++ ``transport`` is ``router`` when this agent runs in WSL and the server is+ the owned Windows llama-server task: WSL localhost is not Windows+ localhost, so proofs run in the owned model-router container against its+ ``llama-server-default`` origin. The vLLM engine (LLM_BACKEND=vllm)+ publishes no host port, so a host agent proves it the same way, against+ the router's ``vllm-default`` origin. Every other runtime is reached+ ``direct``: the container by Docker DNS or its published port, a+ host-native server on loopback."""+ if _vllm_engine(env):+ if os.environ.get("ODS_HOST_INSTALL_DIR"):+ return f"http://{_VLLM_CONTAINER}:8000", "direct"+ return _VLLM_ORIGIN, "router"if _runtime_uses_router_transport(env):return _native_llm_container_origin(env), "router"if _is_windows_host_llama_server(env):@@ -16516,6 +16613,14 @@ def _runtime_endpoint(env: dict) -> tuple[str, str]:return f"http://127.0.0.1:{env.get('OLLAMA_PORT') or '8080'}", "direct"+def _runtime_router_endpoint(env: dict) -> dict:+ """The router endpoint a router-transport proof names (keyword arguments).++ The WSL route keeps the transport's default, ``llama-server-default``.+ """+ return {"endpoint_id": _VLLM_ENDPOINT_ID} if _vllm_engine(env) else {}++def _runtime_api_key(env: dict) -> str:"""Bearer key of a Windows-owned llama-server; containers take none."""if _runtime_uses_router_transport(env) or _is_windows_host_llama_server(env):@@ -16542,6 +16647,7 @@ def _runtime_http(if transport == "router":return _router_transport_request(INSTALL_DIR, origin, path, payload=payload, api_key=api_key, timeout=timeout,+ **_runtime_router_endpoint(env),)command = ["curl", "-s", "--max-time", str(timeout)]if payload is not None:@@ -16599,6 +16705,7 @@ def _runtime_exchange(if transport == "router":return _router_transport_exchange(INSTALL_DIR, origin, path, payload=payload, api_key=api_key, timeout=timeout,+ **_runtime_router_endpoint(env),)headers = {"Content-Type": "application/json"}if api_key:@@ -16995,8 +17102,44 @@ def _profiled_capabilities(capabilities: object, traits: dict | None, model: dic}+def _container_health(container: str) -> str:+ """Docker's health status of ``container`` (starting, healthy, unhealthy, none).++ Empty when Docker cannot inspect it, for example when it does not exist.+ """+ result = subprocess.run(+ ["docker", "inspect", "--type", "container", "--format",+ "{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}", container],+ capture_output=True, text=True, timeout=15,+ )+ return result.stdout.strip().casefold() if result.returncode == 0 else ""+++def _vllm_runtime_health(env: dict) -> str:+ """Return ``ok``, ``loading`` or ``error`` for the vLLM engine.++ vLLM's ``/health`` answers 200 with an empty body once the engine is+ ready, and its server listens only from then on. A cold first start loads+ the weights and compiles kernels for 3 to 15 minutes, during which Docker+ reports the container ``starting`` (docker-compose.vllm.yml allows 60+ minutes): that is ``loading``, not an error. Raises OSError when vLLM is+ unreachable and its container is not starting.+ """+ try:+ status, _body = _runtime_exchange(env, "/health", timeout=5)+ except OSError:+ if _container_health(_VLLM_CONTAINER) == "starting":+ return "loading"+ raise+ if status == 200:+ return "ok"+ return "loading" if _container_health(_VLLM_CONTAINER) == "starting" else "error"++def _runtime_health(env: dict) -> str:- """Return ``ok``, ``loading`` or ``error`` from llama-server ``/health``."""+ """Return ``ok``, ``loading`` or ``error`` from the model server's ``/health``."""+ if _vllm_engine(env):+ return _vllm_runtime_health(env)body = _runtime_http(env, "/health")try:data = json.loads(body or "{}")@@ -17159,6 +17302,24 @@ def _llama_runtime_context_length(env: dict) -> int:return _llama_runtime_props(env)[0]+def _vllm_served_context_length(body: str, runtime_identity: str) -> int:+ """Return the ``max_model_len`` vLLM's /v1/models reports for its served row.++ vLLM has no /props; the row of the model it serves carries the context+ its --max-model-len fixed.+ """+ try:+ data = json.loads(body)+ except (json.JSONDecodeError, TypeError):+ return 0+ rows = data.get("data") if isinstance(data, dict) else None+ for row in rows if isinstance(rows, list) else ():+ if isinstance(row, dict) and str(row.get("id") or "").strip() == runtime_identity:+ value = row.get("max_model_len")+ return value if type(value) is int and value > 0 else 0+ return 0++def _llama_training_context_length(body: str, runtime_identity: str) -> int:"""Return the GGUF training context llama.cpp reports for a loaded row."""try:@@ -17243,11 +17404,11 @@ def _completion_text(data: object, *, include_reasoning: bool = True) -> str:if text.strip():return text[:4096]if include_reasoning:- reasoning_content = (- message.get("reasoning_content")- if isinstance(message, dict)- else choice.get("reasoning_content")- )+ source = message if isinstance(message, dict) else choice+ reasoning_content = source.get("reasoning_content")+ if not isinstance(reasoning_content, str):+ # vLLM's reasoning parsers return the field as ``reasoning``.+ reasoning_content = source.get("reasoning")if isinstance(reasoning_content, str):return reasoning_content[:4096]if isinstance(content, str):@@ -17275,7 +17436,14 @@ _COMPLETION_PROBE_THINKING_SECONDS = 120def _completion_probe_payload(model_name: str, *, disable_thinking: bool,- max_tokens: int = _COMPLETION_PROBE_TOKENS) -> dict:+ max_tokens: int = _COMPLETION_PROBE_TOKENS,+ thinking_off: dict | None = None) -> dict:+ """The bounded readiness probe; ``thinking_off`` replaces enable_thinking=false.++ A vLLM entry names the chat-template arguments that turn its thinking+ down (catalog ``thinking_template_kwargs.off``); GLM's template has no+ enable_thinking at all.+ """payload = {"model": model_name,"messages": [{@@ -17288,10 +17456,21 @@ def _completion_probe_payload(model_name: str, *, disable_thinking: bool,"temperature": 0,}if disable_thinking:- payload["chat_template_kwargs"] = {"enable_thinking": False}+ payload["chat_template_kwargs"] = (+ dict(thinking_off) if thinking_off is not None else {"enable_thinking": False})return payload+def _vllm_probe_thinking_off(env: dict) -> dict | None:+ """The configured vLLM entry's thinking-off template arguments, if any."""+ if not _vllm_engine(env):+ return None+ _model_id, model = _catalog_model_for_current_env(env)+ if not _model_vllm.is_vllm_entry(model):+ return None+ return dict(model["thinking_template_kwargs"]["off"])++def _reasoning_cut_short(response: object) -> bool:"""The answer hit max_tokens inside its reasoning: thinking, but no visible text yet."""choices = response.get("choices") if isinstance(response, dict) else None@@ -17338,6 +17517,7 @@ def _chat_completion_ready(base_url: str = "",disable_thinking: bool = False,require_visible_content: bool = False,+ thinking_off: dict | None = None,) -> bool:"""Require a meaningful completion and, when requested, its model identity."""prefix = "/" + api_prefix.strip("/")@@ -17346,7 +17526,8 @@ def _chat_completion_ready(def probe(max_tokens: int, seconds: int):payload = json.dumps(_completion_probe_payload(- model_name, disable_thinking=disable_thinking, max_tokens=max_tokens))+ model_name, disable_thinking=disable_thinking, max_tokens=max_tokens,+ thinking_off=thinking_off))command = ["curl", "-sf", "--max-time", str(seconds), "--max-filesize", "65536","-X", "POST", url,@@ -17398,9 +17579,11 @@ def _runtime_completion_ready(A reasoning-only answer does not prove the runtime is ready to serveconsumers, so every runtime family must return visible content from the- same model it reported in /v1/models.+ same model it reported in /v1/models. A vLLM entry turns its thinking down+ with its own template arguments (``_vllm_probe_thinking_off``)."""origin, transport = _runtime_endpoint(env)+ thinking_off = _vllm_probe_thinking_off(env)expected = {"expected_model_id": expected_model_id,"expected_gguf_file": expected_gguf_file,@@ -17410,13 +17593,14 @@ def _runtime_completion_ready(return _chat_completion_ready("", "", model_name, "/v1", api_key=_runtime_api_key(env),base_url=origin, disable_thinking=True, require_visible_content=True,- **expected,+ thinking_off=thinking_off, **expected,)try:body = _runtime_http(env,"/v1/chat/completions",- payload=_completion_probe_payload(model_name, disable_thinking=True),+ payload=_completion_probe_payload(+ model_name, disable_thinking=True, thinking_off=thinking_off),timeout=30,)response = json.loads(body or "{}")@@ -17425,7 +17609,8 @@ def _runtime_completion_ready(env,"/v1/chat/completions",payload=_completion_probe_payload(- model_name, disable_thinking=True, max_tokens=_COMPLETION_PROBE_THINKING_TOKENS),+ model_name, disable_thinking=True, max_tokens=_COMPLETION_PROBE_THINKING_TOKENS,+ thinking_off=thinking_off),timeout=_COMPLETION_PROBE_THINKING_SECONDS,)response = json.loads(body or "{}")@@ -17582,7 +17767,10 @@ def _wait_for_model_readiness(requested model, ``/props`` serves its file with an ``n_ctx`` that meetsthe request (llama.cpp may pad a custom size up to the next 256-cellboundary), and one bounded completion with thinking disabled returns- visible content from the same model.+ visible content from the same model. The vLLM engine (LLM_BACKEND=vllm)+ proves the same steps without ``/props``: its ``/v1/models`` row reports+ the served context as ``max_model_len``, and the completion names the+ exact id that row lists (vLLM refuses any other model name).Legacy callers receive a boolean. Identity callers receive the concreteruntime identity. Adapters receive identity, actual context, and proof time,@@ -17608,6 +17796,7 @@ def _wait_for_model_readiness(diagnosis = {}not_ready = {} if return_proof else "" if return_identity else Falseorigin, transport = _runtime_endpoint(env)+ vllm = _vllm_engine(env)completion_model = llm_model_name or gguf_fileexpected_context = _positive_int(env.get("CTX_SIZE") or env.get("MAX_CONTEXT"))@@ -17671,7 +17860,10 @@ def _wait_for_model_readiness(logger.info("Model readiness aborted: route env changed after probing %s", gguf_file)return not_readyif health == "loading":- diagnosis["reason"] = "llama-server is still loading the model"+ diagnosis["reason"] = (+ "vLLM is still starting: a first start loads the weights and compiles "+ "kernels, which can take 15 minutes" if vllm+ else "llama-server is still loading the model")runtime_identity = _llama_loaded_model_identity(body,model_id=model_id,@@ -17679,7 +17871,11 @@ def _wait_for_model_readiness(llm_model_name=llm_model_name,) if body else ""if runtime_identity:- runtime_context, served_file, runtime_vision = _llama_runtime_props(env)+ if vllm:+ runtime_context = _vllm_served_context_length(body, runtime_identity)+ served_file, runtime_vision = "", None+ else:+ runtime_context, served_file, runtime_vision = _llama_runtime_props(env)if served_file and not _runtime_model_identity_matches(served_file, gguf_file=gguf_file,):@@ -17710,7 +17906,7 @@ def _wait_for_model_readiness(runtime_identity = ""if runtime_identity and _runtime_completion_ready(env,- str(completion_model),+ str(runtime_identity if vllm else completion_model),expected_model_id=str(runtime_identity),expected_gguf_file=gguf_file,expected_llm_model_name=llm_model_name,@@ -17878,6 +18074,7 @@ def _render_runtime_config(remote_llm_transport: str = "",remote_llm_base_url: str = "",remote_llm_model: str = "",+ llm_backend: str = "llama-server",) -> bool:renderer = install_dir / "scripts" / "render-runtime-configs.py"if not renderer.exists():@@ -17887,6 +18084,8 @@ def _render_runtime_config(str(renderer),"--surface",surface,+ "--llm-backend",+ llm_backend,"--switchboard-mode",switchboard_mode,"--ods-mode",@@ -17968,6 +18167,11 @@ def _runtime_api_key_env(env: dict) -> str:return "LLAMA_SERVER_API_KEY" if _runtime_api_key(env) else ""+def _render_llm_backend(env: dict) -> str:+ """The renderer's --llm-backend: the .env LLM_BACKEND, llama-server when unset."""+ return str(env.get("LLM_BACKEND") or "").strip().lower() or "llama-server"++def _litellm_render_key(env: dict) -> str:return str(env.get("LITELLM_KEY") or env.get("LITELLM_MASTER_KEY") or "")@@ -17993,6 +18197,9 @@ def _render_model_router_runtime_configs("gpu_backend": env.get("GPU_BACKEND", "nvidia"),"context_length": int(context_length),"switchboard_mode": switchboard_mode,+ # The installed backend, never this process's environment: vllm+ # points model-router at vllm-server with the model's thinking map.+ "llm_backend": _render_llm_backend(env),}surfaces = ["model-router-endpoints"]if str(common["ods_mode"]).strip().lower() != "cloud":diff --git a/ods/config/model-library.json b/ods/config/model-library.jsonindex 1806a478c..62ce71320 100644--- a/ods/config/model-library.json+++ b/ods/config/model-library.json@@ -2832,6 +2832,8 @@"name": "Qwen 3.8 Flash Next NVFP4 (vLLM)","family": "qwen","runtime": "vllm",+ "tools": true,+ "vision": true,"hf_repo": "nvidia/Qwen3.8-Flash-Next-NVFP4","revision": "fc694b54fb0174e0913e6adf86691ef85a4ead47","source_url": "https://huggingface.co/nvidia/Qwen3.8-Flash-Next-NVFP4",@@ -2846,6 +2848,12 @@"min_gpu_mib": 158720,"compute_capability": ["10.0", "10.3"],"reasoning": "medium",+ "thinking_template_kwargs": {+ "off": {"enable_thinking": false},+ "low": {"reasoning_effort": "low"},+ "medium": {"reasoning_effort": "medium"},+ "high": {"reasoning_effort": "xhigh"}+ },"tool_call_parser": "qwen3_coder","reasoning_parser": "qwen3","vllm_args": ["--quantization", "modelopt", "--max-model-len", "131072", "--max-num-seqs", "32", "--max-num-batched-tokens", "8192", "--enable-prefix-caching", "--enable-auto-tool-choice", "--tool-call-parser", "qwen3_coder", "--reasoning-parser", "qwen3", "--trust-remote-code", "--engram-config.cpu_offload=false", "--speculative-config.method=mtp", "--speculative-config.num_speculative_tokens=3"],@@ -2891,6 +2899,19 @@"name": "GLM 5.3 Flash NVFP4 (vLLM)","family": "glm","runtime": "vllm",+ "tools": true,+ "vision": true,+ "app_compatibility": {+ "pixel_agent": {+ "status": "verified",+ "label": "Pixel verified on a DGX Station (vLLM)",+ "reason": "A fresh LLM_BACKEND=vllm install on Workstation06 (DGX Station GB300) served this model and passed Portal visitor rounds 5 and 7: a mood journal that remembers entries, built and checked in 48.6 s; a cited answer that matched GitHub; a CSV download; exact free GPU memory; a follow-up feature; a false bug report investigated without changing code; a three-page site checked page by page; and a clarifying question for an ambiguous request.",+ "evidence": "Workstation06 ~/.local/state/claude-e2e/round5-glm-vllm.json and round7-glm-vllm.json; summarized on Osmantic/ODS#8298",+ "recordedAt": "2026-10-27T22:05:00Z",+ "productSha": "43143c59c1bb031394432af11c120d927e113d5b",+ "hostScope": ["workstation06"]+ }+ },"hf_repo": "nvidia/GLM-5.3-Flash-NVFP4","revision": "da920bb0b9f4a06727223a349e55468e38352348","source_url": "https://huggingface.co/nvidia/GLM-5.3-Flash-NVFP4",@@ -2904,6 +2925,12 @@"min_gpu_mib": 244736,"compute_capability": ["10.0", "10.3"],"reasoning": "medium",+ "thinking_template_kwargs": {+ "off": {"reasoning_effort": "low"},+ "low": {"reasoning_effort": "low"},+ "medium": {"reasoning_effort": "high"},+ "high": {}+ },"tool_call_parser": "glm47","reasoning_parser": "glm47","vllm_args": ["--max-model-len", "131072", "--kv-cache-dtype", "fp8", "--max-num-seqs", "32", "--max-num-batched-tokens", "8192", "--enable-prefix-caching", "--enable-auto-tool-choice", "--tool-call-parser", "glm47", "--reasoning-parser", "glm47", "--trust-remote-code", "--speculative-config.method=mtp", "--speculative-config.num_speculative_tokens=5", "--no-enable-flashinfer-autotune"],diff --git a/ods/extensions/services/dashboard-api/config.py b/ods/extensions/services/dashboard-api/config.pyindex bce267a25..8e35b1b2b 100644--- a/ods/extensions/services/dashboard-api/config.py+++ b/ods/extensions/services/dashboard-api/config.py@@ -321,6 +321,40 @@ def _apply_external_llm_service_override()+VLLM_SERVICE_HOST = "vllm-server"+VLLM_SERVICE_PORT = 8000+VLLM_CONTAINER = "ods-vllm-server"+++def _apply_vllm_service_override(+ services: dict[str, dict[str, Any]],+ environment: Mapping[str, str] | None = None,+) -> None:+ """Route the LLM row's probes to the vLLM engine (LLM_BACKEND=vllm).++ docker-compose.vllm.yml profiles the in-stack llama-server out and runs+ vllm-server on the ODS network: /health answers 200 with an empty body once+ the model is loaded, /v1/models lists the served id. As for a host-native+ or external route, the row keeps the llama-server id that chat, readiness+ and features key on; its container is ods-vllm-server, which the host+ agent's Docker health snapshot reports (a cold start reads as starting),+ and it publishes no host port to link to.+ """+ env = environment if environment is not None else os.environ+ if str(env.get("LLM_BACKEND", "")).strip().lower() != "vllm":+ return+ service = services.get("llama-server")+ if not service:+ return+ service["host"] = VLLM_SERVICE_HOST+ service["port"] = VLLM_SERVICE_PORT+ service["health"] = "/health"+ service["container_name"] = VLLM_CONTAINER+ service["name"] = "vLLM (LLM Inference)"+ service["external_link"] = False+ logger.info("vLLM engine selected; routing LLM probes to %s:%d", VLLM_SERVICE_HOST, VLLM_SERVICE_PORT)++def _read_env_value(key: str) -> str:return (os.environ.get(key) or _read_env_from_file(key)).strip()@@ -655,6 +689,7 @@ if not SERVICES:LLM_BACKEND = normalize_llm_backend(os.environ.get("LLM_BACKEND", ""))_apply_host_native_llm_service_override(SERVICES, GPU_BACKEND)_apply_external_llm_service_override(SERVICES)+_apply_vllm_service_override(SERVICES)# --- Features ---diff --git a/ods/extensions/services/dashboard-api/helpers.py b/ods/extensions/services/dashboard-api/helpers.pyindex 0e8cb64e9..bd4311bb5 100644--- a/ods/extensions/services/dashboard-api/helpers.py+++ b/ods/extensions/services/dashboard-api/helpers.py@@ -684,8 +684,17 @@ def _observe_live_output_slots(payload, sampled_at: float):return round(sum(count - previous["counts"][key] for key, count in counts.items()) / elapsed, 1)+# vLLM's counters (LLM_BACKEND=vllm) under the names the sampler reads. vLLM+# exports no generation-seconds counter, so its throughput reads unavailable+# while its token count still accumulates.+_VLLM_COUNTERS = {+ "vllm:generation_tokens_total": "tokens_predicted_total",+ "vllm:num_requests_running": "requests_processing",+}++def _parse_llama_prometheus(body: str) -> dict:- """Read the llama.cpp counters the dashboard samples from a /metrics body."""+ """Read the llama.cpp (or vLLM) counters the dashboard samples from a /metrics body."""metrics: dict = {}for line in body.split("\n"):line = line.strip()@@ -695,6 +704,12 @@ def _parse_llama_prometheus(body: str) -> dict:if len(parts) < 2:continuemetric_name = parts[0].split("{", 1)[0]+ if metric_name in _VLLM_COUNTERS:+ try:+ metrics[_VLLM_COUNTERS[metric_name]] = float(parts[1])+ except ValueError:+ pass+ continuefor counter in ("requests_processing", "tokens_predicted_total", "tokens_predicted_seconds_total"):if metric_name.endswith(counter):try:@@ -735,7 +750,8 @@ async def _fetch_llama_metrics(model_hint: Optional[str] = None, counter_id: Optmodel_name = model_hint if model_hint is not None else (await get_loaded_model() or "")url = f"http://{host}:{metrics_port}/metrics"params = {"model": model_name} if model_name else {}- slots_url = f"http://{host}:{metrics_port}/slots"+ # /slots is llama.cpp's; vLLM has no per-slot output timing.+ slots_url = None if LLM_BACKEND == "vllm" else f"http://{host}:{metrics_port}/slots"client = await _get_httpx_client()resp = await client.get(url, params=params)resp.raise_for_status()@@ -857,6 +873,8 @@ async def get_llama_context_size(model_hint: Optional[str] = None) -> Optional[ireturn context if type(context) is int and context > 0 else Noneif "llama-server" not in SERVICES:return None+ if LLM_BACKEND == "vllm":+ return await _vllm_context_size(model_hint)try:host = SERVICES["llama-server"]["host"]port = SERVICES["llama-server"]["port"]@@ -873,12 +891,35 @@ async def get_llama_context_size(model_hint: Optional[str] = None) -> Optional[ireturn None+async def _vllm_context_size(model_hint: Optional[str]) -> Optional[int]:+ """The context vLLM serves: ``max_model_len`` of its /v1/models row.++ vLLM has no /props; its --max-model-len fixes the context, and the row of+ the served model reports it.+ """+ try:+ host = SERVICES["llama-server"]["host"]+ port = SERVICES["llama-server"]["port"]+ client = await _get_httpx_client()+ resp = await client.get(f"http://{host}:{port}{_LLM_API_PREFIX}/models")+ payload = resp.json()+ except (httpx.HTTPError, httpx.TimeoutException, ValueError) as e:+ logger.debug("vLLM context size unavailable: %s", e)+ return None+ rows = payload.get("data") if isinstance(payload, dict) else None+ for row in rows if isinstance(rows, list) else ():+ if isinstance(row, dict) and (model_hint is None or row.get("id") == model_hint):+ context = row.get("max_model_len")+ return context if type(context) is int and context > 0 else None+ return None++async def get_llama_vision_support() -> Optional[bool]:"""Whether the active llama-server loaded a vision projector.llama-server reports it as ``/props`` ``modalities.vision``. None when that- cannot be read: the owner's own server, an unreachable runtime, or a build- without the field.+ cannot be read: the owner's own server, an unreachable runtime, a build+ without the field, or the vLLM engine, which reports no modalities."""if _host_native_llm():try:@@ -886,7 +927,7 @@ async def get_llama_vision_support() -> Optional[bool]:except (AgentClientError, OSError, ValueError):return Nonereturn vision if type(vision) is bool else None- if LLM_BACKEND == "external" or "llama-server" not in SERVICES:+ if LLM_BACKEND in {"external", "vllm"} or "llama-server" not in SERVICES:return Nonetry:host = SERVICES["llama-server"]["host"]diff --git a/ods/extensions/services/dashboard-api/main.py b/ods/extensions/services/dashboard-api/main.pyindex 9fd7f3a0a..601893a8d 100644--- a/ods/extensions/services/dashboard-api/main.py+++ b/ods/extensions/services/dashboard-api/main.py@@ -42,7 +42,7 @@ from config import (_detect_container_default_gateway, _running_inside_container,_read_env_from_file,get_env_file_read_error,- normalize_ods_mode, read_live_env_value,+ normalize_ods_mode, read_live_env_value, LLM_BACKEND,)from models import (GPUInfo, ServiceStatus, DiskUsage, ModelInfo, BootstrapStatus,@@ -320,12 +320,21 @@ def _build_readiness_payload(llama_healthy = _service_is_healthy(service_statuses, "llama-server")chat_ready = bool(llama_healthy and loaded_model and context_size)+ # LLM_BACKEND=vllm: the row probes vllm-server (config.py). A cold start+ # loads the model and compiles kernels for up to 15 minutes; a restart+ # would only start that over, so none is suggested while it starts.+ engine = "vllm-server" if LLM_BACKEND == "vllm" else "llama-server"+ llm_row = _service_by_id(service_statuses, "llama-server")+ engine_starting = engine == "vllm-server" and bool(llm_row and llm_row.startup_pending)if chat_ready:chat_detail = f"{loaded_model} loaded with {context_size} context"elif bootstrap_info.active:chat_detail = f"Full model is {_bootstrap_activity(bootstrap_info.phase)}; bootstrap mode may be limited"+ elif engine_starting:+ chat_detail = ("The vLLM engine is starting: a first start loads the model and compiles "+ "kernels, which can take 15 minutes")elif not llama_healthy:- chat_detail = "llama-server is not healthy"+ chat_detail = f"{engine} is not healthy"elif not loaded_model:chat_detail = "No loaded model reported by the inference server"else:@@ -337,7 +346,8 @@ def _build_readiness_payload(ready=chat_ready,status="ready" if chat_ready else "blocked",detail=chat_detail,- repair="ods restart llama-server" if not chat_ready and not bootstrap_info.active else None,+ repair=(f"ods restart {engine}"+ if not chat_ready and not bootstrap_info.active and not engine_starting else None),))webui_ready = _service_is_healthy(service_statuses, "open-webui")diff --git a/ods/extensions/services/dashboard-api/model_vllm.py b/ods/extensions/services/dashboard-api/model_vllm.pyindex 5067b3ac8..54af62e97 100644--- a/ods/extensions/services/dashboard-api/model_vllm.py+++ b/ods/extensions/services/dashboard-api/model_vllm.py@@ -15,6 +15,13 @@ the vLLM engine (``.env`` ``LLM_BACKEND=vllm``), not llama.cpp. They carry nocompute capabilities;* ``reasoning`` (the thinking default), plus ``tool_call_parser`` and``reasoning_parser``, which name the same parsers as ``vllm_args``;+* ``thinking_template_kwargs``: for each ODS thinking level (off, low,+ medium, high) the ``chat_template_kwargs`` that select it in this model's+ own chat template. Templates disagree (GLM-5.3 reads only+ ``reasoning_effort`` low or high and thinks at "max" for any other value;+ Qwen3.8 turns thinking off with ``enable_thinking`` and fails on an+ unknown ``reasoning_effort``), so a level is never passed through+ literally: model-router applies this map on the vllm route;* ``selection``: 0 for every llama.cpp memory class, so no llama.cpp installever picks the entry, and a ``vllm`` priority for ``--llm-runtime vllm``.@@ -55,6 +62,7 @@ _NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*")# splitting, so they hold no whitespace, quotes or shell syntax._ARG = re.compile(r"[A-Za-z0-9_.:/=,+-]+")_GPU_UUID = re.compile(r"[A-Za-z0-9][A-Za-z0-9-]*")+_TEMPLATE_KWARG = re.compile(r"[a-z][a-z0-9_]{0,63}")_GPU_MEMORY = re.compile(r"([0-9]+)(?:\s*MiB)?")_FILE_KEYS = ({"path", "size", "sha256"}, {"path", "size", "git_blob_sha1"})_SELECTION_KEYS = frozenset(LLAMA_MEMORY_CLASSES) | {VLLM_RUNTIME}@@ -226,6 +234,28 @@ def _serving_problems(model: dict[str, Any]) -> list[str]:return problems+def _thinking_problems(value: Any) -> list[str]:+ if not isinstance(value, dict) or set(value) != set(REASONING_LEVELS):+ return [f"thinking_template_kwargs must map each of {', '.join(REASONING_LEVELS)} "+ "to chat_template_kwargs"]+ problems = []+ for level in REASONING_LEVELS:+ kwargs = value[level]+ if not isinstance(kwargs, dict) or not all(+ isinstance(key, str) and _TEMPLATE_KWARG.fullmatch(key)+ and (isinstance(item, bool) or (isinstance(item, str) and _NAME.fullmatch(item)))+ for key, item in kwargs.items()+ ):+ problems.append(f"thinking_template_kwargs.{level} must map template argument names "+ "to booleans or plain words")+ return problems+++def thinking_template_kwargs(model: dict[str, Any], level: str) -> dict[str, Any]:+ """The ``chat_template_kwargs`` that select ODS thinking ``level`` in this model."""+ return dict(model["thinking_template_kwargs"][level])++def _size_problems(model: dict[str, Any]) -> list[str]:total = manifest_bytes(model)problems = []@@ -245,6 +275,7 @@ def entry_problems(model: Any) -> list[str]:*_identity_problems(model),*files,*_serving_problems(model),+ *_thinking_problems(model.get("thinking_template_kwargs")),*_args_problems(model),*_selection_problems(model.get("selection")),*([] if files else _size_problems(model)),diff --git a/ods/extensions/services/dashboard-api/performance_oracle.py b/ods/extensions/services/dashboard-api/performance_oracle.pyindex 89177e5f9..eeafb0afb 100644--- a/ods/extensions/services/dashboard-api/performance_oracle.py+++ b/ods/extensions/services/dashboard-api/performance_oracle.py@@ -29,7 +29,8 @@ from helpers import (get_recorded_model_performance,is_plausible_single_request_tps,)-from model_memory import memory_metadata, required_model_memory_gb+from model_memory import memory_metadata, required_model_memory_gb, vllm_memory_estimate+from model_vllm import is_vllm_entryfrom model_selection import (POLICY as _SHARED_SELECTOR_POLICY,family_allowed as _shared_family_allowed,@@ -1617,15 +1618,71 @@ def _measured_native_activation(model, path, data_root, install_dir, gpu_info, sreturn None+def vllm_model_row(raw: dict[str, Any], loaded_model: Optional[str],+ context_length: Optional[int]) -> dict[str, Any]:+ """The Models page row of the vLLM catalog entry a vLLM install serves.++ LLM_BACKEND=vllm runs one installer-set snapshot (model_vllm.py): the+ page shows its catalog name, size, context and GPU need, marks it+ loaded once vLLM serves it (``served_model_name``) and otherwise+ installed, and labels it managed by the engine (``source: runtime``,+ ``runtime: vllm``) so it offers no download, switch or context change.+ """+ served = {normalize_key(raw.get(key)) for key in ("served_model_name", "llm_model_name", "id")}+ is_loaded = bool(loaded_model) and normalize_key(loaded_model) in served+ size_mb = float(raw.get("size_mb") or 0)+ context = context_length if is_loaded and context_length else int(raw["context_length"])+ return {+ "id": raw["id"],+ "name": raw.get("name") or raw["id"],+ "gguf": None,+ "ggufParts": None,+ "downloadUrl": None,+ "downloadSha256": None,+ "llmModelName": raw.get("llm_model_name"),+ "size": format_size(size_mb),+ "sizeGb": round(size_mb / 1024, 1),+ "vramRequired": round(int(raw["min_gpu_mib"]) / 1024, 1),+ "estimatedRequired": vllm_memory_estimate(raw).device_gib,+ # vLLM serves exactly the context its --max-model-len fixes.+ "contextLength": context,+ "maxContextLength": int(raw["context_length"]),+ "specialty": raw.get("specialty") or "General",+ "description": raw.get("description") or "",+ "quantization": raw.get("quantization"),+ "publisher": model_publisher({**raw, "source_repo": raw.get("hf_repo")}),+ "metadata": {+ "source": "runtime",+ "runtime": "vllm",+ "catalogSource": raw.get("source") or "ods",+ "servedModelName": raw.get("served_model_name"),+ "sourceRepo": raw.get("hf_repo"),+ "sourceRevision": raw.get("revision"),+ "sourceUrl": raw.get("source_url"),+ "license": raw.get("license"),+ "readable": False,+ },+ "appCompatibility": {},+ "status": "loaded" if is_loaded else "downloaded",+ "recommended": False,+ "configured": True,+ "fitsVram": True,+ "activationSupport": None,+ "fitsCurrentVram": None,+ "performance": None,+ }++def build_models_payload(gpu_info: Optional[GPUInfo], loaded_model: Optional[str], live_tps: float,install_dir: str | Path, data_dir: str | Path | None = None,context_length: Optional[int] = None,catalog: list[dict[str, Any]] | None = None,evidence: list[dict[str, Any]] | None = None,downloaded_files_override: dict[str, Any] | None = None) -> dict[str, Any]:+ source_catalog = catalog or load_model_catalog(install_dir)catalog = [model- for model in (normalize_catalog_entry(raw) for raw in (catalog or load_model_catalog(install_dir)))+ for model in (normalize_catalog_entry(raw) for raw in source_catalog)if model is not None]evidence = load_evidence() if evidence is None else evidence@@ -1651,6 +1708,9 @@ def build_models_payload(gpu_info: Optional[GPUInfo], loaded_model: Optional[str])if "turboquant" in normalize_key(runtime_profile_text):runtime = "turboquant"+ if normalize_key(runtime) == "vllm":+ # The installer set a vLLM snapshot up; no GGUF is its selection.+ recommended_entry = Nonedata_root = Path(data_dir) if data_dir is not None else Path(install_dir) / "data"models_dir = model_files_dir(data_root)if downloaded_files_override is not None:@@ -1875,6 +1935,19 @@ def build_models_payload(gpu_info: Optional[GPUInfo], loaded_model: Optional[str}append_model(fallback, path, "downloaded")+ if normalize_key(runtime) == "vllm":+ # A vLLM install serves its LLM_MODEL catalog entry; normalization+ # (GGUF only) leaves vLLM entries out of the list above.+ llm_model = read_env_value("LLM_MODEL", install_dir)+ vllm_entry = next((raw for raw in source_catalog if is_vllm_entry(raw)+ and llm_model in {raw.get("llm_model_name"), raw.get("id")}), None)+ if vllm_entry is not None:+ row = vllm_model_row(vllm_entry, loaded_model, context_length)+ response_models.append(row)+ configured_model_id = row["id"]+ if row["status"] == "loaded":+ current_model_id = row["id"]+if isinstance(loaded_model, str) and loaded_model.strip() and current_model_id is None:# An external runtime can report a model that is neither in our catalog# nor an inspectable local GGUF. Show what is actually running withoutdiff --git a/ods/extensions/services/dashboard-api/routers/models.py b/ods/extensions/services/dashboard-api/routers/models.pyindex c0a8e786e..e8aa4d501 100644--- a/ods/extensions/services/dashboard-api/routers/models.py+++ b/ods/extensions/services/dashboard-api/routers/models.py@@ -54,6 +54,7 @@ from host_agent_client import (request_json as request_agent_json,)from models import ModelLibraryGpu, ModelLibraryResponse+from model_vllm import is_vllm_entryfrom pixel_runtime_state import pixel_stream_activefrom runtime_projection import active_runtime_projectionfrom context_policy import HERMES_MIN_CONTEXT@@ -229,6 +230,16 @@ def _model_activation_mode_denial("model server. Change the model in that server, or rerun the ""installer with an ODS-managed backend to activate downloaded models.")+ elif normalized_backend == "vllm":+ # The vLLM engine serves one installer-set snapshot; activation swaps+ # llama-server GGUFs (the host agent refuses it as well).+ code = "vllm_runtime_installer_managed"+ reason = "vllm_backend_selected"+ message = (+ "This install runs its model on the vLLM engine, and models cannot "+ "be switched here. To use another model, run the ODS installer "+ "again and choose it."+ )elif "unknown" in {effective_mode, configured_mode}:code = "ods_mode_unknown"reason = "mode_unknown"@@ -1517,6 +1528,22 @@ def _hf_requested_projector(details: dict[str, Any], body: dict[str, Any]) -> direturn projector+def _refuse_vllm_entry(model: dict[str, Any], action: str) -> None:+ """409 for a vLLM catalog entry: the installer downloads and runs it.++ Its snapshot is a model folder, not a GGUF the model lifecycle can+ download, switch to or delete (scripts/download-hf-manifest.py and+ LLM_BACKEND=vllm).+ """+ if not is_vllm_entry(model):+ return+ raise HTTPException(status_code=409, detail={+ "error": f"{model.get('name') or model.get('id')} runs only on the vLLM engine. {action}",+ "code": "vllm_model_installer_managed",+ "requestedModelId": model.get("id"),+ })++def _hf_download_payload(record: dict[str, Any]) -> dict[str, Any]:"""The host-agent download request for a model record, projector included."""payload: dict[str, Any] = {@@ -2705,7 +2732,13 @@ def loaded_model_display_name(loaded_model: str | None) -> str | None:"""if not loaded_model:return None- catalog = [entry for entry in map(normalize_catalog_entry, _load_library()) if entry is not None]+ library = _load_library()+ if LLM_BACKEND == "vllm":+ # vLLM serves --served-model-name; its entry has no GGUF to match.+ entry = next((raw for raw in library if is_vllm_entry(raw)+ and raw.get("served_model_name") == loaded_model), None)+ return entry.get("name") if entry else None+ catalog = [entry for entry in map(normalize_catalog_entry, library) if entry is not None]entry = find_catalog_model(catalog, loaded_model, loaded_model)return entry["name"] if entry else None@@ -2901,6 +2934,7 @@ def download_model(model_id: str, api_key: str = Depends(verify_api_key)):model = _find_model_in_library(model_id)if model is None:raise HTTPException(status_code=404, detail=f"Model '{model_id}' not found in library")+ _refuse_vllm_entry(model, "The ODS installer downloads it when it sets that engine up.")bootstrap_conflict = _bootstrap_upgrade_download_conflict()if bootstrap_conflict is not None:@@ -3099,6 +3133,7 @@ def _activate_model(model_id: str, body: dict[str, Any] | None, *, chat_templatemodel = _find_loadable_model(model_id)if model is None:raise HTTPException(status_code=404, detail=f"Model '{model_id}' not found in library or local GGUF files")+ _refuse_vllm_entry(model, "Only the ODS installer sets that engine up and runs it.")requested_context = _requested_activation_context(body)already_active, loaded_model = _already_active_model(model_id, model)@@ -3292,6 +3327,7 @@ def delete_model(model_id: str, api_key: str = Depends(verify_api_key)):model = _find_model_in_library(model_id) or _find_local_gguf_model(model_id)if model is None:raise HTTPException(status_code=404, detail=f"Model '{model_id}' not found in library or local GGUF files")+ _refuse_vllm_entry(model, "The ODS installer manages its files.")payload = {"gguf_file": model["gguf_file"],diff --git a/ods/extensions/services/dashboard-api/setup_chat_route.py b/ods/extensions/services/dashboard-api/setup_chat_route.pyindex 747906c09..8f161d067 100644--- a/ods/extensions/services/dashboard-api/setup_chat_route.py+++ b/ods/extensions/services/dashboard-api/setup_chat_route.py@@ -7,8 +7,10 @@ KEYS = ("LLM_API_URL", "OLLAMA_URL", "LLM_API_BASE_PATH", "LLM_MODEL", "LLM_BACKEND","GGUF_FILE", "ODS_MODEL_SWITCHBOARD", "LITELLM_KEY", "LITELLM_MASTER_KEY","OPEN_WEBUI_LLM_BASE_URL", "OPEN_WEBUI_LLM_API_KEY", "OPEN_WEBUI_TASK_MODEL",- "AMD_INFERENCE_LOCATION",+ "AMD_INFERENCE_LOCATION", "VLLM_SERVED_MODEL",)+LLAMA_SERVER_HOSTS = ("llama-server", "ods-llama-server")+VLLM_HOSTS = ("vllm-server", "ods-vllm-server")def api_base(raw: str, path: str = "/v1") -> str:@@ -45,6 +47,10 @@ def resolve_chat_route(default_url: str) -> tuple[str, str, dict[str, str]]:# A Windows-hosted llama-server requires its key, which only LiteLLM# holds; setup chat reaches it through the gateway.raw = "http://litellm:4000"+ if backend == "vllm" and urlsplit(raw).hostname in LLAMA_SERVER_HOSTS:+ # LLM_BACKEND=vllm turns the in-stack llama-server off; the vLLM+ # engine answers through the gateway (LiteLLM, then model-router).+ raw = "http://litellm:4000"base = api_base(raw, env["LLM_API_BASE_PATH"] or "/v1")target = urlsplit(base)local_gateway = target.scheme == "http" and target.hostname in ("litellm", "ods-litellm") and target.port == 4000@@ -63,7 +69,10 @@ def resolve_chat_route(default_url: str) -> tuple[str, str, dict[str, str]]:elif paired_webui:key = env["OPEN_WEBUI_LLM_API_KEY"]model = env["OPEN_WEBUI_TASK_MODEL"] or model- elif target.hostname in ("llama-server", "ods-llama-server", "host.docker.internal") and backend != "external":+ elif target.hostname in VLLM_HOSTS and backend == "vllm":+ # vLLM serves exactly --served-model-name and refuses any other id.+ model = env["VLLM_SERVED_MODEL"] or model+ elif target.hostname in (*LLAMA_SERVER_HOSTS, "host.docker.internal") and backend != "external":# Every managed llama-server serves its model as --alias <GGUF_FILE>.model = env["GGUF_FILE"] or modelheaders = {"Content-Type": "application/json"}diff --git a/ods/extensions/services/dashboard-api/tests/test_vllm_runtime.py b/ods/extensions/services/dashboard-api/tests/test_vllm_runtime.pynew file mode 100644index 000000000..8f2ea40f2--- /dev/null+++ b/ods/extensions/services/dashboard-api/tests/test_vllm_runtime.py@@ -0,0 +1,313 @@+"""The dashboard on a vLLM install (LLM_BACKEND=vllm, docker-compose.vllm.yml).++vllm-server replaces the in-stack llama-server: the LLM row probes it, its+/v1/models row gives the context (no /props), the Models page shows the+installed vLLM catalog model, readiness names vllm-server (and suggests no+restart during a cold start), setup chat names the served model, and the+model lifecycle refuses vLLM entries with a 409 instead of a 500.+"""++from __future__ import annotations++import json+from pathlib import Path+from unittest.mock import AsyncMock, MagicMock++import pytest++from config import _apply_vllm_service_override+from models import BootstrapStatus, GPUInfo, ServiceStatus++CATALOG = Path(__file__).resolve().parents[4] / "config" / "model-library.json"+SERVED = "GLM-5.3-Flash-NVFP4"+++def _vllm_entry(model_id="glm-5.3-flash-nvfp4-vllm"):+ models = json.loads(CATALOG.read_text(encoding="utf-8"))["models"]+ return next(model for model in models if model["id"] == model_id)+++# --- config: the LLM row probes vllm-server ---+++def test_vllm_route_repoints_the_llm_row_at_vllm_server():+ services = {"llama-server": {"host": "llama-server", "port": 8080, "health": "/health",+ "name": "llama-server (LLM Inference)", "container_name": "ods-llama-server"}}+ _apply_vllm_service_override(services, {"LLM_BACKEND": "vllm"})+ assert services["llama-server"] == {+ "host": "vllm-server", "port": 8000, "health": "/health", "name": "vLLM (LLM Inference)",+ "container_name": "ods-vllm-server", "external_link": False,+ }++[email protected]("backend", ["", "llama-server", "external", "lemonade"])+def test_other_routes_keep_the_llm_row(backend):+ row = {"host": "llama-server", "port": 8080, "health": "/health"}+ services = {"llama-server": dict(row)}+ _apply_vllm_service_override(services, {"LLM_BACKEND": backend})+ assert services["llama-server"] == row+++# --- helpers: context from /v1/models, no /props ---+++def _client(monkeypatch, payload):+ response = MagicMock()+ response.json = MagicMock(return_value=payload)+ client = AsyncMock()+ client.get = AsyncMock(return_value=response)+ monkeypatch.setattr("helpers._get_httpx_client", AsyncMock(return_value=client))+ return client+++async def test_vllm_context_is_the_served_rows_max_model_len(monkeypatch):+ from helpers import get_llama_context_size, get_llama_vision_support++ monkeypatch.setattr("helpers.SERVICES", {"llama-server": {"host": "vllm-server", "port": 8000}})+ monkeypatch.setattr("helpers.LLM_BACKEND", "vllm")+ monkeypatch.setattr("helpers._host_native_llm", lambda: False)+ client = _client(monkeypatch, {"object": "list", "data": [+ {"id": SERVED, "object": "model", "owned_by": "vllm", "max_model_len": 131072}]})++ assert await get_llama_context_size(model_hint=SERVED) == 131072+ assert await get_llama_context_size(model_hint=None) == 131072+ assert [call.args[0] for call in client.get.call_args_list] == ["http://vllm-server:8000/v1/models"] * 2+ # Another id, or a row without a usable length, reports no context.+ assert await get_llama_context_size(model_hint="Other") is None+ _client(monkeypatch, {"data": [{"id": SERVED, "max_model_len": "131072"}]})+ assert await get_llama_context_size(model_hint=SERVED) is None+ _client(monkeypatch, ["not", "an", "object"])+ assert await get_llama_context_size(model_hint=SERVED) is None+ # vLLM reports no modalities: unknown, and nothing asks for /props.+ client = _client(monkeypatch, {})+ assert await get_llama_vision_support() is None+ client.get.assert_not_called()+++VLLM_METRICS = """# HELP vllm:num_requests_running Number of requests in model execution batches.+# TYPE vllm:num_requests_running gauge+vllm:num_requests_running{engine="0",model_name="GLM-5.3-Flash-NVFP4"} 1.0+# TYPE vllm:generation_tokens_total counter+vllm:generation_tokens_total{engine="0",model_name="GLM-5.3-Flash-NVFP4"} 4321.0+vllm:generation_tokens_created{engine="0",model_name="GLM-5.3-Flash-NVFP4"} 1.7e9+vllm:prompt_tokens_total{engine="0",model_name="GLM-5.3-Flash-NVFP4"} 9876.0+"""+++async def test_vllm_metrics_count_tokens_without_claiming_a_rate(monkeypatch, tmp_path):+ import helpers++ monkeypatch.setattr("helpers.SERVICES", {"llama-server": {"host": "vllm-server", "port": 8000}})+ monkeypatch.setattr("helpers.LLM_BACKEND", "vllm")+ monkeypatch.setattr("helpers._host_native_llm", lambda: False)+ monkeypatch.setattr(helpers, "_TOKEN_FILE", tmp_path / "token_counter.json")+ helpers._prev_tokens.clear()+ response = MagicMock()+ response.text = VLLM_METRICS+ response.raise_for_status = MagicMock()+ client = AsyncMock()+ client.get = AsyncMock(return_value=response)+ monkeypatch.setattr("helpers._get_httpx_client", AsyncMock(return_value=client))++ result = await helpers._fetch_llama_metrics(model_hint=SERVED)++ assert result["tokens_per_second"] is None and result["_available"] is False+ assert result["inference_active"] is True+ assert result["_counters"] == (4321.0, None)+ # One /metrics read: vLLM has no llama.cpp /slots to ask while it is busy.+ assert [call.args[0] for call in client.get.call_args_list] == ["http://vllm-server:8000/metrics"]+++# --- readiness: names vllm-server, no restart while it starts ---+++def _readiness(monkeypatch, status, *, startup_pending=False, backend="vllm"):+ import main++ monkeypatch.setattr(main, "LLM_BACKEND", backend)+ statuses = [+ ServiceStatus(id="llama-server", name="vLLM (LLM Inference)", port=8000, external_port=8000,+ status=status, startup_pending=startup_pending),+ ServiceStatus(id="open-webui", name="Open WebUI", port=3000, external_port=3000, status="healthy"),+ ]+ result = main._build_readiness_payload(+ service_statuses=statuses, loaded_model=None if status != "healthy" else SERVED,+ context_size=None if status != "healthy" else 131072, bootstrap_info=BootstrapStatus(active=False),+ host_agent={"available": True}, stt_model_cached=None, stt_model_name="whisper")+ return next(check for check in result["checks"] if check["id"] == "chat")+++def test_readiness_waits_calmly_through_a_vllm_cold_start(monkeypatch):+ chat = _readiness(monkeypatch, "degraded", startup_pending=True)+ assert chat["detail"].startswith("The vLLM engine is starting")+ assert chat.get("repair") is None++ chat = _readiness(monkeypatch, "down")+ assert chat["detail"] == "vllm-server is not healthy"+ assert chat["repair"] == "ods restart vllm-server"++ chat = _readiness(monkeypatch, "healthy")+ assert chat["ready"] is True and chat["detail"] == f"{SERVED} loaded with 131072 context"+++def test_llama_server_readiness_is_unchanged(monkeypatch):+ chat = _readiness(monkeypatch, "degraded", startup_pending=True, backend="llama-server")+ assert chat["detail"] == "llama-server is not healthy"+ assert chat["repair"] == "ods restart llama-server"+++# --- Models page and model lifecycle ---+++def _vllm_install(monkeypatch, tmp_path, env_text):+ import routers.models as models_router++ install_dir = tmp_path / "ods"+ data_dir = install_dir / "data"+ (data_dir / "models").mkdir(parents=True)+ (install_dir / "config").mkdir()+ (install_dir / "config" / "model-library.json").write_text(CATALOG.read_text(encoding="utf-8"),+ encoding="utf-8")+ (install_dir / ".env").write_text(env_text, encoding="utf-8")+ monkeypatch.delenv("LLM_BACKEND", raising=False)+ monkeypatch.delenv("LLM_MODEL", raising=False)+ monkeypatch.setattr(models_router, "INSTALL_DIR", str(install_dir))+ monkeypatch.setattr(models_router, "DATA_DIR", str(data_dir))+ monkeypatch.setattr(models_router, "_LIBRARY_PATH", install_dir / "config" / "model-library.json")+ monkeypatch.setattr(models_router, "_MODELS_DIR", data_dir / "models")+ monkeypatch.setattr(models_router, "_ENV_PATH", install_dir / ".env")+ monkeypatch.setattr(models_router, "ODS_MODE_EFFECTIVE", "local")+ monkeypatch.setattr(models_router, "get_gpu_info", lambda: None)+ monkeypatch.setattr(models_router, "_request_agent_download",+ lambda payload: pytest.fail("a vLLM snapshot reached the GGUF download"))+ monkeypatch.setattr(models_router, "_request_agent_waiting_short_holds",+ lambda *args, **kwargs: pytest.fail("a vLLM snapshot reached the GGUF delete"))+ return models_router, install_dir, data_dir+++VLLM_ENV = ("ODS_MODE=local\nLLM_BACKEND=vllm\nLLM_MODEL=glm-5.3-flash-nvfp4\n"+ "VLLM_SERVED_MODEL=GLM-5.3-Flash-NVFP4\nGGUF_FILE=\nMAX_CONTEXT=131072\n")++[email protected]("method,path", [+ ("post", "/api/models/glm-5.3-flash-nvfp4-vllm/download"),+ ("delete", "/api/models/glm-5.3-flash-nvfp4-vllm"),+ ("post", "/api/models/glm-5.3-flash-nvfp4-vllm/load"),+])+def test_vllm_entries_are_refused_with_a_clear_409(test_client, monkeypatch, tmp_path, method, path):+ models_router, _install, _data = _vllm_install(+ monkeypatch, tmp_path, "ODS_MODE=local\nLLM_BACKEND=llama-server\nGGUF_FILE=a.gguf\n")+ monkeypatch.setattr(models_router, "LLM_BACKEND", "llama-server")+ monkeypatch.setattr(models_router, "_configured_ods_mode", lambda: "local")+ monkeypatch.setattr(models_router, "_windows_hosted_runtime", lambda: False)+ response = getattr(test_client, method)(path, headers=test_client.auth_headers)++ assert response.status_code == 409+ detail = response.json()["detail"]+ assert detail["code"] == "vllm_model_installer_managed"+ assert detail["requestedModelId"] == "glm-5.3-flash-nvfp4-vllm"+ assert detail["error"].startswith("GLM 5.3 Flash NVFP4 (vLLM) runs only on the vLLM engine.")+ assert "ODS installer" in detail["error"]+++def test_a_vllm_install_refuses_model_switches_before_the_host_agent():+ import routers.models as models_router++ denial = models_router._model_activation_mode_denial("local", "local", "vllm")+ assert denial["code"] == "vllm_runtime_installer_managed"+ assert "vLLM engine" in denial["message"] and "installer" in denial["message"]+ assert models_router._model_activation_mode_denial("local", "local", "llama-server") is None+++def _gpu():+ return GPUInfo(name="NVIDIA GB300", memory_used_mb=200_000, memory_total_mb=288_000, memory_percent=69.0,+ utilization_percent=10, temperature_c=40, gpu_backend="nvidia")++[email protected]("loaded,status", [(SERVED, "loaded"), (None, "downloaded")])+def test_models_page_shows_the_installed_vllm_model(test_client, monkeypatch, tmp_path, loaded, status):+ models_router, _install, _data = _vllm_install(monkeypatch, tmp_path, VLLM_ENV)+ monkeypatch.setattr(models_router, "get_gpu_info", _gpu)+ monkeypatch.setattr(models_router, "get_loaded_model", AsyncMock(return_value=loaded))+ monkeypatch.setattr(models_router, "_fetch_llama_loaded_model", AsyncMock(return_value=None))+ monkeypatch.setattr(models_router, "get_llama_metrics", AsyncMock(return_value={"tokens_per_second": 0}))+ monkeypatch.setattr(models_router, "get_llama_context_size",+ AsyncMock(return_value=131072 if loaded else None))+ monkeypatch.setattr(models_router, "LLM_BACKEND", "vllm")+ monkeypatch.setattr(models_router, "record_model_performance",+ lambda *args, **kwargs: pytest.fail("no measured throughput on vLLM"))+ response = test_client.get("/api/models", headers=test_client.auth_headers)++ assert response.status_code == 200+ payload = response.json()+ rows = [row for row in payload["models"] if row["id"] == "glm-5.3-flash-nvfp4-vllm"]+ assert len(rows) == 1+ row = rows[0]+ assert row["status"] == status+ assert row["name"] == "GLM 5.3 Flash NVFP4 (vLLM)"+ assert row["size"] == "199.7 GB" and row["sizeGb"] == 199.7+ assert row["contextLength"] == 131072 and row["maxContextLength"] == 131072+ assert row["quantization"] == "NVFP4"+ assert row["vramRequired"] == 239.0+ assert row["metadata"]["source"] == "runtime" and row["metadata"]["runtime"] == "vllm"+ assert row["metadata"]["servedModelName"] == SERVED+ assert row["gguf"] is None and row["downloadUrl"] is None+ assert payload["configuredModel"] == "glm-5.3-flash-nvfp4-vllm"+ assert payload["llmBackend"] == "vllm"+ assert payload["currentModel"] == ("glm-5.3-flash-nvfp4-vllm" if loaded else None)+ # The served model is the catalog row, never an anonymous runtime row,+ # and no GGUF claims to be the installer's selection.+ assert not [row for row in payload["models"] if row["id"].startswith("runtime-")]+ assert not [row for row in payload["models"] if row["recommended"]]+ # Only the installed vLLM entry is listed; others need the installer.+ assert "qwen3.8-flash-next-nvfp4-vllm" not in {row["id"] for row in payload["models"]}+++def test_status_names_the_served_vllm_model_by_its_catalog_name(monkeypatch, tmp_path):+ models_router, _install, _data = _vllm_install(monkeypatch, tmp_path, VLLM_ENV)+ monkeypatch.setattr(models_router, "LLM_BACKEND", "vllm")+ assert models_router.loaded_model_display_name(SERVED) == "GLM 5.3 Flash NVFP4 (vLLM)"+ assert models_router.loaded_model_display_name("Unknown-Model") is None+ # A llama-server route never names a vLLM entry.+ monkeypatch.setattr(models_router, "LLM_BACKEND", "llama-server")+ assert models_router.loaded_model_display_name(SERVED) is None+++def test_llama_install_never_lists_vllm_entries(monkeypatch, tmp_path):+ from performance_oracle import build_models_payload++ install = tmp_path / "ods"+ install.mkdir()+ (install / ".env").write_text("LLM_BACKEND=llama-server\nLLM_MODEL=glm-5.3-flash-nvfp4\n", encoding="utf-8")+ monkeypatch.delenv("LLM_BACKEND", raising=False)+ monkeypatch.delenv("LLM_MODEL", raising=False)+ payload = build_models_payload(None, None, 0.0, install, install / "data", None,+ catalog=[_vllm_entry()], downloaded_files_override={})+ assert payload["models"] == []+++# --- setup chat ---++[email protected]("values,url,model", [+ # The installer's gateway route: LiteLLM, then model-router, to vLLM.+ ({"LLM_API_URL": "http://litellm:4000", "ODS_MODEL_SWITCHBOARD": "enabled"},+ "http://litellm:4000/v1", "ods/current"),+ # A stale llama-server URL goes through the gateway, never the off container.+ ({"LLM_API_URL": "http://llama-server:8080", "ODS_MODEL_SWITCHBOARD": "enabled"},+ "http://litellm:4000/v1", "ods/current"),+ # Straight to the engine: the exact served id.+ ({"LLM_API_URL": "http://vllm-server:8000"}, "http://vllm-server:8000/v1", SERVED),+])+def test_setup_chat_reaches_the_vllm_engine(monkeypatch, tmp_path, values, url, model):+ import config+ from setup_chat_route import resolve_chat_route++ monkeypatch.setattr(config, "INSTALL_DIR", str(tmp_path))+ values = {"LLM_BACKEND": "vllm", "LLM_MODEL": "glm-5.3-flash-nvfp4", "VLLM_SERVED_MODEL": SERVED,+ "LITELLM_KEY": "gateway-key", **values}+ (tmp_path / ".env").write_text("".join(f"{key}={value}\n" for key, value in values.items()))+ actual_url, actual_model, _headers = resolve_chat_route("http://vllm-server:8000")+ assert (actual_url, actual_model) == (url + "/chat/completions", model)diff --git a/ods/extensions/services/model-router/app/main.py b/ods/extensions/services/model-router/app/main.pyindex 1ce966107..8aeb02c0c 100644--- a/ods/extensions/services/model-router/app/main.py+++ b/ods/extensions/services/model-router/app/main.py@@ -879,10 +879,36 @@ class RouterError(Exception):self.message = message+# ODS thinking levels. A vllm endpoint maps each to the served model's own+# chat-template arguments (config/model-library.json thinking_template_kwargs,+# rendered into endpoints.json as "thinking").+THINKING_LEVELS = ("off", "low", "medium", "high")+# OpenAI reasoning_effort values, read as ODS levels.+_REASONING_EFFORT_LEVELS = {+ "none": "off", "off": "off", "minimal": "low", "low": "low",+ "medium": "medium", "high": "high", "xhigh": "high",+}+++def _valid_thinking(value: Any) -> bool:+ """An endpoint ``thinking`` map: {"default": level, "templateKwargs": {level: {...}}}."""+ if not _has_keys(value, {"default", "templateKwargs"}, {"default", "templateKwargs"}):+ return False+ levels = value["templateKwargs"]+ return (+ value["default"] in THINKING_LEVELS+ and isinstance(levels, dict)+ and set(levels) == set(THINKING_LEVELS)+ and all(isinstance(kwargs, dict) and all(isinstance(key, str) for key in kwargs)+ for kwargs in levels.values())+ )++def _load_endpoints() -> dict[str, dict[str, Any]]:- """Allowlist endpointId -> {baseUrl, apiKeyEnv?}, re-read when the file- changes on disk (activation re-renders it). Retains the last good- allowlist while the file is missing or invalid."""+ """Allowlist endpointId -> {baseUrl, apiKeyEnv?, thinking?}, re-read when+ the file changes on disk (activation re-renders it). Retains the last good+ allowlist while the file is missing or invalid. An endpoint whose thinking+ map is malformed is left out, so its route fails closed."""try:stat = ENDPOINTS_PATH.stat()except OSError as exc:@@ -900,9 +926,14 @@ def _load_endpoints() -> dict[str, dict[str, Any]]:base_url = str(entry.get("baseUrl") or "")if not endpoint_id or not base_url.startswith(("http://", "https://")):continue+ thinking = entry.get("thinking")+ if thinking is not None and not _valid_thinking(thinking):+ logger.error("endpoint %s has a malformed thinking map; it is not allowlisted", endpoint_id)+ continueendpoints[endpoint_id] = {"baseUrl": base_url.rstrip("/"),"apiKeyEnv": str(entry.get("apiKeyEnv") or ""),+ "thinking": thinking,}except (OSError, ValueError) as exc:logger.error("endpoints allowlist unreadable; retaining previous: %s", exc)@@ -1129,6 +1160,7 @@ def _active_route() -> dict[str, Any]:"endpointId": endpoint_id,"baseUrl": endpoint["baseUrl"],"apiKeyEnv": endpoint["apiKeyEnv"],+ "thinking": endpoint.get("thinking"),"queueMode": str(availability.get("mode") or "serve_active") == "queue",}@@ -1146,6 +1178,60 @@ def _llama_tool_compat(route: dict[str, Any]) -> bool:return route["backendKind"] == "llama-server"+def _thinking_level(payload: dict[str, Any], default: str) -> str:+ """The ODS thinking level a chat request asks for.++ A ``reasoning_effort`` names it, top-level (OpenAI's field) or in+ ``chat_template_kwargs`` (vLLM passes both to the template alike);+ otherwise the ODS consumers' on/off switch,+ ``chat_template_kwargs.enable_thinking`` (false is off, true the route's+ default level, at least medium); otherwise the default.+ """+ options = payload.get("chat_template_kwargs")+ options = options if isinstance(options, dict) else {}+ effort = payload.get("reasoning_effort")+ if effort is None:+ effort = options.get("reasoning_effort")+ if effort is not None:+ level = _REASONING_EFFORT_LEVELS.get(effort.strip().lower()) if isinstance(effort, str) else None+ if level is None:+ raise RouterError(400, "invalid_request_error",+ "reasoning_effort must be one of " + ", ".join(_REASONING_EFFORT_LEVELS))+ return level+ enabled = options.get("enable_thinking")+ if enabled is False:+ return "off"+ if enabled is True and default == "off":+ return "medium"+ return default+++def _vllm_thinking_payload(payload: dict[str, Any], thinking: dict[str, Any]) -> tuple[dict[str, Any], str]:+ """The request with its thinking level in the served model's own template terms.++ Chat templates disagree: GLM-5.3 reads only reasoning_effort low or high+ (any other value thinks at "max", about twice as slow) and Qwen3.8 fails+ with HTTP 400 on any reasoning_effort but xhigh, medium and low, so no+ level, reasoning_effort or enable_thinking is forwarded as sent. The+ level's arguments replace them; the other chat_template_kwargs a client+ sends are the model's own and stay. Returns the new payload and the level.+ """+ options = payload.get("chat_template_kwargs")+ if options is not None and not isinstance(options, dict):+ raise RouterError(400, "invalid_request_error", "chat_template_kwargs must be an object")+ level = _thinking_level(payload, thinking["default"])+ kwargs = {+ **thinking["templateKwargs"][level],+ **{key: value for key, value in (options or {}).items()+ if key not in {"enable_thinking", "reasoning_effort"}},+ }+ translated = {key: value for key, value in payload.items()+ if key not in {"reasoning_effort", "chat_template_kwargs"}}+ if kwargs:+ translated["chat_template_kwargs"] = kwargs+ return translated, level++def _offered_tool_evidence(payload: dict[str, Any]) -> dict[str, Any]:"""Fingerprint the outgoing tools, never retain their names or schema text.@@ -2039,6 +2125,15 @@ async def _forward_inner(request: Request, path: str, payload: dict[str, Any],"type": "route_changed", "code": "409"}}, status_code=409), Falsepayload["model"] = route["runtimeModelId"]+ thinking_level = None+ if path == "/v1/chat/completions" and route["backendKind"] == "vllm" and route["thinking"]:+ try:+ payload, thinking_level = _vllm_thinking_payload(payload, route["thinking"])+ except RouterError as exc:+ return JSONResponse(+ {"error": {"message": exc.message, "type": exc.code, "code": str(exc.status)}},+ status_code=exc.status,+ ), False# The stable client alias hides the selected template from agents. Adapt# only outgoing Nemo history after resolving the current route; audit and# response IDs remain unchanged, including across model switches.@@ -2094,6 +2189,8 @@ async def _forward_inner(request: Request, path: str, payload: dict[str, Any],}if thinking_tool_compat:ods_headers["X-ODS-Tool-Stream-Compatibility"] = "thinking-nonstream"+ if thinking_level is not None:+ ods_headers["X-ODS-Thinking-Level"] = thinking_levelurl = route["baseUrl"] + pathclient: httpx.AsyncClient = app.state.httpdiff --git a/ods/extensions/services/model-router/tests/test_router.py b/ods/extensions/services/model-router/tests/test_router.pyindex 94574f5d8..5144d3950 100644--- a/ods/extensions/services/model-router/tests/test_router.py+++ b/ods/extensions/services/model-router/tests/test_router.py@@ -1502,6 +1502,114 @@ class TestVllmRoute:"key": "ok"}+# The two catalog maps (config/model-library.json thinking_template_kwargs),+# as render-runtime-configs.py writes them into the vllm endpoint.+_GLM_THINKING = {"default": "medium", "templateKwargs": {+ "off": {"reasoning_effort": "low"}, "low": {"reasoning_effort": "low"},+ "medium": {"reasoning_effort": "high"}, "high": {}}}+_QWEN_THINKING = {"default": "medium", "templateKwargs": {+ "off": {"enable_thinking": False}, "low": {"reasoning_effort": "low"},+ "medium": {"reasoning_effort": "medium"}, "high": {"reasoning_effort": "xhigh"}}}+++def _write_vllm_thinking_route(tmp_path, write_state, thinking):+ (tmp_path / "endpoints.json").write_text(json.dumps({"endpoints": [+ {"id": "vllm-default", "baseUrl": "http://vllm-server:8000", "thinking": thinking},+ ]}), encoding="utf-8")+ write_state(runtime=_VLLM_MODEL, endpoint="vllm-default",+ mutate=lambda doc: doc["active"]["backend"].update(kind="vllm"))+++class TestVllmThinking:+ """ODS thinking levels reach vLLM in the served model's own template terms."""++ def _send(self, router, tmp_path, thinking, extra, kind="vllm"):+ mod, client, write_state, _calls = router+ if kind == "vllm":+ _write_vllm_thinking_route(tmp_path, write_state, thinking)+ else:+ write_state()+ sent = []++ def handler(request):+ sent.append(json.loads(request.content))+ return httpx.Response(200, json={"model": sent[-1]["model"], "choices": [+ {"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}]})++ _use_upstream(mod, handler)+ response = client.post("/v1/chat/completions", json={+ "model": "ods/current", "messages": [{"role": "user", "content": "hi"}], **extra})+ return response, sent++ @pytest.mark.parametrize("thinking,extra,expected,level", [+ # No hint: the catalog default, medium, which GLM spells "high".+ (_GLM_THINKING, {}, {"reasoning_effort": "high"}, "medium"),+ (_QWEN_THINKING, {}, {"reasoning_effort": "medium"}, "medium"),+ # The ODS consumers' switch: false is off; GLM cannot stop thinking.+ (_GLM_THINKING, {"chat_template_kwargs": {"enable_thinking": False}},+ {"reasoning_effort": "low"}, "off"),+ (_QWEN_THINKING, {"chat_template_kwargs": {"enable_thinking": False}},+ {"enable_thinking": False}, "off"),+ (_QWEN_THINKING, {"chat_template_kwargs": {"enable_thinking": True}},+ {"reasoning_effort": "medium"}, "medium"),+ # OpenAI's reasoning_effort, never forwarded literally: Qwen's+ # template fails on "high" and GLM's thinks at max on "medium".+ (_QWEN_THINKING, {"reasoning_effort": "high"}, {"reasoning_effort": "xhigh"}, "high"),+ (_GLM_THINKING, {"reasoning_effort": "high"}, None, "high"),+ (_GLM_THINKING, {"reasoning_effort": "low"}, {"reasoning_effort": "low"}, "low"),+ (_QWEN_THINKING, {"reasoning_effort": "none"}, {"enable_thinking": False}, "off"),+ (_QWEN_THINKING, {"reasoning_effort": "minimal"}, {"reasoning_effort": "low"}, "low"),+ # vLLM hands chat_template_kwargs.reasoning_effort to the template+ # like the top-level field, so it is translated the same way.+ (_QWEN_THINKING, {"chat_template_kwargs": {"reasoning_effort": "high"}},+ {"reasoning_effort": "xhigh"}, "high"),+ (_GLM_THINKING, {"chat_template_kwargs": {"reasoning_effort": "medium"}},+ {"reasoning_effort": "high"}, "medium"),+ # A client's other template arguments are the model's own and stay.+ (_QWEN_THINKING, {"chat_template_kwargs": {"enable_thinking": True, "preserve_thinking": True}},+ {"reasoning_effort": "medium", "preserve_thinking": True}, "medium"),+ ])+ def test_level_becomes_the_models_template_arguments(+ self, router, tmp_path, thinking, extra, expected, level):+ response, sent = self._send(router, tmp_path, thinking, extra)++ assert response.status_code == 200+ assert len(sent) == 1+ assert sent[0].get("chat_template_kwargs") == expected+ assert "reasoning_effort" not in sent[0]+ assert response.headers["X-ODS-Thinking-Level"] == level++ def test_unknown_reasoning_effort_is_refused_before_vllm(self, router, tmp_path):+ response, sent = self._send(router, tmp_path, _QWEN_THINKING, {"reasoning_effort": "max"})++ assert response.status_code == 400+ assert response.json()["error"]["type"] == "invalid_request_error"+ assert sent == []++ def test_non_object_template_arguments_are_refused(self, router, tmp_path):+ response, sent = self._send(router, tmp_path, _GLM_THINKING, {"chat_template_kwargs": ["x"]})++ assert response.status_code == 400+ assert sent == []++ def test_llama_server_route_forwards_thinking_unchanged(self, router, tmp_path):+ extra = {"chat_template_kwargs": {"enable_thinking": False}, "reasoning_effort": "high"}+ response, sent = self._send(router, tmp_path, None, extra, kind="llama-server")++ assert response.status_code == 200+ assert sent[0]["chat_template_kwargs"] == {"enable_thinking": False}+ assert sent[0]["reasoning_effort"] == "high"+ assert "X-ODS-Thinking-Level" not in response.headers++ def test_malformed_thinking_map_fails_the_route_closed(self, router, tmp_path):+ broken = {"default": "medium", "templateKwargs": {"off": {}, "low": {}, "medium": {}}}+ response, sent = self._send(router, tmp_path, broken, {})++ assert response.status_code == 503+ assert response.json()["error"]["type"] == "endpoint_not_allowlisted"+ assert sent == []++class TestStateTrust:def test_schema_invalid_state_is_not_routable(self, router):mod, client, write_state, calls = routerdiff --git a/ods/extensions/services/pixel-agent/host/pixel_ingress.mjs b/ods/extensions/services/pixel-agent/host/pixel_ingress.mjsindex 9ad17c4e7..e6e3ce374 100644--- a/ods/extensions/services/pixel-agent/host/pixel_ingress.mjs+++ b/ods/extensions/services/pixel-agent/host/pixel_ingress.mjs@@ -169,6 +169,8 @@ const ALLOWED_SERVICES = new Set(["dashboard-api","ods-llama-server","llama-server",+ "ods-vllm-server",+ "vllm-server","ods-searxng","searxng","ods-langfuse",@@ -1786,6 +1788,46 @@ export async function dockerRuntime(deps = defaultDeps) {return { model, context_length: contextLength };}+function commandFlag(command, flag) {+ const index = command.indexOf(flag);+ if (index >= 0) return command[index + 1];+ const joined = command.find((item) => item.startsWith(`${flag}=`));+ return joined === undefined ? undefined : joined.slice(flag.length + 1);+}++// LLM_BACKEND=vllm: ods-vllm-server serves --served-model-name with the+// context its --max-model-len (catalog VLLM_ARGS) fixes; there is no+// ods-llama-server, or only one an earlier install left stopped.+export async function dockerVllmRuntime(deps = defaultDeps) {+ const { stdout } = await execFilePromise(+ deps.execFile,+ "docker",+ ["inspect", "ods-vllm-server", "--format", "{{json .Config.Cmd}}"],+ { timeout: DOCKER_TIMEOUT_MS, maxBuffer: 64 * 1024, windowsHide: true }+ );+ let command;+ try {+ command = JSON.parse(String(stdout || ""));+ } catch {+ return null;+ }+ if (!Array.isArray(command) || !command.every((item) => typeof item === "string")) {+ return null;+ }+ const model = commandFlag(command, "--served-model-name");+ const contextLength = Number(commandFlag(command, "--max-model-len"));+ if (+ typeof model !== "string" ||+ !MODEL_NAME_RE.test(model) ||+ !Number.isInteger(contextLength) ||+ contextLength < 4096 ||+ contextLength > 10_000_000+ ) {+ return null;+ }+ return { model, context_length: contextLength };+}+export async function dockerRouterRuntime(deps = defaultDeps) {const { stdout } = await execFilePromise(deps.execFile,@@ -1884,7 +1926,9 @@ export async function writeStatus(// to label that provider with an unrelated local llama model.if (configuredRuntime === undefined && runtime === null && docker === "ok") {try {- runtime = await dockerRuntime(deps);+ runtime = apps.some(({ name }) => name === "ods-vllm-server")+ ? await dockerVllmRuntime(deps)+ : await dockerRuntime(deps);} catch {runtime = null;}diff --git a/ods/extensions/services/pixel-agent/plugin/projection.mjs b/ods/extensions/services/pixel-agent/plugin/projection.mjsindex 4238c623d..ea04c6633 100644--- a/ods/extensions/services/pixel-agent/plugin/projection.mjs+++ b/ods/extensions/services/pixel-agent/plugin/projection.mjs@@ -50,6 +50,9 @@ registerApp(["ods-whisper", "whisper"], "Whisper", "speech-to-text API", "whisperegisterApp(["ods-embeddings", "embeddings"], "TEI", "embeddings API", "embeddings", "/info");registerApp(["ods-litellm", "litellm"], "LiteLLM", "model gateway", "litellm");registerApp(["ods-llama-server", "llama-server"], "llama-server", "local model API", "llama");+// LLM_BACKEND=vllm: the vLLM engine publishes no host port; model-router+// reaches it on the ODS network, so it has no local URL.+registerApp(["ods-vllm-server", "vllm-server"], "vLLM", "local model engine behind the model router", null);registerApp(["ods-privacy-shield"], "Privacy Shield", "PII protection API", "privacy_shield", "/docs");registerApp(["ods-token-spy"], "Token Spy", "usage monitor", "token_spy", "/dashboard");registerApp(["ods-ape"], "APE", "agent policy engine", "ape");@@ -77,6 +80,8 @@ const ALLOWED_SERVICES = new Set(["dashboard-api","ods-llama-server","llama-server",+ "ods-vllm-server",+ "vllm-server","ods-searxng","searxng","ods-langfuse",@@ -305,13 +310,16 @@ function validAppPorts(value) {function enrichApp(app, appPorts) {const metadata = APP_DIRECTORY.get(app.name);if (!metadata) return { name: app.name, status: app.status };- return {+ const enriched = {name: app.name,status: app.status,display_name: metadata.display_name,purpose: metadata.purpose,- url: `http://localhost:${appPorts[metadata.port_key]}${metadata.url_path}`,};+ if (metadata.port_key !== null) {+ enriched.url = `http://localhost:${appPorts[metadata.port_key]}${metadata.url_path}`;+ }+ return enriched;}// Validate a fully parsed projection object against the strict schema.diff --git a/ods/extensions/services/pixel-agent/plugin/tool-content.mjs b/ods/extensions/services/pixel-agent/plugin/tool-content.mjsindex 842426aa9..b30fa8fd5 100644--- a/ods/extensions/services/pixel-agent/plugin/tool-content.mjs+++ b/ods/extensions/services/pixel-agent/plugin/tool-content.mjs@@ -22,11 +22,16 @@ function freshness(value) {return value ? "stale" : "current";}+function appDescription({ name, status, display_name: displayName, purpose, url }) {+ if (!displayName || !purpose) return `${name} (${status})`;+ // An app without a host port (the vLLM engine) has no local URL.+ return url+ ? `${displayName} [${name}] (${status}; ${purpose}; ${url})`+ : `${displayName} [${name}] (${status}; ${purpose})`;+}+function appList(apps) {- return apps.map(({ name, status, display_name: displayName, purpose, url }) => {- if (!displayName || !purpose || !url) return `${name} (${status})`;- return `${displayName} [${name}] (${status}; ${purpose}; ${url})`;- }).join(", ");+ return apps.map(appDescription).join(", ");}export function statusToolText(projection) {@@ -54,10 +59,7 @@ export function appsToolText(projection) {if (projection.app_count === 0) {return `ODS reports 0 of 0 applications online in its ${freshness(projection.stale)} projection at ${projection.timestamp}. ${EVIDENCE_BOUNDARY}`;}- const first = projection.apps[0];- const firstDescription = first.display_name- ? `${first.display_name} [${first.name}] (${first.status}; ${first.purpose}; ${first.url})`- : `${first.name} (${first.status})`;+ const firstDescription = appDescription(projection.apps[0]);return [`ODS reports ${projection.online_app_count} of ${projection.app_count} applications online in its ${freshness(projection.stale)} projection at ${projection.timestamp}.`,`The first is ${firstDescription}.`,diff --git a/ods/extensions/services/pixel-agent/tests/pixel_ingress.test.mjs b/ods/extensions/services/pixel-agent/tests/pixel_ingress.test.mjsindex 400a408de..e2b229a81 100644--- a/ods/extensions/services/pixel-agent/tests/pixel_ingress.test.mjs+++ b/ods/extensions/services/pixel-agent/tests/pixel_ingress.test.mjs@@ -35,6 +35,7 @@ const {validateConfig,dockerApps,dockerRuntime,+ dockerVllmRuntime,dockerRouterRuntime,writeStatus,start,@@ -1957,6 +1958,77 @@ test("runtime projection rejects paths, malformed contexts, and non-JSON output"}});+// LLM_BACKEND=vllm (docker-compose.vllm.yml): the command Compose builds+// for ods-vllm-server, catalog VLLM_ARGS included.+const VLLM_COMMAND = [+ "--model", "/model", "--served-model-name", "GLM-5.3-Flash-NVFP4", "--host", "0.0.0.0",+ "--port", "8000", "--gpu-memory-utilization", "0.90", "--max-model-len", "131072",+ "--kv-cache-dtype", "fp8", "--speculative-config.method=mtp",+];++test("vLLM runtime projection reads the served name and context from ods-vllm-server", async () => {+ const fakeDocker = (cmd, args, opts, callback) => {+ assert.equal(cmd, "docker");+ assert.deepEqual(args, ["inspect", "ods-vllm-server", "--format", "{{json .Config.Cmd}}"]);+ assert.ok(opts.timeout > 0, "must have explicit timeout");+ callback(null, JSON.stringify(VLLM_COMMAND), "");+ };+ assert.deepEqual(await dockerVllmRuntime({ execFile: fakeDocker }), {+ model: "GLM-5.3-Flash-NVFP4",+ context_length: 131072,+ });+ for (const command of [+ VLLM_COMMAND.filter((item) => item !== "--max-model-len" && item !== "131072"),+ VLLM_COMMAND.map((item) => (item === "131072" ? "2048" : item)),+ VLLM_COMMAND.map((item) => (item === "GLM-5.3-Flash-NVFP4" ? "../secret" : item)),+ ["--model", "/model", "--served-model-name=Joined-Name", "--max-model-len=65536"],+ "not-json",+ ]) {+ const reply = (_cmd, _args, _opts, callback) => {+ callback(null, command === "not-json" ? command : JSON.stringify(command), "");+ };+ const expected = Array.isArray(command) && command.includes("--max-model-len=65536")+ ? { model: "Joined-Name", context_length: 65536 }+ : null;+ assert.deepEqual(await dockerVllmRuntime({ execFile: reply }), expected);+ }+});++test("status on a vLLM install lists the engine and reads its runtime, not a stale llama-server", async () => {+ const statusFile = path.join(DIR, "vllm-runtime-status.json");+ const inspected = [];+ const fakeDocker = (_cmd, args, _opts, callback) => {+ if (args[0] === "ps") {+ callback(null, [+ JSON.stringify({ Names: "ods-vllm-server", Status: "Up 4 minutes (health: starting)" }),+ // An earlier llama-server install left its container behind, stopped.+ JSON.stringify({ Names: "ods-llama-server", State: "exited", Status: "Exited (0) 2 days ago" }),+ "",+ ].join("\n"), "");+ return;+ }+ if (args[0] === "exec") {+ callback(new Error("model-router state unavailable"));+ return;+ }+ inspected.push(args[1]);+ callback(null, JSON.stringify(args[1] === "ods-vllm-server" ? VLLM_COMMAND+ : ["--model", "/models/Stale.gguf", "--ctx-size", "32768"]), "");+ };+ const projection = await writeStatus(true, 18999, statusFile, "2.6.0", {+ fetch: async () => ({ status: 200, body: { cancel: async () => {} } }),+ execFile: fakeDocker,+ setTimeout,+ clearTimeout,+ });+ assert.deepEqual(projection.apps, [+ { name: "ods-llama-server", status: "stopped" },+ { name: "ods-vllm-server", status: "starting" },+ ]);+ assert.deepEqual(projection.runtime, { model: "GLM-5.3-Flash-NVFP4", context_length: 131072 });+ assert.deepEqual(inspected, ["ods-vllm-server"]);+});+test("router runtime projection returns the exact active stable-alias target", async () => {const fakeDocker = (cmd, args, opts, callback) => {assert.equal(cmd, "docker");diff --git a/ods/extensions/services/pixel-agent/tests/projection.test.mjs b/ods/extensions/services/pixel-agent/tests/projection.test.mjsindex 142c89ccd..c436c60a7 100644--- a/ods/extensions/services/pixel-agent/tests/projection.test.mjs+++ b/ods/extensions/services/pixel-agent/tests/projection.test.mjs@@ -196,6 +196,24 @@ test("application payloads expose allowlisted purposes and configured local URLs]);});+test("the vLLM engine is an allowlisted app with no local URL", async () => {+ const raw = JSON.stringify(goodProjection({+ apps: [+ { name: "ods-vllm-server", status: "starting" },+ { name: "vllm-server", status: "healthy" },+ ],+ }));+ const fsImpl = memoryFs({ [FIXED]: makeEntry(raw) });+ const projection = await readProjection(FIXED, fsImpl, Date.now());++ assert.deepEqual(appsPayload(projection).apps, [+ { name: "ods-vllm-server", status: "starting", display_name: "vLLM",+ purpose: "local model engine behind the model router" },+ { name: "vllm-server", status: "healthy", display_name: "vLLM",+ purpose: "local model engine behind the model router" },+ ]);+});+test("starting application status is accepted", async () => {const raw = JSON.stringify(goodProjection({diff --git a/ods/extensions/services/pixel-agent/tests/tool_content.test.mjs b/ods/extensions/services/pixel-agent/tests/tool_content.test.mjsindex 0964a9d72..716a0af79 100644--- a/ods/extensions/services/pixel-agent/tests/tool_content.test.mjs+++ b/ods/extensions/services/pixel-agent/tests/tool_content.test.mjs@@ -43,6 +43,22 @@ test("apps result states the count and first application without JSON parsing",assert.ok(!text.startsWith("{"));});+test("an application without a host port is described without a URL", () => {+ // LLM_BACKEND=vllm: model-router reaches the vLLM engine on the ODS network.+ const vllm = {+ name: "ods-vllm-server",+ status: "starting",+ display_name: "vLLM",+ purpose: "local model engine behind the model router",+ };+ for (const apps of [[vllm, projection.apps[0]], [projection.apps[0], vllm]]) {+ const text = appsToolText({ ...projection, apps });+ assert.match(text, /vLLM \[ods-vllm-server\] \(starting; local model engine behind the model router\)/);+ assert.doesNotMatch(text, /undefined/);+ assert.match(text, /Dashboard \[ods-dashboard\] \(healthy; ODS control center; http:\/\/localhost:3001\/\)/);+ }+});+test("status result states each bounded host fact in natural language", () => {const text = statusToolText(projection);assert.match(text, /Pixel availability: available/);diff --git a/ods/lib/preflight-llm-route.sh b/ods/lib/preflight-llm-route.shindex ace9526e3..65484e289 100644--- a/ods/lib/preflight-llm-route.sh+++ b/ods/lib/preflight-llm-route.sh@@ -7,19 +7,30 @@ ods_preflight_host_native_llm() {[[ -n "${NATIVE_LLM_BASE_URL:-}" ]]}+# True when .env selects the vLLM engine (LLM_BACKEND=vllm,+# docker-compose.vllm.yml): the vllm-server container serves the model on the+# ODS network without a host port, and the in-stack llama-server is off.+# Clients reach it through model-router and LiteLLM.+ods_preflight_vllm_engine() {+ local backend+ backend=$(printf '%s' "${LLM_BACKEND:-}" | tr '[:upper:]' '[:lower:]')+ [[ "$backend" == "vllm" ]]+}+ods_preflight_uses_litellm() {# ods-preflight.sh runs under macOS /bin/bash 3.2, which has no Bash 4 case conversion.local modemode=$(printf '%s' "${ODS_MODE:-local}" | tr '[:upper:]' '[:lower:]')- ods_preflight_host_native_llm || [[ -n "${EXTERNAL_LLM_URL:-}" ]] || [[ "$mode" == "cloud" ]]+ ods_preflight_host_native_llm || ods_preflight_vllm_engine \+ || [[ -n "${EXTERNAL_LLM_URL:-}" ]] || [[ "$mode" == "cloud" ]]}# Status leaves out the in-stack inference services a LiteLLM route does not# start: llama-server always, model-router unless a host-native llama-server-# keeps it for model switching.+# keeps it for model switching or the vLLM engine is served through it.ods_status_skips_managed_inference() {local sid="$1"ods_preflight_uses_litellm || return 1[[ "$sid" == "llama-server" ]] && return 0- [[ "$sid" == "model-router" ]] && ! ods_preflight_host_native_llm+ [[ "$sid" == "model-router" ]] && ! ods_preflight_host_native_llm && ! ods_preflight_vllm_engine}diff --git a/ods/ods-cli b/ods/ods-cliindex 88ad79bdf..8c913d6f5 100755--- a/ods/ods-cli+++ b/ods/ods-cli@@ -807,10 +807,17 @@ load_env() {#=============================================================================. "$SCRIPT_DIR/lib/service-registry.sh"-# Resolve a user-provided service name to a compose service ID+# Resolve a user-provided service name to the compose service ID that serves+# it on this install. With LLM_BACKEND=vllm the vLLM engine serves the model+# and the in-stack llama-server is profiled out (docker-compose.vllm.yml).+# Naming llama-server to Compose would enable its profile and start a second+# engine on the GPU, so "llm" and "llama-server" mean vllm-server there.resolve_service() {- local resolved+ local resolved backend="${LLM_BACKEND:-}"resolved=$(sr_resolve "$1")+ if [[ "$resolved" == "llama-server" && "${backend,,}" == "vllm" ]]; then+ resolved="vllm-server"+ fiecho "$resolved"}@@ -957,6 +964,18 @@ get_compose_flags() {if [[ "$_ods_mode" != "cloud" ]] && [[ "$cached" == *"docker-compose.cloud.yml"* ]]; thenstale=1fi+ # LLM_BACKEND=vllm swaps the in-stack llama-server for the vLLM+ # engine overlay, which (like a host-native route) leaves out the+ # compose.local.yaml waits on llama-server. A cache from the other+ # route starts the wrong engine.+ local _llm_backend="${LLM_BACKEND:-}"+ if [[ "${_llm_backend,,}" == "vllm" ]]; then+ if [[ "$cached" != *"docker-compose.vllm.yml"* || "$cached" == *"compose.local.yaml"* ]]; then+ stale=1+ fi+ elif [[ "$cached" == *"docker-compose.vllm.yml"* ]]; then+ stale=1+ fi# The retired Lemonade overlay can outlive an update that kept# the old file on disk; the stack must be resolved again.if [[ "$cached" == *"docker-compose.lemonade-external.yml"* ]]; then@@ -1452,6 +1471,29 @@ _ods_cli_service_health() {fi}+# The vLLM engine (LLM_BACKEND=vllm) publishes no host port. Its Docker+# healthcheck asks /health, which answers only once the model is loaded, so+# Docker's state is its status: healthy, starting (a cold start loads the+# weights and compiles kernels for up to 15 minutes), unhealthy, or stopped.+_ODS_VLLM_NAME="vLLM (LLM Inference)"+_ods_cli_vllm_health() {+ local inspect state="" health=""+ if ! inspect=$(docker inspect --format \+ '{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' \+ ods-vllm-server 2>&1); then+ printf 'stopped\n'+ return 0+ fi+ read -r state health <<<"$inspect"+ if [[ "$state" != "running" ]]; then+ printf 'stopped\n'+ elif [[ "$health" == "healthy" || "$health" == "starting" ]]; then+ printf '%s\n' "$health"+ else+ printf 'unhealthy\n'+ fi+}+cmd_status() {local json_mode="false"while [[ $# -gt 0 ]]; do@@ -1568,6 +1610,13 @@ cmd_status() {fifidone+ if ods_preflight_vllm_engine; then+ case "$(_ods_cli_vllm_health)" in+ healthy) success "$_ODS_VLLM_NAME: healthy" ;;+ starting) log "$_ODS_VLLM_NAME: starting. A first start loads the model and compiles kernels, which can take 15 minutes. Follow it with: ods logs llm" ;;+ *) warn "$_ODS_VLLM_NAME: not responding. See why with: ods logs llm" ;;+ esac+ fiecho ""@@ -1675,6 +1724,17 @@ cmd_status_json() {--arg containerStatus "$container_status" \'{id: $id, name: $name, category: $category, port: $port, status: $status, containerStatus: $containerStatus}' >> "$tmp"done+ if ods_preflight_vllm_engine; then+ local vllm_status vllm_container="unknown"+ vllm_status="$(_ods_cli_vllm_health)"+ if [[ "$container_state_known" == "true" ]]; then+ vllm_container="stopped"+ grep -Fxq -- vllm-server <<<"$running_services" && vllm_container="running"+ fi+ # No host port: model-router and LiteLLM reach it on the ODS network.+ jq -n --arg name "$_ODS_VLLM_NAME" --arg status "$vllm_status" --arg containerStatus "$vllm_container" \+ '{id: "vllm-server", name: $name, category: "core", port: "", status: $status, containerStatus: $containerStatus}' >> "$tmp"+ fi# GPU summary (optional)local gpu_summary_json="null"@@ -3008,8 +3068,9 @@ if llm_backend:local_warn = llm_backend.get('local_running_warning', False)recovery = llm_backend.get('recovery_hint', '')- is_external = llm_provider != "llama-server"- backend_label = f"{llm_provider} (external)" if is_external else "llama-server (local)"+ # The vLLM engine (vllm-server) is ODS's own local model server, like llama-server.+ is_external = llm_provider not in ("llama-server", "vLLM")+ backend_label = f"{llm_provider} (external)" if is_external else f"{llm_provider} (local)"if llm_status == 'ok':print(f" {GREEN}✓{NC} LLM Backend: {backend_label} — responding")@@ -4155,6 +4216,8 @@ cmd_mode() {check_install; cd "$INSTALL_DIR"load_env || truelocal mode="${1:-}"+ local configured_backend+ configured_backend=$(grep -m1 "^LLM_BACKEND=" .env 2>/dev/null | cut -d= -f2- | tr -d '"\047\r' || true)if [[ -z "$mode" ]]; then# Show current mode@@ -4163,6 +4226,8 @@ cmd_mode() {local api_url=$(grep "^LLM_API_URL=" .env 2>/dev/null | cut -d= -f2 || true)local model=$(grep "^LLM_MODEL=" .env 2>/dev/null | cut -d= -f2 || true)local tier=$(grep "^TIER=" .env 2>/dev/null | cut -d= -f2 || true)+ local engine="llama-server"+ [[ "${configured_backend,,}" != "vllm" ]] || engine="vLLM engine (vllm-server)"echo -e "${BLUE}━━━ ODS Mode ━━━${NC}"echo ""echo -e "Current mode: ${GREEN}${current}${NC}"@@ -4177,11 +4242,11 @@ cmd_mode() {echo " Cost: ~\$0.003-0.06/1K tokens";;local)- echo " LLM: Local llama-server"+ echo " LLM: Local $engine"echo " Cost: \$0 (electricity only)";;hybrid)- echo " LLM: Local llama-server → Cloud fallback on failure"+ echo " LLM: Local $engine → Cloud fallback on failure"echo " Cost: \$0 when local works, cloud rates on fallback";;esac@@ -4201,8 +4266,6 @@ cmd_mode() {*) error "Unknown mode: $mode. Use: local, cloud, hybrid" ;;esac- local configured_backend- configured_backend=$(grep -m1 "^LLM_BACKEND=" .env 2>/dev/null | cut -d= -f2- | tr -d '"\047\r' || true)if [[ "${configured_backend,,}" == "external" ]]; then# Fleet row 4: "installer-managed" did not say what to do or where.echo -e "${RED}✗${NC} ODS is in API mode, which only the installer turns on and off. Nothing was changed." >&2@@ -4210,6 +4273,14 @@ cmd_mode() {echo " Need help? Ask on the ODS Discord: https://discord.gg/4ntNp9MAwC" >&2exit 1fi+ if [[ "${configured_backend,,}" == "vllm" && "$mode" != "local" ]]; then+ # Cloud mode has no local model server, and the resolver refuses the+ # vLLM engine with it; hybrid mode was never proven with vLLM.+ echo -e "${RED}✗${NC} ODS runs this model on the vLLM engine, which works only in local mode. Nothing was changed." >&2+ echo " To use $mode mode, run the ODS installer again without the vLLM engine." >&2+ echo " Need help? Ask on the ODS Discord: https://discord.gg/4ntNp9MAwC" >&2+ exit 1+ fi# Update .env_env_set "ODS_MODE" "$mode"@@ -4219,9 +4290,12 @@ cmd_mode() {local switchboard_mode native_llmswitchboard_mode=$(grep -m1 "^ODS_MODEL_SWITCHBOARD=" .env 2>/dev/null | cut -d= -f2- | tr -d '"\047\r' || true)native_llm=$(grep -m1 "^NATIVE_LLM_BASE_URL=" .env 2>/dev/null | cut -d= -f2- | tr -d '"\047\r' || true)- # A host-native llama-server install keeps the in-stack llama-server- # off; LiteLLM is what reaches the native server (06-directories.sh).- if [[ -n "$native_llm" ]]; then+ # A host-native llama-server install and a vLLM install keep the+ # in-stack llama-server off; LiteLLM is what reaches the model+ # (06-directories.sh), through model-router for vLLM.+ local llama_off="false"+ [[ -n "$native_llm" || "${configured_backend,,}" == "vllm" ]] && llama_off="true"+ if [[ "$llama_off" == "true" ]]; thenapi_url="http://litellm:4000"elseapi_url="http://llama-server:8080"@@ -4230,7 +4304,7 @@ cmd_mode() {if [[ "${switchboard_mode,,}" == "enabled" ]]; then_env_set "HERMES_LLM_BASE_URL" "http://model-router:9099/v1"_env_set "HERMES_LLM_API_KEY" "no-key"- elif [[ -n "$native_llm" ]]; then+ elif [[ "$llama_off" == "true" ]]; then_env_set "HERMES_LLM_BASE_URL" "http://litellm:4000/v1"if [[ -n "${LITELLM_KEY:-}" ]]; then_env_set "HERMES_LLM_API_KEY" "$LITELLM_KEY"diff --git a/ods/scripts/mode-switch.sh b/ods/scripts/mode-switch.shindex c94e62a30..ecbb15f65 100755--- a/ods/scripts/mode-switch.sh+++ b/ods/scripts/mode-switch.sh@@ -80,17 +80,26 @@ switch_mode() {if [[ "$configured_backend" == "external" ]]; thenerror "External LLM routing is installer-managed. Run './install.sh --no-external-llm' first, then retry the mode switch."fi+ if [[ "$configured_backend" == "vllm" && "$mode" != "local" ]]; then+ # Cloud mode has no local model server, and the resolver refuses the+ # vLLM engine with it; hybrid mode was never proven with vLLM.+ error "ODS runs this model on the vLLM engine, which works only in local mode. Nothing was changed. To use $mode mode, run the ODS installer again without the vLLM engine. Need help? Ask on the ODS Discord: https://discord.gg/4ntNp9MAwC"+ fi# Update .envenv_set "ODS_MODE" "$mode"if [[ "$mode" == "local" ]]; then- local switchboard_mode native_llm+ local switchboard_mode native_llm llama_off="false"switchboard_mode=$(grep -m1 "^ODS_MODEL_SWITCHBOARD=" "$ENV_FILE" 2>/dev/null | cut -d= -f2- | tr -d '"\047\r' || true)native_llm=$(grep -m1 "^NATIVE_LLM_BASE_URL=" "$ENV_FILE" 2>/dev/null | cut -d= -f2- | tr -d '"\047\r' || true)- # A host-native llama-server install keeps the in-stack llama-server- # off; LiteLLM is what reaches the native server (06-directories.sh).- if [[ -n "$native_llm" ]]; then+ # A host-native llama-server install and a vLLM install keep the+ # in-stack llama-server off; LiteLLM is what reaches the model+ # (06-directories.sh), through model-router for vLLM.+ if [[ -n "$native_llm" || "$configured_backend" == "vllm" ]]; then+ llama_off="true"+ fi+ if [[ "$llama_off" == "true" ]]; thenenv_set "LLM_API_URL" "http://litellm:4000"elseenv_set "LLM_API_URL" "http://llama-server:8080"@@ -99,7 +108,7 @@ switch_mode() {if [[ "$switchboard_mode" == "enabled" ]]; thenenv_set "HERMES_LLM_BASE_URL" "http://model-router:9099/v1"env_set "HERMES_LLM_API_KEY" "no-key"- elif [[ -n "$native_llm" ]]; then+ elif [[ "$llama_off" == "true" ]]; thenenv_set "HERMES_LLM_BASE_URL" "http://litellm:4000/v1"local litellm_keylitellm_key=$(grep -m1 "^LITELLM_KEY=" "$ENV_FILE" 2>/dev/null | cut -d= -f2- | tr -d '"\047\r' || true)diff --git a/ods/scripts/ods-doctor.sh b/ods/scripts/ods-doctor.shindex 6930596be..268ca815a 100755--- a/ods/scripts/ods-doctor.sh+++ b/ods/scripts/ods-doctor.sh@@ -445,11 +445,57 @@ _doctor_check_llama_server() {fi}+# LLM_BACKEND=vllm (docker-compose.vllm.yml): vllm-server serves the model on+# the ODS network without a host port, and the in-stack llama-server is off.+# Docker's healthcheck asks vLLM's /health, which answers only once the model+# is loaded, so its state is the engine's status. A cold start loads the+# weights and compiles kernels for up to 15 minutes: that is starting, not a+# fault. A llama-server container left by an earlier install still running+# would compete for the GPU.+_doctor_check_vllm_engine() {+ local inspect="" state="" health=""+ LLM_PROVIDER="vLLM"+ LLM_URL="http://vllm-server:8000 (ODS network only)"+ LLM_MODEL="${VLLM_SERVED_MODEL:-}"+ LLM_RECOVERY=""+ LLM_LOCAL_WARNING="false"+ if [[ "$DOCKER_DAEMON" == "true" ]] && inspect=$(docker inspect --format \+ '{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' \+ ods-vllm-server 2>&1); then+ read -r state health <<<"$inspect"+ fi+ if [[ "$state" == "running" && "$health" == "healthy" ]]; then+ LLM_STATUS="ok"+ log_ok "LLM backend: vLLM (local) — responding"+ [[ -z "$LLM_MODEL" ]] || log_ok " Model : $LLM_MODEL"+ elif [[ "$state" == "running" && "$health" == "starting" ]]; then+ LLM_STATUS="starting"+ log_info "LLM backend: vLLM (local) — starting: a first start loads the model and compiles kernels, which can take 15 minutes"+ log_info " Follow : ods logs llm"+ else+ LLM_STATUS="fail"+ if [[ "$state" == "running" ]]; then+ LLM_RECOVERY="ods restart vllm-server"+ log_fail "LLM backend: vLLM (local) — container running but not healthy"+ else+ LLM_RECOVERY="ods start vllm-server"+ log_fail "LLM backend: vLLM (local) — container not running"+ fi+ log_info " Recovery : $LLM_RECOVERY (see why with: ods logs llm)"+ fi+ if [[ "$DOCKER_DAEMON" == "true" ]] && docker ps --format '{{.Names}}' 2>/dev/null | grep -qx 'ods-llama-server'; then+ LLM_LOCAL_WARNING="true"+ log_warn " Note : A llama-server container from an earlier install is running beside vLLM; stop it with: docker stop ods-llama-server"+ fi+}+_doctor_check_llm_backend() {local ext_url="${EXTERNAL_LLM_URL:-}"local ext_provider="${EXTERNAL_LLM_PROVIDER:-}"local ext_model="${EXTERNAL_LLM_MODEL:-}"local mode="${ODS_MODE:-local}"+ local backend+ backend=$(printf '%s' "${LLM_BACKEND:-}" | tr '[:upper:]' '[:lower:]')if [ -n "$ext_url" ]; then# External LLM mode — skip llama-server check@@ -457,6 +503,8 @@ _doctor_check_llm_backend() {"$ROOT_DIR/config/litellm/external-upstream.key"elif [[ -n "${NATIVE_LLM_BASE_URL:-}" ]]; then_doctor_check_host_native_llm+ elif [[ "$backend" == "vllm" && "$mode" != "cloud" ]]; then+ _doctor_check_vllm_engineelif [[ "$mode" == "cloud" ]]; thenlocal cloud_url="${LLM_API_URL:-}"if [ -n "$cloud_url" ]; then@@ -1114,6 +1162,14 @@ def _looks_like_local_llama_route(value):}+def _url_hostname(value):+ """The lower-case host of a URL, or "" when it names none."""+ try:+ return (parse.urlsplit(value.strip()).hostname or "").lower() if "://" in value else ""+ except ValueError:+ return ""++def _source(path):try:return path.relative_to(root_dir).as_posix()@@ -1194,8 +1250,12 @@ def _collect_inference_contract():generic_external = bool(external_llm_url.strip()) or llm_backend.strip().lower() == "external"external_inference = ods_mode == "cloud" or generic_external+ # LLM_BACKEND=vllm: vllm-server serves the model behind model-router;+ # llama-server is off and clients use the LiteLLM gateway.+ vllm_engine = llm_backend.strip().lower() == "vllm" and not external_inference+ vllm_overlay = _has_compose_file(compose_files, "docker-compose.vllm.yml")expected_owner = "external" if external_inference else "ods"- expected_gateway = "litellm" if external_inference or host_native else "llama-server"+ expected_gateway = "litellm" if external_inference or host_native or vllm_engine else "llama-server"issues = []if lemonade_era:@@ -1293,7 +1353,27 @@ def _collect_inference_contract():))- if ods_mode == "local" and not host_native and not generic_external and not lemonade_era:+ if vllm_engine:+ if compose_flags_exists and not vllm_overlay:+ issues.append(+ _inference_issue(+ "ODS-RUNTIME-VLLM-OVERLAY-MISSING",+ "blocker",+ ".compose-flags",+ "LLM_BACKEND=vllm but docker-compose.vllm.yml is not in the resolved compose stack.",+ )+ )+ if _url_hostname(llm_api_url) in {"llama-server", "ods-llama-server"}:+ issues.append(+ _inference_issue(+ "ODS-RUNTIME-VLLM-LLAMA-ROUTE",+ "blocker",+ ".env",+ f"The vLLM engine serves the model, but LLM_API_URL points at the in-stack llama-server ({llm_api_url}), which is off.",+ )+ )++ if ods_mode == "local" and not host_native and not generic_external and not lemonade_era and not vllm_engine:if compose_flags_exists and cloud_overlay:issues.append(_inference_issue(@@ -1327,6 +1407,8 @@ def _collect_inference_contract():"ODS-RUNTIME-HOST-NATIVE-KEY-MISSING": "The Windows llama-server route has no API key","ODS-RUNTIME-LOCAL-CLOUD-OVERLAY": "Local mode still has the cloud compose overlay","ODS-RUNTIME-LOCAL-LITELLM-ROUTE": "Local mode routes through LiteLLM unexpectedly",+ "ODS-RUNTIME-VLLM-OVERLAY-MISSING": "The vLLM route is missing its compose overlay",+ "ODS-RUNTIME-VLLM-LLAMA-ROUTE": "The vLLM route still sends clients to the in-stack llama-server",}next_steps = {"ODS-RUNTIME-CLOUD-OVERLAY-MISSING": [@@ -1368,6 +1450,12 @@ def _collect_inference_contract():"ODS-RUNTIME-MODE-UNKNOWN": ["Set ODS_MODE to local, cloud, or hybrid.",],+ "ODS-RUNTIME-VLLM-OVERLAY-MISSING": [+ "Run `ods restart`: the CLI resolves the compose stack again from LLM_BACKEND=vllm.",+ ],+ "ODS-RUNTIME-VLLM-LLAMA-ROUTE": [+ "Run `ods mode local` to point LLM_API_URL at the LiteLLM gateway, then `ods restart`.",+ ],}for issue in issues:diagnoses.append(@@ -1398,6 +1486,8 @@ def _collect_inference_contract():"host_native_overlay": host_native_overlay,"local_inference_overlay": local_inference_overlay,"native_key_configured": native_key_configured,+ "vllm_engine": vllm_engine,+ "vllm_overlay": vllm_overlay,},"issues": issues,"issue_counts": {@@ -1778,14 +1868,20 @@ llm_local_warn = os.environ.get("LLM_LOCAL_WARNING", "false") == "true"llm_failure = os.environ.get("LLM_FAILURE", "")if llm_status == "fail" and llm_recovery:- if llm_provider != "llama-server" and llm_failure in ("key-refused", "key-required"):+ if llm_provider == "vLLM":+ fix_hints.append(f"The vLLM engine (vllm-server) is not serving the model. Hint: {llm_recovery}")+ elif llm_provider != "llama-server" and llm_failure in ("key-refused", "key-required"):fix_hints.append(f"External LLM backend ({llm_provider}) answered but did not accept a stored API key. Hint: {llm_recovery}")elif llm_provider != "llama-server":fix_hints.append(f"External LLM backend ({llm_provider}) is unreachable. Hint: {llm_recovery}")else:fix_hints.append(f"Local llama-server is unreachable. Hint: {llm_recovery}")-if llm_local_warn:+if llm_local_warn and llm_provider == "vLLM":+ # `ods stop llama-server` names vllm-server on this route; stop the container itself.+ fix_hints.append("A llama-server container from an earlier install is running beside the vLLM engine "+ "and competes for its GPU. Stop it with: docker stop ods-llama-server")+elif llm_local_warn:fix_hints.append("External LLM is configured but the local llama-server container is still running. Stop it with: ods stop llama-server")# Rootless daemon without (enough) subordinate IDs: images fail to extract# with "uid/gid map out of range" long after doctor reports healthy Docker.diff --git a/ods/scripts/ods-test.sh b/ods/scripts/ods-test.shindex 0a31767df..71599c4d4 100755--- a/ods/scripts/ods-test.sh+++ b/ods/scripts/ods-test.sh@@ -46,6 +46,20 @@ fiLLM_HOST="${LLM_HOST:-localhost}"LLM_PORT="${LLM_PORT:-${SERVICE_PORTS[llama-server]:-11434}}"LLM_URL="http://${LLM_HOST}:${LLM_PORT}"+LLM_HEALTH_PATH="/health"+LLM_ENGINE="llama-server"+LLM_GATEWAY_KEY=""+# LLM_BACKEND=vllm: vllm-server publishes no host port. ODS clients reach it+# through LiteLLM as ods/current (model-router adapts the request to vLLM), so+# the LLM checks go there too, with the gateway key (_llm_chat).+_ods_test_backend="$(printf '%s' "${LLM_BACKEND:-}" | tr '[:upper:]' '[:lower:]')"+if [[ "$_ods_test_backend" == "vllm" ]]; then+ LLM_URL="http://${LLM_HOST}:${LITELLM_PORT:-4000}"+ LLM_HEALTH_PATH="/health/readiness"+ LLM_ENGINE="vllm-server"+ LLM_GATEWAY_KEY="${LITELLM_KEY:-${LITELLM_MASTER_KEY:-}}"+fi+unset _ods_test_backendWHISPER_HOST="${WHISPER_HOST:-localhost}"WHISPER_PORT="${WHISPER_PORT:-${SERVICE_PORTS[whisper]:-9000}}"TTS_HOST="${TTS_HOST:-localhost}"@@ -108,11 +122,32 @@ _now_ms() {# /v1/models as test_llm always has; "local" when the listing fails, and the# request that follows then reports the fault._llm_model_id() {+ # The gateway serves the active model under its stable alias.+ if [[ -n "$LLM_GATEWAY_KEY" ]]; then+ echo "ods/current"+ return 0+ filocal model_idmodel_id=$(curl -s --max-time 10 "$LLM_URL/v1/models" 2>/dev/null | grep -o '"id":"[^"]*"' | head -1 | cut -d'"' -f4) || model_id=""echo "${model_id:-local}"}+# POST one chat completion ($2) to the LLM route within $1 seconds. A gateway+# key travels on curl's stdin (-H @-), never in argv.+_llm_chat() {+ local seconds="$1" payload="$2"+ if [[ -n "$LLM_GATEWAY_KEY" ]]; then+ printf 'Authorization: Bearer %s\n' "$LLM_GATEWAY_KEY" \+ | curl -s --max-time "$seconds" -X POST "$LLM_URL/v1/chat/completions" \+ -H "Content-Type: application/json" -H @- \+ -d "$payload" 2>/dev/null+ else+ curl -s --max-time "$seconds" -X POST "$LLM_URL/v1/chat/completions" \+ -H "Content-Type: application/json" \+ -d "$payload" 2>/dev/null+ fi+}+# Judge a chat completion (read on stdin) by its JSON structure, not by# grepping for a word: a reply with "tool_calls": null and no call contains the# word too. $1 is "text" (an assistant answer; a thinking model may spend a@@ -340,13 +375,16 @@ test_gpu() {}test_llm() {- print_section "LLM Inference (llama-server)"+ print_section "LLM Inference ($LLM_ENGINE)"# A failed check is already recorded; return 0 so the short-circuit only# skips this section's dependent checks instead of aborting the whole run# under set -e (which lost the summary and the --json document).- test_http "LLM Health" "$LLM_URL/health" "200" || return 0- test_http "LLM Models API" "$LLM_URL/v1/models" "200" || log "LLM models API check failed"+ test_http "LLM Health" "$LLM_URL$LLM_HEALTH_PATH" "200" || return 0+ # The gateway lists only its aliases, and only with its key.+ if [[ -z "$LLM_GATEWAY_KEY" ]]; then+ test_http "LLM Models API" "$LLM_URL/v1/models" "200" || log "LLM models API check failed"+ fiif [[ "$QUICK_MODE" == "true" ]]; thenrecord_result "LLM Inference" "skip" "quick mode"@@ -360,10 +398,7 @@ test_llm() {local payload="{\"model\": \"$model_id\", \"messages\": [{\"role\": \"user\", \"content\": \"Say hello\"}], \"max_tokens\": 10}"local response- response=$(curl -s --max-time 30 \- -X POST "$LLM_URL/v1/chat/completions" \- -H "Content-Type: application/json" \- -d "$payload" 2>/dev/null) || response=""+ response=$(_llm_chat 30 "$payload") || response=""if printf '%s' "$response" | _completion_ok text; thenlocal tokens_used@@ -393,11 +428,8 @@ test_tool_calling() {local payload="{\"model\": \"$model_id\", \"messages\": [{\"role\": \"user\", \"content\": \"What is the weather in Tokyo?\"}], \"tools\": $tools, \"tool_choice\": \"auto\", \"max_tokens\": 100}"local response- response=$(curl -s --max-time 30 \- -X POST "$LLM_URL/v1/chat/completions" \- -H "Content-Type: application/json" \- -d "$payload" 2>/dev/null) || response=""-+ response=$(_llm_chat 30 "$payload") || response=""+if printf '%s' "$response" | _completion_ok tool; thenrecord_result "Tool Calling" "pass" "function called"print_test "Tool Calling" "pass" "function called"@@ -495,7 +527,7 @@ test_voice_roundtrip() {if curl -s --max-time 5 "http://${TTS_HOST}:${TTS_PORT}/v1/audio/voices" &>/dev/null; thentts_ready=truefi- if curl -s --max-time 5 "$LLM_URL/health" &>/dev/null; then+ if curl -s --max-time 5 "$LLM_URL$LLM_HEALTH_PATH" &>/dev/null; thenllm_ready=truefi@@ -522,10 +554,7 @@ test_voice_roundtrip() {model_id=$(_llm_model_id)local llm_payload="{\"model\": \"$model_id\", \"messages\": [{\"role\": \"user\", \"content\": \"What is the weather today?\"}], \"max_tokens\": 50}"local llm_response- llm_response=$(curl -s --max-time 15 \- -X POST "$LLM_URL/v1/chat/completions" \- -H "Content-Type: application/json" \- -d "$llm_payload" 2>/dev/null) || llm_response=""+ llm_response=$(_llm_chat 15 "$llm_payload") || llm_response=""if ! printf '%s' "$llm_response" | _completion_ok text; thenrecord_result "Voice Round-Trip" "fail" "LLM step failed"@@ -669,14 +698,14 @@ _print_text_summary() {echo "Actionable fixes:"if [[ "${RESULTS_STATUS[0]:-}" == "fail" ]] && [[ "${RESULTS_NAMES[0]:-}" == *"LLM"* ]]; then- echo " - LLM not responding - check: docker logs ods-llama-server"+ echo " - LLM not responding - check: docker logs ods-${LLM_ENGINE}"filocal ifor i in "${!RESULTS_NAMES[@]}"; doif [[ "${RESULTS_STATUS[$i]}" == "fail" ]]; thencase "${RESULTS_NAMES[$i]}" in- "Tool Calling") echo " - Tool calling failed - check llama-server tool support" ;;+ "Tool Calling") echo " - Tool calling failed - check $LLM_ENGINE tool support" ;;"Whisper Port") echo " - Whisper not running - start: docker compose up whisper" ;;"TTS Port") echo " - TTS not running - start: docker compose up kokoro-tts" ;;esacdiff --git a/ods/scripts/preserve-active-model.py b/ods/scripts/preserve-active-model.pyindex 202404195..2170ce60b 100644--- a/ods/scripts/preserve-active-model.py+++ b/ods/scripts/preserve-active-model.py@@ -13,6 +13,9 @@ installs wrote. Every mode emits a small allowlisted dotenv fragment without``eval``. The default mode also keeps the settings model activation owns forthe same model on this host's llama-server container: an import's visionprojector and a fixed chat template, each only while it still checks out.+On a vLLM install (``LLM_BACKEND=vllm``) the default mode keeps the vLLM+route, its catalog snapshot and GPU instead (``preserved_vllm_contract``),+and exits 2 rather than let the rerun fall back to a llama.cpp model.Round F serves every model as ``--alias <GGUF_FILE>``, so a served model id isthe GGUF filename. For one release the retired Lemonade forms of the same id@@ -548,6 +551,100 @@ def repaired_served_model_contract(return env, served_id, contract+def is_vllm_runtime(env: dict[str, str]) -> bool:+ """A ``.env`` whose model the vLLM engine serves (LLM_BACKEND=vllm)."""+ return env.get("LLM_BACKEND", "").strip().lower() == "vllm"+++_VLLM_GPU_UUID = re.compile(r"[A-Za-z0-9][A-Za-z0-9-]*")+_VLLM_MEMORY_SHARE = re.compile(r"(?:0(?:\.[0-9]+)?|1(?:\.0+)?)")+++def vllm_snapshot_installed(models_dir: Path, folder: str, files: list[dict[str, Any]]) -> bool:+ """Every manifest file is in ``models_dir/folder`` under its final name, at its size.++ scripts/download-hf-manifest.py renames a file into place only after its+ digest matched, so a complete set of final names at the catalog sizes is a+ complete verified download. Links are refused, as the GGUF check does.+ """+ try:+ root = models_dir.resolve()+ snapshot = models_dir / folder+ if snapshot.is_symlink() or not snapshot.is_dir() or snapshot.resolve().parent != root:+ return False+ for item in files:+ info = (snapshot / item["path"]).lstat()+ if not stat.S_ISREG(info.st_mode) or info.st_size != item["size"]:+ return False+ if not (snapshot / item["path"]).resolve().is_relative_to(snapshot.resolve()):+ return False+ except (OSError, RuntimeError):+ return False+ return True+++def preserved_vllm_contract(args: argparse.Namespace) -> dict[str, str] | None:+ """The retained vLLM route, while its catalog snapshot is still installed.++ The route stays on vLLM (LLM_BACKEND=vllm) with the same model and GPU, and+ clears the GGUF keys so no llama.cpp model is downloaded or started. The+ model folder and served id must be the catalog entry's (a new revision is+ a new download, which this never starts). VLLM_ARGS comes from this+ release's catalog entry for that snapshot: its flags are measured with the+ release's vLLM image, which the installer pins separately (VLLM_IMAGE).+ The GPU and its memory share are the host's and stay as installed.+ """+ env = parse_dotenv(args.env)+ if (+ not is_vllm_runtime(env)+ or env.get("ODS_MODE", "local").strip().lower() != "local"+ or env.get("EXTERNAL_LLM_URL")+ or env.get("NATIVE_LLM_BASE_URL")+ ):+ return None+ sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "extensions/services/dashboard-api"))+ import model_vllm++ llm_model = env.get("LLM_MODEL", "").strip()+ matches = [+ item for item in load_records(args.catalog, None)+ if model_vllm.is_vllm_entry(item) and item.get("llm_model_name") == llm_model+ ]+ if len(matches) != 1 or model_vllm.entry_problems(matches[0]):+ return None+ model = matches[0]+ folder = model_vllm.model_dir_name(model)+ gpu_uuid = env.get("VLLM_GPU_UUID", "")+ share = env.get("VLLM_GPU_MEMORY_UTILIZATION", "")+ if (+ env.get("VLLM_MODEL_DIR") != folder+ or env.get("VLLM_SERVED_MODEL") != model["served_model_name"]+ or not _VLLM_GPU_UUID.fullmatch(gpu_uuid)+ or (share and (not _VLLM_MEMORY_SHARE.fullmatch(share) or float(share) <= 0))+ or not vllm_snapshot_installed(args.models_dir, folder, model["files"])+ ):+ return None+ source = env.get("MODEL_SELECTION_SOURCE", "")+ contract = {+ "LLM_BACKEND": "vllm",+ "LLM_MODEL": str(model["llm_model_name"]),+ # A vLLM route has no GGUF; empty values override the tier map's.+ "GGUF_FILE": "",+ "GGUF_URL": "",+ "GGUF_SHA256": "",+ "MAX_CONTEXT": str(model["context_length"]),+ "LLM_MODEL_SIZE_MB": str(round(model_vllm.manifest_bytes(model) / 1_000_000)),+ "MODEL_SELECTION_SOURCE": source if source in {"installer", "dashboard", "operator"} else "preserved-local",+ "VLLM_MODEL_DIR": folder,+ "VLLM_SERVED_MODEL": str(model["served_model_name"]),+ "VLLM_ARGS": model_vllm.vllm_args_text(model),+ "VLLM_GPU_UUID": gpu_uuid,+ }+ if share:+ contract["VLLM_GPU_MEMORY_UTILIZATION"] = share+ return contract++def is_local_runtime(env: dict[str, str]) -> bool:"""A ``.env`` whose model this host's managed llama-server serves."""if env.get("EXTERNAL_LLM_URL", "") or env.get("LEMONADE_EXTERNAL", "false").lower() == "true":@@ -981,6 +1078,16 @@ def main() -> int:)print(f"Repairing the saved description of the served model ({served_id}): "f"{changes}. The served model is unchanged.", file=sys.stderr)+ elif not (args.native_llm or args.external_lemonade) and is_vllm_runtime(parse_dotenv(args.env)):+ contract = preserved_vllm_contract(args)+ if contract is None:+ # Falling through to the recommendation would turn the vLLM+ # install into a llama.cpp one and download a GGUF.+ print("This installation runs its model on the vLLM engine, but its saved model is not "+ "a complete vLLM catalog snapshot for the GPU it names; refusing to replace it. "+ "Run the installer again and choose the vLLM model, or use --reselect-model.",+ file=sys.stderr)+ return 2else:contract = preserved_contract(args)if contract is None:diff --git a/ods/scripts/render-runtime-configs.py b/ods/scripts/render-runtime-configs.pyindex 397e3edc2..38ee8233d 100644--- a/ods/scripts/render-runtime-configs.py+++ b/ods/scripts/render-runtime-configs.py@@ -16,9 +16,9 @@ import statimport sysimport tempfileimport time-from dataclasses import asdict, dataclass+from dataclasses import asdict, dataclass, replacefrom pathlib import Path-from typing import Callable+from typing import Any, Callablefrom urllib.parse import urlsplit@@ -136,6 +136,9 @@ class RenderInputs:switchboard_mode: str = "enabled"# Model server behind model-router: llama-server | vllm (model_server()).llm_backend: str = "llama-server"+ # The vllm route's thinking map for model-router (vllm_thinking()):+ # {"default": <level>, "templateKwargs": {<level>: chat_template_kwargs}}.+ vllm_thinking: dict[str, Any] | None = None@dataclass(frozen=True)@@ -172,6 +175,30 @@ def model_server(llm_backend: str) -> str:return "vllm" if llm_backend == "vllm" else "llama-server"+def vllm_thinking(model: str, catalog_path: Path = ROOT / "config" / "model-library.json") -> dict[str, Any]:+ """The thinking map model-router applies on the vllm route for ``model``.++ ``model`` is .env LLM_MODEL: the vLLM catalog entry's llm_model_name (or+ its id). vLLM serves one model, so its endpoint carries that model's+ default level (``reasoning``) and the chat-template arguments of each+ level (``thinking_template_kwargs``); see model_vllm.py.+ """+ sys.path.insert(0, str(ROOT / "extensions" / "services" / "dashboard-api"))+ import model_vllm++ catalog = json.loads(catalog_path.read_text(encoding="utf-8"))+ matches = [+ entry for entry in catalog.get("models", [])+ if model_vllm.is_vllm_entry(entry) and model in {entry.get("llm_model_name"), entry.get("id")}+ ]+ if len(matches) != 1:+ raise ValueError(+ f"LLM_BACKEND=vllm needs --model to name one vLLM catalog entry (its llm_model_name); got {model!r}"+ )+ entry = model_vllm.require_valid(matches[0])+ return {"default": entry["reasoning"], "templateKwargs": entry["thinking_template_kwargs"]}++def hermes_model_id(inputs: RenderInputs) -> str:if inputs.switchboard_mode == "enabled":return "ods/current"@@ -669,8 +696,12 @@ def render_model_router_endpoints(inputs: RenderInputs) -> RenderedFile:if inputs.llm_backend == "vllm":# The in-stack vllm-server container on its internal port. It takes no- # key; the llama-server origin and key inputs do not describe it.- endpoint = {"id": VLLM_ENDPOINT_ID, "baseUrl": VLLM_SERVER_ORIGIN}+ # key; the llama-server origin and key inputs do not describe it. The+ # router translates a request's thinking level with the served+ # model's own map, which only this endpoint knows.+ if inputs.vllm_thinking is None:+ raise ValueError("the vllm-server endpoint needs its model's thinking map (vllm_thinking)")+ endpoint = {"id": VLLM_ENDPOINT_ID, "baseUrl": VLLM_SERVER_ORIGIN, "thinking": inputs.vllm_thinking}else:# Otherwise the model server is upstream llama-server: the in-stack# container, or a host-native one reached at@@ -921,15 +952,15 @@ def render(args: argparse.Namespace) -> dict[str, object]:"litellm-switchboard is local-runtime-only and cannot be rendered ""for ODS_MODE=cloud")- files = [- RENDERERS[name](inputs)- for name in select_surfaces(- args.surface,- inputs.ods_mode,- inputs.switchboard_mode,- inputs.remote_llm_enabled,- )- ]+ surfaces = select_surfaces(+ args.surface,+ inputs.ods_mode,+ inputs.switchboard_mode,+ inputs.remote_llm_enabled,+ )+ if inputs.llm_backend == "vllm" and "model-router-endpoints" in surfaces:+ inputs = replace(inputs, vllm_thinking=vllm_thinking(inputs.model))+ files = [RENDERERS[name](inputs) for name in surfaces]written: list[str] = []if args.write:output_root = Path(args.output_root)diff --git a/ods/tests/ci-suite.txt b/ods/tests/ci-suite.txtindex f4a9e000f..9c8c9f3a6 100644--- a/ods/tests/ci-suite.txt+++ b/ods/tests/ci-suite.txt@@ -63,6 +63,8 @@ tests/test-comfyui-user-dir-persisted.shtests/test-comfyui-cuda-cache-persisted.shtests/test-llama-cuda-cache-persisted.shtests/test-vllm-engine-service.sh+tests/test-cli-vllm-engine.sh+tests/test-doctor-vllm-engine.shtests/test_comfyui_extension_contract.pytests/test_extension_assets.pytests/test-compose-failure-evidence.sh@@ -145,6 +147,9 @@ tests/test-model-library-coverage.pytests/test-model-library-verdicts.pytests/test-model-library-vllm.pytests/test_select_model_vllm.py+tests/test_host_agent_vllm.py+tests/test_preserve_active_model_vllm.py+tests/test_doctor_vllm_route.pytests/test-native-llm-phase02.shtests/test-network-security.shtests/test-network-timeouts.shdiff --git a/ods/tests/test-cli-vllm-engine.sh b/ods/tests/test-cli-vllm-engine.shnew file mode 100644index 000000000..6dc313c33--- /dev/null+++ b/ods/tests/test-cli-vllm-engine.sh@@ -0,0 +1,198 @@+#!/usr/bin/env bash+# The CLI on a vLLM install (LLM_BACKEND=vllm, docker-compose.vllm.yml): the+# vllm-server container serves the model and the in-stack llama-server is+# profiled out. A saved Compose file list from the other route is stale; "llm"+# and "llama-server" name vllm-server for logs and lifecycle commands (naming+# llama-server to Compose would enable its profile and start a second engine);+# status reports vllm-server from Docker's health (it has no host port) and no+# llama-server fault; ods mode keeps the route local and never points clients+# back at llama-server. Docker, curl and the resolver are stubs.+set -euo pipefail++ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"+FIXTURE="$(mktemp -d)"+trap 'rm -rf "$FIXTURE"' EXIT+install_dir="$FIXTURE/install"+mkdir -p "$install_dir/lib" "$install_dir/extensions/services" "$install_dir/scripts" "$FIXTURE/bin"+cp "$ROOT_DIR/ods-cli" "$install_dir/ods-cli"+cp "$ROOT_DIR/scripts/mode-switch.sh" "$install_dir/scripts/mode-switch.sh"+for library in safe-env.sh service-registry.sh python-cmd.sh preflight-llm-route.sh; do+ cp "$ROOT_DIR/lib/$library" "$install_dir/lib/"+done+for compose in docker-compose.base.yml docker-compose.nvidia.yml docker-compose.vllm.yml; do+ printf 'services: {}\n' > "$install_dir/$compose"+done++fail() { printf 'FAIL: %s\n' "$*" >&2; exit 1; }+pass() { printf 'PASS: %s\n' "$*"; }++for entry in 'llama-server core 8080 ods-llama-server [llm]' 'model-router core 9099 ods-model-router []' \+ 'dashboard core 3001 ods-dashboard []'; do+ read -r service category port container aliases <<< "$entry"+ mkdir -p "$install_dir/extensions/services/$service"+ cat > "$install_dir/extensions/services/$service/manifest.yaml" <<MANIFEST+schema_version: ods.services.v1+service:+ id: $service+ name: $service+ aliases: $aliases+ container_name: $container+ category: $category+ port: $port+ health: /health+MANIFEST+done++VLLM_FLAGS='-f docker-compose.base.yml -f docker-compose.nvidia.yml -f docker-compose.vllm.yml'+# The resolver selects the vLLM overlay from the install's LLM_BACKEND.+cat > "$install_dir/scripts/resolve-compose-stack.sh" <<'STUB'+#!/usr/bin/env bash+set -euo pipefail+printf 'resolve\n' >> "$CLI_CALLS"+if grep -qx 'LLM_BACKEND=vllm' "$INSTALL_DIR/.env"; then+ printf '%s\n' '-f docker-compose.base.yml -f docker-compose.nvidia.yml -f docker-compose.vllm.yml'+else+ printf '%s\n' '-f docker-compose.base.yml -f docker-compose.nvidia.yml'+fi+STUB+chmod +x "$install_dir/scripts/resolve-compose-stack.sh"+cat > "$FIXTURE/bin/docker" <<'STUB'+#!/usr/bin/env bash+set -euo pipefail+printf 'docker %s\n' "$*" >> "$CLI_CALLS"+case "$*" in+ info*) printf '16\n' ;;+ 'inspect --format '*' ods-vllm-server')+ [[ "$VLLM_STATE" == missing ]] && { printf 'Error: No such object: ods-vllm-server\n' >&2; exit 1; }+ printf '%s\n' "$VLLM_STATE" ;;+ compose*'ps --format {{.Service}}') printf 'vllm-server\nmodel-router\ndashboard\n' ;;+ compose*) ;;+ *) exit 64 ;;+esac+STUB+cat > "$FIXTURE/bin/curl" <<'STUB'+#!/usr/bin/env bash+printf 'curl %s\n' "$*" >> "$CLI_CALLS"+exit 0+STUB+chmod +x "$FIXTURE/bin/"*++export CLI_CALLS="$FIXTURE/calls" VLLM_STATE="running healthy"+write_env() {+ printf '%s\n' 'GPU_BACKEND=nvidia' 'ODS_MODE=local' 'LLM_BACKEND=vllm' \+ 'LLM_MODEL=glm-5.3-flash-nvfp4' 'VLLM_SERVED_MODEL=GLM-5.3-Flash-NVFP4' \+ 'ODS_MODEL_SWITCHBOARD=enabled' 'LLM_API_URL=http://litellm:4000' \+ 'HERMES_DASHBOARD_SESSION_TOKEN=fixture-session' "$@" > "$install_dir/.env"+}+cli() {+ : > "$CLI_CALLS"+ INSTALL_DIR="$install_dir" ODS_HOME="$install_dir" HOME="$FIXTURE" PATH="$FIXTURE/bin:$PATH" \+ bash "$install_dir/ods-cli" "$@"+}++# 1. A Compose file list saved for llama-server is stale on the vLLM route,+# and the model service of logs is vllm-server.+write_env+printf '%s\n' '-f docker-compose.base.yml -f docker-compose.nvidia.yml' > "$install_dir/.compose-flags"+cli logs llm 5 > /dev/null+grep -qx 'resolve' "$CLI_CALLS" || fail 'a llama-server stack cache was reused on the vLLM route'+grep -qx "docker compose $VLLM_FLAGS logs -f --tail 5 vllm-server" "$CLI_CALLS" \+ || fail "ods logs llm did not follow vllm-server: $(cat "$CLI_CALLS")"+pass 'a llama-server stack cache is re-resolved and ods logs llm shows vllm-server'++printf '%s\n' "$VLLM_FLAGS -f extensions/services/hermes/compose.local.yaml" > "$install_dir/.compose-flags"+cli logs llama-server 5 > /dev/null+grep -qx 'resolve' "$CLI_CALLS" || fail 'a cache with llama-server waits was reused on the vLLM route'+grep -q ' logs -f --tail 5 vllm-server$' "$CLI_CALLS" || fail 'ods logs llama-server reached llama-server'+pass 'compose.local.yaml waits on llama-server make a vLLM cache stale'++printf '%s\n' "$VLLM_FLAGS" > "$install_dir/.compose-flags"+cli logs llm 5 > /dev/null+grep -q '^resolve$' "$CLI_CALLS" && fail 'a current vLLM cache was resolved again'+pass 'a current vLLM cache is reused'++write_env 'LLM_BACKEND=llama-server'+sed -i '/^LLM_BACKEND=vllm$/d' "$install_dir/.env"+printf '%s\n' "$VLLM_FLAGS" > "$install_dir/.compose-flags"+cli logs llm 5 > /dev/null+grep -qx 'resolve' "$CLI_CALLS" || fail 'a vLLM cache was reused on the llama-server route'+grep -q ' logs -f --tail 5 llama-server$' "$CLI_CALLS" || fail 'a llama-server install lost its llm alias'+pass 'a llama-server install re-resolves a vLLM cache and keeps llm on llama-server'++# 2. Lifecycle commands never name llama-server to Compose.+write_env+printf '%s\n' "$VLLM_FLAGS" > "$install_dir/.compose-flags"+cli restart llm > /dev/null+grep -q 'up -d --force-recreate --no-build --pull never vllm-server$' "$CLI_CALLS" \+ || fail "ods restart llm did not recreate vllm-server: $(cat "$CLI_CALLS")"+cli stop llm > /dev/null+grep -q ' stop vllm-server$' "$CLI_CALLS" || fail 'ods stop llm did not stop vllm-server'+cli start llama-server > /dev/null+grep -q ' up -d vllm-server$' "$CLI_CALLS" || fail 'ods start llama-server did not start vllm-server'+for command in restart stop start; do+ cli "$command" llm > /dev/null+ if grep -qE '^docker compose .* llama-server$' "$CLI_CALLS"; then+ fail "ods $command named llama-server to Compose"+ fi+done+pass 'restart, stop and start of llm or llama-server reach vllm-server only'++# 3. Status: vllm-server from Docker's health, no llama-server fault.+for state in 'running healthy:healthy' 'running starting:starting' 'running unhealthy:unhealthy' \+ 'exited none:stopped' 'missing:stopped'; do+ VLLM_STATE="${state%%:*}"+ cli status --json > "$FIXTURE/status.json"+ jq -e --arg status "${state##*:}" '.services[] | select(.id == "vllm-server")+ | .status == $status and .containerStatus == "running" and .port == ""' "$FIXTURE/status.json" >/dev/null \+ || fail "vllm-server should be ${state##*:} for '$VLLM_STATE': $(cat "$FIXTURE/status.json")"+ jq -e 'all(.services[]; .id != "llama-server") and any(.services[]; .id == "model-router")' \+ "$FIXTURE/status.json" >/dev/null || fail 'status JSON reported llama-server or dropped model-router'+done+VLLM_STATE="running starting"+cli status > "$FIXTURE/status.txt" 2>&1+grep -q 'vLLM (LLM Inference): starting. A first start loads the model' "$FIXTURE/status.txt" \+ || fail "a cold start did not read as starting: $(cat "$FIXTURE/status.txt")"+grep -q 'llama-server' "$FIXTURE/status.txt" && fail "status mentioned llama-server: $(cat "$FIXTURE/status.txt")"+grep -q '127.0.0.1:8080/health' "$CLI_CALLS" && fail 'status probed the profiled-out llama-server port'+VLLM_STATE="running healthy"+cli status > "$FIXTURE/status.txt" 2>&1+grep -q 'vLLM (LLM Inference): healthy' "$FIXTURE/status.txt" || fail 'a ready vLLM engine was not healthy'+pass 'status reports vllm-server from Docker health, starting during a cold start, and no llama-server'++# 4. ods mode and mode-switch.sh keep the vLLM route local.+for mode in cloud hybrid; do+ write_env+ cp "$install_dir/.env" "$FIXTURE/before.env"+ if cli mode "$mode" > "$FIXTURE/mode.out" 2>&1; then fail "ods mode $mode was accepted on vLLM"; fi+ grep -q 'vLLM engine, which works only in local mode. Nothing was changed.' "$FIXTURE/mode.out" \+ || fail "ods mode $mode refusal: $(cat "$FIXTURE/mode.out")"+ grep -q 'discord.gg' "$FIXTURE/mode.out" || fail 'the refusal names no help'+ cmp -s "$FIXTURE/before.env" "$install_dir/.env" || fail "ods mode $mode changed .env"+ if (cd "$install_dir/scripts" && bash ./mode-switch.sh "$mode") > "$FIXTURE/mode.out" 2>&1; then+ fail "mode-switch.sh $mode was accepted on vLLM"+ fi+ grep -q 'works only in local mode' "$FIXTURE/mode.out" || fail "mode-switch.sh $mode refusal"+ cmp -s "$FIXTURE/before.env" "$install_dir/.env" || fail "mode-switch.sh $mode changed .env"+done+write_env 'LLM_API_URL=http://llama-server:8080' 'ODS_MODE=hybrid'+cli mode local > /dev/null+grep -qx 'LLM_API_URL=http://litellm:4000' "$install_dir/.env" || fail 'ods mode local pointed back at llama-server'+grep -qx 'HERMES_LLM_BASE_URL=http://model-router:9099/v1' "$install_dir/.env" || fail 'Hermes left model-router'+write_env 'LLM_API_URL=http://llama-server:8080'+(cd "$install_dir/scripts" && bash ./mode-switch.sh local) > /dev/null+grep -qx 'LLM_API_URL=http://litellm:4000' "$install_dir/.env" || fail 'mode-switch.sh local pointed back at llama-server'+cli mode > "$FIXTURE/mode.out" 2>&1+grep -q 'LLM: Local vLLM engine (vllm-server)' "$FIXTURE/mode.out" || fail 'ods mode did not name the vLLM engine'+pass 'ods mode refuses cloud and hybrid on vLLM and local keeps clients off llama-server'++# 5. ods doctor's summary names the vLLM engine as ODS's own local model server.+awk '/python3 - "\$report_file" <<'"'"'PY'"'"'/ {take=1; next} take && /^PY$/ {exit} take {print}' "$ROOT_DIR/ods-cli" > "$FIXTURE/doctor-view.py"+[[ -s "$FIXTURE/doctor-view.py" ]] || fail 'the doctor report renderer was not found in ods-cli'+printf '%s\n' '{"runtime": {"llm_backend": {"status": "ok", "provider": "vLLM", "url": "http://vllm-server:8000 (ODS network only)"}}}' > "$FIXTURE/doctor.json"+# The renderer exits 1 when this partial report lacks the Docker checks; only its LLM line matters here.+view_status=0+PYTHONIOENCODING=utf-8 python3 "$FIXTURE/doctor-view.py" "$FIXTURE/doctor.json" > "$FIXTURE/doctor.out" 2>&1 || view_status=$?+[[ "$view_status" -le 1 ]] || fail "the doctor report renderer crashed ($view_status): $(cat "$FIXTURE/doctor.out")"+grep -q 'LLM Backend: vLLM (local) — responding' "$FIXTURE/doctor.out" \+ || fail "ods doctor must call the vLLM engine local: $(cat "$FIXTURE/doctor.out")"+pass 'ods doctor names the vLLM engine as a local model server'diff --git a/ods/tests/test-doctor-vllm-engine.sh b/ods/tests/test-doctor-vllm-engine.shnew file mode 100644index 000000000..dee990766--- /dev/null+++ b/ods/tests/test-doctor-vllm-engine.sh@@ -0,0 +1,98 @@+#!/usr/bin/env bash+# ods doctor on a vLLM install (LLM_BACKEND=vllm, docker-compose.vllm.yml):+# vllm-server has no host port and llama-server is off, so the LLM check reads+# Docker's health of ods-vllm-server. A cold start (weights load plus kernel+# compiles, up to 15 minutes) reads as starting, not a fault; a missing+# llama-server is no fault; a llama-server container an earlier install left+# running is named, with a stop command that does not stop vLLM.+# Extracted production functions consume the fixture globals below.+# shellcheck disable=SC2034+set -euo pipefail++root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"+if (( BASH_VERSINFO[0] < 4 )); then+ for candidate in /opt/homebrew/bin/bash /usr/local/bin/bash; do+ if [[ -x "$candidate" ]]; then exec "$candidate" "$0" "$@"; fi+ done+ echo 'Bash 4+ is required' >&2+ exit 1+fi+extract() {+ awk -v name="$1" '$0 ~ "^" name "[(][)]" { active=1 }+ active {print} active && /^}/ {exit}' "$root/scripts/ods-doctor.sh"+}+eval "$(extract _doctor_check_vllm_engine)"+eval "$(extract _doctor_check_llm_backend)"+fail() { printf '[FAIL] %s\n' "$*" >&2; exit 1; }+pass() { printf '[PASS] %s\n' "$*"; }+messages=()+log_ok() { messages+=("ok:$*"); }+log_fail() { messages+=("fail:$*"); }+log_info() { messages+=("info:$*"); }+log_warn() { messages+=("warn:$*"); }+_doctor_check_llama_server() { LLM_PROVIDER="llama-server"; LLM_STATUS="llama-checked"; }+ROOT_DIR="$root"+vllm_state="running healthy"+llama_running=false+docker() {+ case "$1" in+ inspect)+ [[ "${*: -1}" == ods-vllm-server ]] || fail "inspected ${*: -1}"+ [[ "$vllm_state" != missing ]] || { printf 'Error: No such object: ods-vllm-server\n'; return 1; }+ printf '%s\n' "$vllm_state" ;;+ ps) [[ "$llama_running" == false ]] || printf 'ods-llama-server\n' ;;+ *) fail "unexpected docker $*" ;;+ esac+}+DOCKER_DAEMON=true+ODS_MODE=local+VLLM_SERVED_MODEL=GLM-5.3-Flash-NVFP4+unset EXTERNAL_LLM_URL NATIVE_LLM_BASE_URL++for backend in vllm VLLM; do+ LLM_BACKEND="$backend"+ _doctor_check_llm_backend+ [[ "$LLM_PROVIDER" == vLLM && "$LLM_STATUS" == ok && "$LLM_MODEL" == GLM-5.3-Flash-NVFP4 ]] \+ || fail "LLM_BACKEND=$backend did not check the vLLM engine: $LLM_PROVIDER $LLM_STATUS"+done+pass 'LLM_BACKEND=vllm checks vllm-server, never llama-server'++messages=()+vllm_state="running starting"+_doctor_check_llm_backend+[[ "$LLM_STATUS" == starting && -z "$LLM_RECOVERY" ]] || fail "a cold start was not starting: $LLM_STATUS"+said="$(printf '%s\n' "${messages[@]}")"+grep -q '^info:LLM backend: vLLM (local) — starting: a first start loads the model' <<<"$said" \+ || fail "cold start message: ${messages[*]}"+if grep -q '^fail:' <<<"$said"; then fail "a cold start reported a fault: ${messages[*]}"; fi+pass 'a cold start reads as starting, with no fault and no restart advice'++for state in 'running unhealthy:ods restart vllm-server' 'exited none:ods start vllm-server' \+ 'missing:ods start vllm-server'; do+ vllm_state="${state%%:*}"+ _doctor_check_llm_backend+ [[ "$LLM_STATUS" == fail && "$LLM_RECOVERY" == "${state##*:}" ]] \+ || fail "'$vllm_state' should fail with ${state##*:}: $LLM_STATUS $LLM_RECOVERY"+done+pass 'an unhealthy or stopped engine fails with a vllm-server recovery'++messages=()+vllm_state="running healthy"+llama_running=true+_doctor_check_llm_backend+[[ "$LLM_STATUS" == ok && "$LLM_LOCAL_WARNING" == true ]] || fail 'a running stale llama-server was not noted'+said="$(printf '%s\n' "${messages[@]}")"+grep -q 'docker stop ods-llama-server' <<<"$said" || fail "stale llama-server note: ${messages[*]}"+grep -q 'Stop it with: docker stop ods-llama-server' "$root/scripts/ods-doctor.sh" \+ || fail 'the fix hint must stop the container, since ods stop llama-server names vllm-server on this route'+pass 'a llama-server left running beside vLLM is named with a stop that keeps vLLM'++LLM_BACKEND=llama-server+llama_running=false+_doctor_check_llm_backend+[[ "$LLM_STATUS" == llama-checked ]] || fail 'a llama-server install left the llama-server check'+LLM_BACKEND=vllm+ODS_MODE=cloud+_doctor_check_llm_backend+[[ "$LLM_PROVIDER" != vLLM ]] || fail 'cloud mode checked the vLLM engine'+pass 'llama-server and cloud installs keep their checks'diff --git a/ods/tests/test-model-library-vllm.py b/ods/tests/test-model-library-vllm.pyindex d29d638c4..1058fedb7 100644--- a/ods/tests/test-model-library-vllm.py+++ b/ods/tests/test-model-library-vllm.py@@ -159,6 +159,11 @@ def _mutate(path: str, value):("selection.discrete", 50, "selection.discrete must be 0"),("selection.vllm", KeyError, "selection.vllm must be a whole number"),("served_model_name", "GLM 5.3", "served_model_name must be letters, digits and ._-"),+ ("thinking_template_kwargs", KeyError, "thinking_template_kwargs must map each of off, low, medium, high"),+ ("thinking_template_kwargs.high", KeyError, "thinking_template_kwargs must map each of"),+ ("thinking_template_kwargs.medium", "high", "thinking_template_kwargs.medium must map"),+ ("thinking_template_kwargs.off", {"reasoning_effort": "lo w"}, "thinking_template_kwargs.off must map"),+ ("thinking_template_kwargs.low", {"Reasoning-Effort": "low"}, "thinking_template_kwargs.low must map"),])def test_the_schema_catches_broken_entries(path, value, expected):problems = entry_problems(_mutate(path, value))@@ -213,6 +218,55 @@ def test_the_entries_keep_the_flags_measured_on_a_dgx_station(model_id):assert switches == MEASURED_SWITCHES[model_id], model_id+# Each ODS level in the model's own chat-template arguments, read from the+# templates on the DGX Station (10-27). GLM-5.3's template has no+# enable_thinking and accepts only reasoning_effort low or high: any other+# value (none, medium, xhigh) thinks at "max", so "low" is the nearest to off+# and high omits the field. Qwen3.8's turns thinking off with enable_thinking+# false and answers HTTP 400 for any reasoning_effort but xhigh (its+# default), medium and low. Measured (GB300, temperature 0.6): GLM max 7.5 s+# vs high 3.4 s and low 2.8 s; Qwen xhigh 5.0 s, medium 3.7 s, low 3.4 s.+THINKING = {+ "glm-5.3-flash-nvfp4-vllm": {+ "off": {"reasoning_effort": "low"}, "low": {"reasoning_effort": "low"},+ "medium": {"reasoning_effort": "high"}, "high": {},+ },+ "qwen3.8-flash-next-nvfp4-vllm": {+ "off": {"enable_thinking": False}, "low": {"reasoning_effort": "low"},+ "medium": {"reasoning_effort": "medium"}, "high": {"reasoning_effort": "xhigh"},+ },+}+TEMPLATE_EFFORTS = {+ "glm-5.3-flash-nvfp4-vllm": {"low", "high"},+ "qwen3.8-flash-next-nvfp4-vllm": {"low", "medium", "xhigh"},+}++[email protected]("model_id", sorted(THINKING))+def test_each_level_uses_the_models_own_template_arguments(model_id):+ model = {item["id"]: item for item in _vllm_models()}[model_id]+ assert model["thinking_template_kwargs"] == THINKING[model_id]+ # No level hands the template a reasoning_effort it does not accept.+ for level, kwargs in model["thinking_template_kwargs"].items():+ assert kwargs.get("reasoning_effort", "low") in TEMPLATE_EFFORTS[model_id], (model_id, level)+++def test_every_vllm_entry_has_a_thinking_map():+ assert {model["id"] for model in _vllm_models()} == set(THINKING)++[email protected]("model_id", sorted(MEASURED_FLAGS))+def test_the_entries_declare_tools_and_vision(model_id):+ # The host agent's route proof publishes these catalog booleans as the+ # route's capabilities (model-state, Pixel connection metadata). Both+ # checkpoints are image-text-to-text and run vLLM's tool parser; on the+ # DGX Station GB300 GLM made tool calls (streamed and not) and read an+ # image through model-router. Without them the route claimed neither.+ model = {item["id"]: item for item in _vllm_models()}[model_id]+ assert model.get("tools") is True and model.get("vision") is True, model_id+ assert "--enable-auto-tool-choice" in model["vllm_args"], model_id++def test_the_broken_entry_cases_edit_what_they_name():# "repeats README.md" needs README.md after files[0]; "sha256 must be# lowercase hex" needs files[8] to be an LFS file.diff --git a/ods/tests/test-ods-preflight-llm-route.sh b/ods/tests/test-ods-preflight-llm-route.shindex c6438f816..415e60e1f 100644--- a/ods/tests/test-ods-preflight-llm-route.sh+++ b/ods/tests/test-ods-preflight-llm-route.sh@@ -45,6 +45,32 @@ assert_skipped skipped model-router cloud ''assert_skipped skipped llama-server local http://localhost:8080assert_skipped shown model-router local http://localhost:8080+# LLM_BACKEND=vllm: vllm-server (no host port) serves the model through+# model-router and LiteLLM; the in-stack llama-server is off.+for backend in vllm VLLM; do+ if ! (export LLM_BACKEND="$backend" ODS_MODE=local EXTERNAL_LLM_URL='' NATIVE_LLM_BASE_URL=''+ ods_preflight_vllm_engine && ods_preflight_uses_litellm); then+ printf 'FAIL LLM_BACKEND=%s should select the gateway route\n' "$backend" >&2+ exit 1+ fi+ if ! (export LLM_BACKEND="$backend" ODS_MODE=local EXTERNAL_LLM_URL='' NATIVE_LLM_BASE_URL=''+ ods_status_skips_managed_inference llama-server); then+ printf 'FAIL LLM_BACKEND=%s status should leave out llama-server\n' "$backend" >&2+ exit 1+ fi+ if (export LLM_BACKEND="$backend" ODS_MODE=local EXTERNAL_LLM_URL='' NATIVE_LLM_BASE_URL=''+ ods_status_skips_managed_inference model-router); then+ printf 'FAIL LLM_BACKEND=%s status must keep model-router, which reaches vllm-server\n' "$backend" >&2+ exit 1+ fi+done+for backend in '' llama-server external; do+ if (export LLM_BACKEND="$backend"; ods_preflight_vllm_engine); then+ printf 'FAIL LLM_BACKEND=%s is not the vLLM engine\n' "$backend" >&2+ exit 1+ fi+done+# ods-preflight.sh is #!/bin/bash, which is Bash 3.2 on macOS: the route# selector must not use Bash 4 case conversion (${var,,} aborts the preflight).if grep -nE '\$\{[A-Za-z_][A-Za-z_0-9]*(,,?|\^\^?)\}' "$ROOT_DIR/lib/preflight-llm-route.sh"; thendiff --git a/ods/tests/test-ods-test-tool-calling.sh b/ods/tests/test-ods-test-tool-calling.shindex 9c17b0f8f..dccb55773 100755--- a/ods/tests/test-ods-test-tool-calling.sh+++ b/ods/tests/test-ods-test-tool-calling.sh@@ -105,7 +105,8 @@ class Handler(BaseHTTPRequestHandler):self.wfile.write(data)def do_GET(self):- if self.path == "/health":+ # /health/readiness: LiteLLM's, which a vLLM install's checks ask.+ if self.path in ("/health", "/health/readiness"):self.reply(200, {"status": "ok"})elif self.path == "/v1/models":self.reply(200, {"object": "list", "data": [{"id": MODEL, "object": "model"}]})@@ -122,7 +123,7 @@ class Handler(BaseHTTPRequestHandler):request = Noneif request is not None:with open(os.path.join(WORK, "requests.jsonl"), "a") as log:- log.write(json.dumps(request) + "\n")+ log.write(json.dumps({**request, "_authorization": self.headers.get("Authorization")}) + "\n")elif mode in WORKING:self.reply(400, {"error": {"code": 400, "type": "invalid_request_error","message": "Failed to parse the request body as JSON"}})@@ -205,3 +206,19 @@ expect "Tool Calling reads the install's .env by default (empty HOME)" call toolexpect "LLM Inference passes on an assistant answer" text llm "LLM Inference" passexpect "LLM Inference passes when a thinking model only reasoned" reasoning llm "LLM Inference" passexpect "LLM Inference fails on an error body" error llm "LLM Inference" fail++# LLM_BACKEND=vllm: vllm-server publishes no host port, so the LLM checks go+# through the LiteLLM gateway (its readiness, then ods/current with its key),+# as ODS clients reach the engine.+printf 'LLM_BACKEND=vllm\nLITELLM_PORT=%s\nLITELLM_KEY=sk-gateway-fixture\n' "$(cat "$TMP_DIR/port")" > "$INSTALL/.env"+expect "Tool Calling on a vLLM install goes through the gateway" call tool-calling "Tool Calling" pass+python3 - "$TMP_DIR/requests.jsonl" <<'PY' || fail "the vLLM check did not ask the gateway for ods/current with its key"+import json, sys+with open(sys.argv[1]) as stream:+ requests = [json.loads(line) for line in stream if line.strip()]+assert requests, "no parseable chat request reached the gateway"+assert requests[-1]["model"] == "ods/current", requests[-1]["model"]+assert requests[-1]["_authorization"] == "Bearer sk-gateway-fixture", requests[-1]["_authorization"]+PY+pass "the vLLM check names ods/current and sends the gateway key"+expect "LLM Inference on a vLLM install passes through the gateway" text llm "LLM Inference" passdiff --git a/ods/tests/test-render-runtime-configs.py b/ods/tests/test-render-runtime-configs.pyindex 60a864897..d1d47bca8 100644--- a/ods/tests/test-render-runtime-configs.py+++ b/ods/tests/test-render-runtime-configs.py@@ -528,8 +528,20 @@ def test_llama_server_backends_render_todays_bytes() -> None:assert flagged["files"] == baseline["files"], value+# The vllm endpoint carries the served model's thinking map (catalog+# reasoning + thinking_template_kwargs): model-router translates a request's+# level with it, since GLM-5.3 and Qwen3.8 read different template arguments.+VLLM_MODEL = ("--model", "glm-5.3-flash-nvfp4")+GLM_THINKING = {"default": "medium", "templateKwargs": {+ "off": {"reasoning_effort": "low"}, "low": {"reasoning_effort": "low"},+ "medium": {"reasoning_effort": "high"}, "high": {}}}+QWEN_THINKING = {"default": "medium", "templateKwargs": {+ "off": {"enable_thinking": False}, "low": {"reasoning_effort": "low"},+ "medium": {"reasoning_effort": "medium"}, "high": {"reasoning_effort": "xhigh"}}}++def test_vllm_backend_routes_model_router_to_vllm_server() -> None:- surface = ("--surface", "model-router-endpoints")+ surface = ("--surface", "model-router-endpoints", *VLLM_MODEL)for payload in (run_renderer(*surface, "--llm-backend", "vllm", env=llm_backend_env()),run_renderer(*surface, env=llm_backend_env("vllm")),@@ -541,14 +553,42 @@ def test_vllm_backend_routes_model_router_to_vllm_server() -> None:assert payload["inputs"]["llm_backend"] == "vllm"content = file_by_surface(payload, "model-router-endpoints")["content"]assert json.loads(content) == {- "endpoints": [{"id": "vllm-default", "baseUrl": "http://vllm-server:8000"}],+ "endpoints": [{"id": "vllm-default", "baseUrl": "http://vllm-server:8000",+ "thinking": GLM_THINKING}],}+def test_vllm_endpoint_carries_each_models_own_thinking_map() -> None:+ surface = ("--surface", "model-router-endpoints", "--llm-backend", "vllm")+ for model, thinking in (("glm-5.3-flash-nvfp4", GLM_THINKING),+ ("qwen3.8-flash-next-nvfp4", QWEN_THINKING),+ # The catalog id names the same entry.+ ("qwen3.8-flash-next-nvfp4-vllm", QWEN_THINKING)):+ payload = run_renderer(*surface, "--model", model, env=llm_backend_env())+ endpoint = json.loads(file_by_surface(payload, "model-router-endpoints")["content"])["endpoints"][0]+ assert endpoint["thinking"] == thinking, model+++def test_vllm_endpoint_refuses_a_model_without_a_vllm_catalog_entry() -> None:+ for model in ("qwen3.5-9b", "no-such-model"):+ proc = subprocess.run(+ [sys.executable, str(SCRIPT), "--surface", "model-router-endpoints",+ "--llm-backend", "vllm", "--model", model],+ cwd=ROOT, env=llm_backend_env(), text=True, capture_output=True,+ )+ assert proc.returncode == 2, model+ assert "needs --model to name one vLLM catalog entry" in proc.stderr, proc.stderr+ # Only the router endpoint needs the entry; other surfaces render as before.+ switchboard = run_renderer("--surface", "litellm-switchboard", "--llm-backend", "vllm",+ "--litellm-key", "sk-test-litellm", env=llm_backend_env())+ assert switchboard["inputs"]["vllm_thinking"] is None++def test_vllm_backend_changes_only_the_router_endpoint_and_env_backend() -> None:- baseline = run_renderer(*ALL_ENABLED, env=llm_backend_env())- vllm = run_renderer(*ALL_ENABLED, "--llm-backend", "vllm", env=llm_backend_env())- assert file_by_surface(vllm, "env")["content"] == LLAMA_SERVER_ENV.replace(+ baseline = run_renderer(*ALL_ENABLED, *VLLM_MODEL, env=llm_backend_env())+ vllm = run_renderer(*ALL_ENABLED, *VLLM_MODEL, "--llm-backend", "vllm", env=llm_backend_env())+ assert "LLM_BACKEND=vllm\n" in file_by_surface(vllm, "env")["content"]+ assert file_by_surface(vllm, "env")["content"] == file_by_surface(baseline, "env")["content"].replace("LLM_BACKEND=llama-server\n", "LLM_BACKEND=vllm\n")changed = {after["surface"] for after, before in zip(vllm["files"], baseline["files"])if after != before}@@ -934,6 +974,8 @@ def main() -> int:test_router_endpoints_strip_trailing_v1,test_llama_server_backends_render_todays_bytes,test_vllm_backend_routes_model_router_to_vllm_server,+ test_vllm_endpoint_carries_each_models_own_thinking_map,+ test_vllm_endpoint_refuses_a_model_without_a_vllm_catalog_entry,test_vllm_backend_changes_only_the_router_endpoint_and_env_backend,test_retired_lemonade_flags_are_accepted_and_ignored,test_litellm_lemonade_surface_is_retired,diff --git a/ods/tests/test-router-transport.py b/ods/tests/test-router-transport.pyindex ccc9e5112..f6821d584 100644--- a/ods/tests/test-router-transport.py+++ b/ods/tests/test-router-transport.py@@ -64,6 +64,20 @@ class OwnershipTests(unittest.TestCase):self.assertEqual(calls[2].kwargs["timeout"], 15)self.assertNotIn(".Config.Env", calls[1].args[0][5])+ def test_the_vllm_endpoint_is_named_and_no_other_is_accepted(self):+ # LLM_BACKEND=vllm: vllm-server has no host port; the router container+ # reaches it at the vllm-default endpoint's origin.+ result, calls = self.run_request(endpoint_id="vllm-default", response=b"")+ self.assertEqual(result, "")+ self.assertEqual(json.loads(calls[2].kwargs["input"])["endpoint_id"], "vllm-default")+ for endpoint_id in ("lemonade-default", "keyed", ""):+ with self.subTest(endpoint_id=endpoint_id), \+ patch.object(transport.subprocess, "run", side_effect=AssertionError("docker ran")):+ with self.assertRaises(ValueError):+ transport.request(self.root, ORIGIN, "/health", endpoint_id=endpoint_id)+ with self.assertRaises(ValueError):+ transport.exchange(self.root, ORIGIN, "/health", endpoint_id=endpoint_id)+def test_exchange_asks_for_an_envelope_and_returns_status_and_text(self):envelope = json.dumps({"status": 400, "body": "Unable to generate parser for this template."}).encode()results = [self.process((CONTAINER_ID + "\n").encode()), self.process(json.dumps(self.info).encode()),diff --git a/ods/tests/test_doctor_vllm_route.py b/ods/tests/test_doctor_vllm_route.pynew file mode 100644index 000000000..542fc546f--- /dev/null+++ b/ods/tests/test_doctor_vllm_route.py@@ -0,0 +1,93 @@+"""ods doctor's inference contract on a vLLM install (LLM_BACKEND=vllm).++The route keeps llama-server off and sends clients through LiteLLM to+model-router and vllm-server, so the doctor expects the LiteLLM gateway and+the docker-compose.vllm.yml overlay, flags an LLM_API_URL that still names+the in-stack llama-server, and no longer warns that local mode routes through+LiteLLM. The contract functions are compiled from scripts/ods-doctor.sh's+embedded Python without running the doctor.+"""++from __future__ import annotations++import ast+import json+import os+import pathlib+import re+import shlex+import sys+from urllib import error, parse, request++import pytest++DOCTOR = pathlib.Path(__file__).resolve().parents[1] / "scripts" / "ods-doctor.sh"+NAMES = {+ "_clean_env", "_read_text", "_parse_kv_lines", "_truthy", "_env_file_values", "_compose_files_from_flags",+ "_has_compose_file", "_looks_like_litellm_route", "_looks_like_local_llama_route",+ "_url_hostname", "_source", "_evidence", "_diagnosis", "_inference_issue",+ "_collect_inference_contract",+}+++def _contract(root: pathlib.Path):+ text = DOCTOR.read_text(encoding="utf-8")+ start = text.index("<<'PY'\n") + len("<<'PY'\n")+ source = text[start:text.index("\nPY\n", start)]+ tree = ast.parse(source)+ module = ast.Module(body=[node for node in tree.body+ if isinstance(node, ast.FunctionDef) and node.name in NAMES], type_ignores=[])+ namespace = {"root_dir": root, "os": os, "re": re, "json": json, "shlex": shlex, "sys": sys,+ "parse": parse, "error": error, "request": request, "pathlib": pathlib}+ exec(compile(module, str(DOCTOR), "exec"), namespace)+ assert NAMES <= set(namespace), NAMES - set(namespace)+ return namespace["_collect_inference_contract"]+++VLLM_FLAGS = "-f docker-compose.base.yml -f docker-compose.nvidia.yml -f docker-compose.vllm.yml"+++def doctor(tmp_path, monkeypatch):+ for key in ("ODS_MODE", "LLM_BACKEND", "LLM_API_URL", "EXTERNAL_LLM_URL", "NATIVE_LLM_BASE_URL",+ "HERMES_LLM_BASE_URL", "GPU_BACKEND"):+ monkeypatch.delenv(key, raising=False)++ def run(env: dict[str, str], flags: str | None = VLLM_FLAGS):+ (tmp_path / ".env").write_text("".join(f"{k}={v}\n" for k, v in env.items()), encoding="utf-8")+ if flags is None:+ (tmp_path / ".compose-flags").unlink(missing_ok=True)+ else:+ (tmp_path / ".compose-flags").write_text(flags + "\n", encoding="utf-8")+ return _contract(tmp_path)()+ return run+++VLLM_ENV = {"ODS_MODE": "local", "GPU_BACKEND": "nvidia", "LLM_BACKEND": "vllm",+ "LLM_API_URL": "http://litellm:4000", "HERMES_LLM_BASE_URL": "http://model-router:9099/v1"}+++def test_a_vllm_install_expects_the_gateway_and_has_no_issue(doctor):+ report = doctor(VLLM_ENV)+ assert report["expected_gateway"] == "litellm"+ assert report["expected_inference_owner"] == "ods"+ assert report["signals"]["vllm_engine"] is True and report["signals"]["vllm_overlay"] is True+ assert report["issues"] == []+++def test_a_missing_overlay_or_a_llama_server_url_is_a_blocker(doctor):+ report = doctor(VLLM_ENV, flags="-f docker-compose.base.yml -f docker-compose.nvidia.yml")+ assert [issue["id"] for issue in report["issues"]] == ["ODS-RUNTIME-VLLM-OVERLAY-MISSING"]+ for url in ("http://llama-server:8080", "http://ods-llama-server:8080/v1"):+ report = doctor({**VLLM_ENV, "LLM_API_URL": url})+ assert [issue["id"] for issue in report["issues"]] == ["ODS-RUNTIME-VLLM-LLAMA-ROUTE"], url+ assert report["diagnoses"][0]["severity"] == "blocker"+ assert "ods mode local" in report["diagnoses"][0]["next_steps"][0]+++def test_llama_server_installs_keep_their_contract(doctor):+ llama = {**VLLM_ENV, "LLM_BACKEND": "llama-server"}+ report = doctor(llama, flags="-f docker-compose.base.yml -f docker-compose.nvidia.yml")+ assert report["expected_gateway"] == "llama-server"+ assert [issue["id"] for issue in report["issues"]] == ["ODS-RUNTIME-LOCAL-LITELLM-ROUTE"]+ assert report["signals"]["vllm_engine"] is Falsediff --git a/ods/tests/test_hf_manifest_download.py b/ods/tests/test_hf_manifest_download.pyindex 5cf8b0205..8a773c48a 100644--- a/ods/tests/test_hf_manifest_download.py+++ b/ods/tests/test_hf_manifest_download.py@@ -68,6 +68,7 @@ def write_catalog(path: Path, files: dict[str, bytes], **overrides) -> Path:"served_model_name": "Test-Model-NVFP4", "llm_model_name": "test-model-nvfp4","context_length": 8192, "min_gpu_mib": 1024, "compute_capability": ["10.0"],"reasoning": "off", "tool_call_parser": "hermes", "reasoning_parser": "qwen3",+ "thinking_template_kwargs": {"off": {"enable_thinking": False}, "low": {}, "medium": {}, "high": {}},"vllm_args": ["--max-model-len", "8192", "--enable-auto-tool-choice","--tool-call-parser", "hermes", "--reasoning-parser", "qwen3"],"selection": {"discrete": 0, "unified": 0, "cpu": 0, "vllm": 1},diff --git a/ods/tests/test_host_agent_vllm.py b/ods/tests/test_host_agent_vllm.pynew file mode 100644index 000000000..06236f4dc--- /dev/null+++ b/ods/tests/test_host_agent_vllm.py@@ -0,0 +1,278 @@+"""The host agent on a vLLM install (LLM_BACKEND=vllm, docker-compose.vllm.yml).++vllm-server serves the one model on the ODS network without a host port, so+the agent proves it through the owned model-router container against the+router's vllm-default endpoint: /health is an empty 200, /v1/models lists the+served id with max_model_len (vLLM has no /props), and the completion names+that exact id. A cold start reads as loading while Docker reports the+container starting. Model switching, profile checks and every path that+would name llama-server to Compose refuse with a clear message instead.+No Docker, network or real install is used.+"""++from __future__ import annotations++import importlib.util+import io+import json+import shutil+import subprocess+import sys+from pathlib import Path++import pytest++ROOT = Path(__file__).resolve().parents[1]+_spec = importlib.util.spec_from_file_location("ods_host_agent_vllm_test", ROOT / "bin" / "ods-host-agent.py")+agent = importlib.util.module_from_spec(_spec)+sys.modules["ods_host_agent_vllm_test"] = agent+_spec.loader.exec_module(agent)++SERVED = "GLM-5.3-Flash-NVFP4"+ENV = {+ "ODS_MODE": "local", "GPU_BACKEND": "nvidia", "LLM_BACKEND": "vllm",+ "LLM_MODEL": "glm-5.3-flash-nvfp4", "VLLM_SERVED_MODEL": SERVED,+ "VLLM_MODEL_DIR": "nvidia--GLM-5.3-Flash-NVFP4@da920bb0",+ "GGUF_FILE": "", "CTX_SIZE": "131072", "MAX_CONTEXT": "131072",+}+# GLM-5.3's template has no enable_thinking; its catalog "off" is this.+GLM_OFF = {"reasoning_effort": "low"}+++def models(model=SERVED, context=131072):+ return json.dumps({"object": "list", "data": [{+ "id": model, "object": "model", "owned_by": "vllm", "root": "/model", "max_model_len": context}]})+++def completion(model=SERVED, content="READY"):+ return json.dumps({"model": model, "choices": [{"message": {"role": "assistant", "content": content},+ "finish_reason": "stop"}]})+++class FakeVllm:+ """vllm-server as the router transport reaches it."""++ def __init__(self, *, health=200, listed=None, answer=None, container="healthy"):+ self.health, self.listed, self.answer, self.container = health, listed, answer, container+ self.calls = []++ def request(self, install_dir, origin, path, **kwargs):+ self.calls.append((origin, path, kwargs))+ if path == "/v1/models":+ return self.listed or models()+ if path == "/v1/chat/completions":+ return self.answer or completion()+ # vLLM has no /props: the transport fails an HTTP 404.+ raise OSError(f"Docker runtime transport failed (exit 1): runtime returned HTTP 404 for {path}")++ def exchange(self, install_dir, origin, path, **kwargs):+ self.calls.append((origin, path, kwargs))+ if isinstance(self.health, Exception):+ raise self.health+ return self.health, ""+++def vllm(monkeypatch, tmp_path):+ fake = FakeVllm()+ monkeypatch.setattr(agent, "_router_transport_request", fake.request)+ monkeypatch.setattr(agent, "_router_transport_exchange", fake.exchange)+ monkeypatch.setattr(agent, "_container_health", lambda container: fake.container)+ monkeypatch.setattr(agent, "INSTALL_DIR", tmp_path)+ monkeypatch.delenv("ODS_HOST_INSTALL_DIR", raising=False)+ (tmp_path / "config").mkdir()+ shutil.copyfile(ROOT / "config" / "model-library.json", tmp_path / "config" / "model-library.json")+ return fake+++def prove(env=ENV, **kwargs):+ return agent._wait_for_model_readiness(+ env, model_id="glm-5.3-flash-nvfp4-vllm", gguf_file=SERVED, llm_model_name=env["LLM_MODEL"],+ attempts=kwargs.pop("attempts", 1), initial_delay=0, interval=0, return_proof=True, **kwargs)+++def test_route_backend_and_endpoint_follow_llm_backend():+ assert agent._initial_switchboard_backend(ENV) == ("vllm", "vllm-default", None)+ assert agent._switchboard_state.default_endpoint_id("vllm") == "vllm-default"+ for backend in ("", "llama-server", "external", "lemonade"):+ assert agent._initial_switchboard_backend({**ENV, "LLM_BACKEND": backend}) == (+ "llama-server", "llama-server-default", None), backend+++def test_vllm_is_proven_through_the_routers_vllm_endpoint(vllm, monkeypatch):+ monkeypatch.setattr(agent.subprocess, "run", lambda *a, **k: pytest.fail("no host port to probe"))+ proof = prove()++ assert proof["identity"] == SERVED+ assert proof["contextLength"] == 131072 and proof["contextVerified"] is True+ assert "vision" not in proof+ assert [call[1] for call in vllm.calls] == ["/health", "/v1/models", "/v1/chat/completions"]+ assert all(call[0] == "http://vllm-server:8000" for call in vllm.calls)+ assert all(call[2]["endpoint_id"] == "vllm-default" and call[2]["api_key"] == "" for call in vllm.calls)+ payload = vllm.calls[-1][2]["payload"]+ # vLLM answers 404 for any name but the served one.+ assert payload["model"] == SERVED+ assert payload["chat_template_kwargs"] == GLM_OFF+++def test_qwen_turns_thinking_off_with_its_own_template_argument(vllm):+ env = {**ENV, "LLM_MODEL": "qwen3.8-flash-next-nvfp4", "VLLM_SERVED_MODEL": "Qwen3.8-Flash-Next-NVFP4"}+ vllm.listed = models("Qwen3.8-Flash-Next-NVFP4")+ vllm.answer = completion("Qwen3.8-Flash-Next-NVFP4")+ assert prove(env)["identity"] == "Qwen3.8-Flash-Next-NVFP4"+ assert vllm.calls[-1][2]["payload"]["chat_template_kwargs"] == {"enable_thinking": False}+++def test_a_short_or_missing_served_context_never_proves_the_route(vllm):+ for listed in (models(context=32768), json.dumps({"data": [{"id": SERVED}]}), models("Other-Model")):+ vllm.listed, vllm.calls = listed, []+ diagnosis = {}+ assert prove(diagnosis=diagnosis) == {}+ assert "/v1/chat/completions" not in [call[1] for call in vllm.calls]+++def test_cold_start_reads_as_loading_while_docker_reports_starting(vllm):+ vllm.health = OSError("Docker runtime transport failed (exit 1): runtime transport failed: URLError")+ vllm.container = "starting"+ assert agent._runtime_health(ENV) == "loading"+ diagnosis = {}+ assert prove(diagnosis=diagnosis) == {}+ assert diagnosis["reason"].startswith("vLLM is still starting")+ assert "final" not in diagnosis++ vllm.health = 503+ assert agent._runtime_health(ENV) == "loading"+ # Past its start period Docker says unhealthy: an error, not loading.+ vllm.container = "unhealthy"+ assert agent._runtime_health(ENV) == "error"+ vllm.health = OSError("connection refused")+ with pytest.raises(OSError):+ agent._runtime_health(ENV)+ vllm.health, vllm.container = 200, "healthy"+ assert agent._runtime_health(ENV) == "ok"+++def test_llama_server_health_is_unchanged(monkeypatch):+ monkeypatch.setattr(agent, "_runtime_http", lambda env, path, **kwargs: "")+ monkeypatch.setattr(agent, "_router_transport_exchange", lambda *a, **k: pytest.fail("vLLM probe"))+ assert agent._runtime_health({"LLM_BACKEND": "llama-server"}) == "error"+++def test_catalog_entry_comes_from_llm_model_not_a_leftover_gguf(vllm):+ leftover = {**ENV, "GGUF_FILE": "Qwen3.5-9B-Q4_K_M.gguf"}+ model_id, model = agent._catalog_model_for_current_env(leftover)+ assert model_id == "glm-5.3-flash-nvfp4-vllm" and model["runtime"] == "vllm"+ # A llama-server route never resolves to a vLLM entry.+ llama = {"LLM_BACKEND": "llama-server", "LLM_MODEL": "glm-5.3-flash-nvfp4", "GGUF_FILE": ""}+ assert agent._catalog_model_for_current_env(llama) == ("glm-5.3-flash-nvfp4", {})+ assert agent._vllm_probe_thinking_off(ENV) == GLM_OFF+ assert agent._vllm_probe_thinking_off(llama) is None+++def test_compose_resolution_reads_llm_backend_from_the_install(monkeypatch, tmp_path):+ assert "LLM_BACKEND" in agent._HOST_LLM_COMPOSE_SELECTORS+ assert "LLM_BACKEND" in agent._PREPARE_COMPOSE_SELECTORS+ (tmp_path / "scripts").mkdir()+ (tmp_path / "scripts" / "resolve-compose-stack.sh").write_text("", encoding="utf-8")+ (tmp_path / ".env").write_text("LLM_BACKEND=vllm\nODS_MODE=local\n", encoding="utf-8")+ monkeypatch.setattr(agent, "INSTALL_DIR", tmp_path)+ monkeypatch.setattr(agent, "_find_usable_bash", lambda: "bash")+ monkeypatch.setattr(agent.platform, "system", lambda: "Linux")+ # A stale value exported when the agent started must not win.+ monkeypatch.setenv("LLM_BACKEND", "llama-server")+ seen = {}++ def run(cmd, **kwargs):+ seen.update(kwargs["env"])+ return subprocess.CompletedProcess(cmd, 0, stdout="-f docker-compose.base.yml", stderr="")++ monkeypatch.setattr(agent.subprocess, "run", run)+ agent._run_compose_resolver()+ assert seen["LLM_BACKEND"] == "vllm"+++def test_router_configs_render_with_the_installed_backend(monkeypatch, tmp_path):+ (tmp_path / "scripts").mkdir()+ (tmp_path / "scripts" / "render-runtime-configs.py").write_text("", encoding="utf-8")+ commands = []++ def run(cmd, **kwargs):+ commands.append(cmd)+ return subprocess.CompletedProcess(cmd, 0, stdout="", stderr="")++ monkeypatch.setattr(agent.subprocess, "run", run)+ monkeypatch.setenv("LLM_BACKEND", "llama-server")+ agent._render_model_router_runtime_configs(+ tmp_path, {**ENV, "ODS_MODEL_SWITCHBOARD": "enabled"},+ model=ENV["LLM_MODEL"], gguf_file="", context_length=131072)+ assert commands and all(cmd[cmd.index("--llm-backend") + 1] == "vllm" for cmd in commands)+ assert all(cmd[cmd.index("--model") + 1] == "glm-5.3-flash-nvfp4" for cmd in commands)+ commands.clear()+ agent._render_model_router_runtime_configs(+ tmp_path, {"ODS_MODEL_SWITCHBOARD": "enabled", "GGUF_FILE": "m.gguf"},+ model="m", gguf_file="m.gguf", context_length=8192)+ assert all(cmd[cmd.index("--llm-backend") + 1] == "llama-server" for cmd in commands)+++class Handler:+ def __init__(self):+ self.wfile = io.BytesIO()+ self.response_code = None++ def send_response(self, code):+ self.response_code = code++ def send_header(self, name, value):+ pass++ def end_headers(self):+ pass++ def body(self):+ return json.loads(self.wfile.getvalue().decode("utf-8"))+++def _install(tmp_path, monkeypatch, env_text):+ (tmp_path / ".env").write_text(env_text, encoding="utf-8")+ (tmp_path / "config").mkdir(exist_ok=True)+ shutil.copyfile(ROOT / "config" / "model-library.json", tmp_path / "config" / "model-library.json")+ monkeypatch.setattr(agent, "INSTALL_DIR", tmp_path)+ monkeypatch.setattr(agent, "STARTUP_ODS_MODE", "local")+ monkeypatch.setattr(agent, "_managed_wsl_runtime", lambda env: {"managed": False})+ monkeypatch.setattr(agent, "_recreate_llama_server", lambda *a, **k: pytest.fail("llama-server recreated"))+ monkeypatch.setattr(agent, "_compose_restart_llama_server", lambda *a, **k: pytest.fail("llama-server started"))+++def test_a_vllm_install_refuses_model_switches_before_any_change(monkeypatch, tmp_path):+ _install(tmp_path, monkeypatch, "ODS_MODE=local\nLLM_BACKEND=vllm\nLLM_MODEL=glm-5.3-flash-nvfp4\n")+ monkeypatch.setattr(agent, "_load_model_library_records", lambda: pytest.fail("refused before lookup"))+ before = (tmp_path / ".env").read_bytes()+ handler = Handler()+ agent.AgentHandler._do_model_activate(handler, "qwen3.5-9b-q4")+ assert handler.response_code == 409+ body = handler.body()+ assert body["code"] == "vllm_runtime_installer_managed"+ assert "vLLM engine" in body["error"] and "installer" in body["hint"]+ assert "discord.gg" in body["hint"]+ assert (tmp_path / ".env").read_bytes() == before+++def test_a_llama_install_refuses_a_vllm_catalog_model(monkeypatch, tmp_path):+ _install(tmp_path, monkeypatch, "ODS_MODE=local\nLLM_BACKEND=llama-server\nGGUF_FILE=a.gguf\n")+ handler = Handler()+ agent.AgentHandler._do_model_activate(handler, "glm-5.3-flash-nvfp4-vllm")+ assert handler.response_code == 409+ body = handler.body()+ assert body["code"] == "vllm_model_installer_only"+ assert body["requestedModelId"] == "glm-5.3-flash-nvfp4-vllm"+ assert body["error"].startswith("GLM 5.3 Flash NVFP4 (vLLM) runs only on the vLLM engine")+++def test_llama_server_is_never_named_to_compose_on_a_vllm_install(monkeypatch, tmp_path):+ (tmp_path / ".env").write_text("LLM_BACKEND=vllm\n", encoding="utf-8")+ monkeypatch.setattr(agent, "INSTALL_DIR", tmp_path)+ monkeypatch.setattr(agent, "CORE_SERVICE_IDS", {"llama-server", "litellm", "vllm-server"})+ monkeypatch.setattr(agent.subprocess, "run", lambda *a, **k: pytest.fail("compose ran"))+ ok, error = agent.docker_compose_recreate(["litellm", "llama-server"])+ assert not ok and "vllm-server" in error+ assert "vllm-server" in agent.ALWAYS_ON_SERVICESdiff --git a/ods/tests/test_preserve_active_model_vllm.py b/ods/tests/test_preserve_active_model_vllm.pynew file mode 100644index 000000000..96abfdfe7--- /dev/null+++ b/ods/tests/test_preserve_active_model_vllm.py@@ -0,0 +1,164 @@+"""An installer rerun (ods update) keeps a vLLM install on vLLM.++scripts/preserve-active-model.py's default mode used to keep only a+llama-server GGUF route, so a .env with LLM_BACKEND=vllm got no contract and+the rerun fell back to the tier's GGUF. It now keeps the vLLM route while the+catalog snapshot is installed, clears the GGUF keys, and exits 2 rather than+fall back when it cannot. Runs the helper with this Python; no installer,+Docker or network.+"""++from __future__ import annotations++import hashlib+import json+import shlex+import subprocess+import sys+from pathlib import Path++import pytest++ROOT = Path(__file__).resolve().parents[1]+HELPER = ROOT / "scripts" / "preserve-active-model.py"+sys.path.insert(0, str(ROOT / "extensions" / "services" / "dashboard-api"))+import model_vllm # noqa: E402++REVISION = "0123456789abcdef0123456789abcdef01234567"+FILES = {"config.json": b'{"model_type": "test"}', "model-00001-of-00001.safetensors": b"w" * 4096,+ "model.safetensors.index.json": b"{}", "tokenizer_config.json": b"{}"}+LFS = {"model-00001-of-00001.safetensors", "model.safetensors.index.json"}+ARGS = ["--max-model-len", "65536", "--enable-auto-tool-choice", "--tool-call-parser", "glm47",+ "--reasoning-parser", "glm47", "--speculative-config.method=mtp"]+++def blob_sha1(content: bytes) -> str:+ return hashlib.sha1(b"blob %d\0" % len(content) + content).hexdigest()+++def entry() -> dict:+ files = [{"path": path, "size": len(data), "sha256": hashlib.sha256(data).hexdigest()} if path in LFS+ else {"path": path, "size": len(data), "git_blob_sha1": blob_sha1(data)}+ for path, data in FILES.items()]+ return model_vllm.require_valid({+ "id": "test-model-nvfp4-vllm", "name": "Test Model NVFP4 (vLLM)", "runtime": "vllm",+ "hf_repo": "nvidia/Test-Model-NVFP4", "revision": REVISION, "files": files,+ "served_model_name": "Test-Model-NVFP4", "llm_model_name": "test-model-nvfp4",+ "context_length": 65536, "min_gpu_mib": 1024, "compute_capability": ["10.0"],+ "reasoning": "medium", "tool_call_parser": "glm47", "reasoning_parser": "glm47",+ "thinking_template_kwargs": {"off": {"reasoning_effort": "low"}, "low": {"reasoning_effort": "low"},+ "medium": {"reasoning_effort": "high"}, "high": {}},+ "vllm_args": ARGS, "selection": {"discrete": 0, "unified": 0, "cpu": 0, "vllm": 1},+ })+++FOLDER = "nvidia--Test-Model-NVFP4@01234567"+VLLM_ENV = {+ "ODS_MODE": "local", "LLM_BACKEND": "vllm", "LLM_MODEL": "test-model-nvfp4",+ "GGUF_FILE": "", "MAX_CONTEXT": "65536", "CTX_SIZE": "65536",+ "VLLM_MODEL_DIR": FOLDER, "VLLM_SERVED_MODEL": "Test-Model-NVFP4",+ # The args an older release measured; a rerun takes this release's.+ "VLLM_ARGS": "--max-model-len 65536 --old-flag", "VLLM_GPU_UUID": "GPU-0f1e2d3c-aaaa-bbbb-cccc-0123456789ab",+ "VLLM_GPU_MEMORY_UTILIZATION": "0.85", "MODEL_SELECTION_SOURCE": "installer",+}+++def install(tmp_path):+ gguf = {"id": "tiny-q4", "name": "Tiny", "llm_model_name": "tiny", "gguf_file": "tiny.gguf",+ "gguf_url": "https://huggingface.co/a/b/resolve/main/tiny.gguf"}+ (tmp_path / "model-library.json").write_text(json.dumps({"models": [gguf, entry()]}), encoding="utf-8")+ snapshot = tmp_path / "data" / "models" / FOLDER+ snapshot.mkdir(parents=True)+ for path, data in FILES.items():+ (snapshot / path).write_bytes(data)+ return tmp_path+++def run(install: Path, env: dict[str, str]) -> subprocess.CompletedProcess:+ (install / ".env").write_text("".join(f"{key}={value}\n" for key, value in env.items()), encoding="utf-8")+ return subprocess.run(+ [sys.executable, str(HELPER), "--env", str(install / ".env"),+ "--catalog", str(install / "model-library.json"), "--models-dir", str(install / "data" / "models"),+ "--state", str(install / "data" / "model-state.json"), "--backend", "nvidia", "--local-model"],+ capture_output=True, text=True, timeout=60)+++def contract(stdout: str) -> dict[str, str]:+ return {key: shlex.split(value)[0] if value != '""' else ""+ for key, _, value in (line.partition("=") for line in stdout.splitlines())}+++def test_an_installed_vllm_route_is_kept_and_no_gguf_is_named(install):+ result = run(install, VLLM_ENV)++ assert result.returncode == 0, result.stderr+ assert contract(result.stdout) == {+ "LLM_BACKEND": "vllm",+ "LLM_MODEL": "test-model-nvfp4",+ "GGUF_FILE": "", "GGUF_URL": "", "GGUF_SHA256": "",+ "MAX_CONTEXT": "65536",+ "LLM_MODEL_SIZE_MB": "0",+ "MODEL_SELECTION_SOURCE": "installer",+ "VLLM_MODEL_DIR": FOLDER,+ "VLLM_SERVED_MODEL": "Test-Model-NVFP4",+ "VLLM_ARGS": " ".join(ARGS),+ "VLLM_GPU_UUID": VLLM_ENV["VLLM_GPU_UUID"],+ "VLLM_GPU_MEMORY_UTILIZATION": "0.85",+ }+ # Empty GGUF keys are written (they replace the tier map's), unlike the+ # optional llama.cpp settings the helper omits.+ assert 'GGUF_FILE=""' in result.stdout.splitlines()+++def test_backend_value_is_read_case_insensitively(install):+ result = run(install, {**VLLM_ENV, "LLM_BACKEND": "VLLM"})+ assert result.returncode == 0, result.stderr+ assert contract(result.stdout)["LLM_BACKEND"] == "vllm"++[email protected]("damage", ["missing", "short", "partial-only", "linked-folder"])+def test_an_incomplete_snapshot_stops_the_rerun_instead_of_falling_back(install, damage):+ snapshot = install / "data" / "models" / FOLDER+ weights = snapshot / "model-00001-of-00001.safetensors"+ if damage == "missing":+ weights.unlink()+ elif damage == "short":+ weights.write_bytes(b"w" * 10)+ elif damage == "partial-only":+ weights.rename(weights.with_name(weights.name + ".partial"))+ else:+ real = install / "elsewhere"+ snapshot.rename(real)+ try:+ snapshot.symlink_to(real, target_is_directory=True)+ except OSError:+ pytest.skip("this host cannot create directory symlinks")+ result = run(install, VLLM_ENV)++ assert result.returncode == 2+ assert result.stdout == ""+ assert "vLLM engine" in result.stderr and "refusing to replace it" in result.stderr++[email protected]("change", [+ {"VLLM_MODEL_DIR": "nvidia--Test-Model-NVFP4@fedcba98"},+ {"VLLM_SERVED_MODEL": "Other-Model"},+ {"LLM_MODEL": "tiny"},+ {"VLLM_GPU_UUID": ""},+ {"VLLM_GPU_UUID": "GPU 0; rm -rf /"},+ {"VLLM_GPU_MEMORY_UTILIZATION": "1.5"},+ {"VLLM_GPU_MEMORY_UTILIZATION": "0"},+ {"ODS_MODE": "cloud"},+])+def test_a_route_that_is_not_the_catalog_snapshot_is_never_rewritten(install, change):+ result = run(install, {**VLLM_ENV, **change})+ assert result.returncode == 2+ assert result.stdout == ""+++def test_llama_server_routes_keep_their_behaviour(install):+ # No vLLM route and no installed GGUF: nothing is preserved, as before.+ result = run(install, {"ODS_MODE": "local", "LLM_BACKEND": "llama-server",+ "LLM_MODEL": "tiny", "GGUF_FILE": "tiny.gguf"})+ assert (result.returncode, result.stdout) == (0, "")diff --git a/ods/tests/test_select_model_vllm.py b/ods/tests/test_select_model_vllm.pyindex d309386c9..6d8200f69 100644--- a/ods/tests/test_select_model_vllm.py+++ b/ods/tests/test_select_model_vllm.py@@ -115,7 +115,8 @@ def test_dgx_station_gets_the_top_priority_vllm_model_on_its_gb300():assert env["MAX_CONTEXT"] == str(best["context_length"]) == "131072"assert env["LLM_MODEL_SIZE_MB"] == str(best["size_mb"])assert env["GGUF_FILE"] == env["GGUF_URL"] == env["GGUF_SHA256"] == ""- assert env["PIXEL_AGENT_MODEL_READY"] == "false"+ pixel = ((best.get("app_compatibility") or {}).get("pixel_agent") or {}).get("status")+ assert env["PIXEL_AGENT_MODEL_READY"] == ("true" if pixel == "verified" else "false")assert env["MODEL_RECOMMENDATION_POLICY"] == "vllm-gpu-fit-v1"assert env["MODEL_RECOMMENDATION_SOURCE"] == "catalog_vllm_pre_download"assert "GPU-test-gb300 (compute capability 10.3, 256703 MiB)" in env["MODEL_RECOMMENDATION_REASON"]