ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.
Python ai-agentsamdcomfyuidockerllama-cppllmlocal-ain8nnvidiaopen-webuiragself-hostedspeech-to-textstrix-halotext-to-speechworkflow-automation
feat(models): add the vllm backend kind and its router endpoint (#8288)
Mike Bradleya2c9c62d0d20
diff --git a/ods/bin/model_switchboard/__init__.py b/ods/bin/model_switchboard/__init__.pyindex da0782e68..f7fe018f4 100644--- a/ods/bin/model_switchboard/__init__.py+++ b/ods/bin/model_switchboard/__init__.py@@ -11,6 +11,7 @@ from .state import ( # noqa: F401SCHEMA_VERSION,StateError,atomic_write_state,+ default_endpoint_id,initial_state,initialize_if_missing,is_legacy_route,@@ -25,6 +26,7 @@ __all__ = ["SCHEMA_VERSION","StateError","atomic_write_state",+ "default_endpoint_id","initial_state","initialize_if_missing","is_legacy_route",diff --git a/ods/bin/model_switchboard/state.py b/ods/bin/model_switchboard/state.pyindex 1696909e6..597339aed 100644--- a/ods/bin/model_switchboard/state.py+++ b/ods/bin/model_switchboard/state.py@@ -33,13 +33,23 @@ HISTORY_LIMIT = 10PUBLIC_MODEL_DEFAULT = "ods/current"STATE_FILE_MODE = 0o644-_WRITABLE_BACKEND_KINDS = {"llama-server", "hipfire", "unknown"}+DEFAULT_ENDPOINT_ID = "llama-server-default"+VLLM_ENDPOINT_ID = "vllm-default"+# Every writable backend kind and the model-router endpoint (endpoints.json+# id) it routes to when the caller names none. hipfire and unknown routes+# have always used the llama-server endpoint; vLLM serves from vllm-server.+_DEFAULT_ENDPOINT_IDS = {+ "llama-server": DEFAULT_ENDPOINT_ID,+ "hipfire": DEFAULT_ENDPOINT_ID,+ "unknown": DEFAULT_ENDPOINT_ID,+ "vllm": VLLM_ENDPOINT_ID,+}+_WRITABLE_BACKEND_KINDS = set(_DEFAULT_ENDPOINT_IDS)# Readable for one release only: records written before round F carry the# retired Lemonade route until the host agent rewrites and re-proves them._LEGACY_BACKEND_KINDS = {"lemonade"}_LEGACY_ENDPOINT_IDS = {"lemonade-default"}_BACKEND_KINDS = _WRITABLE_BACKEND_KINDS | _LEGACY_BACKEND_KINDS-DEFAULT_ENDPOINT_ID = "llama-server-default"_OPERATION_PHASES = {"requested", "staging", "verifying", "publishing","flipping", "serving", "failed", "rolling_back",@@ -505,25 +515,26 @@ def is_legacy_route(active: Any) -> bool:)+def default_endpoint_id(backend_kind: str) -> str:+ """The router endpoint id a writable backend kind uses by default."""+ return _DEFAULT_ENDPOINT_IDS[backend_kind]++def migrate_env_identity(env: dict[str, str]) -> dict[str, Any] | None:"""Derive a best-effort runtime identity from ``.env`` values.- Every managed runtime is upstream llama-server serving ``--alias- <GGUF_FILE>``, so the runtime identity is the GGUF filename (or the model- name when no file is recorded). A retired ``LEMONADE_MODEL`` line is never- read: a leftover key must not relabel the route. Returns ``None`` when the- environment carries no local model identity (e.g. cloud-only).+ A llama-server runtime serves ``--alias <GGUF_FILE>``, so its identity is+ the GGUF filename (or the model name when no file is recorded). A vLLM+ runtime (``LLM_BACKEND=vllm``) serves ``--served-model-name+ <VLLM_SERVED_MODEL>`` (or the model name when none is recorded); a+ ``GGUF_FILE`` left by an earlier llama-server install never names it. A+ retired ``LEMONADE_MODEL`` line is never read: a leftover key must not+ relabel the route. Returns ``None`` when the environment carries no local+ model identity (e.g. cloud-only)."""if str(env.get("ODS_MODE") or "").strip().casefold() == "cloud":return None- gguf = str(env.get("GGUF_FILE") or "").strip()- llm_model = str(env.get("LLM_MODEL") or "").strip()-- runtime_id = gguf or llm_model- if not runtime_id:- return None-backend = str(env.get("LLM_BACKEND") or "").strip().lower()if backend in {"", "lemonade"}:# A pre-migration LLM_BACKEND=lemonade install now serves through@@ -532,6 +543,16 @@ def migrate_env_identity(env: dict[str, str]) -> dict[str, Any] | None:if backend not in _WRITABLE_BACKEND_KINDS:backend = "unknown"+ llm_model = str(env.get("LLM_MODEL") or "").strip()+ if backend == "vllm":+ served = str(env.get("VLLM_SERVED_MODEL") or "").strip()+ else:+ served = str(env.get("GGUF_FILE") or "").strip()++ runtime_id = served or llm_model+ if not runtime_id:+ return None+catalog_guess = llm_modelif not catalog_guess:stem = runtime_id@@ -565,8 +586,9 @@ def initialize_if_missing() -> dict[str, Any] | None:"""One-time startup reconstruction when no v1 state was ever committed.- Existing files — valid or malformed — are never overwritten here. Returns- the written doc, or ``None`` when nothing was written.+ Existing files — valid or malformed — are never overwritten here. Without+ an ``endpoint_id`` the route uses its backend kind's default endpoint.+ Returns the written doc, or ``None`` when nothing was written."""if Path(path).exists():return None@@ -574,7 +596,7 @@ def initialize_if_missing(if identity is None:return Noneif endpoint_id is None:- endpoint_id = DEFAULT_ENDPOINT_ID+ endpoint_id = default_endpoint_id(identity["backendKind"])return record_verified_route(path,catalog_id=identity["catalogId"],diff --git a/ods/config/model-state.schema.v1.json b/ods/config/model-state.schema.v1.jsonindex fbce172ac..174b584f3 100644--- a/ods/config/model-state.schema.v1.json+++ b/ods/config/model-state.schema.v1.json@@ -57,7 +57,7 @@"additionalProperties": false,"properties": {"kind": {- "enum": ["llama-server", "lemonade", "hipfire", "unknown"],+ "enum": ["llama-server", "vllm", "lemonade", "hipfire", "unknown"],"description": "lemonade names a pre-round-F route: readable for one release so the host agent can replace it, never written"},"endpointId": { "type": "string", "minLength": 1 },diff --git a/ods/extensions/services/dashboard-api/builtin_source_recipes.py b/ods/extensions/services/dashboard-api/builtin_source_recipes.pyindex ed9dfb395..c275f6ddf 100644--- a/ods/extensions/services/dashboard-api/builtin_source_recipes.py+++ b/ods/extensions/services/dashboard-api/builtin_source_recipes.py@@ -35,7 +35,7 @@ _CONTEXT_BUILDS = {"ape": ("ape", None, "Dockerfile", "ffa14f37d9bca3fc9aa35582b5f3a4b9adaa1e75222a9c29dc9d55d9cfa49805"),"brave-search": ("brave-search", "ods-brave-search:local", "Dockerfile", "f1be25450cae210788c29d02dc497ff64f3fe7f9184cc73332c08a734e709e95"),"privacy-shield": ("privacy-shield", None, "Dockerfile", "deb8da4c40e7eac01c2576c33bd11f0df21c2332bf929bc6ba0272669ffa64b8"),- "token-spy": ("token-spy", None, "Dockerfile", "a437422b3bd3eeb0ce35631137dc78665769489cd7d7f6b6bb219586273f2f60"),+ "token-spy": ("token-spy", None, "Dockerfile", "98e93c93cdff9ff63bed7569a8dec9e45171c59e60516878e097a43274277eff"),}_MAX_CONTEXT_FILE_BYTES = 8 * 1024 * 1024_MAX_CONTEXT_FILES = 512diff --git a/ods/extensions/services/dashboard-api/tests/test_model_state.py b/ods/extensions/services/dashboard-api/tests/test_model_state.pyindex d42ba75e5..a3efe54da 100644--- a/ods/extensions/services/dashboard-api/tests/test_model_state.py+++ b/ods/extensions/services/dashboard-api/tests/test_model_state.py@@ -49,6 +49,40 @@ class TestStateModule:schema = json.loads(_SCHEMA_PATH.read_text(encoding="utf-8"))jsonschema.validate(read, schema)+ def test_vllm_route_roundtrips_and_matches_schema(self, tmp_path):+ path = tmp_path / "model-state.json"+ doc = _record(path, model="glm-5.3-flash-nvfp4", runtime="GLM-5.3-Flash-NVFP4",+ backend="vllm", endpoint="vllm-default", context=131072)+ assert doc["active"]["backend"] == {+ "kind": "vllm", "endpointId": "vllm-default", "nativeRoute": None,+ }+ read, errors = sb.read_state(path)+ assert errors == [] and read == doc+ assert sb.validate_state(read) == []+ jsonschema = pytest.importorskip("jsonschema")+ schema = json.loads(_SCHEMA_PATH.read_text(encoding="utf-8"))+ jsonschema.validate(read, schema)++ def test_backend_kinds_agree_with_schema(self):+ schema = json.loads(_SCHEMA_PATH.read_text(encoding="utf-8"))+ active = next(option for option in schema["properties"]["active"]["oneOf"]+ if option.get("type") == "object")+ kinds = active["properties"]["backend"]["properties"]["kind"]["enum"]+ assert set(kinds) == sb._BACKEND_KINDS++ @pytest.mark.parametrize("kind,endpoint", [+ ("llama-server", "llama-server-default"),+ ("hipfire", "llama-server-default"),+ ("unknown", "llama-server-default"),+ ("vllm", "vllm-default"),+ ])+ def test_default_endpoint_per_backend_kind(self, kind, endpoint):+ assert sb.default_endpoint_id(kind) == endpoint++ def test_retired_kind_has_no_default_endpoint(self):+ with pytest.raises(KeyError):+ sb.default_endpoint_id("lemonade")+def test_state_file_is_readable_by_model_router_user(self, tmp_path):path = tmp_path / "model-state.json"_record(path)@@ -183,6 +217,32 @@ class TestStateModule:"runtimeModelId": "stale.gguf","backendKind": "llama-server",}),+ # vLLM serves --served-model-name; a GGUF_FILE left by an earlier+ # llama-server install never names the route.+ ({+ "LLM_BACKEND": "vllm",+ "VLLM_SERVED_MODEL": "GLM-5.3-Flash-NVFP4",+ "LLM_MODEL": "glm-5.3-flash-nvfp4",+ "GGUF_FILE": "Qwen3.5-9B-Q4_K_M.gguf",+ "MAX_CONTEXT": "131072",+ }, {+ "catalogId": "glm-5.3-flash-nvfp4",+ "runtimeModelId": "GLM-5.3-Flash-NVFP4",+ "backendKind": "vllm",+ "contextLength": 131072,+ }),+ ({"LLM_BACKEND": "vllm", "LLM_MODEL": "glm-5.3-flash-nvfp4",+ "GGUF_FILE": "Qwen3.5-9B-Q4_K_M.gguf"},+ {"catalogId": "glm-5.3-flash-nvfp4", "runtimeModelId": "glm-5.3-flash-nvfp4",+ "backendKind": "vllm"}),+ ({"LLM_BACKEND": " VLLM ", "VLLM_SERVED_MODEL": "Served-Model",+ "CTX_SIZE": "65536", "MAX_CONTEXT": "131072"},+ {"catalogId": "Served-Model", "runtimeModelId": "Served-Model",+ "backendKind": "vllm", "contextLength": 65536}),+ # VLLM_SERVED_MODEL names only a vllm route.+ ({"LLM_BACKEND": "llama-server", "VLLM_SERVED_MODEL": "Served-Model",+ "GGUF_FILE": "M.gguf"},+ {"catalogId": "M", "runtimeModelId": "M.gguf", "backendKind": "llama-server"}),],)def test_migrate_env_forms(self, env, expected):@@ -190,6 +250,11 @@ class TestStateModule:for key, value in expected.items():assert got[key] == value+ def test_migrate_vllm_env_never_takes_the_gguf_identity(self):+ assert sb.migrate_env_identity({+ "LLM_BACKEND": "vllm", "GGUF_FILE": "Qwen3.5-9B-Q4_K_M.gguf",+ }) is None+def test_migrate_cloud_only_yields_none(self):env = {"ODS_MODE": "cloud",@@ -258,6 +323,46 @@ class TestStateModule:}assert doc["active"]["runtimeModelId"] == "Model.gguf"+ @pytest.mark.parametrize("env,kind,endpoint", [+ ({"GGUF_FILE": "M.gguf"}, "llama-server", "llama-server-default"),+ ({"LLM_BACKEND": "hipfire", "LLM_MODEL": "Qwen3.5-2B"}, "hipfire", "llama-server-default"),+ ({"LLM_BACKEND": "external", "LLM_MODEL": "lan-model"}, "unknown", "llama-server-default"),+ ({"LLM_BACKEND": "vllm", "VLLM_SERVED_MODEL": "GLM-5.3-Flash-NVFP4"}, "vllm", "vllm-default"),+ ])+ def test_initialize_uses_the_backend_kinds_default_endpoint(self, tmp_path, env, kind, endpoint):+ doc = sb.initialize_if_missing(tmp_path / "model-state.json", env)+ assert doc["active"]["backend"] == {"kind": kind, "endpointId": endpoint, "nativeRoute": None}+ assert sb.validate_state(doc) == []++ def test_initialize_for_a_vllm_env_reconstructs_the_served_model(self, tmp_path):+ path = tmp_path / "model-state.json"+ doc = sb.initialize_if_missing(path, {+ "LLM_BACKEND": "vllm",+ "VLLM_SERVED_MODEL": "GLM-5.3-Flash-NVFP4",+ "LLM_MODEL": "glm-5.3-flash-nvfp4",+ "GGUF_FILE": "Qwen3.5-9B-Q4_K_M.gguf",+ "MAX_CONTEXT": "131072",+ })+ assert doc["active"]["catalogId"] == "glm-5.3-flash-nvfp4"+ assert doc["active"]["runtimeModelId"] == "GLM-5.3-Flash-NVFP4"+ assert doc["active"]["contextLength"] == 131072+ assert doc["active"]["reconstructed"] is True+ assert doc["active"]["proof"] == {"identity": "GLM-5.3-Flash-NVFP4", "completion": False}+ read, errors = sb.read_state(path)+ assert errors == [] and read == doc+ jsonschema = pytest.importorskip("jsonschema")+ jsonschema.validate(read, json.loads(_SCHEMA_PATH.read_text(encoding="utf-8")))++ def test_initialize_keeps_a_caller_named_endpoint(self, tmp_path):+ doc = sb.initialize_if_missing(+ tmp_path / "model-state.json",+ {"LLM_BACKEND": "vllm", "VLLM_SERVED_MODEL": "GLM-5.3-Flash-NVFP4"},+ endpoint_id="vllm-native",+ )+ assert doc["active"]["backend"] == {+ "kind": "vllm", "endpointId": "vllm-native", "nativeRoute": None,+ }+def test_legacy_lemonade_route_is_readable_but_never_written(self, tmp_path):path = tmp_path / "model-state.json"doc = _record(path, runtime="Model.gguf")diff --git a/ods/extensions/services/model-router/app/main.py b/ods/extensions/services/model-router/app/main.pyindex 8500cc1c9..1ce966107 100644--- a/ods/extensions/services/model-router/app/main.py+++ b/ods/extensions/services/model-router/app/main.py@@ -1005,7 +1005,7 @@ def _validate_state_schema(doc: Any) -> bool:# A pre-round-F state names the retired "lemonade" kind. It stays# readable for one release so the host agent can replace it (contract# section 6.7); no route path treats it specially.- if backend["kind"] not in {"llama-server", "lemonade", "hipfire", "unknown"}:+ if backend["kind"] not in {"llama-server", "vllm", "lemonade", "hipfire", "unknown"}:return Falseif not isinstance(backend["endpointId"], str) or not backend["endpointId"]:return False@@ -1133,6 +1133,19 @@ def _active_route() -> dict[str, Any]:}+def _llama_tool_compat(route: dict[str, Any]) -> bool:+ """Whether the router's llama.cpp tool-call workarounds apply to a route.++ Only llama-server routes get them: buffered tool streams, the non-stream+ request for thinking-enabled tool streams, bounded tool-protocol repair+ and native tool-markup cleanup (tool_id_compat keeps its Nemo tool-id+ rewrite llama-server-only too). Other kinds are forwarded unchanged; a+ vllm route parses tool calls and reasoning itself with its own+ --tool-call-parser and --reasoning-parser.+ """+ return route["backendKind"] == "llama-server"++def _offered_tool_evidence(payload: dict[str, Any]) -> dict[str, Any]:"""Fingerprint the outgoing tools, never retain their names or schema text.@@ -2040,7 +2053,7 @@ async def _forward_inner(request: Request, path: str, payload: dict[str, Any],completed_tool_stream = (is_streamand path == "/v1/chat/completions"- and route["backendKind"] == "llama-server"+ and _llama_tool_compat(route)and isinstance(payload.get("tools"), list)and bool(payload["tools"]))@@ -2361,7 +2374,7 @@ async def _forward_inner(request: Request, path: str, payload: dict[str, Any],# model decision, not an executed action; no host/tool operation is# replayed here. Every rejected decision is logged by shape, never values.if (path == "/v1/chat/completions"- and route["backendKind"] == "llama-server"+ and _llama_tool_compat(route)and _repairable_native_tool_request(payload)and isinstance(payload.get("messages"), list)and 200 <= upstream.status_code < 300):@@ -2491,7 +2504,7 @@ async def _forward_inner(request: Request, path: str, payload: dict[str, Any],if (parsed is not None and 200 <= upstream.status_code < 300and path == "/v1/chat/completions"- and route["backendKind"] == "llama-server"+ and _llama_tool_compat(route)and isinstance(payload.get("tools"), list) and payload["tools"]and _normalize_native_tool_markup(parsed, payload)):content = json.dumps(parsed).encode("utf-8")diff --git a/ods/extensions/services/model-router/tests/test_router.py b/ods/extensions/services/model-router/tests/test_router.pyindex 8d9069460..94574f5d8 100644--- a/ods/extensions/services/model-router/tests/test_router.py+++ b/ods/extensions/services/model-router/tests/test_router.py@@ -1344,6 +1344,164 @@ class TestForwarding:assert mod._inflight == 0+_VLLM_MODEL = "GLM-5.3-Flash-NVFP4"+_LOOKUP_TOOL = {"type": "function", "function": {"name": "lookup", "parameters": {+ "type": "object", "additionalProperties": False, "required": ["key"],+ "properties": {"key": {"type": "string"}}}}}+_LOOKUP_MARKUP = ("<tool_call>\n<function=lookup>\n"+ "<parameter=key>\nok\n</parameter>\n</function>\n</tool_call>")+++def _write_route(tmp_path, write_state, kind):+ """Publish a verified route of ``kind``; a vllm one at vllm-default."""+ if kind == "llama-server":+ write_state()+ return "Concrete.gguf"+ # The allowlist the renderer writes for LLM_BACKEND=vllm, plus the+ # fixture's llama-server row so a wrong endpoint choice would still route.+ (tmp_path / "endpoints.json").write_text(json.dumps({"endpoints": [+ {"id": "llama-server-default", "baseUrl": "http://upstream:8080"},+ {"id": "vllm-default", "baseUrl": "http://vllm-server:8000"},+ ]}), encoding="utf-8")+ write_state(runtime=_VLLM_MODEL, endpoint="vllm-default",+ mutate=lambda doc: doc["active"]["backend"].update(kind=kind))+ return _VLLM_MODEL+++def _use_upstream(mod, handler):+ asyncio.run(mod.app.state.http.aclose())+ mod.app.state.http = httpx.AsyncClient(transport=httpx.MockTransport(handler))+++class TestVllmRoute:+ """A vllm route is forwarded unchanged: vLLM parses tools and reasoning."""++ def test_vllm_route_reaches_vllm_server_under_the_alias(self, router, tmp_path):+ mod, client, write_state, calls = router+ _write_route(tmp_path, write_state, "vllm")+ recorder = _RecordingTelemetry()+ mod.app.state.telemetry = recorder++ resp = client.post("/v1/chat/completions", json={+ "model": "ods/current", "messages": [{"role": "user", "content": "hi"}],+ })++ assert resp.status_code == 200+ assert calls[-1]["url"] == "http://vllm-server:8000/v1/chat/completions"+ assert calls[-1]["model"] == _VLLM_MODEL+ assert calls[-1]["auth"] is None+ assert resp.json()["model"] == "ods/current"+ assert resp.headers["X-ODS-Backend"] == "vllm"+ assert resp.headers["X-ODS-Routed-Model"] == _VLLM_MODEL+ assert client.get("/v1/models").json()["ods"]["backend"] == "vllm"+ assert [event["provider_name"] for event in recorder.events] == ["vllm"]++ @pytest.mark.parametrize("thinking", [False, True])+ def test_vllm_tool_stream_is_forwarded_live(self, router, tmp_path, thinking):+ mod, client, write_state, _calls = router+ _write_route(tmp_path, write_state, "vllm")+ sent = []+ # A llama-server route buffers this unfinished tool decision and+ # rejects it (test_incomplete_tool_stream_fails_without_nonstream_retry).+ partial = {"model": _VLLM_MODEL, "choices": [{"index": 0, "delta": {+ "role": "assistant", "tool_calls": [{+ "index": 0, "id": "call-1", "type": "function",+ "function": {"name": "lookup", "arguments": '{"key":"ok"}'}}]},+ "finish_reason": None}]}++ def handler(request):+ sent.append(json.loads(request.content))+ return httpx.Response(200, content=b"data: " + json.dumps(partial).encode() + b"\n\n",+ headers={"content-type": "text/event-stream"})++ _use_upstream(mod, handler)+ body = {"model": "ods/current", "stream": True,+ "messages": [{"role": "user", "content": "look up ok"}],+ "tools": [_LOOKUP_TOOL]}+ if thinking:+ body["chat_template_kwargs"] = {"enable_thinking": True}+ response = client.post("/v1/chat/completions", json=body)++ assert response.status_code == 200+ assert len(sent) == 1 and sent[0]["stream"] is True+ assert "stream_options" not in sent[0]+ assert sent[0].get("chat_template_kwargs") == body.get("chat_template_kwargs")+ for header in ("x-ods-tool-stream-fallback", "x-ods-tool-stream-compatibility",+ "x-ods-tool-stream-repair"):+ assert header not in response.headers+ frames = [json.loads(item.removeprefix("data: "))+ for item in response.text.split("\n\n") if item]+ assert frames == [{**partial, "model": "ods/current"}]++ @pytest.mark.parametrize("kind,upstream_calls", [("llama-server", 2), ("vllm", 1)])+ def test_only_llama_server_repairs_a_rejected_tool_decision(+ self, router, tmp_path, kind, upstream_calls):+ mod, client, write_state, _calls = router+ runtime = _write_route(tmp_path, write_state, kind)+ sent = []+ # Names a function that was not offered; the offered one is lookup.+ rejected = _LOOKUP_MARKUP.replace("function=lookup", "function=pixel_ods_lookup")++ def handler(request):+ sent.append(json.loads(request.content))+ message = {"role": "assistant", "content": rejected}+ reason = "stop"+ if len(sent) > 1:+ message = {"role": "assistant", "content": None, "tool_calls": [{+ "id": "call-1", "type": "function",+ "function": {"name": "lookup", "arguments": '{"key":"ok"}'}}]}+ reason = "tool_calls"+ return httpx.Response(200, json={"model": runtime, "choices": [+ {"index": 0, "message": message, "finish_reason": reason}]})++ _use_upstream(mod, handler)+ response = client.post("/v1/chat/completions", json={+ "model": "ods/current", "stream": False,+ "messages": [{"role": "user", "content": "look up ok"}],+ "tools": [_LOOKUP_TOOL],+ })++ assert response.status_code == 200+ assert len(sent) == upstream_calls+ choice = response.json()["choices"][0]+ if kind == "vllm":+ assert choice == {"index": 0, "message": {"role": "assistant", "content": rejected},+ "finish_reason": "stop"}+ else:+ assert choice["message"]["tool_calls"][0]["function"]["name"] == "lookup"++ @pytest.mark.parametrize("kind", ["llama-server", "vllm"])+ def test_only_llama_server_converts_native_tool_markup(self, router, tmp_path, kind):+ mod, client, write_state, _calls = router+ runtime = _write_route(tmp_path, write_state, kind)+ sent = []++ def handler(request):+ sent.append(json.loads(request.content))+ return httpx.Response(200, json={"model": runtime, "choices": [{"index": 0,+ "message": {"role": "assistant", "content": _LOOKUP_MARKUP},+ "finish_reason": "stop"}]})++ _use_upstream(mod, handler)+ response = client.post("/v1/chat/completions", json={+ "model": "ods/current", "stream": False,+ "messages": [{"role": "user", "content": "look up ok"}],+ "tools": [_LOOKUP_TOOL],+ })++ assert response.status_code == 200+ assert len(sent) == 1+ choice = response.json()["choices"][0]+ if kind == "vllm":+ assert choice["finish_reason"] == "stop"+ assert choice["message"] == {"role": "assistant", "content": _LOOKUP_MARKUP}+ else:+ assert choice["finish_reason"] == "tool_calls"+ assert choice["message"]["content"] is None+ assert json.loads(choice["message"]["tool_calls"][0]["function"]["arguments"]) == {+ "key": "ok"}++class TestStateTrust:def test_schema_invalid_state_is_not_routable(self, router):mod, client, write_state, calls = router@@ -1354,6 +1512,16 @@ class TestStateTrust:assert resp.status_code == 503assert calls == []+ def test_unlisted_backend_kind_is_not_routable(self, router):+ mod, client, write_state, calls = router+ write_state(mutate=lambda doc: doc["active"]["backend"].update(kind="sglang"))+ resp = client.post("/v1/chat/completions", json={+ "model": "ods/current", "messages": [],+ })+ assert resp.status_code == 503+ assert resp.json()["error"]["type"] == "no_active_route"+ assert calls == []+@pytest.mark.parametrize("mutation", [lambda doc: doc["active"].update({"reconstructed": True}),lambda doc: doc["active"]["proof"].update({"completion": False}),diff --git a/ods/extensions/services/model-router/tests/test_tool_id_compat.py b/ods/extensions/services/model-router/tests/test_tool_id_compat.pyindex bad5285f1..7c932b257 100644--- a/ods/extensions/services/model-router/tests/test_tool_id_compat.py+++ b/ods/extensions/services/model-router/tests/test_tool_id_compat.py@@ -49,6 +49,14 @@ class DetectionTests(unittest.TestCase):tool_id_compat.route_requires_strict9_tool_ids(NEMO_ID, "hipfire"))+ def test_negative_vllm_backend(self):+ # vLLM's own tool parser owns tool-call ids on a vllm route.+ self.assertFalse(+ tool_id_compat.route_requires_strict9_tool_ids(+ "mistralai/Mistral-Nemo-Instruct-2407", "vllm"+ )+ )+def test_negative_caller_alias(self):self.assertFalse(tool_id_compat.route_requires_strict9_tool_ids("ods/current", "llama-server")diff --git a/ods/extensions/services/token-spy/tests/test_routed_telemetry.py b/ods/extensions/services/token-spy/tests/test_routed_telemetry.pyindex 4a5588217..37b81bb6b 100644--- a/ods/extensions/services/token-spy/tests/test_routed_telemetry.py+++ b/ods/extensions/services/token-spy/tests/test_routed_telemetry.py@@ -142,3 +142,20 @@ def test_hipfire_backend_is_accounted_as_local_runtime():assert usage["provider_name"] == "local"assert usage["cost_source"] == "local_zero_cost"+++def test_vllm_backend_is_accounted_as_local_runtime():+ telemetry = _load(+ TOKEN_SPY_DIR / "routed_telemetry.py", "routed_telemetry"+ )++ # model-router reports a vllm route's backend kind as its provider.+ usage = telemetry.routed_event_to_usage(+ telemetry.validate_routed_event(+ _event(provider_name="vllm", model="GLM-5.3-Flash-NVFP4")+ )+ )++ assert usage["provider_name"] == "local"+ assert usage["cost_source"] == "local_zero_cost"+ assert usage["model"] == "GLM-5.3-Flash-NVFP4"diff --git a/ods/scripts/render-runtime-configs.py b/ods/scripts/render-runtime-configs.pyindex 36d0f65a5..397e3edc2 100644--- a/ods/scripts/render-runtime-configs.py+++ b/ods/scripts/render-runtime-configs.py@@ -38,6 +38,14 @@ LEGACY_ODS_MODES = {"lemonade": "local"}PUBLIC_MODEL_ALIAS = "ods/current"REMOTE_PROVIDER_EGRESS_BASE_URL = "http://remote-provider-egress:8091/v1"REMOTE_MODEL_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/+-]{0,255}$")+# .env LLM_BACKEND values. vllm routes model-router to the vllm-server+# container; every other value keeps today's llama-server routes (external+# installs replace the router through their overlay, hipfire routes share the+# llama-server endpoint, and lemonade and litellm are retired values still+# accepted for upgrade compatibility).+LLM_BACKEND_VALUES = ("llama-server", "vllm", "external", "hipfire", "lemonade", "litellm")+VLLM_ENDPOINT_ID = "vllm-default"+VLLM_SERVER_ORIGIN = "http://vllm-server:8000"def atomic_write_text(target: Path, content: str, *, file_mode: int | None = None) -> None:@@ -126,6 +134,8 @@ class RenderInputs:llm_api_key_env: str = ""# Switchboard rollout mode: legacy | observe | enabled (plan section 8)switchboard_mode: str = "enabled"+ # Model server behind model-router: llama-server | vllm (model_server()).+ llm_backend: str = "llama-server"@dataclass(frozen=True)@@ -157,6 +167,11 @@ def remote_route_enabled(inputs: RenderInputs) -> bool:return inputs.remote_llm_enabled+def model_server(llm_backend: str) -> str:+ """The model server an .env LLM_BACKEND value routes to."""+ return "vllm" if llm_backend == "vllm" else "llama-server"++def hermes_model_id(inputs: RenderInputs) -> str:if inputs.switchboard_mode == "enabled":return "ods/current"@@ -555,7 +570,7 @@ def render_env(inputs: RenderInputs) -> RenderedFile:lines = [f"ODS_MODE={inputs.ods_mode}",f"ODS_MODEL_SWITCHBOARD={inputs.switchboard_mode}",- "LLM_BACKEND=llama-server",+ f"LLM_BACKEND={inputs.llm_backend}",f"LLM_MODEL={inputs.model}",f"GGUF_FILE={inputs.gguf_file}",f"GPU_BACKEND={inputs.gpu_backend}",@@ -652,13 +667,19 @@ def render_model_router_endpoints(inputs: RenderInputs) -> RenderedFile:raise ValueError("model router endpoint origin must be an HTTP URL without embedded credentials")return base- # Every managed runtime is upstream llama-server; a host-native one is- # reached at http://host.docker.internal:<port> with its key, which the- # router reads from the named env var (never from this file).- endpoint = {"id": "llama-server-default",- "baseUrl": _origin_base(inputs.llm_base_url, "http://llama-server:8080")}- if inputs.llm_api_key_env:- endpoint["apiKeyEnv"] = inputs.llm_api_key_env+ if inputs.llm_backend == "vllm":+ # The in-stack vllm-server container on its internal port. It takes no+ # key; the llama-server origin and key inputs do not describe it.+ endpoint = {"id": VLLM_ENDPOINT_ID, "baseUrl": VLLM_SERVER_ORIGIN}+ else:+ # Otherwise the model server is upstream llama-server: the in-stack+ # container, or a host-native one reached at+ # http://host.docker.internal:<port> with its key, which the router+ # reads from the named env var (never from this file).+ endpoint = {"id": "llama-server-default",+ "baseUrl": _origin_base(inputs.llm_base_url, "http://llama-server:8080")}+ if inputs.llm_api_key_env:+ endpoint["apiKeyEnv"] = inputs.llm_api_key_envcontent = json.dumps({"endpoints": [endpoint]}, indent=2) + "\n"return RenderedFile("model-router-endpoints", "config/model-router/endpoints.json", content@@ -737,6 +758,15 @@ def parse_args(argv: list[str]) -> argparse.Namespace:"--ods-mode", choices=["local", "cloud", "hybrid", *LEGACY_ODS_MODES], default="local",)parser.add_argument("--llm-base-url", default="http://llama-server:8080/v1")+ llm_backend = parser.add_argument(+ "--llm-backend",+ choices=LLM_BACKEND_VALUES,+ default=(os.environ.get("LLM_BACKEND") or "").strip().lower() or "llama-server",+ help=(+ "The .env LLM_BACKEND value. vllm points model-router at vllm-server; "+ "every other value keeps the llama-server routes."+ ),+ )parser.add_argument("--llm-api-key-env", default="",help=(@@ -789,7 +819,7 @@ def parse_args(argv: list[str]) -> argparse.Namespace:# environment. A typo must not silently select a different routing mode# and overwrite working runtime configs. Validate effective values so an# explicit valid CLI override can still repair a bad environment default.- for action in (switchboard_mode, remote_enabled, remote_transport):+ for action in (switchboard_mode, remote_enabled, remote_transport, llm_backend):value = getattr(args, action.dest)if value not in action.choices:parser.error(f"{action.option_strings[0]}: invalid choice {value!r}; choose from {action.choices}")@@ -883,6 +913,7 @@ def render(args: argparse.Namespace) -> dict[str, object]:remote_llm_model=args.remote_llm_model,external_llm_authenticated=args.external_llm_authenticated,llm_api_key_env=getattr(args, "llm_api_key_env", ""),+ llm_backend=model_server(getattr(args, "llm_backend", "llama-server")),)validate_render_inputs(inputs)if args.surface == "litellm-switchboard" and inputs.ods_mode == "cloud":diff --git a/ods/tests/test-render-runtime-configs.py b/ods/tests/test-render-runtime-configs.pyindex de1be54f1..60a864897 100644--- a/ods/tests/test-render-runtime-configs.py+++ b/ods/tests/test-render-runtime-configs.py@@ -27,10 +27,11 @@ def load_renderer_module() -> ModuleType:return module-def run_renderer(*args: str) -> dict[str, object]:+def run_renderer(*args: str, env: dict[str, str] | None = None) -> dict[str, object]:proc = subprocess.run([sys.executable, str(SCRIPT), *args],cwd=ROOT,+ env=env,text=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE,@@ -39,6 +40,28 @@ def run_renderer(*args: str) -> dict[str, object]:return json.loads(proc.stdout)+def llm_backend_env(value: str | None = None) -> dict[str, str]:+ """This environment with LLM_BACKEND unset, or set to ``value``."""+ env = {key: item for key, item in os.environ.items() if key != "LLM_BACKEND"}+ if value is not None:+ env["LLM_BACKEND"] = value+ return env+++# Byte for byte what the renderer wrote before LLM_BACKEND became an input.+LLAMA_SERVER_ENDPOINTS = (+ '{\n "endpoints": [\n {\n "id": "llama-server-default",\n'+ ' "baseUrl": "http://llama-server:8080"\n }\n ]\n}\n'+)+LLAMA_SERVER_ENV = (+ "ODS_MODE=local\nODS_MODEL_SWITCHBOARD=enabled\nLLM_BACKEND=llama-server\n"+ "LLM_MODEL=qwen3.5-9b\nGGUF_FILE=Qwen3.5-9B-Q4_K_M.gguf\nGPU_BACKEND=nvidia\n"+ "LLM_API_URL=http://llama-server:8080/v1\nCTX_SIZE=131072\nMAX_CONTEXT=131072\n"+ "OPEN_WEBUI_LLM_BASE_URL=http://litellm:4000\nOPEN_WEBUI_LLM_API_KEY=sk-test-litellm\n"+)+ALL_ENABLED = ("--surface", "all", "--switchboard-mode", "enabled", "--litellm-key", "sk-test-litellm")++def file_by_surface(payload: dict[str, object], surface: str) -> dict[str, str]:for item in payload["files"]:if item["surface"] == surface:@@ -491,6 +514,47 @@ def test_router_endpoints_strip_trailing_v1() -> None:assert content == {"endpoints": [{"id": "llama-server-default", "baseUrl": "http://llama-server:8080"}]}+def test_llama_server_backends_render_todays_bytes() -> None:+ baseline = run_renderer(*ALL_ENABLED, env=llm_backend_env())+ assert baseline["inputs"]["llm_backend"] == "llama-server"+ assert file_by_surface(baseline, "model-router-endpoints")["content"] == LLAMA_SERVER_ENDPOINTS+ assert file_by_surface(baseline, "env")["content"] == LLAMA_SERVER_ENV+ # Every other .env LLM_BACKEND value, by flag or environment, renders+ # every surface exactly as before.+ for value in ("", "llama-server", "external", "hipfire", "lemonade", "litellm"):+ assert run_renderer(*ALL_ENABLED, env=llm_backend_env(value))["files"] == baseline["files"], value+ if value:+ flagged = run_renderer(*ALL_ENABLED, "--llm-backend", value, env=llm_backend_env())+ assert flagged["files"] == baseline["files"], value+++def test_vllm_backend_routes_model_router_to_vllm_server() -> None:+ surface = ("--surface", "model-router-endpoints")+ for payload in (+ run_renderer(*surface, "--llm-backend", "vllm", env=llm_backend_env()),+ run_renderer(*surface, env=llm_backend_env("vllm")),+ # The llama-server origin and key inputs do not describe vllm-server.+ run_renderer(*surface, "--llm-backend", "vllm",+ "--llm-base-url", "http://host.docker.internal:18080/v1",+ "--llm-api-key-env", "LLAMA_SERVER_API_KEY", env=llm_backend_env()),+ ):+ assert payload["inputs"]["llm_backend"] == "vllm"+ content = file_by_surface(payload, "model-router-endpoints")["content"]+ assert json.loads(content) == {+ "endpoints": [{"id": "vllm-default", "baseUrl": "http://vllm-server:8000"}],+ }+++def test_vllm_backend_changes_only_the_router_endpoint_and_env_backend() -> None:+ baseline = run_renderer(*ALL_ENABLED, env=llm_backend_env())+ vllm = run_renderer(*ALL_ENABLED, "--llm-backend", "vllm", env=llm_backend_env())+ assert file_by_surface(vllm, "env")["content"] == LLAMA_SERVER_ENV.replace(+ "LLM_BACKEND=llama-server\n", "LLM_BACKEND=vllm\n")+ changed = {after["surface"] for after, before in zip(vllm["files"], baseline["files"])+ if after != before}+ assert changed == {"env", "model-router-endpoints"}++def test_retired_lemonade_flags_are_accepted_and_ignored() -> None:payload = run_renderer("--surface",@@ -868,6 +932,9 @@ def main() -> int:test_enabled_hermes_uses_stable_switchboard_alias,test_enabled_opencode_uses_stable_switchboard_alias,test_router_endpoints_strip_trailing_v1,+ test_llama_server_backends_render_todays_bytes,+ test_vllm_backend_routes_model_router_to_vllm_server,+ test_vllm_backend_changes_only_the_router_endpoint_and_env_backend,test_retired_lemonade_flags_are_accepted_and_ignored,test_litellm_lemonade_surface_is_retired,test_amd_local_env_has_no_lemonade_keys,diff --git a/ods/tests/test-switchboard-install-proof.py b/ods/tests/test-switchboard-install-proof.pyindex 82e5fa01c..18c2abe14 100644--- a/ods/tests/test-switchboard-install-proof.py+++ b/ods/tests/test-switchboard-install-proof.py@@ -1,10 +1,15 @@"""Exercise host route scheduling without starting a real host service."""import astfrom pathlib import Path+import sys+import tempfileimport threadingimport unittestfrom unittest.mock import Mock+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'bin'))+from model_switchboard import state as switchboard_state # noqa: E402+class InstallProof(unittest.TestCase):def test_idle_schedules_proof_but_active_lifecycle_cancels_it(self):@@ -82,6 +87,59 @@ class InstallProof(unittest.TestCase):self.assertFalse(ns['_publish_verified_initial_switchboard_route'](reason='startup'))state.record_verified_route.assert_not_called()+ def test_vllm_install_route_reconstructs_then_publishes_through_the_state_module(self):+ tree = ast.parse((Path(__file__).parents[1] / 'bin/ods-host-agent.py').read_text(encoding='utf-8'))+ names = {'_prepare_initial_switchboard_verification', '_publish_verified_initial_switchboard_route'}+ module = ast.Module(body=[node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name in names], type_ignores=[])+ env = {'LLM_BACKEND': 'vllm', 'VLLM_SERVED_MODEL': 'GLM-5.3-Flash-NVFP4',+ 'LLM_MODEL': 'glm-5.3-flash-nvfp4', 'MAX_CONTEXT': '131072'}+ vllm_backend = {'kind': 'vllm', 'endpointId': 'vllm-default', 'nativeRoute': None}+ with tempfile.TemporaryDirectory() as tmp:+ state_path = Path(tmp) / 'model-state.json'+ # Startup reconstruction from .env, as the host agent runs it.+ reconstructed = switchboard_state.initialize_if_missing(state_path, env)+ self.assertEqual(reconstructed['active']['backend'], vllm_backend)+ self.assertIs(reconstructed['active']['reconstructed'], True)+ ns = dict(+ _switchboard_state=switchboard_state,+ _switchboard_initial_verify_cancel=threading.Event(),+ _model_lifecycle_state_lock=threading.Lock(),+ _model_lifecycle_operation=None,+ _pixel_model_transition_pending_for_route_proof=lambda: False,+ _switchboard_state_path=lambda: state_path,+ _switchboard_state_needs_current_env_verification=lambda path, env=None: True,+ # What the host agent serves without a GGUF_FILE: the identity.+ _current_runtime_model_inputs=lambda env, identity: (+ identity['runtimeModelId'], identity['catalogId']),+ _catalog_model_for_current_env=lambda env: ('glm-5.3-flash-nvfp4', {'tools': True}),+ # Stubbed: the route the host agent publishes for LLM_BACKEND=vllm.+ _initial_switchboard_backend=lambda env: (+ 'vllm', switchboard_state.default_endpoint_id('vllm'), None),+ _wait_for_model_readiness=lambda *args, **kwargs: {+ 'identity': 'GLM-5.3-Flash-NVFP4', 'contextLength': 131072},+ _initial_switchboard_route_env_matches=lambda env: True,+ _runtime_model_identity_matches=lambda *args, **kwargs: True,+ _model_agent_viable=lambda model, context: True,+ _model_profiles_mode=lambda env: 'observe',+ _SWITCHBOARD_ROUTE_ENV_KEYS=(),+ load_env=lambda path: dict(env),+ INSTALL_DIR=Path(tmp),+ logger=Mock(),+ )+ exec(compile(module, '<host proof functions>', 'exec'), ns)+ self.assertTrue(ns['_publish_verified_initial_switchboard_route'](reason='install'))+ doc, errors = switchboard_state.read_state(state_path)+ self.assertEqual(errors, [])+ active = doc['active']+ self.assertEqual(active['backend'], vllm_backend)+ self.assertEqual(active['catalogId'], 'glm-5.3-flash-nvfp4')+ self.assertEqual(active['runtimeModelId'], 'GLM-5.3-Flash-NVFP4')+ self.assertEqual(active['contextLength'], 131072)+ self.assertEqual(active['proof'], {'identity': 'GLM-5.3-Flash-NVFP4', 'completion': True})+ self.assertNotIn('reconstructed', active)+ # Proving the reconstructed route keeps it: no new route, no history.+ self.assertEqual((doc['seq'], doc['routeSeq'], doc['history']), (2, 1, []))+if __name__ == '__main__':unittest.main()diff --git a/ods/tests/test_renderer_environment_modes.py b/ods/tests/test_renderer_environment_modes.pyindex 00a57f3bc..8c4dc93a8 100644--- a/ods/tests/test_renderer_environment_modes.py+++ b/ods/tests/test_renderer_environment_modes.py@@ -12,6 +12,7 @@ import pytest("ODS_MODEL_SWITCHBOARD", "--switchboard-mode", "enabledd", "enabled"),("REMOTE_LLM_ENABLED", "--remote-llm-enabled", "tru", "false"),("REMOTE_LLM_TRANSPORT", "--remote-llm-transport", "sshh", ""),+ ("LLM_BACKEND", "--llm-backend", "vlm", "llama-server"),])def test_invalid_environment_mode_cannot_write_configs(tmp_path, key, option, bad, valid):script = Path(__file__).resolve().parents[1] / "scripts/render-runtime-configs.py"