Skip to content

armgit / ODS

Public Imported from GitHub
0

ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.

feat(models): add the vllm backend kind and its router endpoint (#8288)

Mike Bradleya2c9c62d0d20
diff --git a/ods/bin/model_switchboard/__init__.py b/ods/bin/model_switchboard/__init__.py
index da0782e68..f7fe018f4 100644
--- a/ods/bin/model_switchboard/__init__.py
+++ b/ods/bin/model_switchboard/__init__.py
@@ -11,6 +11,7 @@ from .state import ( # noqa: F401
SCHEMA_VERSION,
StateError,
atomic_write_state,
+ default_endpoint_id,
initial_state,
initialize_if_missing,
is_legacy_route,
@@ -25,6 +26,7 @@ __all__ = [
"SCHEMA_VERSION",
"StateError",
"atomic_write_state",
+ "default_endpoint_id",
"initial_state",
"initialize_if_missing",
"is_legacy_route",
diff --git a/ods/bin/model_switchboard/state.py b/ods/bin/model_switchboard/state.py
index 1696909e6..597339aed 100644
--- a/ods/bin/model_switchboard/state.py
+++ b/ods/bin/model_switchboard/state.py
@@ -33,13 +33,23 @@ HISTORY_LIMIT = 10
PUBLIC_MODEL_DEFAULT = "ods/current"
STATE_FILE_MODE = 0o644
-_WRITABLE_BACKEND_KINDS = {"llama-server", "hipfire", "unknown"}
+DEFAULT_ENDPOINT_ID = "llama-server-default"
+VLLM_ENDPOINT_ID = "vllm-default"
+# Every writable backend kind and the model-router endpoint (endpoints.json
+# id) it routes to when the caller names none. hipfire and unknown routes
+# have always used the llama-server endpoint; vLLM serves from vllm-server.
+_DEFAULT_ENDPOINT_IDS = {
+ "llama-server": DEFAULT_ENDPOINT_ID,
+ "hipfire": DEFAULT_ENDPOINT_ID,
+ "unknown": DEFAULT_ENDPOINT_ID,
+ "vllm": VLLM_ENDPOINT_ID,
+}
+_WRITABLE_BACKEND_KINDS = set(_DEFAULT_ENDPOINT_IDS)
# Readable for one release only: records written before round F carry the
# retired Lemonade route until the host agent rewrites and re-proves them.
_LEGACY_BACKEND_KINDS = {"lemonade"}
_LEGACY_ENDPOINT_IDS = {"lemonade-default"}
_BACKEND_KINDS = _WRITABLE_BACKEND_KINDS | _LEGACY_BACKEND_KINDS
-DEFAULT_ENDPOINT_ID = "llama-server-default"
_OPERATION_PHASES = {
"requested", "staging", "verifying", "publishing",
"flipping", "serving", "failed", "rolling_back",
@@ -505,25 +515,26 @@ def is_legacy_route(active: Any) -> bool:
)
+def default_endpoint_id(backend_kind: str) -> str:
+ """The router endpoint id a writable backend kind uses by default."""
+ return _DEFAULT_ENDPOINT_IDS[backend_kind]
+
+
def migrate_env_identity(env: dict[str, str]) -> dict[str, Any] | None:
"""Derive a best-effort runtime identity from ``.env`` values.
- Every managed runtime is upstream llama-server serving ``--alias
- <GGUF_FILE>``, so the runtime identity is the GGUF filename (or the model
- name when no file is recorded). A retired ``LEMONADE_MODEL`` line is never
- read: a leftover key must not relabel the route. Returns ``None`` when the
- environment carries no local model identity (e.g. cloud-only).
+ A llama-server runtime serves ``--alias <GGUF_FILE>``, so its identity is
+ the GGUF filename (or the model name when no file is recorded). A vLLM
+ runtime (``LLM_BACKEND=vllm``) serves ``--served-model-name
+ <VLLM_SERVED_MODEL>`` (or the model name when none is recorded); a
+ ``GGUF_FILE`` left by an earlier llama-server install never names it. A
+ retired ``LEMONADE_MODEL`` line is never read: a leftover key must not
+ relabel the route. Returns ``None`` when the environment carries no local
+ model identity (e.g. cloud-only).
"""
if str(env.get("ODS_MODE") or "").strip().casefold() == "cloud":
return None
- gguf = str(env.get("GGUF_FILE") or "").strip()
- llm_model = str(env.get("LLM_MODEL") or "").strip()
-
- runtime_id = gguf or llm_model
- if not runtime_id:
- return None
-
backend = str(env.get("LLM_BACKEND") or "").strip().lower()
if backend in {"", "lemonade"}:
# A pre-migration LLM_BACKEND=lemonade install now serves through
@@ -532,6 +543,16 @@ def migrate_env_identity(env: dict[str, str]) -> dict[str, Any] | None:
if backend not in _WRITABLE_BACKEND_KINDS:
backend = "unknown"
+ llm_model = str(env.get("LLM_MODEL") or "").strip()
+ if backend == "vllm":
+ served = str(env.get("VLLM_SERVED_MODEL") or "").strip()
+ else:
+ served = str(env.get("GGUF_FILE") or "").strip()
+
+ runtime_id = served or llm_model
+ if not runtime_id:
+ return None
+
catalog_guess = llm_model
if not catalog_guess:
stem = runtime_id
@@ -565,8 +586,9 @@ def initialize_if_missing(
) -> dict[str, Any] | None:
"""One-time startup reconstruction when no v1 state was ever committed.
- Existing files — valid or malformed — are never overwritten here. Returns
- the written doc, or ``None`` when nothing was written.
+ Existing files — valid or malformed — are never overwritten here. Without
+ an ``endpoint_id`` the route uses its backend kind's default endpoint.
+ Returns the written doc, or ``None`` when nothing was written.
"""
if Path(path).exists():
return None
@@ -574,7 +596,7 @@ def initialize_if_missing(
if identity is None:
return None
if endpoint_id is None:
- endpoint_id = DEFAULT_ENDPOINT_ID
+ endpoint_id = default_endpoint_id(identity["backendKind"])
return record_verified_route(
path,
catalog_id=identity["catalogId"],
diff --git a/ods/config/model-state.schema.v1.json b/ods/config/model-state.schema.v1.json
index fbce172ac..174b584f3 100644
--- a/ods/config/model-state.schema.v1.json
+++ b/ods/config/model-state.schema.v1.json
@@ -57,7 +57,7 @@
"additionalProperties": false,
"properties": {
"kind": {
- "enum": ["llama-server", "lemonade", "hipfire", "unknown"],
+ "enum": ["llama-server", "vllm", "lemonade", "hipfire", "unknown"],
"description": "lemonade names a pre-round-F route: readable for one release so the host agent can replace it, never written"
},
"endpointId": { "type": "string", "minLength": 1 },
diff --git a/ods/extensions/services/dashboard-api/builtin_source_recipes.py b/ods/extensions/services/dashboard-api/builtin_source_recipes.py
index ed9dfb395..c275f6ddf 100644
--- a/ods/extensions/services/dashboard-api/builtin_source_recipes.py
+++ b/ods/extensions/services/dashboard-api/builtin_source_recipes.py
@@ -35,7 +35,7 @@ _CONTEXT_BUILDS = {
"ape": ("ape", None, "Dockerfile", "ffa14f37d9bca3fc9aa35582b5f3a4b9adaa1e75222a9c29dc9d55d9cfa49805"),
"brave-search": ("brave-search", "ods-brave-search:local", "Dockerfile", "f1be25450cae210788c29d02dc497ff64f3fe7f9184cc73332c08a734e709e95"),
"privacy-shield": ("privacy-shield", None, "Dockerfile", "deb8da4c40e7eac01c2576c33bd11f0df21c2332bf929bc6ba0272669ffa64b8"),
- "token-spy": ("token-spy", None, "Dockerfile", "a437422b3bd3eeb0ce35631137dc78665769489cd7d7f6b6bb219586273f2f60"),
+ "token-spy": ("token-spy", None, "Dockerfile", "98e93c93cdff9ff63bed7569a8dec9e45171c59e60516878e097a43274277eff"),
}
_MAX_CONTEXT_FILE_BYTES = 8 * 1024 * 1024
_MAX_CONTEXT_FILES = 512
diff --git a/ods/extensions/services/dashboard-api/tests/test_model_state.py b/ods/extensions/services/dashboard-api/tests/test_model_state.py
index d42ba75e5..a3efe54da 100644
--- a/ods/extensions/services/dashboard-api/tests/test_model_state.py
+++ b/ods/extensions/services/dashboard-api/tests/test_model_state.py
@@ -49,6 +49,40 @@ class TestStateModule:
schema = json.loads(_SCHEMA_PATH.read_text(encoding="utf-8"))
jsonschema.validate(read, schema)
+ def test_vllm_route_roundtrips_and_matches_schema(self, tmp_path):
+ path = tmp_path / "model-state.json"
+ doc = _record(path, model="glm-5.3-flash-nvfp4", runtime="GLM-5.3-Flash-NVFP4",
+ backend="vllm", endpoint="vllm-default", context=131072)
+ assert doc["active"]["backend"] == {
+ "kind": "vllm", "endpointId": "vllm-default", "nativeRoute": None,
+ }
+ read, errors = sb.read_state(path)
+ assert errors == [] and read == doc
+ assert sb.validate_state(read) == []
+ jsonschema = pytest.importorskip("jsonschema")
+ schema = json.loads(_SCHEMA_PATH.read_text(encoding="utf-8"))
+ jsonschema.validate(read, schema)
+
+ def test_backend_kinds_agree_with_schema(self):
+ schema = json.loads(_SCHEMA_PATH.read_text(encoding="utf-8"))
+ active = next(option for option in schema["properties"]["active"]["oneOf"]
+ if option.get("type") == "object")
+ kinds = active["properties"]["backend"]["properties"]["kind"]["enum"]
+ assert set(kinds) == sb._BACKEND_KINDS
+
+ @pytest.mark.parametrize("kind,endpoint", [
+ ("llama-server", "llama-server-default"),
+ ("hipfire", "llama-server-default"),
+ ("unknown", "llama-server-default"),
+ ("vllm", "vllm-default"),
+ ])
+ def test_default_endpoint_per_backend_kind(self, kind, endpoint):
+ assert sb.default_endpoint_id(kind) == endpoint
+
+ def test_retired_kind_has_no_default_endpoint(self):
+ with pytest.raises(KeyError):
+ sb.default_endpoint_id("lemonade")
+
def test_state_file_is_readable_by_model_router_user(self, tmp_path):
path = tmp_path / "model-state.json"
_record(path)
@@ -183,6 +217,32 @@ class TestStateModule:
"runtimeModelId": "stale.gguf",
"backendKind": "llama-server",
}),
+ # vLLM serves --served-model-name; a GGUF_FILE left by an earlier
+ # llama-server install never names the route.
+ ({
+ "LLM_BACKEND": "vllm",
+ "VLLM_SERVED_MODEL": "GLM-5.3-Flash-NVFP4",
+ "LLM_MODEL": "glm-5.3-flash-nvfp4",
+ "GGUF_FILE": "Qwen3.5-9B-Q4_K_M.gguf",
+ "MAX_CONTEXT": "131072",
+ }, {
+ "catalogId": "glm-5.3-flash-nvfp4",
+ "runtimeModelId": "GLM-5.3-Flash-NVFP4",
+ "backendKind": "vllm",
+ "contextLength": 131072,
+ }),
+ ({"LLM_BACKEND": "vllm", "LLM_MODEL": "glm-5.3-flash-nvfp4",
+ "GGUF_FILE": "Qwen3.5-9B-Q4_K_M.gguf"},
+ {"catalogId": "glm-5.3-flash-nvfp4", "runtimeModelId": "glm-5.3-flash-nvfp4",
+ "backendKind": "vllm"}),
+ ({"LLM_BACKEND": " VLLM ", "VLLM_SERVED_MODEL": "Served-Model",
+ "CTX_SIZE": "65536", "MAX_CONTEXT": "131072"},
+ {"catalogId": "Served-Model", "runtimeModelId": "Served-Model",
+ "backendKind": "vllm", "contextLength": 65536}),
+ # VLLM_SERVED_MODEL names only a vllm route.
+ ({"LLM_BACKEND": "llama-server", "VLLM_SERVED_MODEL": "Served-Model",
+ "GGUF_FILE": "M.gguf"},
+ {"catalogId": "M", "runtimeModelId": "M.gguf", "backendKind": "llama-server"}),
],
)
def test_migrate_env_forms(self, env, expected):
@@ -190,6 +250,11 @@ class TestStateModule:
for key, value in expected.items():
assert got[key] == value
+ def test_migrate_vllm_env_never_takes_the_gguf_identity(self):
+ assert sb.migrate_env_identity({
+ "LLM_BACKEND": "vllm", "GGUF_FILE": "Qwen3.5-9B-Q4_K_M.gguf",
+ }) is None
+
def test_migrate_cloud_only_yields_none(self):
env = {
"ODS_MODE": "cloud",
@@ -258,6 +323,46 @@ class TestStateModule:
}
assert doc["active"]["runtimeModelId"] == "Model.gguf"
+ @pytest.mark.parametrize("env,kind,endpoint", [
+ ({"GGUF_FILE": "M.gguf"}, "llama-server", "llama-server-default"),
+ ({"LLM_BACKEND": "hipfire", "LLM_MODEL": "Qwen3.5-2B"}, "hipfire", "llama-server-default"),
+ ({"LLM_BACKEND": "external", "LLM_MODEL": "lan-model"}, "unknown", "llama-server-default"),
+ ({"LLM_BACKEND": "vllm", "VLLM_SERVED_MODEL": "GLM-5.3-Flash-NVFP4"}, "vllm", "vllm-default"),
+ ])
+ def test_initialize_uses_the_backend_kinds_default_endpoint(self, tmp_path, env, kind, endpoint):
+ doc = sb.initialize_if_missing(tmp_path / "model-state.json", env)
+ assert doc["active"]["backend"] == {"kind": kind, "endpointId": endpoint, "nativeRoute": None}
+ assert sb.validate_state(doc) == []
+
+ def test_initialize_for_a_vllm_env_reconstructs_the_served_model(self, tmp_path):
+ path = tmp_path / "model-state.json"
+ doc = sb.initialize_if_missing(path, {
+ "LLM_BACKEND": "vllm",
+ "VLLM_SERVED_MODEL": "GLM-5.3-Flash-NVFP4",
+ "LLM_MODEL": "glm-5.3-flash-nvfp4",
+ "GGUF_FILE": "Qwen3.5-9B-Q4_K_M.gguf",
+ "MAX_CONTEXT": "131072",
+ })
+ assert doc["active"]["catalogId"] == "glm-5.3-flash-nvfp4"
+ assert doc["active"]["runtimeModelId"] == "GLM-5.3-Flash-NVFP4"
+ assert doc["active"]["contextLength"] == 131072
+ assert doc["active"]["reconstructed"] is True
+ assert doc["active"]["proof"] == {"identity": "GLM-5.3-Flash-NVFP4", "completion": False}
+ read, errors = sb.read_state(path)
+ assert errors == [] and read == doc
+ jsonschema = pytest.importorskip("jsonschema")
+ jsonschema.validate(read, json.loads(_SCHEMA_PATH.read_text(encoding="utf-8")))
+
+ def test_initialize_keeps_a_caller_named_endpoint(self, tmp_path):
+ doc = sb.initialize_if_missing(
+ tmp_path / "model-state.json",
+ {"LLM_BACKEND": "vllm", "VLLM_SERVED_MODEL": "GLM-5.3-Flash-NVFP4"},
+ endpoint_id="vllm-native",
+ )
+ assert doc["active"]["backend"] == {
+ "kind": "vllm", "endpointId": "vllm-native", "nativeRoute": None,
+ }
+
def test_legacy_lemonade_route_is_readable_but_never_written(self, tmp_path):
path = tmp_path / "model-state.json"
doc = _record(path, runtime="Model.gguf")
diff --git a/ods/extensions/services/model-router/app/main.py b/ods/extensions/services/model-router/app/main.py
index 8500cc1c9..1ce966107 100644
--- a/ods/extensions/services/model-router/app/main.py
+++ b/ods/extensions/services/model-router/app/main.py
@@ -1005,7 +1005,7 @@ def _validate_state_schema(doc: Any) -> bool:
# A pre-round-F state names the retired "lemonade" kind. It stays
# readable for one release so the host agent can replace it (contract
# section 6.7); no route path treats it specially.
- if backend["kind"] not in {"llama-server", "lemonade", "hipfire", "unknown"}:
+ if backend["kind"] not in {"llama-server", "vllm", "lemonade", "hipfire", "unknown"}:
return False
if not isinstance(backend["endpointId"], str) or not backend["endpointId"]:
return False
@@ -1133,6 +1133,19 @@ def _active_route() -> dict[str, Any]:
}
+def _llama_tool_compat(route: dict[str, Any]) -> bool:
+ """Whether the router's llama.cpp tool-call workarounds apply to a route.
+
+ Only llama-server routes get them: buffered tool streams, the non-stream
+ request for thinking-enabled tool streams, bounded tool-protocol repair
+ and native tool-markup cleanup (tool_id_compat keeps its Nemo tool-id
+ rewrite llama-server-only too). Other kinds are forwarded unchanged; a
+ vllm route parses tool calls and reasoning itself with its own
+ --tool-call-parser and --reasoning-parser.
+ """
+ return route["backendKind"] == "llama-server"
+
+
def _offered_tool_evidence(payload: dict[str, Any]) -> dict[str, Any]:
"""Fingerprint the outgoing tools, never retain their names or schema text.
@@ -2040,7 +2053,7 @@ async def _forward_inner(request: Request, path: str, payload: dict[str, Any],
completed_tool_stream = (
is_stream
and path == "/v1/chat/completions"
- and route["backendKind"] == "llama-server"
+ and _llama_tool_compat(route)
and isinstance(payload.get("tools"), list)
and bool(payload["tools"])
)
@@ -2361,7 +2374,7 @@ async def _forward_inner(request: Request, path: str, payload: dict[str, Any],
# model decision, not an executed action; no host/tool operation is
# replayed here. Every rejected decision is logged by shape, never values.
if (path == "/v1/chat/completions"
- and route["backendKind"] == "llama-server"
+ and _llama_tool_compat(route)
and _repairable_native_tool_request(payload)
and isinstance(payload.get("messages"), list)
and 200 <= upstream.status_code < 300):
@@ -2491,7 +2504,7 @@ async def _forward_inner(request: Request, path: str, payload: dict[str, Any],
if (parsed is not None and 200 <= upstream.status_code < 300
and path == "/v1/chat/completions"
- and route["backendKind"] == "llama-server"
+ and _llama_tool_compat(route)
and isinstance(payload.get("tools"), list) and payload["tools"]
and _normalize_native_tool_markup(parsed, payload)):
content = json.dumps(parsed).encode("utf-8")
diff --git a/ods/extensions/services/model-router/tests/test_router.py b/ods/extensions/services/model-router/tests/test_router.py
index 8d9069460..94574f5d8 100644
--- a/ods/extensions/services/model-router/tests/test_router.py
+++ b/ods/extensions/services/model-router/tests/test_router.py
@@ -1344,6 +1344,164 @@ class TestForwarding:
assert mod._inflight == 0
+_VLLM_MODEL = "GLM-5.3-Flash-NVFP4"
+_LOOKUP_TOOL = {"type": "function", "function": {"name": "lookup", "parameters": {
+ "type": "object", "additionalProperties": False, "required": ["key"],
+ "properties": {"key": {"type": "string"}}}}}
+_LOOKUP_MARKUP = ("<tool_call>\n<function=lookup>\n"
+ "<parameter=key>\nok\n</parameter>\n</function>\n</tool_call>")
+
+
+def _write_route(tmp_path, write_state, kind):
+ """Publish a verified route of ``kind``; a vllm one at vllm-default."""
+ if kind == "llama-server":
+ write_state()
+ return "Concrete.gguf"
+ # The allowlist the renderer writes for LLM_BACKEND=vllm, plus the
+ # fixture's llama-server row so a wrong endpoint choice would still route.
+ (tmp_path / "endpoints.json").write_text(json.dumps({"endpoints": [
+ {"id": "llama-server-default", "baseUrl": "http://upstream:8080"},
+ {"id": "vllm-default", "baseUrl": "http://vllm-server:8000"},
+ ]}), encoding="utf-8")
+ write_state(runtime=_VLLM_MODEL, endpoint="vllm-default",
+ mutate=lambda doc: doc["active"]["backend"].update(kind=kind))
+ return _VLLM_MODEL
+
+
+def _use_upstream(mod, handler):
+ asyncio.run(mod.app.state.http.aclose())
+ mod.app.state.http = httpx.AsyncClient(transport=httpx.MockTransport(handler))
+
+
+class TestVllmRoute:
+ """A vllm route is forwarded unchanged: vLLM parses tools and reasoning."""
+
+ def test_vllm_route_reaches_vllm_server_under_the_alias(self, router, tmp_path):
+ mod, client, write_state, calls = router
+ _write_route(tmp_path, write_state, "vllm")
+ recorder = _RecordingTelemetry()
+ mod.app.state.telemetry = recorder
+
+ resp = client.post("/v1/chat/completions", json={
+ "model": "ods/current", "messages": [{"role": "user", "content": "hi"}],
+ })
+
+ assert resp.status_code == 200
+ assert calls[-1]["url"] == "http://vllm-server:8000/v1/chat/completions"
+ assert calls[-1]["model"] == _VLLM_MODEL
+ assert calls[-1]["auth"] is None
+ assert resp.json()["model"] == "ods/current"
+ assert resp.headers["X-ODS-Backend"] == "vllm"
+ assert resp.headers["X-ODS-Routed-Model"] == _VLLM_MODEL
+ assert client.get("/v1/models").json()["ods"]["backend"] == "vllm"
+ assert [event["provider_name"] for event in recorder.events] == ["vllm"]
+
+ @pytest.mark.parametrize("thinking", [False, True])
+ def test_vllm_tool_stream_is_forwarded_live(self, router, tmp_path, thinking):
+ mod, client, write_state, _calls = router
+ _write_route(tmp_path, write_state, "vllm")
+ sent = []
+ # A llama-server route buffers this unfinished tool decision and
+ # rejects it (test_incomplete_tool_stream_fails_without_nonstream_retry).
+ partial = {"model": _VLLM_MODEL, "choices": [{"index": 0, "delta": {
+ "role": "assistant", "tool_calls": [{
+ "index": 0, "id": "call-1", "type": "function",
+ "function": {"name": "lookup", "arguments": '{"key":"ok"}'}}]},
+ "finish_reason": None}]}
+
+ def handler(request):
+ sent.append(json.loads(request.content))
+ return httpx.Response(200, content=b"data: " + json.dumps(partial).encode() + b"\n\n",
+ headers={"content-type": "text/event-stream"})
+
+ _use_upstream(mod, handler)
+ body = {"model": "ods/current", "stream": True,
+ "messages": [{"role": "user", "content": "look up ok"}],
+ "tools": [_LOOKUP_TOOL]}
+ if thinking:
+ body["chat_template_kwargs"] = {"enable_thinking": True}
+ response = client.post("/v1/chat/completions", json=body)
+
+ assert response.status_code == 200
+ assert len(sent) == 1 and sent[0]["stream"] is True
+ assert "stream_options" not in sent[0]
+ assert sent[0].get("chat_template_kwargs") == body.get("chat_template_kwargs")
+ for header in ("x-ods-tool-stream-fallback", "x-ods-tool-stream-compatibility",
+ "x-ods-tool-stream-repair"):
+ assert header not in response.headers
+ frames = [json.loads(item.removeprefix("data: "))
+ for item in response.text.split("\n\n") if item]
+ assert frames == [{**partial, "model": "ods/current"}]
+
+ @pytest.mark.parametrize("kind,upstream_calls", [("llama-server", 2), ("vllm", 1)])
+ def test_only_llama_server_repairs_a_rejected_tool_decision(
+ self, router, tmp_path, kind, upstream_calls):
+ mod, client, write_state, _calls = router
+ runtime = _write_route(tmp_path, write_state, kind)
+ sent = []
+ # Names a function that was not offered; the offered one is lookup.
+ rejected = _LOOKUP_MARKUP.replace("function=lookup", "function=pixel_ods_lookup")
+
+ def handler(request):
+ sent.append(json.loads(request.content))
+ message = {"role": "assistant", "content": rejected}
+ reason = "stop"
+ if len(sent) > 1:
+ message = {"role": "assistant", "content": None, "tool_calls": [{
+ "id": "call-1", "type": "function",
+ "function": {"name": "lookup", "arguments": '{"key":"ok"}'}}]}
+ reason = "tool_calls"
+ return httpx.Response(200, json={"model": runtime, "choices": [
+ {"index": 0, "message": message, "finish_reason": reason}]})
+
+ _use_upstream(mod, handler)
+ response = client.post("/v1/chat/completions", json={
+ "model": "ods/current", "stream": False,
+ "messages": [{"role": "user", "content": "look up ok"}],
+ "tools": [_LOOKUP_TOOL],
+ })
+
+ assert response.status_code == 200
+ assert len(sent) == upstream_calls
+ choice = response.json()["choices"][0]
+ if kind == "vllm":
+ assert choice == {"index": 0, "message": {"role": "assistant", "content": rejected},
+ "finish_reason": "stop"}
+ else:
+ assert choice["message"]["tool_calls"][0]["function"]["name"] == "lookup"
+
+ @pytest.mark.parametrize("kind", ["llama-server", "vllm"])
+ def test_only_llama_server_converts_native_tool_markup(self, router, tmp_path, kind):
+ mod, client, write_state, _calls = router
+ runtime = _write_route(tmp_path, write_state, kind)
+ sent = []
+
+ def handler(request):
+ sent.append(json.loads(request.content))
+ return httpx.Response(200, json={"model": runtime, "choices": [{"index": 0,
+ "message": {"role": "assistant", "content": _LOOKUP_MARKUP},
+ "finish_reason": "stop"}]})
+
+ _use_upstream(mod, handler)
+ response = client.post("/v1/chat/completions", json={
+ "model": "ods/current", "stream": False,
+ "messages": [{"role": "user", "content": "look up ok"}],
+ "tools": [_LOOKUP_TOOL],
+ })
+
+ assert response.status_code == 200
+ assert len(sent) == 1
+ choice = response.json()["choices"][0]
+ if kind == "vllm":
+ assert choice["finish_reason"] == "stop"
+ assert choice["message"] == {"role": "assistant", "content": _LOOKUP_MARKUP}
+ else:
+ assert choice["finish_reason"] == "tool_calls"
+ assert choice["message"]["content"] is None
+ assert json.loads(choice["message"]["tool_calls"][0]["function"]["arguments"]) == {
+ "key": "ok"}
+
+
class TestStateTrust:
def test_schema_invalid_state_is_not_routable(self, router):
mod, client, write_state, calls = router
@@ -1354,6 +1512,16 @@ class TestStateTrust:
assert resp.status_code == 503
assert calls == []
+ def test_unlisted_backend_kind_is_not_routable(self, router):
+ mod, client, write_state, calls = router
+ write_state(mutate=lambda doc: doc["active"]["backend"].update(kind="sglang"))
+ resp = client.post("/v1/chat/completions", json={
+ "model": "ods/current", "messages": [],
+ })
+ assert resp.status_code == 503
+ assert resp.json()["error"]["type"] == "no_active_route"
+ assert calls == []
+
@pytest.mark.parametrize("mutation", [
lambda doc: doc["active"].update({"reconstructed": True}),
lambda doc: doc["active"]["proof"].update({"completion": False}),
diff --git a/ods/extensions/services/model-router/tests/test_tool_id_compat.py b/ods/extensions/services/model-router/tests/test_tool_id_compat.py
index bad5285f1..7c932b257 100644
--- a/ods/extensions/services/model-router/tests/test_tool_id_compat.py
+++ b/ods/extensions/services/model-router/tests/test_tool_id_compat.py
@@ -49,6 +49,14 @@ class DetectionTests(unittest.TestCase):
tool_id_compat.route_requires_strict9_tool_ids(NEMO_ID, "hipfire")
)
+ def test_negative_vllm_backend(self):
+ # vLLM's own tool parser owns tool-call ids on a vllm route.
+ self.assertFalse(
+ tool_id_compat.route_requires_strict9_tool_ids(
+ "mistralai/Mistral-Nemo-Instruct-2407", "vllm"
+ )
+ )
+
def test_negative_caller_alias(self):
self.assertFalse(
tool_id_compat.route_requires_strict9_tool_ids("ods/current", "llama-server")
diff --git a/ods/extensions/services/token-spy/tests/test_routed_telemetry.py b/ods/extensions/services/token-spy/tests/test_routed_telemetry.py
index 4a5588217..37b81bb6b 100644
--- a/ods/extensions/services/token-spy/tests/test_routed_telemetry.py
+++ b/ods/extensions/services/token-spy/tests/test_routed_telemetry.py
@@ -142,3 +142,20 @@ def test_hipfire_backend_is_accounted_as_local_runtime():
assert usage["provider_name"] == "local"
assert usage["cost_source"] == "local_zero_cost"
+
+
+def test_vllm_backend_is_accounted_as_local_runtime():
+ telemetry = _load(
+ TOKEN_SPY_DIR / "routed_telemetry.py", "routed_telemetry"
+ )
+
+ # model-router reports a vllm route's backend kind as its provider.
+ usage = telemetry.routed_event_to_usage(
+ telemetry.validate_routed_event(
+ _event(provider_name="vllm", model="GLM-5.3-Flash-NVFP4")
+ )
+ )
+
+ assert usage["provider_name"] == "local"
+ assert usage["cost_source"] == "local_zero_cost"
+ assert usage["model"] == "GLM-5.3-Flash-NVFP4"
diff --git a/ods/scripts/render-runtime-configs.py b/ods/scripts/render-runtime-configs.py
index 36d0f65a5..397e3edc2 100644
--- a/ods/scripts/render-runtime-configs.py
+++ b/ods/scripts/render-runtime-configs.py
@@ -38,6 +38,14 @@ LEGACY_ODS_MODES = {"lemonade": "local"}
PUBLIC_MODEL_ALIAS = "ods/current"
REMOTE_PROVIDER_EGRESS_BASE_URL = "http://remote-provider-egress:8091/v1"
REMOTE_MODEL_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/+-]{0,255}$")
+# .env LLM_BACKEND values. vllm routes model-router to the vllm-server
+# container; every other value keeps today's llama-server routes (external
+# installs replace the router through their overlay, hipfire routes share the
+# llama-server endpoint, and lemonade and litellm are retired values still
+# accepted for upgrade compatibility).
+LLM_BACKEND_VALUES = ("llama-server", "vllm", "external", "hipfire", "lemonade", "litellm")
+VLLM_ENDPOINT_ID = "vllm-default"
+VLLM_SERVER_ORIGIN = "http://vllm-server:8000"
def atomic_write_text(target: Path, content: str, *, file_mode: int | None = None) -> None:
@@ -126,6 +134,8 @@ class RenderInputs:
llm_api_key_env: str = ""
# Switchboard rollout mode: legacy | observe | enabled (plan section 8)
switchboard_mode: str = "enabled"
+ # Model server behind model-router: llama-server | vllm (model_server()).
+ llm_backend: str = "llama-server"
@dataclass(frozen=True)
@@ -157,6 +167,11 @@ def remote_route_enabled(inputs: RenderInputs) -> bool:
return inputs.remote_llm_enabled
+def model_server(llm_backend: str) -> str:
+ """The model server an .env LLM_BACKEND value routes to."""
+ return "vllm" if llm_backend == "vllm" else "llama-server"
+
+
def hermes_model_id(inputs: RenderInputs) -> str:
if inputs.switchboard_mode == "enabled":
return "ods/current"
@@ -555,7 +570,7 @@ def render_env(inputs: RenderInputs) -> RenderedFile:
lines = [
f"ODS_MODE={inputs.ods_mode}",
f"ODS_MODEL_SWITCHBOARD={inputs.switchboard_mode}",
- "LLM_BACKEND=llama-server",
+ f"LLM_BACKEND={inputs.llm_backend}",
f"LLM_MODEL={inputs.model}",
f"GGUF_FILE={inputs.gguf_file}",
f"GPU_BACKEND={inputs.gpu_backend}",
@@ -652,13 +667,19 @@ def render_model_router_endpoints(inputs: RenderInputs) -> RenderedFile:
raise ValueError("model router endpoint origin must be an HTTP URL without embedded credentials")
return base
- # Every managed runtime is upstream llama-server; a host-native one is
- # reached at http://host.docker.internal:<port> with its key, which the
- # router reads from the named env var (never from this file).
- endpoint = {"id": "llama-server-default",
- "baseUrl": _origin_base(inputs.llm_base_url, "http://llama-server:8080")}
- if inputs.llm_api_key_env:
- endpoint["apiKeyEnv"] = inputs.llm_api_key_env
+ if inputs.llm_backend == "vllm":
+ # The in-stack vllm-server container on its internal port. It takes no
+ # key; the llama-server origin and key inputs do not describe it.
+ endpoint = {"id": VLLM_ENDPOINT_ID, "baseUrl": VLLM_SERVER_ORIGIN}
+ else:
+ # Otherwise the model server is upstream llama-server: the in-stack
+ # container, or a host-native one reached at
+ # http://host.docker.internal:<port> with its key, which the router
+ # reads from the named env var (never from this file).
+ endpoint = {"id": "llama-server-default",
+ "baseUrl": _origin_base(inputs.llm_base_url, "http://llama-server:8080")}
+ if inputs.llm_api_key_env:
+ endpoint["apiKeyEnv"] = inputs.llm_api_key_env
content = json.dumps({"endpoints": [endpoint]}, indent=2) + "\n"
return RenderedFile(
"model-router-endpoints", "config/model-router/endpoints.json", content
@@ -737,6 +758,15 @@ def parse_args(argv: list[str]) -> argparse.Namespace:
"--ods-mode", choices=["local", "cloud", "hybrid", *LEGACY_ODS_MODES], default="local",
)
parser.add_argument("--llm-base-url", default="http://llama-server:8080/v1")
+ llm_backend = parser.add_argument(
+ "--llm-backend",
+ choices=LLM_BACKEND_VALUES,
+ default=(os.environ.get("LLM_BACKEND") or "").strip().lower() or "llama-server",
+ help=(
+ "The .env LLM_BACKEND value. vllm points model-router at vllm-server; "
+ "every other value keeps the llama-server routes."
+ ),
+ )
parser.add_argument(
"--llm-api-key-env", default="",
help=(
@@ -789,7 +819,7 @@ def parse_args(argv: list[str]) -> argparse.Namespace:
# environment. A typo must not silently select a different routing mode
# and overwrite working runtime configs. Validate effective values so an
# explicit valid CLI override can still repair a bad environment default.
- for action in (switchboard_mode, remote_enabled, remote_transport):
+ for action in (switchboard_mode, remote_enabled, remote_transport, llm_backend):
value = getattr(args, action.dest)
if value not in action.choices:
parser.error(f"{action.option_strings[0]}: invalid choice {value!r}; choose from {action.choices}")
@@ -883,6 +913,7 @@ def render(args: argparse.Namespace) -> dict[str, object]:
remote_llm_model=args.remote_llm_model,
external_llm_authenticated=args.external_llm_authenticated,
llm_api_key_env=getattr(args, "llm_api_key_env", ""),
+ llm_backend=model_server(getattr(args, "llm_backend", "llama-server")),
)
validate_render_inputs(inputs)
if args.surface == "litellm-switchboard" and inputs.ods_mode == "cloud":
diff --git a/ods/tests/test-render-runtime-configs.py b/ods/tests/test-render-runtime-configs.py
index de1be54f1..60a864897 100644
--- a/ods/tests/test-render-runtime-configs.py
+++ b/ods/tests/test-render-runtime-configs.py
@@ -27,10 +27,11 @@ def load_renderer_module() -> ModuleType:
return module
-def run_renderer(*args: str) -> dict[str, object]:
+def run_renderer(*args: str, env: dict[str, str] | None = None) -> dict[str, object]:
proc = subprocess.run(
[sys.executable, str(SCRIPT), *args],
cwd=ROOT,
+ env=env,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
@@ -39,6 +40,28 @@ def run_renderer(*args: str) -> dict[str, object]:
return json.loads(proc.stdout)
+def llm_backend_env(value: str | None = None) -> dict[str, str]:
+ """This environment with LLM_BACKEND unset, or set to ``value``."""
+ env = {key: item for key, item in os.environ.items() if key != "LLM_BACKEND"}
+ if value is not None:
+ env["LLM_BACKEND"] = value
+ return env
+
+
+# Byte for byte what the renderer wrote before LLM_BACKEND became an input.
+LLAMA_SERVER_ENDPOINTS = (
+ '{\n "endpoints": [\n {\n "id": "llama-server-default",\n'
+ ' "baseUrl": "http://llama-server:8080"\n }\n ]\n}\n'
+)
+LLAMA_SERVER_ENV = (
+ "ODS_MODE=local\nODS_MODEL_SWITCHBOARD=enabled\nLLM_BACKEND=llama-server\n"
+ "LLM_MODEL=qwen3.5-9b\nGGUF_FILE=Qwen3.5-9B-Q4_K_M.gguf\nGPU_BACKEND=nvidia\n"
+ "LLM_API_URL=http://llama-server:8080/v1\nCTX_SIZE=131072\nMAX_CONTEXT=131072\n"
+ "OPEN_WEBUI_LLM_BASE_URL=http://litellm:4000\nOPEN_WEBUI_LLM_API_KEY=sk-test-litellm\n"
+)
+ALL_ENABLED = ("--surface", "all", "--switchboard-mode", "enabled", "--litellm-key", "sk-test-litellm")
+
+
def file_by_surface(payload: dict[str, object], surface: str) -> dict[str, str]:
for item in payload["files"]:
if item["surface"] == surface:
@@ -491,6 +514,47 @@ def test_router_endpoints_strip_trailing_v1() -> None:
assert content == {"endpoints": [{"id": "llama-server-default", "baseUrl": "http://llama-server:8080"}]}
+def test_llama_server_backends_render_todays_bytes() -> None:
+ baseline = run_renderer(*ALL_ENABLED, env=llm_backend_env())
+ assert baseline["inputs"]["llm_backend"] == "llama-server"
+ assert file_by_surface(baseline, "model-router-endpoints")["content"] == LLAMA_SERVER_ENDPOINTS
+ assert file_by_surface(baseline, "env")["content"] == LLAMA_SERVER_ENV
+ # Every other .env LLM_BACKEND value, by flag or environment, renders
+ # every surface exactly as before.
+ for value in ("", "llama-server", "external", "hipfire", "lemonade", "litellm"):
+ assert run_renderer(*ALL_ENABLED, env=llm_backend_env(value))["files"] == baseline["files"], value
+ if value:
+ flagged = run_renderer(*ALL_ENABLED, "--llm-backend", value, env=llm_backend_env())
+ assert flagged["files"] == baseline["files"], value
+
+
+def test_vllm_backend_routes_model_router_to_vllm_server() -> None:
+ surface = ("--surface", "model-router-endpoints")
+ for payload in (
+ run_renderer(*surface, "--llm-backend", "vllm", env=llm_backend_env()),
+ run_renderer(*surface, env=llm_backend_env("vllm")),
+ # The llama-server origin and key inputs do not describe vllm-server.
+ run_renderer(*surface, "--llm-backend", "vllm",
+ "--llm-base-url", "http://host.docker.internal:18080/v1",
+ "--llm-api-key-env", "LLAMA_SERVER_API_KEY", env=llm_backend_env()),
+ ):
+ assert payload["inputs"]["llm_backend"] == "vllm"
+ content = file_by_surface(payload, "model-router-endpoints")["content"]
+ assert json.loads(content) == {
+ "endpoints": [{"id": "vllm-default", "baseUrl": "http://vllm-server:8000"}],
+ }
+
+
+def test_vllm_backend_changes_only_the_router_endpoint_and_env_backend() -> None:
+ baseline = run_renderer(*ALL_ENABLED, env=llm_backend_env())
+ vllm = run_renderer(*ALL_ENABLED, "--llm-backend", "vllm", env=llm_backend_env())
+ assert file_by_surface(vllm, "env")["content"] == LLAMA_SERVER_ENV.replace(
+ "LLM_BACKEND=llama-server\n", "LLM_BACKEND=vllm\n")
+ changed = {after["surface"] for after, before in zip(vllm["files"], baseline["files"])
+ if after != before}
+ assert changed == {"env", "model-router-endpoints"}
+
+
def test_retired_lemonade_flags_are_accepted_and_ignored() -> None:
payload = run_renderer(
"--surface",
@@ -868,6 +932,9 @@ def main() -> int:
test_enabled_hermes_uses_stable_switchboard_alias,
test_enabled_opencode_uses_stable_switchboard_alias,
test_router_endpoints_strip_trailing_v1,
+ test_llama_server_backends_render_todays_bytes,
+ test_vllm_backend_routes_model_router_to_vllm_server,
+ test_vllm_backend_changes_only_the_router_endpoint_and_env_backend,
test_retired_lemonade_flags_are_accepted_and_ignored,
test_litellm_lemonade_surface_is_retired,
test_amd_local_env_has_no_lemonade_keys,
diff --git a/ods/tests/test-switchboard-install-proof.py b/ods/tests/test-switchboard-install-proof.py
index 82e5fa01c..18c2abe14 100644
--- a/ods/tests/test-switchboard-install-proof.py
+++ b/ods/tests/test-switchboard-install-proof.py
@@ -1,10 +1,15 @@
"""Exercise host route scheduling without starting a real host service."""
import ast
from pathlib import Path
+import sys
+import tempfile
import threading
import unittest
from unittest.mock import Mock
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'bin'))
+from model_switchboard import state as switchboard_state # noqa: E402
+
class InstallProof(unittest.TestCase):
def test_idle_schedules_proof_but_active_lifecycle_cancels_it(self):
@@ -82,6 +87,59 @@ class InstallProof(unittest.TestCase):
self.assertFalse(ns['_publish_verified_initial_switchboard_route'](reason='startup'))
state.record_verified_route.assert_not_called()
+ def test_vllm_install_route_reconstructs_then_publishes_through_the_state_module(self):
+ tree = ast.parse((Path(__file__).parents[1] / 'bin/ods-host-agent.py').read_text(encoding='utf-8'))
+ names = {'_prepare_initial_switchboard_verification', '_publish_verified_initial_switchboard_route'}
+ module = ast.Module(body=[node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name in names], type_ignores=[])
+ env = {'LLM_BACKEND': 'vllm', 'VLLM_SERVED_MODEL': 'GLM-5.3-Flash-NVFP4',
+ 'LLM_MODEL': 'glm-5.3-flash-nvfp4', 'MAX_CONTEXT': '131072'}
+ vllm_backend = {'kind': 'vllm', 'endpointId': 'vllm-default', 'nativeRoute': None}
+ with tempfile.TemporaryDirectory() as tmp:
+ state_path = Path(tmp) / 'model-state.json'
+ # Startup reconstruction from .env, as the host agent runs it.
+ reconstructed = switchboard_state.initialize_if_missing(state_path, env)
+ self.assertEqual(reconstructed['active']['backend'], vllm_backend)
+ self.assertIs(reconstructed['active']['reconstructed'], True)
+ ns = dict(
+ _switchboard_state=switchboard_state,
+ _switchboard_initial_verify_cancel=threading.Event(),
+ _model_lifecycle_state_lock=threading.Lock(),
+ _model_lifecycle_operation=None,
+ _pixel_model_transition_pending_for_route_proof=lambda: False,
+ _switchboard_state_path=lambda: state_path,
+ _switchboard_state_needs_current_env_verification=lambda path, env=None: True,
+ # What the host agent serves without a GGUF_FILE: the identity.
+ _current_runtime_model_inputs=lambda env, identity: (
+ identity['runtimeModelId'], identity['catalogId']),
+ _catalog_model_for_current_env=lambda env: ('glm-5.3-flash-nvfp4', {'tools': True}),
+ # Stubbed: the route the host agent publishes for LLM_BACKEND=vllm.
+ _initial_switchboard_backend=lambda env: (
+ 'vllm', switchboard_state.default_endpoint_id('vllm'), None),
+ _wait_for_model_readiness=lambda *args, **kwargs: {
+ 'identity': 'GLM-5.3-Flash-NVFP4', 'contextLength': 131072},
+ _initial_switchboard_route_env_matches=lambda env: True,
+ _runtime_model_identity_matches=lambda *args, **kwargs: True,
+ _model_agent_viable=lambda model, context: True,
+ _model_profiles_mode=lambda env: 'observe',
+ _SWITCHBOARD_ROUTE_ENV_KEYS=(),
+ load_env=lambda path: dict(env),
+ INSTALL_DIR=Path(tmp),
+ logger=Mock(),
+ )
+ exec(compile(module, '<host proof functions>', 'exec'), ns)
+ self.assertTrue(ns['_publish_verified_initial_switchboard_route'](reason='install'))
+ doc, errors = switchboard_state.read_state(state_path)
+ self.assertEqual(errors, [])
+ active = doc['active']
+ self.assertEqual(active['backend'], vllm_backend)
+ self.assertEqual(active['catalogId'], 'glm-5.3-flash-nvfp4')
+ self.assertEqual(active['runtimeModelId'], 'GLM-5.3-Flash-NVFP4')
+ self.assertEqual(active['contextLength'], 131072)
+ self.assertEqual(active['proof'], {'identity': 'GLM-5.3-Flash-NVFP4', 'completion': True})
+ self.assertNotIn('reconstructed', active)
+ # Proving the reconstructed route keeps it: no new route, no history.
+ self.assertEqual((doc['seq'], doc['routeSeq'], doc['history']), (2, 1, []))
+
if __name__ == '__main__':
unittest.main()
diff --git a/ods/tests/test_renderer_environment_modes.py b/ods/tests/test_renderer_environment_modes.py
index 00a57f3bc..8c4dc93a8 100644
--- a/ods/tests/test_renderer_environment_modes.py
+++ b/ods/tests/test_renderer_environment_modes.py
@@ -12,6 +12,7 @@ import pytest
("ODS_MODEL_SWITCHBOARD", "--switchboard-mode", "enabledd", "enabled"),
("REMOTE_LLM_ENABLED", "--remote-llm-enabled", "tru", "false"),
("REMOTE_LLM_TRANSPORT", "--remote-llm-transport", "sshh", ""),
+ ("LLM_BACKEND", "--llm-backend", "vlm", "llama-server"),
])
def test_invalid_environment_mode_cannot_write_configs(tmp_path, key, option, bad, valid):
script = Path(__file__).resolve().parents[1] / "scripts/render-runtime-configs.py"