Skip to content

armgit / ODS

Public Imported from GitHub
0

ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.

13.2 KB
Raw Blame
"""The host agent on a vLLM install (LLM_BACKEND=vllm, docker-compose.vllm.yml).

vllm-server serves the one model on the ODS network without a host port, so
the agent proves it through the owned model-router container against the
router's vllm-default endpoint: /health is an empty 200, /v1/models lists the
served id with max_model_len (vLLM has no /props), and the completion names
that exact id. A cold start reads as loading while Docker reports the
container starting. Model switching, profile checks and every path that
would name llama-server to Compose refuse with a clear message instead.
No Docker, network or real install is used.
"""

from __future__ import annotations

import importlib.util
import io
import json
import shutil
import subprocess
import sys
from pathlib import Path

import pytest

ROOT = Path(__file__).resolve().parents[1]
_spec = importlib.util.spec_from_file_location("ods_host_agent_vllm_test", ROOT / "bin" / "ods-host-agent.py")
agent = importlib.util.module_from_spec(_spec)
sys.modules["ods_host_agent_vllm_test"] = agent
_spec.loader.exec_module(agent)

SERVED = "GLM-5.3-Flash-NVFP4"
ENV = {
    "ODS_MODE": "local", "GPU_BACKEND": "nvidia", "LLM_BACKEND": "vllm",
    "LLM_MODEL": "glm-5.3-flash-nvfp4", "VLLM_SERVED_MODEL": SERVED,
    "VLLM_MODEL_DIR": "nvidia--GLM-5.3-Flash-NVFP4@da920bb0",
    "GGUF_FILE": "", "CTX_SIZE": "131072", "MAX_CONTEXT": "131072",
}
# GLM-5.3's template has no enable_thinking; its catalog "off" is this.
GLM_OFF = {"reasoning_effort": "low"}


def models(model=SERVED, context=131072):
    return json.dumps({"object": "list", "data": [{
        "id": model, "object": "model", "owned_by": "vllm", "root": "/model", "max_model_len": context}]})


def completion(model=SERVED, content="READY"):
    return json.dumps({"model": model, "choices": [{"message": {"role": "assistant", "content": content},
                                                    "finish_reason": "stop"}]})


class FakeVllm:
    """vllm-server as the router transport reaches it."""

    def __init__(self, *, health=200, listed=None, answer=None, container="healthy"):
        self.health, self.listed, self.answer, self.container = health, listed, answer, container
        self.calls = []

    def request(self, install_dir, origin, path, **kwargs):
        self.calls.append((origin, path, kwargs))
        if path == "/v1/models":
            return self.listed or models()
        if path == "/v1/chat/completions":
            return self.answer or completion()
        # vLLM has no /props: the transport fails an HTTP 404.
        raise OSError(f"Docker runtime transport failed (exit 1): runtime returned HTTP 404 for {path}")

    def exchange(self, install_dir, origin, path, **kwargs):
        self.calls.append((origin, path, kwargs))
        if isinstance(self.health, Exception):
            raise self.health
        return self.health, ""


@pytest.fixture()
def vllm(monkeypatch, tmp_path):
    fake = FakeVllm()
    monkeypatch.setattr(agent, "_router_transport_request", fake.request)
    monkeypatch.setattr(agent, "_router_transport_exchange", fake.exchange)
    monkeypatch.setattr(agent, "_container_health", lambda container: fake.container)
    monkeypatch.setattr(agent, "INSTALL_DIR", tmp_path)
    monkeypatch.delenv("ODS_HOST_INSTALL_DIR", raising=False)
    (tmp_path / "config").mkdir()
    shutil.copyfile(ROOT / "config" / "model-library.json", tmp_path / "config" / "model-library.json")
    return fake


def prove(env=ENV, **kwargs):
    return agent._wait_for_model_readiness(
        env, model_id="glm-5.3-flash-nvfp4-vllm", gguf_file=SERVED, llm_model_name=env["LLM_MODEL"],
        attempts=kwargs.pop("attempts", 1), initial_delay=0, interval=0, return_proof=True, **kwargs)


def test_route_backend_and_endpoint_follow_llm_backend():
    assert agent._initial_switchboard_backend(ENV) == ("vllm", "vllm-default", None)
    assert agent._switchboard_state.default_endpoint_id("vllm") == "vllm-default"
    for backend in ("", "llama-server", "external", "lemonade"):
        assert agent._initial_switchboard_backend({**ENV, "LLM_BACKEND": backend}) == (
            "llama-server", "llama-server-default", None), backend


def test_vllm_is_proven_through_the_routers_vllm_endpoint(vllm, monkeypatch):
    monkeypatch.setattr(agent.subprocess, "run", lambda *a, **k: pytest.fail("no host port to probe"))
    proof = prove()

    assert proof["identity"] == SERVED
    assert proof["contextLength"] == 131072 and proof["contextVerified"] is True
    assert "vision" not in proof
    assert [call[1] for call in vllm.calls] == ["/health", "/v1/models", "/v1/chat/completions"]
    assert all(call[0] == "http://vllm-server:8000" for call in vllm.calls)
    assert all(call[2]["endpoint_id"] == "vllm-default" and call[2]["api_key"] == "" for call in vllm.calls)
    payload = vllm.calls[-1][2]["payload"]
    # vLLM answers 404 for any name but the served one.
    assert payload["model"] == SERVED
    assert payload["chat_template_kwargs"] == GLM_OFF


def test_qwen_turns_thinking_off_with_its_own_template_argument(vllm):
    env = {**ENV, "LLM_MODEL": "qwen3.8-flash-next-nvfp4", "VLLM_SERVED_MODEL": "Qwen3.8-Flash-Next-NVFP4"}
    vllm.listed = models("Qwen3.8-Flash-Next-NVFP4")
    vllm.answer = completion("Qwen3.8-Flash-Next-NVFP4")
    assert prove(env)["identity"] == "Qwen3.8-Flash-Next-NVFP4"
    assert vllm.calls[-1][2]["payload"]["chat_template_kwargs"] == {"enable_thinking": False}


def test_a_short_or_missing_served_context_never_proves_the_route(vllm):
    for listed in (models(context=32768), json.dumps({"data": [{"id": SERVED}]}), models("Other-Model")):
        vllm.listed, vllm.calls = listed, []
        diagnosis = {}
        assert prove(diagnosis=diagnosis) == {}
        assert "/v1/chat/completions" not in [call[1] for call in vllm.calls]


def test_cold_start_reads_as_loading_while_docker_reports_starting(vllm):
    vllm.health = OSError("Docker runtime transport failed (exit 1): runtime transport failed: URLError")
    vllm.container = "starting"
    assert agent._runtime_health(ENV) == "loading"
    diagnosis = {}
    assert prove(diagnosis=diagnosis) == {}
    assert diagnosis["reason"].startswith("vLLM is still starting")
    assert "final" not in diagnosis

    vllm.health = 503
    assert agent._runtime_health(ENV) == "loading"
    # Past its start period Docker says unhealthy: an error, not loading.
    vllm.container = "unhealthy"
    assert agent._runtime_health(ENV) == "error"
    vllm.health = OSError("connection refused")
    with pytest.raises(OSError):
        agent._runtime_health(ENV)
    vllm.health, vllm.container = 200, "healthy"
    assert agent._runtime_health(ENV) == "ok"


def test_llama_server_health_is_unchanged(monkeypatch):
    monkeypatch.setattr(agent, "_runtime_http", lambda env, path, **kwargs: "")
    monkeypatch.setattr(agent, "_router_transport_exchange", lambda *a, **k: pytest.fail("vLLM probe"))
    assert agent._runtime_health({"LLM_BACKEND": "llama-server"}) == "error"


def test_catalog_entry_comes_from_llm_model_not_a_leftover_gguf(vllm):
    leftover = {**ENV, "GGUF_FILE": "Qwen3.5-9B-Q4_K_M.gguf"}
    model_id, model = agent._catalog_model_for_current_env(leftover)
    assert model_id == "glm-5.3-flash-nvfp4-vllm" and model["runtime"] == "vllm"
    # A llama-server route never resolves to a vLLM entry.
    llama = {"LLM_BACKEND": "llama-server", "LLM_MODEL": "glm-5.3-flash-nvfp4", "GGUF_FILE": ""}
    assert agent._catalog_model_for_current_env(llama) == ("glm-5.3-flash-nvfp4", {})
    assert agent._vllm_probe_thinking_off(ENV) == GLM_OFF
    assert agent._vllm_probe_thinking_off(llama) is None


def test_compose_resolution_reads_llm_backend_from_the_install(monkeypatch, tmp_path):
    assert "LLM_BACKEND" in agent._HOST_LLM_COMPOSE_SELECTORS
    assert "LLM_BACKEND" in agent._PREPARE_COMPOSE_SELECTORS
    (tmp_path / "scripts").mkdir()
    (tmp_path / "scripts" / "resolve-compose-stack.sh").write_text("", encoding="utf-8")
    (tmp_path / ".env").write_text("LLM_BACKEND=vllm\nODS_MODE=local\n", encoding="utf-8")
    monkeypatch.setattr(agent, "INSTALL_DIR", tmp_path)
    monkeypatch.setattr(agent, "_find_usable_bash", lambda: "bash")
    monkeypatch.setattr(agent.platform, "system", lambda: "Linux")
    # A stale value exported when the agent started must not win.
    monkeypatch.setenv("LLM_BACKEND", "llama-server")
    seen = {}

    def run(cmd, **kwargs):
        seen.update(kwargs["env"])
        return subprocess.CompletedProcess(cmd, 0, stdout="-f docker-compose.base.yml", stderr="")

    monkeypatch.setattr(agent.subprocess, "run", run)
    agent._run_compose_resolver()
    assert seen["LLM_BACKEND"] == "vllm"


def test_router_configs_render_with_the_installed_backend(monkeypatch, tmp_path):
    (tmp_path / "scripts").mkdir()
    (tmp_path / "scripts" / "render-runtime-configs.py").write_text("", encoding="utf-8")
    commands = []

    def run(cmd, **kwargs):
        commands.append(cmd)
        return subprocess.CompletedProcess(cmd, 0, stdout="", stderr="")

    monkeypatch.setattr(agent.subprocess, "run", run)
    monkeypatch.setenv("LLM_BACKEND", "llama-server")
    agent._render_model_router_runtime_configs(
        tmp_path, {**ENV, "ODS_MODEL_SWITCHBOARD": "enabled"},
        model=ENV["LLM_MODEL"], gguf_file="", context_length=131072)
    assert commands and all(cmd[cmd.index("--llm-backend") + 1] == "vllm" for cmd in commands)
    assert all(cmd[cmd.index("--model") + 1] == "glm-5.3-flash-nvfp4" for cmd in commands)
    commands.clear()
    agent._render_model_router_runtime_configs(
        tmp_path, {"ODS_MODEL_SWITCHBOARD": "enabled", "GGUF_FILE": "m.gguf"},
        model="m", gguf_file="m.gguf", context_length=8192)
    assert all(cmd[cmd.index("--llm-backend") + 1] == "llama-server" for cmd in commands)


class Handler:
    def __init__(self):
        self.wfile = io.BytesIO()
        self.response_code = None

    def send_response(self, code):
        self.response_code = code

    def send_header(self, name, value):
        pass

    def end_headers(self):
        pass

    def body(self):
        return json.loads(self.wfile.getvalue().decode("utf-8"))


def _install(tmp_path, monkeypatch, env_text):
    (tmp_path / ".env").write_text(env_text, encoding="utf-8")
    (tmp_path / "config").mkdir(exist_ok=True)
    shutil.copyfile(ROOT / "config" / "model-library.json", tmp_path / "config" / "model-library.json")
    monkeypatch.setattr(agent, "INSTALL_DIR", tmp_path)
    monkeypatch.setattr(agent, "STARTUP_ODS_MODE", "local")
    monkeypatch.setattr(agent, "_managed_wsl_runtime", lambda env: {"managed": False})
    monkeypatch.setattr(agent, "_recreate_llama_server", lambda *a, **k: pytest.fail("llama-server recreated"))
    monkeypatch.setattr(agent, "_compose_restart_llama_server", lambda *a, **k: pytest.fail("llama-server started"))


def test_a_vllm_install_refuses_model_switches_before_any_change(monkeypatch, tmp_path):
    _install(tmp_path, monkeypatch, "ODS_MODE=local\nLLM_BACKEND=vllm\nLLM_MODEL=glm-5.3-flash-nvfp4\n")
    monkeypatch.setattr(agent, "_load_model_library_records", lambda: pytest.fail("refused before lookup"))
    before = (tmp_path / ".env").read_bytes()
    handler = Handler()
    agent.AgentHandler._do_model_activate(handler, "qwen3.5-9b-q4")
    assert handler.response_code == 409
    body = handler.body()
    assert body["code"] == "vllm_runtime_installer_managed"
    assert "vLLM engine" in body["error"] and "installer" in body["hint"]
    assert "discord.gg" in body["hint"]
    assert (tmp_path / ".env").read_bytes() == before


def test_a_llama_install_refuses_a_vllm_catalog_model(monkeypatch, tmp_path):
    _install(tmp_path, monkeypatch, "ODS_MODE=local\nLLM_BACKEND=llama-server\nGGUF_FILE=a.gguf\n")
    handler = Handler()
    agent.AgentHandler._do_model_activate(handler, "glm-5.3-flash-nvfp4-vllm")
    assert handler.response_code == 409
    body = handler.body()
    assert body["code"] == "vllm_model_installer_only"
    assert body["requestedModelId"] == "glm-5.3-flash-nvfp4-vllm"
    assert body["error"].startswith("GLM 5.3 Flash NVFP4 (vLLM) runs only on the vLLM engine")
    # The entry's own GPU requirement, not one wording for every vLLM model.
    assert body["hint"].startswith("It needs one GPU with compute capability 10.0 or 10.3 and at least 244736 MiB.")


def test_the_refusal_names_each_entrys_own_gpu_requirement():
    rtx = next(model for model in json.loads((ROOT / "config" / "model-library.json").read_text(encoding="utf-8"))["models"]
               if model["id"] == "qwen3.6-35b-a3b-nvfp4-vllm")
    assert agent._vllm_model_refusal(rtx)["hint"].startswith(
        "It needs one GPU with compute capability 12.0 and at least 51200 MiB.")
    broken = {**rtx, "min_gpu_mib": 0}
    assert agent._vllm_model_refusal(broken)["hint"].startswith("It needs an NVIDIA GPU the vLLM engine supports.")


def test_llama_server_is_never_named_to_compose_on_a_vllm_install(monkeypatch, tmp_path):
    (tmp_path / ".env").write_text("LLM_BACKEND=vllm\n", encoding="utf-8")
    monkeypatch.setattr(agent, "INSTALL_DIR", tmp_path)
    monkeypatch.setattr(agent, "CORE_SERVICE_IDS", {"llama-server", "litellm", "vllm-server"})
    monkeypatch.setattr(agent.subprocess, "run", lambda *a, **k: pytest.fail("compose ran"))
    ok, error = agent.docker_compose_recreate(["litellm", "llama-server"])
    assert not ok and "vllm-server" in error
    assert "vllm-server" in agent.ALWAYS_ON_SERVICES