Skip to content

armgit / ODS

Public Imported from GitHub
0

ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.

18.8 KB
Raw Blame
"""scripts/select-model.py --llm-runtime vllm, and proof that GGUF picks are unchanged.

vLLM catalog entries (``"runtime": "vllm"``) are eligible only when the
selector is told the runtime is vllm and a GPU named by --gpu has one of the
entry's compute capabilities and at least its min_gpu_mib. Without
--llm-runtime vllm the selector's output over every installer envelope
(tests/fixtures/model-selection-envelopes.json) is byte-identical to the
output for the same catalog without its vLLM entries.
"""

from __future__ import annotations

import contextlib
import functools
import importlib.util
import io
import json
import re
import shlex
import subprocess
import sys
from pathlib import Path

import pytest


ROOT = Path(__file__).resolve().parents[1]
CATALOG = ROOT / "config" / "model-library.json"
SELECTOR = ROOT / "scripts" / "select-model.py"
ENVELOPES = ROOT / "tests" / "fixtures" / "model-selection-envelopes.json"
TIER_MAP = ROOT / "installers" / "lib" / "tier-map.sh"
SIMULATOR = ROOT / "scripts" / "simulate-model-selection.py"

# nvidia-smi --query-gpu=uuid,compute_cap,memory.total --format=csv,noheader,nounits rows.
# GB300 and RTX PRO 6000 memory: the DGX Station Workstation06 (2026-10-27).
GB300 = "GPU-test-gb300, 10.3, 256703"
RTX_PRO_6000 = "GPU-test-rtx-pro-6000, 12.0, 97887"
RTX_5090 = "GPU-test-rtx-5090, 12.0, 32607"
B200 = "GPU-test-b200, 10.0, 183359"
H200 = "GPU-test-h200, 9.0, 143771"
# The installer's selector arguments on the DGX Station, plus the vLLM runtime.
VLLM = ("--backend", "nvidia", "--memory-type", "discrete", "--vram-mb", "354590", "--ram-gb", "494",
        "--profile", "qwen", "--tier", "NV_ULTRA", "--max-size-mb", "0", "--host-arch", "arm64",
        "--installable-only", "--min-context", "65536", "--llm-runtime", "vllm")
VLLM_ENV_KEYS = {
    "LLM_MODEL", "GGUF_FILE", "GGUF_URL", "GGUF_SHA256", "MAX_CONTEXT", "LLM_MODEL_SIZE_MB",
    "MODEL_RECOMMENDATION_SOURCE", "MODEL_RECOMMENDATION_POLICY", "MODEL_RECOMMENDATION_CONFIDENCE",
    "MODEL_RECOMMENDATION_REASON", "MODEL_RECOMMENDED_ALTERNATIVES", "PIXEL_AGENT_MODEL_READY",
    "VLLM_MODEL_DIR", "VLLM_SERVED_MODEL", "VLLM_ARGS", "VLLM_GPU_UUID", "VLLM_GPU_MEMORY_UTILIZATION",
}


def _load(name: str, path: Path):
    spec = importlib.util.spec_from_file_location(name, path)
    module = importlib.util.module_from_spec(spec)
    sys.modules[spec.name] = module
    spec.loader.exec_module(module)
    return module


@functools.lru_cache(maxsize=1)
def _selector():
    return _load("ods_select_model_vllm_test", SELECTOR)


def select(*args: str, catalog: Path = CATALOG) -> tuple[int, str, str]:
    out, err = io.StringIO(), io.StringIO()
    with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
        status = _selector().main(["--catalog", str(catalog), *args])
    return status, out.getvalue(), err.getvalue()


def env_values(stdout: str) -> dict[str, str]:
    """Decode --env lines the way lib/safe-env.sh load_env_from_output does."""
    values = {}
    for line in stdout.splitlines():
        match = re.fullmatch(r'([A-Za-z_][A-Za-z0-9_]*)="(.*)"', line)
        assert match, line
        values[match.group(1)] = (
            match.group(2).replace('\\"', '"').replace("\\$", "$").replace("\\`", "`").replace("\\\\", "\\")
        )
    return values


def _catalog() -> dict:
    return json.loads(CATALOG.read_text(encoding="utf-8"))


def _vllm_entries() -> list[dict]:
    return [model for model in _catalog()["models"] if model.get("runtime") == "vllm"]


def _fits(capability: str, memory_mib: int) -> list[dict]:
    return [model for model in _vllm_entries()
            if capability in model["compute_capability"] and model["min_gpu_mib"] <= memory_mib]


def _best_fitting(capability: str, memory_mib: int) -> tuple[dict, list[dict]]:
    fitting = _fits(capability, memory_mib)
    assert fitting
    return max(fitting, key=lambda model: model["selection"]["vllm"]), fitting


def _gpu_args(*rows: str) -> list[str]:
    return [arg for row in rows for arg in ("--gpu", row)]


def test_dgx_station_gets_the_top_priority_vllm_model_on_its_gb300():
    status, out, err = select(*VLLM, *_gpu_args(RTX_PRO_6000, GB300), "--env")

    assert status == 0, err
    env = env_values(out)
    best, fitting = _best_fitting("10.3", 256703)
    assert set(env) == VLLM_ENV_KEYS
    assert env["LLM_MODEL"] == best["llm_model_name"]
    assert env["VLLM_GPU_UUID"] == "GPU-test-gb300"
    assert env["VLLM_MODEL_DIR"] == f"{best['hf_repo'].replace('/', '--')}@{best['revision'][:8]}"
    assert env["VLLM_SERVED_MODEL"] == best["served_model_name"]
    assert shlex.split(env["VLLM_ARGS"]) == best["vllm_args"]
    assert env["MAX_CONTEXT"] == str(best["context_length"]) == "131072"
    assert env["LLM_MODEL_SIZE_MB"] == str(best["size_mb"])
    assert env["GGUF_FILE"] == env["GGUF_URL"] == env["GGUF_SHA256"] == ""
    pixel = ((best.get("app_compatibility") or {}).get("pixel_agent") or {}).get("status")
    assert env["PIXEL_AGENT_MODEL_READY"] == ("true" if pixel == "verified" else "false")
    assert env["MODEL_RECOMMENDATION_POLICY"] == "vllm-gpu-fit-v1"
    assert env["MODEL_RECOMMENDATION_SOURCE"] == "catalog_vllm_pre_download"
    # An entry without its own share still sends vLLM's 0.9, so a newly
    # picked model never inherits an earlier model's .env share.
    assert "gpu_memory_utilization" not in best and env["VLLM_GPU_MEMORY_UTILIZATION"] == "0.9"
    assert "GPU-test-gb300 (compute capability 10.3, 256703 MiB)" in env["MODEL_RECOMMENDATION_REASON"]
    # The alternatives are every entry that fits either GPU (the RTX PRO 6000's own one too).
    alternatives = [item.split(":")[0] for item in env["MODEL_RECOMMENDED_ALTERNATIVES"].split(";")]
    either = {model["id"] for model in fitting + _fits("12.0", 97887)}
    assert alternatives[0] == best["id"] and set(alternatives) == either


def test_the_gb300_fits_the_datacenter_entries_and_the_rtx_pro_6000_its_own():
    _, fitting = _best_fitting("10.3", 256703)
    assert {model["id"] for model in fitting} == {"qwen3.8-flash-next-nvfp4-vllm", "glm-5.3-flash-nvfp4-vllm"}
    assert [model["id"] for model in _fits("12.0", 97887)] == ["qwen3.6-35b-a3b-nvfp4-vllm"]
    assert _fits("12.0", 32607) == []


def test_a_lone_rtx_pro_6000_gets_its_class_entry_at_half_the_card():
    # A workstation's only GPU also runs ComfyUI and voice, so this entry
    # takes half of it; the selector hands that share to the installer.
    status, out, err = select(*VLLM, *_gpu_args(RTX_PRO_6000), "--env")

    assert status == 0, err
    env = env_values(out)
    best, _ = _best_fitting("12.0", 97887)
    assert set(env) == VLLM_ENV_KEYS
    assert env["LLM_MODEL"] == best["llm_model_name"] == "qwen3.6-35b-a3b-nvfp4"
    assert env["VLLM_GPU_UUID"] == "GPU-test-rtx-pro-6000"
    assert env["VLLM_GPU_MEMORY_UTILIZATION"] == "0.5" == f"{best['gpu_memory_utilization']:g}"
    assert shlex.split(env["VLLM_ARGS"]) == best["vllm_args"]
    assert "--gpu-memory-utilization" not in env["VLLM_ARGS"]
    reason = env["MODEL_RECOMMENDATION_REASON"]
    # 97887 MiB * 0.5 / 1024 - 21.85 GiB of weights.
    assert "leave about 25.9 GiB of vLLM's 0.50 share of that GPU" in reason, reason


def test_a_b200_gets_only_what_fits_its_memory():
    status, out, err = select(*VLLM, *_gpu_args(B200), "--env")

    assert status == 0, err
    env = env_values(out)
    best, fitting = _best_fitting("10.0", 183359)
    assert env["LLM_MODEL"] == best["llm_model_name"]
    assert "glm-5.3-flash-nvfp4-vllm" not in {model["id"] for model in fitting}
    assert "glm-5.3-flash-nvfp4-vllm" not in env["MODEL_RECOMMENDED_ALTERNATIVES"]


@pytest.mark.parametrize("rows,expected", [
    ((RTX_5090,), "needs one GPU with compute capability 10.0 or 10.3 and at least"),
    ((RTX_5090,), "needs one GPU with compute capability 12.0 and at least 51200 MiB"),
    ((H200,), "GPU-test-h200 (compute capability 9.0, 143771 MiB)"),
    ((), "pass each one as --gpu"),
    (("GPU-test-gb10, 12.1, [N/A]",), "is not UUID,COMPUTE_CAP,MEMORY_MIB"),
])
def test_no_fitting_gpu_fails_instead_of_falling_back(rows, expected):
    status, out, err = select(*VLLM, *_gpu_args(*rows), "--env")

    assert status == 2
    assert out == ""
    assert err.startswith("error: ") and expected in err


def test_the_exit_status_reaches_the_installer():
    result = subprocess.run(
        [sys.executable, str(SELECTOR), "--catalog", str(CATALOG), *VLLM, *_gpu_args(RTX_5090), "--env"],
        capture_output=True, text=True, check=False,
    )
    assert (result.returncode, result.stdout) == (2, "")
    assert "no vLLM catalog model fits this host" in result.stderr


def test_vllm_needs_an_nvidia_backend():
    status, out, err = select("--backend", "amd", "--llm-runtime", "vllm", *_gpu_args(GB300), "--env")
    assert (status, out) == (2, "")
    assert "need NVIDIA GPUs" in err


def test_check_fit_is_refused_for_vllm():
    status, out, err = select(*VLLM, *_gpu_args(GB300), "--check-fit",
                              "--model-id", "glm-5.3-flash-nvfp4-vllm", "--context", "65536")
    assert (status, out) == (1, "")
    assert "--check-fit checks llama.cpp models" in err


def test_context_floor_is_a_preference_unless_required():
    status, out, err = select(*VLLM, *_gpu_args(GB300), "--min-context", "262144", "--env")
    assert status == 0, err
    assert "No vLLM catalog model serves the 262144 context floor" in env_values(out)["MODEL_RECOMMENDATION_REASON"]

    status, out, err = select(*VLLM, *_gpu_args(GB300), "--min-context", "262144", "--require-min-context", "--env")
    assert (status, out) == (2, "")
    assert "at the 262144 context floor" in err


def test_the_largest_fitting_gpu_is_the_one_vllm_owns():
    status, out, err = select(*VLLM, *_gpu_args(GB300, "GPU-test-b300, 10.3, 275040", GB300), "--env")
    assert status == 0, err
    assert env_values(out)["VLLM_GPU_UUID"] == "GPU-test-b300"


# A fresh install adds nvidia-smi memory.free as a fourth field. vLLM refuses
# to start unless its share of the card is free once its CUDA context exists:
# half an RTX PRO 6000 is ceil(0.5 * 97887) = 48944 MiB, plus START_MARGIN_MIB
# (1024) makes 49968 MiB free.
def test_a_gpu_without_room_for_vllm_is_not_given_to_vllm():
    # tower2 (2026-10-27): another model server held about 84 GB of each card.
    rows = ("GPU-test-busy-0, 12.0, 97887, 13278", "GPU-test-busy-1, 12.0, 97887, 13298")
    status, out, err = select(*VLLM, *_gpu_args(*rows), "--env")

    assert (status, out) == (4, "")
    assert err == (
        "error: other programs use the GPU memory the vLLM engine needs: Qwen 3.6 35B A3B NVFP4 (vLLM) "
        "needs 49968 MiB free when it starts (0.50 of the GPU plus 1024 MiB for its CUDA context), "
        "and the freest GPU it fits, GPU-test-busy-1, has 13298 MiB free\n"
    )


def test_vllm_takes_a_gpu_with_room_to_start_and_says_why():
    status, out, err = select(*VLLM, *_gpu_args("GPU-test-busy, 12.0, 97887, 13278",
                                                "GPU-test-exact, 12.0, 97887, 49968"), "--env")
    assert status == 0, err
    env = env_values(out)
    assert env["VLLM_GPU_UUID"] == "GPU-test-exact"
    assert env["MODEL_RECOMMENDATION_REASON"].endswith(
        " Qwen 3.6 35B A3B NVFP4 (vLLM) would run on GPU GPU-test-busy, but other programs use the memory "
        "it needs there.")

    status, _, err = select(*VLLM, *_gpu_args("GPU-test-short, 12.0, 97887, 49967"), "--env")
    assert status == 4 and "has 49967 MiB free" in err


def test_a_busy_gb300_gives_vllm_the_idle_rtx_pro_6000_and_says_why():
    # A DGX Station whose GB300 runs another model server: the RTX PRO 6000's
    # own entry runs instead of the GB300's top pick, and the reason says so.
    best_gb300, _ = _best_fitting("10.3", 256703)
    status, out, err = select(*VLLM, *_gpu_args(RTX_PRO_6000 + ", 97000", GB300 + ", 30000"), "--env")

    assert status == 0, err
    env = env_values(out)
    assert (env["VLLM_GPU_UUID"], env["LLM_MODEL"]) == ("GPU-test-rtx-pro-6000", "qwen3.6-35b-a3b-nvfp4")
    assert (f" {best_gb300['name']} would run on GPU GPU-test-gb300, but other programs use the memory it "
            f"needs there.") in env["MODEL_RECOMMENDATION_REASON"]

    # Idle, the GB300 keeps its top pick and the reason no note.
    status, out, err = select(*VLLM, *_gpu_args(RTX_PRO_6000 + ", 97000", GB300 + ", 255000"), "--env")
    assert status == 0, err
    env = env_values(out)
    assert (env["VLLM_GPU_UUID"], env["LLM_MODEL"]) == ("GPU-test-gb300", best_gb300["llm_model_name"])
    assert "other programs" not in env["MODEL_RECOMMENDATION_REASON"]

    status, out, err = select(*VLLM, *_gpu_args(RTX_PRO_6000 + ", 97000"))
    assert status == 0, err
    assert json.loads(out)["gpu"] == {"uuid": "GPU-test-rtx-pro-6000", "compute_capability": "12.0",
                                      "memory_mib": 97887, "free_mib": 97000}


def test_equal_gpus_go_to_the_one_with_more_memory_free():
    rows = ("GPU-test-desktop, 12.0, 97887, 95000", "GPU-test-idle, 12.0, 97887, 97000")
    status, out, err = select(*VLLM, *_gpu_args(*rows), "--env")
    assert status == 0, err
    assert env_values(out)["VLLM_GPU_UUID"] == "GPU-test-idle"


def test_free_memory_never_lets_a_gpu_fit_that_its_hardware_does_not():
    # A free H200 still lacks a compute capability any entry names: no fit, not busy.
    status, out, err = select(*VLLM, *_gpu_args(H200 + ", 143000"), "--env")
    assert (status, out) == (2, "")
    assert "GPU-test-h200 (compute capability 9.0, 143771 MiB, 143000 MiB free)" in err


def test_an_unreadable_free_field_leaves_that_gpu_unknown():
    # [N/A] on one GPU must not take vLLM from the whole host.
    status, out, err = select(*VLLM, *_gpu_args(RTX_PRO_6000 + ", [N/A]", GB300 + ", 250000"), "--env")
    assert status == 0, err
    assert env_values(out)["VLLM_GPU_UUID"] == "GPU-test-gb300"

    status, out, err = select(*VLLM, *_gpu_args(RTX_PRO_6000 + ", [N/A]"), "--env")
    assert status == 0, err
    assert env_values(out)["VLLM_GPU_UUID"] == "GPU-test-rtx-pro-6000"

    # The other fields stay required.
    status, out, err = select(*VLLM, *_gpu_args("GPU-test-x, 12.0, [N/A], 1000"), "--env")
    assert (status, out) == (2, "")
    assert "is not UUID,COMPUTE_CAP,MEMORY_MIB[,FREE_MIB]" in err


def test_vllms_default_share_is_the_memory_estimates():
    selector = _selector()
    assert selector.memory_share({}) == selector.vllm_memory_share({}) == 0.9
    assert selector.memory_share({"gpu_memory_utilization": 0.5}) \
        == selector.vllm_memory_share({"gpu_memory_utilization": 0.5}) == 0.5


def test_json_payload_names_the_gpu_folder_and_estimate():
    status, out, err = select(*VLLM, *_gpu_args(GB300))

    assert status == 0, err
    payload = json.loads(out)
    best, _ = _best_fitting("10.3", 256703)
    assert payload["runtime"] == "vllm"
    assert payload["gpu"] == {"uuid": "GPU-test-gb300", "compute_capability": "10.3", "memory_mib": 256703}
    assert payload["selected"]["id"] == best["id"]
    assert payload["selected"]["file_count"] == len(best["files"])
    assert "files" not in payload["selected"]
    assert payload["selected"]["estimate"]["method"] == "vllm"


def _write_catalog(tmp_path: Path, catalog: dict) -> Path:
    path = tmp_path / "model-library.json"
    path.write_text(json.dumps(catalog), encoding="utf-8")
    return path


def test_selection_priority_decides_between_fitting_entries(tmp_path):
    catalog = _catalog()
    for model in catalog["models"]:
        if model.get("runtime") == "vllm":
            model["selection"]["vllm"] = 100 if model["id"].startswith("glm") else 90
    path = _write_catalog(tmp_path, catalog)

    status, out, err = select(*VLLM, *_gpu_args(GB300), "--env", catalog=path)
    assert status == 0, err
    assert env_values(out)["LLM_MODEL"] == "glm-5.3-flash-nvfp4"
    status, out, err = select(*VLLM, *_gpu_args(B200), "--env", catalog=path)
    assert env_values(out)["LLM_MODEL"] == "qwen3.8-flash-next-nvfp4"

    for model in catalog["models"]:
        if model.get("runtime") == "vllm":
            model["selection"]["vllm"] = 0
    path = _write_catalog(tmp_path, catalog)
    assert select(*VLLM, *_gpu_args(GB300), "--env", catalog=path)[:2] == (2, "")


def test_install_recommendation_false_keeps_an_entry_out_of_installer_picks(tmp_path):
    catalog = _catalog()
    for model in catalog["models"]:
        if model.get("runtime") == "vllm":
            model["install_recommendation"] = False
    path = _write_catalog(tmp_path, catalog)
    assert select(*VLLM, *_gpu_args(GB300), "--env", catalog=path)[:2] == (2, "")


def test_llama_server_selection_never_returns_a_vllm_entry_on_a_gb300_sized_gpu():
    status, out, err = select("--backend", "nvidia", "--memory-type", "discrete", "--vram-mb", "256703",
                              "--ram-gb", "494", "--tier", "NV_ULTRA", "--host-arch", "arm64",
                              "--installable-only", "--min-context", "65536", *_gpu_args(GB300), "--env")
    assert status == 0, err
    env = env_values(out)
    assert env["GGUF_FILE"].endswith(".gguf")
    assert not any(key.startswith("VLLM_") for key in env)
    assert env["LLM_MODEL"] not in {model["llm_model_name"] for model in _vllm_entries()}


def _envelope_args(envelope: dict, simulator) -> list[str]:
    """The arguments scripts/simulate-model-selection.py passes, as the installers do."""
    ceiling = envelope.get("ceiling")
    ceiling = (simulator.tier_map_size_mb(TIER_MAP, str(envelope["tier"]), str(envelope["host_arch"]))
               if ceiling == "tier-map" else int(ceiling or 0))
    return [
        "--backend", envelope["backend"], "--memory-type", envelope["memory_type"],
        "--vram-mb", str(envelope["vram_mb"]), "--ram-gb", str(envelope["ram_gb"]),
        "--profile", "qwen", "--tier", str(envelope["tier"]), "--max-size-mb", str(ceiling),
        "--host-arch", envelope["host_arch"], "--installable-only", "--min-context", "65536",
    ]


def test_gguf_selection_is_byte_identical_with_or_without_vllm_entries(tmp_path):
    catalog = _catalog()
    assert len(_vllm_entries()) == 3
    without = _write_catalog(tmp_path, {
        **catalog, "models": [model for model in catalog["models"] if model.get("runtime") != "vllm"],
    })
    simulator = _load("ods_simulate_model_selection_vllm_test", SIMULATOR)
    envelopes = json.loads(ENVELOPES.read_text(encoding="utf-8"))["envelopes"]
    compared = picked = 0
    for envelope in envelopes:
        args = _envelope_args(envelope, simulator)
        for extra in ([], ["--env"], ["--agent-ready-only", "--env"]):
            today = select(*args, *extra, catalog=without)
            case = (envelope["id"], extra)
            assert select(*args, *extra) == today, case
            assert select(*args, *extra, "--llm-runtime", "llama-server", *_gpu_args(GB300)) == today, case
            compared += 1
            picked += today[0] == 0 and bool(today[1])
    assert compared == 3 * len(envelopes) == 195
    # Not vacuous: every envelope picks a model (the Pixel-verdict route some).
    assert picked >= 2 * len(envelopes)


if __name__ == "__main__":
    raise SystemExit(pytest.main([__file__, "-q"]))