ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.
Python ai-agentsamdcomfyuidockerllama-cppllmlocal-ain8nnvidiaopen-webuiragself-hostedspeech-to-textstrix-halotext-to-speechworkflow-automation
feat(vllm): run the RTX PRO 6000 class on vLLM, at half the GPU (#8303)
Mike Bradley07e551621def
diff --git a/ods/.env.schema.json b/ods/.env.schema.jsonindex c1e071424..b72357dc1 100644--- a/ods/.env.schema.json+++ b/ods/.env.schema.json@@ -170,7 +170,7 @@},"LLM_BACKEND": {"type": "string",- "description": "Inference backend. Current installers write llama-server (every local GPU, AMD included), external, or vllm: the vLLM engine (docker-compose.vllm.yml) instead of llama-server, which the Linux installer chooses when a vLLM catalog model fits one NVIDIA datacenter GPU (install.sh --llm-backend vllm|llama-server overrides the choice). Legacy values such as lemonade and litellm remain accepted for upgrade compatibility.",+ "description": "Inference backend. Current installers write llama-server (every local GPU, AMD included), external, or vllm: the vLLM engine (docker-compose.vllm.yml) instead of llama-server, which the Linux installer chooses when a vLLM catalog model fits one NVIDIA GPU, datacenter Blackwell or the RTX PRO 6000 class (install.sh --llm-backend vllm|llama-server overrides the choice). Legacy values such as lemonade and litellm remain accepted for upgrade compatibility.","default": "llama-server"},"VLLM_IMAGE": {@@ -191,11 +191,11 @@},"VLLM_GPU_UUID": {"type": "string",- "description": "UUID of the one NVIDIA GPU the vLLM engine owns, picked at install (the largest compute capability 10.x device). Required when LLM_BACKEND=vllm."+ "description": "UUID of the one NVIDIA GPU the vLLM engine owns, picked at install (the largest GPU the chosen catalog model fits). Required when LLM_BACKEND=vllm."},"VLLM_GPU_MEMORY_UTILIZATION": {"type": "number",- "description": "Share of the vLLM GPU's memory that vLLM pre-allocates for weights and KV cache (--gpu-memory-utilization). Lower it to leave room for other GPU services on the same GPU.",+ "description": "Share of the vLLM GPU's memory that vLLM pre-allocates for weights and KV cache (--gpu-memory-utilization). The installer writes the catalog model's own share when it sets one (0.5 for the RTX PRO 6000 class, so other GPU services fit beside it); without one, 0.9. Lower it to leave room for other GPU services on the same GPU.","minimum": 0.01,"maximum": 1,"default": 0.9diff --git a/ods/bin/ods-host-agent.py b/ods/bin/ods-host-agent.pyindex 131dd8f8f..0b5130c6c 100755--- a/ods/bin/ods-host-agent.py+++ b/ods/bin/ods-host-agent.py@@ -809,11 +809,14 @@ def _vllm_activation_refusal(model_id: str) -> dict:def _vllm_model_refusal(model: dict) -> dict:"""409 body for a vLLM catalog entry asked of llama-server's model lifecycle."""+ # The entry's own GPU requirement; a malformed record names none.+ needs = ("an NVIDIA GPU the vLLM engine supports" if _model_vllm.entry_problems(model)+ else _model_vllm.requirement_text(model))return {"error": f"{model.get('name') or model.get('id')} runs only on the vLLM engine, ""which only the ODS installer sets up.","code": "vllm_model_installer_only",- "hint": ("It needs one NVIDIA datacenter Blackwell GPU. Run the ODS installer and "+ "hint": (f"It needs {needs}. Run the ODS installer and "f"choose the vLLM engine to use it. {_ODS_HELP}"),"requestedModelId": model.get("id"),}diff --git a/ods/config/model-library.json b/ods/config/model-library.jsonindex 62ce71320..17f8d02b5 100644--- a/ods/config/model-library.json+++ b/ods/config/model-library.json@@ -2989,6 +2989,65 @@{"path": "tokenizer.json", "size": 20217442, "sha256": "19e773648cb4e65de8660ea6365e10acca112d42a854923df93db4a6f333a82d"},{"path": "tokenizer_config.json", "size": 761, "git_blob_sha1": "e375fa0a4e16660ce0e2026e39374d35dad4c079"}]+ },+ {+ "id": "qwen3.6-35b-a3b-nvfp4-vllm",+ "name": "Qwen 3.6 35B A3B NVFP4 (vLLM)",+ "family": "qwen",+ "runtime": "vllm",+ "tools": true,+ "vision": true,+ "hf_repo": "nvidia/Qwen3.6-35B-A3B-NVFP4",+ "revision": "1355db6a052410cfd62085d94b58866fd0f2c3c5",+ "source_url": "https://huggingface.co/nvidia/Qwen3.6-35B-A3B-NVFP4",+ "license": "apache-2.0",+ "size_bytes": 23462477857,+ "size_mb": 23462,+ "served_model_name": "Qwen3.6-35B-A3B-NVFP4",+ "llm_model_name": "qwen3.6-35b-a3b-nvfp4",+ "context_length": 131072,+ "max_context_length": 262144,+ "min_gpu_mib": 51200,+ "gpu_memory_utilization": 0.5,+ "compute_capability": ["12.0"],+ "reasoning": "medium",+ "thinking_template_kwargs": {+ "off": {"enable_thinking": false},+ "low": {"enable_thinking": true},+ "medium": {"enable_thinking": true},+ "high": {"enable_thinking": true}+ },+ "tool_call_parser": "qwen3_coder",+ "reasoning_parser": "qwen3",+ "vllm_args": ["--max-model-len", "131072", "--max-num-seqs", "32", "--max-num-batched-tokens", "8192", "--enable-prefix-caching", "--enable-auto-tool-choice", "--tool-call-parser", "qwen3_coder", "--reasoning-parser", "qwen3", "--trust-remote-code", "--speculative-config.method=mtp", "--speculative-config.num_speculative_tokens=3"],+ "selection": {+ "discrete": 0,+ "unified": 0,+ "cpu": 0,+ "vllm": 80+ },+ "quantization": "NVFP4",+ "specialty": "Speed",+ "description": "NVIDIA's NVFP4 build of Qwen 3.6 35B A3B for the vLLM engine (not llama.cpp): a mixture-of-experts model with 35B parameters, 3B active, plus an MTP head. It reads text, images and video. It needs one RTX PRO 6000-class GPU (Blackwell, compute capability 12.0) with at least 50 GiB of memory, and 24 GB of disk. vLLM takes half of that GPU, so image generation and voice still fit beside it on a workstation's only GPU. Served at 128K context with thinking on and MTP speculative decoding. On a DGX Station's RTX PRO 6000 it wrote about 540 tokens/s on one coding stream (llama.cpp's Qwen 3.6 Q4: 250) and read prompts at about 36,000 tokens/s (10,300).",+ "files": [+ {"path": ".gitattributes", "size": 1635, "git_blob_sha1": "aa7aacd0134a92c3c1943fdecc75cd8b7420cce6"},+ {"path": ".quant_summary.txt", "size": 4752286, "git_blob_sha1": "b97d531c8e2bd7cbd1c42cbcb49f029090010251"},+ {"path": "README.md", "size": 9936, "git_blob_sha1": "0ecb54c1554791cedb4615b7ced45826b9817d19"},+ {"path": "chat_template.jinja", "size": 7764, "git_blob_sha1": "a8755d827c0a7b614c246c4060dfd58ab352a8ff"},+ {"path": "config.json", "size": 58110, "git_blob_sha1": "ec1f82c006fd5db6d0b5fa1a59e16fbda9a5c1dd"},+ {"path": "configuration.json", "size": 58, "git_blob_sha1": "d24dba949ee1fe70cc810e4c4709a0bddf4e06ba"},+ {"path": "generation_config.json", "size": 202, "git_blob_sha1": "023756cfadf88e5bf69eefeee3e172f38c448d64"},+ {"path": "hf_quant_config.json", "size": 35085, "git_blob_sha1": "8deefe44cc69cfab994141ea6b9493b03c864061"},+ {"path": "model-00001-of-00003.safetensors", "size": 10006877608, "sha256": "07141c2db92e47bc08777132cd1a0323faf300eab3a7d7c111bc2bf075fda050"},+ {"path": "model-00002-of-00003.safetensors", "size": 10003595752, "sha256": "6dea9c759a0f941cf9e1cc1501216b0e107966a66c90425050e212efbd053f02"},+ {"path": "model-00003-of-00003.safetensors", "size": 3413864960, "sha256": "9758875fc55e49561165f4a44342b654c123c3e25c6811b34abb83553fb1a164"},+ {"path": "model.safetensors.index.json", "size": 13726227, "sha256": "d67403a4e9793c0ba8a136baf14b3b76ec7b32c822267978084895e07ebd8a3e"},+ {"path": "preprocessor_config.json", "size": 390, "git_blob_sha1": "2ea84a437d448ff71b08df68fdd949d5cc4ebb64"},+ {"path": "tokenizer.json", "size": 12807982, "sha256": "5f9e4d4901a92b997e463c1f46055088b6cca5ca61a6522d1b9f64c4bb81cb42"},+ {"path": "tokenizer_config.json", "size": 16718, "git_blob_sha1": "28d96ff303d1d20350185caf4bf037045916ed35"},+ {"path": "video_preprocessor_config.json", "size": 385, "git_blob_sha1": "3ba673a5ad7d4d13f54155ecd38b2a94a6dac8fe"},+ {"path": "vocab.json", "size": 6722759, "git_blob_sha1": "0aa0ce0658d60ac4a5d609f4eadb0e8e43514176"}+ ]}]}diff --git a/ods/docker-compose.vllm.yml b/ods/docker-compose.vllm.ymlindex e57632867..7b1e6d82f 100644--- a/ods/docker-compose.vllm.yml+++ b/ods/docker-compose.vllm.yml@@ -1,8 +1,10 @@# ODS - vLLM engine overlay## Used when .env sets LLM_BACKEND=vllm: vLLM serves the model on one big-# NVIDIA GPU (datacenter Blackwell, compute capability 10.x) instead of the-# in-stack llama-server. As on the host-native route, llama-server is off and+# NVIDIA GPU (datacenter Blackwell, compute capability 10.x, or the RTX PRO+# 6000 class, 12.0) instead of the in-stack llama-server. It preallocates+# VLLM_GPU_MEMORY_UTILIZATION of that GPU, the catalog model's own share+# when it sets one. As on the host-native route, llama-server is off and# model-router stays; endpoints.json points model-router at vllm-server:8000.## This file is intentionally not standalone. scripts/resolve-compose-stack.shdiff --git a/ods/extensions/services/dashboard-api/model_memory.py b/ods/extensions/services/dashboard-api/model_memory.pyindex dfe1bf45e..ce9fb46af 100644--- a/ods/extensions/services/dashboard-api/model_memory.py+++ b/ods/extensions/services/dashboard-api/model_memory.py@@ -24,8 +24,9 @@ There are two estimate paths:Entries served by vLLM (``"runtime": "vllm"``, see model_vllm.py) takeneither path: llama.cpp's KV, recurrent-state and checkpoint maths do notapply to them. Their weights are the sum of the ``files`` manifest, and the-device need is weights / :data:`VLLM_GPU_MEMORY_UTILIZATION`, the GPU size at-which the share vLLM preallocates just holds the weights (method ``vllm``).+device need is weights / the entry's share (:func:`vllm_memory_share`, its+``gpu_memory_utilization`` or :data:`VLLM_GPU_MEMORY_UTILIZATION`), the GPU+size at which the share vLLM preallocates just holds the weights (method ``vllm``).vLLM fits its KV cache and activations into whatever that share leaves, sothis is a floor for callers that ask; the catalog's ``min_gpu_mib`` (weightsplus about 25%) is what the installer admits a GPU on.@@ -50,14 +51,14 @@ MEMORY_METADATA_KEYS = ("size_bytes", "max_context_length","sliding_window", "sliding_window_layer_count", "sliding_window_head_count_kv","sliding_window_key_length", "sliding_window_value_length",- "runtime", "files", "vllm_args",+ "runtime", "files", "vllm_args", "gpu_memory_utilization",)# A catalog entry with this ``runtime`` is served by vLLM (model_vllm.py).VLLM_RUNTIME = "vllm"# The share of its GPU vLLM preallocates (--gpu-memory-utilization; .env# VLLM_GPU_MEMORY_UTILIZATION defaults to it) for weights, activations and-# its KV cache.+# its KV cache, unless the entry sets its own gpu_memory_utilization.VLLM_GPU_MEMORY_UTILIZATION = 0.90MIB = 1024.0 ** 2@@ -454,6 +455,11 @@ def _vllm_kv_cache_dtype(model: dict[str, Any]) -> str:return "auto"+def vllm_memory_share(model: dict[str, Any]) -> float:+ """The share of its GPU a vLLM entry preallocates: its ``gpu_memory_utilization``, else 0.90."""+ return _positive_number(model.get("gpu_memory_utilization")) or VLLM_GPU_MEMORY_UTILIZATION++def vllm_memory_estimate(model: dict[str, Any],*,@@ -463,12 +469,12 @@ def vllm_memory_estimate("""The documented estimate for a vLLM entry (see the module docstring).Weights are the ``files`` manifest total (or ``weight_size_mb``, a measured- size on disk). ``device_gib`` is weights / VLLM_GPU_MEMORY_UTILIZATION; KV+ size on disk). ``device_gib`` is weights / :func:`vllm_memory_share`; KVcache, recurrent state, overhead and host checkpoints are 0 because vLLMkeeps them inside its preallocated share. The context does not change it."""weights = _vllm_weights_bytes(model, weight_size_mb) / GIB- device = round(weights / VLLM_GPU_MEMORY_UTILIZATION, 2)+ device = round(weights / vllm_memory_share(model), 2)cache = _vllm_kv_cache_dtype(model)return MemoryEstimate(context_length=_context(model, context_length),diff --git a/ods/extensions/services/dashboard-api/model_vllm.py b/ods/extensions/services/dashboard-api/model_vllm.pyindex 54af62e97..2d62bcb1b 100644--- a/ods/extensions/services/dashboard-api/model_vllm.py+++ b/ods/extensions/services/dashboard-api/model_vllm.py@@ -13,6 +13,11 @@ the vLLM engine (``.env`` ``LLM_BACKEND=vllm``), not llama.cpp. They carry no* ``min_gpu_mib`` and ``compute_capability``: the one GPU vLLM runs on needsat least this much memory (nvidia-smi ``memory.total``) and one of thesecompute capabilities;+* optional ``gpu_memory_utilization``: the share of that GPU vLLM+ preallocates (``.env`` VLLM_GPU_MEMORY_UTILIZATION, the vllm-server+ service's ``--gpu-memory-utilization``; vLLM's 0.90 when absent). An entry+ for a GPU that is often a workstation's only one (RTX PRO 6000 class) takes+ less, so ComfyUI and voice still fit beside it;* ``reasoning`` (the thinking default), plus ``tool_call_parser`` and``reasoning_parser``, which name the same parsers as ``vllm_args``;* ``thinking_template_kwargs``: for each ODS thinking level (off, low,@@ -46,9 +51,14 @@ from typing import Any, Iterable, NamedTupleVLLM_RUNTIME = "vllm"REASONING_LEVELS = ("off", "low", "medium", "high")LLAMA_MEMORY_CLASSES = ("discrete", "unified", "cpu")-# The vllm-server compose service passes these itself:-# --model /model --served-model-name ${VLLM_SERVED_MODEL} ${VLLM_ARGS}.-COMPOSE_FLAGS = frozenset({"--model", "--served-model-name", "--host", "--port"})+# The vllm-server compose service passes these itself: --model /model+# --served-model-name ${VLLM_SERVED_MODEL} --gpu-memory-utilization+# ${VLLM_GPU_MEMORY_UTILIZATION} ${VLLM_ARGS}. An entry sets its share with+# ``gpu_memory_utilization``, never in vllm_args, where it would silently+# override the owner's .env value.+COMPOSE_FLAGS = frozenset({"--model", "--served-model-name", "--host", "--port", "--gpu-memory-utilization"})+# The range an entry's gpu_memory_utilization may take.+MIN_MEMORY_SHARE, MAX_MEMORY_SHARE = 0.10, 0.95# A file downloads under <name><PARTIAL_SUFFIX> until it is verified.PARTIAL_SUFFIX = ".partial"GPU_QUERY = "nvidia-smi --query-gpu=uuid,compute_cap,memory.total --format=csv,noheader,nounits"@@ -171,7 +181,8 @@ def _args_problems(model: dict[str, Any]) -> list[str]:):return ["vllm_args must be a list of plain strings (letters, digits and _.:/=,+-)"]problems = []- owned = sorted({arg.split("=", 1)[0] for arg in args} & COMPOSE_FLAGS)+ # vLLM's parser reads --gpu_memory_utilization as --gpu-memory-utilization.+ owned = sorted({arg.split("=", 1)[0].replace("_", "-") for arg in args if arg.startswith("--")} & COMPOSE_FLAGS)if owned:problems.append(f"vllm_args may not set {', '.join(owned)}: the vllm-server service sets them")for flag, key in (("--max-model-len", "context_length"), ("--tool-call-parser", "tool_call_parser"),@@ -221,6 +232,10 @@ def _serving_problems(model: dict[str, Any]) -> list[str]:problems.append("max_context_length (config.json max_position_embeddings) must be at least context_length")if not _whole(model.get("min_gpu_mib"), 1):problems.append("min_gpu_mib must be a positive whole number of MiB")+ share = model.get("gpu_memory_utilization", MAX_MEMORY_SHARE)+ if (not isinstance(share, (int, float)) or isinstance(share, bool)+ or not MIN_MEMORY_SHARE <= share <= MAX_MEMORY_SHARE):+ problems.append(f"gpu_memory_utilization must be a number from {MIN_MEMORY_SHARE} to {MAX_MEMORY_SHARE}")capabilities = model.get("compute_capability")if (not isinstance(capabilities, list) or not capabilitiesor len(set(map(str, capabilities))) != len(capabilities)diff --git a/ods/extensions/services/dashboard-api/tests/test_model_memory.py b/ods/extensions/services/dashboard-api/tests/test_model_memory.pyindex 9af1db61f..cf61b9993 100644--- a/ods/extensions/services/dashboard-api/tests/test_model_memory.py+++ b/ods/extensions/services/dashboard-api/tests/test_model_memory.py@@ -594,6 +594,13 @@ class TestVllmEstimate:def test_measured_weights_override_the_manifest(self):assert estimate_model_memory(VLLM_ENTRY, weight_size_mb=92160).device_gib == 100.0+ def test_an_entry_share_sets_the_device_need(self):+ # The RTX PRO 6000-class entry takes half its GPU, so it needs a GPU twice its weights.+ entry = {**VLLM_ENTRY, "gpu_memory_utilization": 0.5}+ weights = 90 + 30 / 1024+ assert estimate_model_memory(entry).device_gib == round(weights / 0.5, 2) == 180.06+ assert estimate_model_memory(VLLM_ENTRY).device_gib == round(weights / 0.90, 2)+def test_kv_cache_dtype_defaults_to_auto(self):entry = {**VLLM_ENTRY, "vllm_args": ["--max-model-len", "131072"]}assert estimate_model_memory(entry).cache_type_k == "auto"@@ -601,21 +608,21 @@ class TestVllmEstimate:assert estimate_model_memory(entry).cache_type_v == "fp8_e4m3"def test_normalized_records_keep_what_the_estimate_reads(self):- kept = memory_metadata(VLLM_ENTRY)- assert {key: kept[key] for key in ("runtime", "files", "vllm_args")} == {- key: VLLM_ENTRY[key] for key in ("runtime", "files", "vllm_args")- }- assert estimate_model_memory(kept).device_gib == estimate_model_memory(VLLM_ENTRY).device_gib+ entry = {**VLLM_ENTRY, "gpu_memory_utilization": 0.5}+ kept = memory_metadata(entry)+ keys = ("runtime", "files", "vllm_args", "gpu_memory_utilization")+ assert {key: kept[key] for key in keys} == {key: entry[key] for key in keys}+ assert estimate_model_memory(kept).device_gib == estimate_model_memory(entry).device_gib == 180.06def test_catalog_vllm_entries_get_a_sane_number_below_their_gpu_minimum(self):entries = [raw for raw in _catalog_entries() if raw.get("runtime") == "vllm"]- assert len(entries) == 2+ assert len(entries) == 3select_model = _load_select_model()for raw in entries:weights = sum(item["size"] for item in raw["files"]) / 1024 ** 3estimate = estimate_model_memory(raw)assert estimate.method == "vllm"assert estimate.weights_gib == round(weights, 3)- assert estimate.device_gib == round(weights / 0.90, 2)+ assert estimate.device_gib == round(weights / raw.get("gpu_memory_utilization", 0.90), 2)assert estimate.device_gib * 1024 < raw["min_gpu_mib"]assert select_model.selector_required_memory_gb(raw) == estimate.device_gibdiff --git a/ods/install-core.sh b/ods/install-core.shindex 6272d254d..8bacc25c7 100755--- a/ods/install-core.sh+++ b/ods/install-core.sh@@ -319,7 +319,7 @@ Options:--reselect-model Replace a valid active local model with the current installer recommendation--llm-backend vllm|llama-serverModel server. Without it, a new Linux install runs the vLLM engine- when a vLLM catalog model fits one NVIDIA datacenter GPU, and+ when a vLLM catalog model fits one NVIDIA GPU, andllama.cpp (llama-server) otherwise; a rerun keeps its engine.vllm stops when no vLLM model fits; llama-server never uses vLLM--voice Enable voice services (Whisper + Kokoro)diff --git a/ods/installers/lib/llm-backend.sh b/ods/installers/lib/llm-backend.shindex d04fcfdbe..be90bd441 100644--- a/ods/installers/lib/llm-backend.sh+++ b/ods/installers/lib/llm-backend.sh@@ -5,8 +5,9 @@# Part of: installers/lib/# Purpose: The rules phase 02 uses to choose the model server. The vLLM# engine (docker-compose.vllm.yml) serves a vLLM catalog model on one-# NVIDIA datacenter GPU; llama.cpp (llama-server) serves a GGUF-# everywhere else. Which vLLM model fits which GPU is+# NVIDIA GPU (datacenter Blackwell or the RTX PRO 6000 class);+# llama.cpp (llama-server) serves a GGUF everywhere else. Which vLLM+# model fits which GPU is# scripts/select-model.py --llm-runtime vllm's decision.## Expects: nothing (arguments only)diff --git a/ods/installers/macos/install-macos.sh b/ods/installers/macos/install-macos.shindex 8b11fc22f..ba57eebf6 100755--- a/ods/installers/macos/install-macos.sh+++ b/ods/installers/macos/install-macos.sh@@ -171,7 +171,7 @@ while [[ $# -gt 0 ]]; docase "${2:-}" inllama-server) shift 2 ;;vllm)- echo "--llm-backend vllm needs Linux with an NVIDIA datacenter GPU; macOS runs llama.cpp. Nothing was changed." >&2+ echo "--llm-backend vllm needs Linux with an NVIDIA GPU; macOS runs llama.cpp. Nothing was changed." >&2echo "Rerun without --llm-backend, or with --llm-backend llama-server." >&2echo "Need help? Ask on the ODS Discord: https://discord.gg/4ntNp9MAwC" >&2exit 1 ;;diff --git a/ods/installers/phases/02-detection.sh b/ods/installers/phases/02-detection.shindex 0892d2234..49c5dfa49 100755--- a/ods/installers/phases/02-detection.sh+++ b/ods/installers/phases/02-detection.sh@@ -23,8 +23,9 @@# LLM_MODEL_SIZE_MB, AMD_GFX_TARGET, AMD_INFERENCE_BACKEND,# LLM_BACKEND (llama-server or vllm; exported for vllm and when a# run leaves vllm); with vllm also VLLM_IMAGE, VLLM_MODEL_DIR,-# VLLM_SERVED_MODEL, VLLM_ARGS, VLLM_GPU_UUID, and on a rerun that-# keeps a vLLM model the VLLM_GPU_MEMORY_UTILIZATION it was given+# VLLM_SERVED_MODEL, VLLM_ARGS, VLLM_GPU_UUID and+# VLLM_GPU_MEMORY_UTILIZATION (the pick's own share; on a rerun that+# keeps a vLLM model, the share it was given)## Modder notes:# Change tier auto-detection thresholds or add new hardware classes here.@@ -577,8 +578,9 @@ fi# Model server: llama.cpp (llama-server) or the vLLM engine# -----------------------------------------------------------------------------# The vLLM engine (docker-compose.vllm.yml) serves a vLLM catalog model,-# NVIDIA's NVFP4 builds, on one NVIDIA datacenter GPU (compute capability 10.x-# with room for the model). Only Linux (not WSL) with an NVIDIA GPU, in local+# NVIDIA's NVFP4 builds, on one NVIDIA GPU that entry names (datacenter+# Blackwell, compute capability 10.x, or the RTX PRO 6000 class, 12.0, with+# room for the model). Only Linux (not WSL) with an NVIDIA GPU, in local# mode and with no external, host-native or cloud model route, can run it# (installers/lib/llm-backend.sh).#@@ -855,7 +857,8 @@ _phase02_vllm_keep_refused() {# served id and GPU, which scripts/preserve-active-model.py prints from .env# (LLM_BACKEND=vllm, the VLLM_* keys, empty GGUF_* keys). The vLLM pick above# stays the recommendation. The VLLM_* keys are cleared first, so a key the-# loader dropped cannot leave the pick's value in place.+# loader dropped cannot leave the pick's value in place, and the kept model+# never takes the pick's memory share: only its own .env share is kept._phase02_keep_vllm_model() {local python status=0 contractpython="$(ods_model_selector_python)"@@ -874,7 +877,7 @@ _phase02_keep_vllm_model() {--local-model \2>>"$LOG_FILE")" || status=$?[[ "$status" -eq 0 && -n "$contract" ]] || _phase02_vllm_keep_refused "$status"- unset VLLM_MODEL_DIR VLLM_SERVED_MODEL VLLM_ARGS VLLM_GPU_UUID+ unset VLLM_MODEL_DIR VLLM_SERVED_MODEL VLLM_ARGS VLLM_GPU_UUID VLLM_GPU_MEMORY_UTILIZATIONload_model_selector_env_from_output <<< "$contract"if [[ "$LLM_BACKEND" != "vllm" || -z "${VLLM_MODEL_DIR:-}" || -z "${VLLM_SERVED_MODEL:-}" \|| -z "${VLLM_ARGS:-}" || -z "${VLLM_GPU_UUID:-}" ]]; thendiff --git a/ods/lib/safe-env.sh b/ods/lib/safe-env.shindex 229be987c..3a884b25c 100755--- a/ods/lib/safe-env.sh+++ b/ods/lib/safe-env.sh@@ -153,7 +153,8 @@ load_env_from_output_allowlist() {# LLAMA_ARG_CHAT_TEMPLATE_FILE; model activation owns them). Only a vLLM pick# (select-model.py --llm-runtime vllm) and a vLLM installation's preserved# contract carry the VLLM_* keys; only that contract carries LLM_BACKEND-# (always vllm) and VLLM_GPU_MEMORY_UTILIZATION.+# (always vllm). VLLM_GPU_MEMORY_UTILIZATION is the pick's own share, or on+# a rerun that keeps its model, the share that installation was given.load_model_selector_env_from_output() {load_env_from_output_allowlist \LLM_MODEL \diff --git a/ods/scripts/select-model.py b/ods/scripts/select-model.pyindex ae795c7b8..8851bc315 100644--- a/ods/scripts/select-model.py+++ b/ods/scripts/select-model.py@@ -22,13 +22,13 @@ from typing import Anysys.path.insert(0, str(Path(__file__).resolve().parents[1] / "extensions/services/dashboard-api"))from model_memory import ( # noqa: E402- VLLM_GPU_MEMORY_UTILIZATION,context_fitting_model as context_fitting_model,estimate_model_memory,estimated_context_kv_gb as estimated_context_kv_gb,estimated_param_billions as estimated_param_billions,memory_metadata,required_model_memory_gb,+ vllm_memory_share,)from model_selection import ( # noqa: E402POLICY,@@ -470,13 +470,14 @@ def _check_fit_main(args: argparse.Namespace, catalog: list[dict[str, Any]],def _vllm_reason(candidate: VllmCandidate, min_context: int) -> str:model, gpu = candidate.model, candidate.gpuweights_gib = estimate_model_memory(model).weights_gib- room_gib = gpu.memory_mib * VLLM_GPU_MEMORY_UTILIZATION / 1024.0 - weights_gib+ share = vllm_memory_share(model)+ room_gib = gpu.memory_mib * share / 1024.0 - weights_gibreason = (f"Curated vLLM fit ({VLLM_POLICY}): {model['name']} is the highest-priority vLLM "f"catalog model for GPU {gpu.uuid} (compute capability {gpu.compute_capability}, "f"{gpu.memory_mib} MiB), which meets its {model['min_gpu_mib']} MiB minimum. Its "f"{weights_gib:.1f} GiB of weights leave about {room_gib:.1f} GiB of vLLM's "- f"{VLLM_GPU_MEMORY_UTILIZATION:.2f} share of that GPU for KV cache and activations at "+ f"{share:.2f} share of that GPU for KV cache and activations at "f"{model['context_length'] // 1024}K context. Throughput requires a local benchmark "f"after first launch.")@@ -535,7 +536,13 @@ def _vllm_payload(ranked: list[VllmCandidate], reason: str, min_context: int, hodef _vllm_env(ranked: list[VllmCandidate], reason: str) -> dict[str, Any]:- """The installer contract for a vLLM pick (VLLM_* are .env keys as named)."""+ """The installer contract for a vLLM pick (VLLM_* are .env keys as named).++ VLLM_GPU_MEMORY_UTILIZATION is always the pick's own share (its entry's,+ else 0.9), so a newly picked model never inherits the share an earlier+ model left in .env. A rerun that keeps its model keeps its .env share+ instead (phase 02's keep path).+ """model, gpu = ranked[0].model, ranked[0].gpureturn {"LLM_MODEL": model["llm_model_name"],@@ -558,6 +565,7 @@ def _vllm_env(ranked: list[VllmCandidate], reason: str) -> dict[str, Any]:"VLLM_SERVED_MODEL": model["served_model_name"],"VLLM_ARGS": vllm_args_text(model),"VLLM_GPU_UUID": gpu.uuid,+ "VLLM_GPU_MEMORY_UTILIZATION": f"{vllm_memory_share(model):g}",}diff --git a/ods/tests/test-model-library-coverage.py b/ods/tests/test-model-library-coverage.pyindex ec4237e33..173ba11d9 100644--- a/ods/tests/test-model-library-coverage.py+++ b/ods/tests/test-model-library-coverage.py@@ -77,7 +77,7 @@ def test_low_vram_catalog_has_six_agent_viable_downloadable_models():low_vram = [modelfor model in models- # vLLM entries need a datacenter GPU (min_gpu_mib) and download a+ # vLLM entries need a large NVIDIA GPU (min_gpu_mib) and download a# snapshot, not a GGUF; they are never low-VRAM downloads.if model.get("runtime") != "vllm"and int(model.get("vram_required_gb") or 0) <= 8diff --git a/ods/tests/test-model-library-vllm.py b/ods/tests/test-model-library-vllm.pyindex 1058fedb7..4c64147a7 100644--- a/ods/tests/test-model-library-vllm.py+++ b/ods/tests/test-model-library-vllm.py@@ -8,10 +8,11 @@ checks keep the shipped entries well formed and their numbers consistent:* the schema itself (model_vllm.entry_problems), and that it catches themistakes it is meant to catch;-* ``min_gpu_mib`` covers the weights plus 25% (the authoring rule) and so- also model_memory's device need, weights / 0.90;+* ``min_gpu_mib`` covers the weights plus 25% (the authoring rule), scaled+ by 0.90 / ``gpu_memory_utilization`` for an entry that takes a smaller+ share of its GPU, and so also model_memory's device need, weights / share;* every shard a safetensors set names is in the manifest;-* the two first entries keep the revisions and licences reviewed for them.+* the entries keep the revisions, licences and GPUs reviewed for them.Network-free: verify a new manifest against the Hub's``/api/models/<repo>/revision/<sha>?blobs=true`` before adding it.@@ -31,7 +32,7 @@ ROOT = Path(__file__).resolve().parents[1]CATALOG = ROOT / "config" / "model-library.json"sys.path.insert(0, str(ROOT / "extensions" / "services" / "dashboard-api"))-from model_memory import estimate_model_memory # noqa: E402+from model_memory import VLLM_GPU_MEMORY_UTILIZATION, estimate_model_memory, vllm_memory_share # noqa: E402from model_vllm import entry_problems, manifest_bytes, model_dir_name # noqa: E402MIB = 1024 ** 2@@ -41,13 +42,21 @@ REVIEWED = {"hf_repo": "nvidia/Qwen3.8-Flash-Next-NVFP4","revision": "fc694b54fb0174e0913e6adf86691ef85a4ead47","license": "other", "license_name": "nvidia-open-model-license",- "size_bytes": 132734506847, "files": 25,+ "size_bytes": 132734506847, "files": 25, "compute_capability": ["10.0", "10.3"],},"glm-5.3-flash-nvfp4-vllm": {"hf_repo": "nvidia/GLM-5.3-Flash-NVFP4","revision": "da920bb0b9f4a06727223a349e55468e38352348","license": "mit", "license_name": None,- "size_bytes": 204476278199, "files": 44,+ "size_bytes": 204476278199, "files": 44, "compute_capability": ["10.0", "10.3"],+ },+ # The RTX PRO 6000 class (Blackwell workstation, sm_120), measured on the+ # DGX Station's RTX PRO 6000 (10-27). It takes half of that GPU.+ "qwen3.6-35b-a3b-nvfp4-vllm": {+ "hf_repo": "nvidia/Qwen3.6-35B-A3B-NVFP4",+ "revision": "1355db6a052410cfd62085d94b58866fd0f2c3c5",+ "license": "apache-2.0", "license_name": None,+ "size_bytes": 23462477857, "files": 17, "compute_capability": ["12.0"],},}@@ -75,7 +84,7 @@ def test_reviewed_entries_are_in_the_catalog():assert model.get("license_name") == expected["license_name"], model_idassert model["size_bytes"] == expected["size_bytes"], model_idassert len(model["files"]) == expected["files"], model_id- assert model["compute_capability"] == ["10.0", "10.3"], model_id+ assert model["compute_capability"] == expected["compute_capability"], model_idassert model["reasoning"] == "medium", model_id@@ -86,10 +95,11 @@ def test_entries_follow_the_schema(model):@pytest.mark.parametrize("model", _vllm_models(), ids=_ids(_vllm_models()))def test_min_gpu_memory_covers_the_weights_plus_a_quarter(model):- weights_mib = manifest_bytes(model) / MIB- assert model["min_gpu_mib"] >= weights_mib * 1.25, model["id"]+ # At vLLM's usual 0.90 share; a smaller share needs a proportionally larger GPU.+ needed_mib = manifest_bytes(model) / MIB * 1.25 * VLLM_GPU_MEMORY_UTILIZATION / vllm_memory_share(model)+ assert model["min_gpu_mib"] >= needed_mib, model["id"]# Rounded up to a whole GiB, not padded further.- assert model["min_gpu_mib"] % 1024 == 0 and model["min_gpu_mib"] - weights_mib * 1.25 < 1024, model["id"]+ assert model["min_gpu_mib"] % 1024 == 0 and model["min_gpu_mib"] - needed_mib < 1024, model["id"]device_gib = estimate_model_memory(model).device_gibassert device_gib * 1024 <= model["min_gpu_mib"], (model["id"], device_gib)@@ -154,6 +164,12 @@ def _mutate(path: str, value):("vllm_args", ["--model", "/x"], "vllm_args may not set --model"),("vllm_args", ["--speculative-config", '{"method": "mtp"}'], "vllm_args must be a list of plain strings"),("min_gpu_mib", 0, "min_gpu_mib must be a positive whole number"),+ ("gpu_memory_utilization", 0.05, "gpu_memory_utilization must be a number from 0.1 to 0.95"),+ ("gpu_memory_utilization", 1, "gpu_memory_utilization must be a number from 0.1 to 0.95"),+ ("gpu_memory_utilization", True, "gpu_memory_utilization must be a number from 0.1 to 0.95"),+ ("gpu_memory_utilization", "0.5", "gpu_memory_utilization must be a number from 0.1 to 0.95"),+ ("vllm_args", ["--gpu-memory-utilization=0.5"], "vllm_args may not set --gpu-memory-utilization"),+ ("vllm_args", ["--gpu_memory_utilization", "0.5"], "vllm_args may not set --gpu-memory-utilization"),("compute_capability", ["10"], 'compute_capability must list distinct "major.minor" strings'),("reasoning", "on", "reasoning must be one of off, low, medium, high"),("selection.discrete", 50, "selection.discrete must be 0"),@@ -186,6 +202,8 @@ def _dotted_config(args: list[str], flag: str) -> dict:# Measured on the DGX Station GB300 (vLLM v0.32.0, 10-27): MTP drafts lifted one-stream# writing from about 200 to 313 tokens/s on Qwen and from 145 to 210 on GLM, and Qwen's# n-gram embedding table on the GPU (not offloaded to the CPU) is part of that best setup.+# On its RTX PRO 6000, MTP lifted Qwen 3.6's one-stream coding from 272 to 537 tokens/s+# (llama.cpp's Qwen 3.6 Q4 there: 250) and 16 streams from 2,404 to 3,797 (llama.cpp: 462).MEASURED_FLAGS = {"qwen3.8-flash-next-nvfp4-vllm": {"--speculative-config": {"method": "mtp", "num_speculative_tokens": 3},@@ -195,6 +213,10 @@ MEASURED_FLAGS = {"--speculative-config": {"method": "mtp", "num_speculative_tokens": 5},"--engram-config": {},},+ "qwen3.6-35b-a3b-nvfp4-vllm": {+ "--speculative-config": {"method": "mtp", "num_speculative_tokens": 3},+ "--engram-config": {},+ },}@@ -203,9 +225,12 @@ MEASURED_FLAGS = {# 156 s instead of 392 s at the same speed (1/4/16 streams: 349/965/2,276 tok/s against 349/961/2,246).# GLM fits only sm_103 GPUs (GB300/B300), where that was measured. Qwen3.8 also showed no loss on the# GB300 (226 s -> 155 s), but its main fit is B200-class sm_100, which is unmeasured, so it keeps it.+# On the RTX PRO 6000 (sm_120) Qwen 3.6's autotune reads its cache in under a second: a restart was+# ready in 105 s either way, and with it off one-stream coding fell from 537 to 464 tokens/s, so it stays.MEASURED_SWITCHES = {"qwen3.8-flash-next-nvfp4-vllm": set(),"glm-5.3-flash-nvfp4-vllm": {"--no-enable-flashinfer-autotune"},+ "qwen3.6-35b-a3b-nvfp4-vllm": set(),}@@ -226,6 +251,10 @@ def test_the_entries_keep_the_flags_measured_on_a_dgx_station(model_id):# false and answers HTTP 400 for any reasoning_effort but xhigh (its# default), medium and low. Measured (GB300, temperature 0.6): GLM max 7.5 s# vs high 3.4 s and low 2.8 s; Qwen xhigh 5.0 s, medium 3.7 s, low 3.4 s.+# Qwen3.6's template reads only enable_thinking (thinking stays on unless it+# is false) and ignores reasoning_effort, so off turns thinking off and every+# other level keeps it on. Measured (RTX PRO 6000): off 3.0 s with no+# reasoning, on 4.6 s, and a stray reasoning_effort high answered normally.THINKING = {"glm-5.3-flash-nvfp4-vllm": {"off": {"reasoning_effort": "low"}, "low": {"reasoning_effort": "low"},@@ -235,10 +264,15 @@ THINKING = {"off": {"enable_thinking": False}, "low": {"reasoning_effort": "low"},"medium": {"reasoning_effort": "medium"}, "high": {"reasoning_effort": "xhigh"},},+ "qwen3.6-35b-a3b-nvfp4-vllm": {+ "off": {"enable_thinking": False}, "low": {"enable_thinking": True},+ "medium": {"enable_thinking": True}, "high": {"enable_thinking": True},+ },}TEMPLATE_EFFORTS = {"glm-5.3-flash-nvfp4-vllm": {"low", "high"},"qwen3.8-flash-next-nvfp4-vllm": {"low", "medium", "xhigh"},+ "qwen3.6-35b-a3b-nvfp4-vllm": set(),}@@ -248,7 +282,8 @@ def test_each_level_uses_the_models_own_template_arguments(model_id):assert model["thinking_template_kwargs"] == THINKING[model_id]# No level hands the template a reasoning_effort it does not accept.for level, kwargs in model["thinking_template_kwargs"].items():- assert kwargs.get("reasoning_effort", "low") in TEMPLATE_EFFORTS[model_id], (model_id, level)+ if "reasoning_effort" in kwargs:+ assert kwargs["reasoning_effort"] in TEMPLATE_EFFORTS[model_id], (model_id, level)def test_every_vllm_entry_has_a_thinking_map():@@ -258,10 +293,11 @@ def test_every_vllm_entry_has_a_thinking_map():@pytest.mark.parametrize("model_id", sorted(MEASURED_FLAGS))def test_the_entries_declare_tools_and_vision(model_id):# The host agent's route proof publishes these catalog booleans as the- # route's capabilities (model-state, Pixel connection metadata). Both+ # route's capabilities (model-state, Pixel connection metadata). The# checkpoints are image-text-to-text and run vLLM's tool parser; on the# DGX Station GB300 GLM made tool calls (streamed and not) and read an- # image through model-router. Without them the route claimed neither.+ # image through model-router, as Qwen 3.6 did on its RTX PRO 6000.+ # Without them the route claimed neither.model = {item["id"]: item for item in _vllm_models()}[model_id]assert model.get("tools") is True and model.get("vision") is True, model_idassert "--enable-auto-tool-choice" in model["vllm_args"], model_iddiff --git a/ods/tests/test-vllm-installer-phases.sh b/ods/tests/test-vllm-installer-phases.shindex 410971f5a..0dd9d8c04 100644--- a/ods/tests/test-vllm-installer-phases.sh+++ b/ods/tests/test-vllm-installer-phases.sh@@ -263,6 +263,17 @@ render owner "$tmp/vllm.fixture"|| fail "an owner's own vision server must be kept"pass "a rerun moves llama-server URLs to LiteLLM and model-router, turns the switchboard on and keeps owner settings"+# A newly picked model writes its own memory share (the selector always sends+# one), never the share an earlier model left in .env.+mkdir -p "$tmp/p06-pickshare"+printf '%s\n' 'LLM_BACKEND=vllm' 'VLLM_GPU_MEMORY_UTILIZATION=0.85' > "$tmp/p06-pickshare/.env"+{ cat "$tmp/vllm.fixture"; printf '%s\n' 'VLLM_GPU_MEMORY_UTILIZATION=0.5'; } > "$tmp/pickshare.fixture"+render pickshare "$tmp/pickshare.fixture"+[[ "$(envv pickshare VLLM_GPU_MEMORY_UTILIZATION)" == 0.5 \+ && "$(grep -c '^VLLM_GPU_MEMORY_UTILIZATION=' "$tmp/p06-pickshare/.env")" == 1 ]] \+ || fail "the pick's own share must replace an earlier model's, once: $(grep '^VLLM_GPU' "$tmp/p06-pickshare/.env")"+pass "a newly picked vLLM model writes its own memory share over an earlier model's"+mkdir -p "$tmp/p06-noimage"grep -v '^VLLM_IMAGE=' "$tmp/vllm.fixture" > "$tmp/noimage.fixture"! ods_phase06_render_env "$PHASE06" "$ROOT_DIR" "$tmp/p06-noimage" "$tmp/noimage.fixture" \diff --git a/ods/tests/test-vllm-installer-route.sh b/ods/tests/test-vllm-installer-route.shindex 0ad2794b7..8b767cbd3 100644--- a/ods/tests/test-vllm-installer-route.sh+++ b/ods/tests/test-vllm-installer-route.sh@@ -70,7 +70,7 @@ macos_parse() ( eval "$macos_parser"; printf 'parsed\n' )[[ "$(macos_parse --llm-backend llama-server)" == parsed ]] || fail "macOS must accept --llm-backend llama-server"rc=0macos_parse --llm-backend vllm >"$tmp/out" 2>"$tmp/err" || rc=$?-[[ "$rc" -ne 0 ]] && grep -q 'needs Linux with an NVIDIA datacenter GPU' "$tmp/err" && grep -q 'discord.gg' "$tmp/err" \+[[ "$rc" -ne 0 ]] && grep -q 'needs Linux with an NVIDIA GPU;' "$tmp/err" && grep -q 'discord.gg' "$tmp/err" \|| fail "macOS must refuse --llm-backend vllm with the reason and the Discord link"pass "the macOS installer accepts llama-server and refuses vllm with a reason"@@ -323,9 +323,31 @@ grep -q 'LOG Model server: llama.cpp (no vLLM catalog model fits this host' "$tm[[ ! -s "$tmp/rtx-auto/err" ]] || fail "a host without a vLLM fit must not print anything: $(cat "$tmp/rtx-auto/err")"decide spark-auto "export FAKE_GPU_ROWS='GPU-eeeeeeee-gb10, 12.1, [N/A]\n'"expect_llama spark-auto+decide rtx5090-auto "export FAKE_GPU_ROWS='GPU-gggggggg-5090, 12.0, 32607\n'"+expect_llama rtx5090-auto+pass "without a vLLM fit (RTX 4090, DGX Spark, RTX 5090) the llama.cpp path is unchanged and quiet"++# A workstation whose only GPU is an RTX PRO 6000 gets the catalog's RTX PRO+# 6000-class vLLM model and that entry's own memory share, which leaves room+# for ComfyUI and voice on the same GPU.decide pro6000-auto "export FAKE_GPU_ROWS='GPU-ffffffff-rtx6000, 12.0, 97887\n'"-expect_llama pro6000-auto-pass "without a vLLM fit (RTX 4090, DGX Spark, RTX PRO 6000) the llama.cpp path is unchanged and quiet"+read -r p6k_model p6k_share < <("$real_python" - "$ROOT_DIR" <<'PY'+import json, sys+from pathlib import Path+root = Path(sys.argv[1])+sys.path.insert(0, str(root / "extensions/services/dashboard-api"))+from model_vllm import Gpu, rank_entries+models = json.loads((root / "config/model-library.json").read_text(encoding="utf-8"))["models"]+best = rank_entries(models, [Gpu("GPU-ffffffff-rtx6000", "12.0", 97887)])[0].model+print(best["llm_model_name"], f"{best['gpu_memory_utilization']:g}")+PY+)+p6k_share="${p6k_share%$'\r'}" # a Windows Python ends the line with CRLF+[[ "$(rc_of pro6000-auto)" == 0 && "$(value pro6000-auto BACKEND)" == vllm \+ && "$(value pro6000-auto MODEL)" == "$p6k_model" && "$(value pro6000-auto VLLM_GPU_UUID)" == GPU-ffffffff-rtx6000 \+ && "$(value pro6000-auto VLLM_SHARE)" == "$p6k_share" && "$p6k_share" == 0.5 ]] \+ || fail "one RTX PRO 6000 must choose vLLM with its entry's memory share: $(cat "$tmp/pro6000-auto/out" "$tmp/pro6000-auto/err")"+pass "a lone RTX PRO 6000 chooses vLLM ($p6k_model) at its entry's $p6k_share memory share"decide rtx-forced "export FAKE_GPU_ROWS='$RTX_ROWS'" 'LLM_BACKEND_REQUESTED=vllm'expect_refused rtx-forced 'no vLLM catalog model fits this host'@@ -416,6 +438,23 @@ grep -qF "LOG Kept this installation's vLLM model across installer rerun: glm-5."$tmp/rerun-vllm/install.log" || fail "keeping the vLLM model must reach the install log"pass "a rerun of a vLLM install keeps its model, folder, GPU and memory share; the fresh pick stays the recommendation"+# The kept model never takes the fresh pick's memory share: kept without a+# share of its own, it stays on compose's default even where the fresh pick+# (the RTX PRO 6000 entry here) sets one.+share_line='VLLM_GPU_MEMORY_UTILIZATION="0.85"\n'+GLM_CONTRACT_NO_SHARE="${GLM_CONTRACT%"$share_line"}"+[[ "$GLM_CONTRACT_NO_SHARE" != "$GLM_CONTRACT" ]] || fail "the contract fixture must end with the share line"+name=rerun-vllm-own-share; mkdir -p "$tmp/$name/install"+retained 'LLM_BACKEND=vllm' 'LLM_MODEL=glm-5.3-flash-nvfp4' "VLLM_MODEL_DIR=$GLM_DIR" \+ 'VLLM_SERVED_MODEL=GLM-5.3-Flash-NVFP4' 'VLLM_GPU_UUID=GPU-aaaaaaaa-gb300'+decide rerun-vllm-own-share "export FAKE_GPU_ROWS='GPU-ffffffff-rtx6000, 12.0, 97887\n'" \+ "export FAKE_PRESERVE_OUT='$GLM_CONTRACT_NO_SHARE'"+[[ "$(rc_of rerun-vllm-own-share)" == 0 && "$(value rerun-vllm-own-share MODEL)" == glm-5.3-flash-nvfp4 \+ && -z "$(value rerun-vllm-own-share VLLM_SHARE)" \+ && "$(value rerun-vllm-own-share RECOMMENDED)" == "$p6k_model" ]] \+ || fail "a kept vLLM model must not take the fresh pick's memory share: $(cat "$tmp/rerun-vllm-own-share/out" "$tmp/rerun-vllm-own-share/err")"+pass "a kept vLLM model keeps only its own memory share, never the fresh pick's"+name=rerun-vllm-incomplete; mkdir -p "$tmp/$name/install"; vllm_installdecide rerun-vllm-incomplete 'export FAKE_PRESERVE_STATUS=2' \"export FAKE_PRESERVE_STDERR='This installation runs its model on the vLLM engine, but its saved model is not a complete vLLM catalog snapshot for the GPU it names; refusing to replace it.'"diff --git a/ods/tests/test_host_agent_vllm.py b/ods/tests/test_host_agent_vllm.pyindex 06236f4dc..1a5916209 100644--- a/ods/tests/test_host_agent_vllm.py+++ b/ods/tests/test_host_agent_vllm.py@@ -266,6 +266,17 @@ def test_a_llama_install_refuses_a_vllm_catalog_model(monkeypatch, tmp_path):assert body["code"] == "vllm_model_installer_only"assert body["requestedModelId"] == "glm-5.3-flash-nvfp4-vllm"assert body["error"].startswith("GLM 5.3 Flash NVFP4 (vLLM) runs only on the vLLM engine")+ # The entry's own GPU requirement, not one wording for every vLLM model.+ assert body["hint"].startswith("It needs one GPU with compute capability 10.0 or 10.3 and at least 244736 MiB.")+++def test_the_refusal_names_each_entrys_own_gpu_requirement():+ rtx = next(model for model in json.loads((ROOT / "config" / "model-library.json").read_text(encoding="utf-8"))["models"]+ if model["id"] == "qwen3.6-35b-a3b-nvfp4-vllm")+ assert agent._vllm_model_refusal(rtx)["hint"].startswith(+ "It needs one GPU with compute capability 12.0 and at least 51200 MiB.")+ broken = {**rtx, "min_gpu_mib": 0}+ assert agent._vllm_model_refusal(broken)["hint"].startswith("It needs an NVIDIA GPU the vLLM engine supports.")def test_llama_server_is_never_named_to_compose_on_a_vllm_install(monkeypatch, tmp_path):diff --git a/ods/tests/test_model_library_native_context.py b/ods/tests/test_model_library_native_context.pyindex a7a7c25e6..44af66e27 100644--- a/ods/tests/test_model_library_native_context.py+++ b/ods/tests/test_model_library_native_context.py@@ -87,6 +87,7 @@ NATIVE_CONTEXT = {# vLLM entries: config.json text_config.max_position_embeddings (2026-10-27)."qwen3.8-flash-next-nvfp4-vllm": 262144,"glm-5.3-flash-nvfp4-vllm": 1048576,+ "qwen3.6-35b-a3b-nvfp4-vllm": 262144,}diff --git a/ods/tests/test_select_model_vllm.py b/ods/tests/test_select_model_vllm.pyindex 6d8200f69..a711b5913 100644--- a/ods/tests/test_select_model_vllm.py+++ b/ods/tests/test_select_model_vllm.py@@ -35,6 +35,7 @@ SIMULATOR = ROOT / "scripts" / "simulate-model-selection.py"# GB300 and RTX PRO 6000 memory: the DGX Station Workstation06 (2026-10-27).GB300 = "GPU-test-gb300, 10.3, 256703"RTX_PRO_6000 = "GPU-test-rtx-pro-6000, 12.0, 97887"+RTX_5090 = "GPU-test-rtx-5090, 12.0, 32607"B200 = "GPU-test-b200, 10.0, 183359"H200 = "GPU-test-h200, 9.0, 143771"# The installer's selector arguments on the DGX Station, plus the vLLM runtime.@@ -45,7 +46,7 @@ VLLM_ENV_KEYS = {"LLM_MODEL", "GGUF_FILE", "GGUF_URL", "GGUF_SHA256", "MAX_CONTEXT", "LLM_MODEL_SIZE_MB","MODEL_RECOMMENDATION_SOURCE", "MODEL_RECOMMENDATION_POLICY", "MODEL_RECOMMENDATION_CONFIDENCE","MODEL_RECOMMENDATION_REASON", "MODEL_RECOMMENDED_ALTERNATIVES", "PIXEL_AGENT_MODEL_READY",- "VLLM_MODEL_DIR", "VLLM_SERVED_MODEL", "VLLM_ARGS", "VLLM_GPU_UUID",+ "VLLM_MODEL_DIR", "VLLM_SERVED_MODEL", "VLLM_ARGS", "VLLM_GPU_UUID", "VLLM_GPU_MEMORY_UTILIZATION",}@@ -89,9 +90,13 @@ def _vllm_entries() -> list[dict]:return [model for model in _catalog()["models"] if model.get("runtime") == "vllm"]+def _fits(capability: str, memory_mib: int) -> list[dict]:+ return [model for model in _vllm_entries()+ if capability in model["compute_capability"] and model["min_gpu_mib"] <= memory_mib]++def _best_fitting(capability: str, memory_mib: int) -> tuple[dict, list[dict]]:- fitting = [model for model in _vllm_entries()- if capability in model["compute_capability"] and model["min_gpu_mib"] <= memory_mib]+ fitting = _fits(capability, memory_mib)assert fittingreturn max(fitting, key=lambda model: model["selection"]["vllm"]), fitting@@ -119,16 +124,40 @@ def test_dgx_station_gets_the_top_priority_vllm_model_on_its_gb300():assert env["PIXEL_AGENT_MODEL_READY"] == ("true" if pixel == "verified" else "false")assert env["MODEL_RECOMMENDATION_POLICY"] == "vllm-gpu-fit-v1"assert env["MODEL_RECOMMENDATION_SOURCE"] == "catalog_vllm_pre_download"+ # An entry without its own share still sends vLLM's 0.9, so a newly+ # picked model never inherits an earlier model's .env share.+ assert "gpu_memory_utilization" not in best and env["VLLM_GPU_MEMORY_UTILIZATION"] == "0.9"assert "GPU-test-gb300 (compute capability 10.3, 256703 MiB)" in env["MODEL_RECOMMENDATION_REASON"]+ # The alternatives are every entry that fits either GPU (the RTX PRO 6000's own one too).alternatives = [item.split(":")[0] for item in env["MODEL_RECOMMENDED_ALTERNATIVES"].split(";")]- assert alternatives[0] == best["id"] and set(alternatives) == {model["id"] for model in fitting}+ either = {model["id"] for model in fitting + _fits("12.0", 97887)}+ assert alternatives[0] == best["id"] and set(alternatives) == either-def test_both_entries_fit_the_gb300_and_neither_the_rtx_pro_6000():+def test_the_gb300_fits_the_datacenter_entries_and_the_rtx_pro_6000_its_own():_, fitting = _best_fitting("10.3", 256703)assert {model["id"] for model in fitting} == {"qwen3.8-flash-next-nvfp4-vllm", "glm-5.3-flash-nvfp4-vllm"}- status, out, _ = select(*VLLM, *_gpu_args(RTX_PRO_6000), "--env")- assert (status, out) == (2, "")+ assert [model["id"] for model in _fits("12.0", 97887)] == ["qwen3.6-35b-a3b-nvfp4-vllm"]+ assert _fits("12.0", 32607) == []+++def test_a_lone_rtx_pro_6000_gets_its_class_entry_at_half_the_card():+ # A workstation's only GPU also runs ComfyUI and voice, so this entry+ # takes half of it; the selector hands that share to the installer.+ status, out, err = select(*VLLM, *_gpu_args(RTX_PRO_6000), "--env")++ assert status == 0, err+ env = env_values(out)+ best, _ = _best_fitting("12.0", 97887)+ assert set(env) == VLLM_ENV_KEYS+ assert env["LLM_MODEL"] == best["llm_model_name"] == "qwen3.6-35b-a3b-nvfp4"+ assert env["VLLM_GPU_UUID"] == "GPU-test-rtx-pro-6000"+ assert env["VLLM_GPU_MEMORY_UTILIZATION"] == "0.5" == f"{best['gpu_memory_utilization']:g}"+ assert shlex.split(env["VLLM_ARGS"]) == best["vllm_args"]+ assert "--gpu-memory-utilization" not in env["VLLM_ARGS"]+ reason = env["MODEL_RECOMMENDATION_REASON"]+ # 97887 MiB * 0.5 / 1024 - 21.85 GiB of weights.+ assert "leave about 25.9 GiB of vLLM's 0.50 share of that GPU" in reason, reasondef test_a_b200_gets_only_what_fits_its_memory():@@ -143,7 +172,8 @@ def test_a_b200_gets_only_what_fits_its_memory():@pytest.mark.parametrize("rows,expected", [- ((RTX_PRO_6000,), "needs one GPU with compute capability 10.0 or 10.3 and at least"),+ ((RTX_5090,), "needs one GPU with compute capability 10.0 or 10.3 and at least"),+ ((RTX_5090,), "needs one GPU with compute capability 12.0 and at least 51200 MiB"),((H200,), "GPU-test-h200 (compute capability 9.0, 143771 MiB)"),((), "pass each one as --gpu"),(("GPU-test-gb10, 12.1, [N/A]",), "is not UUID,COMPUTE_CAP,MEMORY_MIB"),@@ -158,7 +188,7 @@ def test_no_fitting_gpu_fails_instead_of_falling_back(rows, expected):def test_the_exit_status_reaches_the_installer():result = subprocess.run(- [sys.executable, str(SELECTOR), "--catalog", str(CATALOG), *VLLM, *_gpu_args(RTX_PRO_6000), "--env"],+ [sys.executable, str(SELECTOR), "--catalog", str(CATALOG), *VLLM, *_gpu_args(RTX_5090), "--env"],capture_output=True, text=True, check=False,)assert (result.returncode, result.stdout) == (2, "")@@ -269,7 +299,7 @@ def _envelope_args(envelope: dict, simulator) -> list[str]:def test_gguf_selection_is_byte_identical_with_or_without_vllm_entries(tmp_path):catalog = _catalog()- assert len(_vllm_entries()) == 2+ assert len(_vllm_entries()) == 3without = _write_catalog(tmp_path, {**catalog, "models": [model for model in catalog["models"] if model.get("runtime") != "vllm"],})