ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.
Python ai-agentsamdcomfyuidockerllama-cppllmlocal-ain8nnvidiaopen-webuiragself-hostedspeech-to-textstrix-halotext-to-speechworkflow-automation
fix(installer): give vLLM only a GPU with its share free on a fresh install (#8306)
Mike Bradleybffd6064ba3b
diff --git a/ods/extensions/services/dashboard-api/model_vllm.py b/ods/extensions/services/dashboard-api/model_vllm.pyindex 2d62bcb1b..2df9bd800 100644--- a/ods/extensions/services/dashboard-api/model_vllm.py+++ b/ods/extensions/services/dashboard-api/model_vllm.py@@ -12,7 +12,10 @@ the vLLM engine (``.env`` ``LLM_BACKEND=vllm``), not llama.cpp. They carry no``context_length`` (equal to ``--max-model-len``) and ``vllm_args``;* ``min_gpu_mib`` and ``compute_capability``: the one GPU vLLM runs on needsat least this much memory (nvidia-smi ``memory.total``) and one of these- compute capabilities;+ compute capabilities. When a GPU's free memory is known (a fresh install+ passes nvidia-smi ``memory.free``), it fits only if vLLM's share of its+ total memory is free, plus START_MARGIN_MIB for the CUDA context vLLM+ creates before it checks: vLLM refuses to start with less;* optional ``gpu_memory_utilization``: the share of that GPU vLLMpreallocates (``.env`` VLLM_GPU_MEMORY_UTILIZATION, the vllm-serverservice's ``--gpu-memory-utilization``; vLLM's 0.90 when absent). An entry@@ -43,6 +46,7 @@ model_memory.py.from __future__ import annotations+import mathimport reimport shlexfrom typing import Any, Iterable, NamedTuple@@ -59,9 +63,16 @@ LLAMA_MEMORY_CLASSES = ("discrete", "unified", "cpu")COMPOSE_FLAGS = frozenset({"--model", "--served-model-name", "--host", "--port", "--gpu-memory-utilization"})# The range an entry's gpu_memory_utilization may take.MIN_MEMORY_SHARE, MAX_MEMORY_SHARE = 0.10, 0.95+# vLLM's own --gpu-memory-utilization default, for an entry that sets none.+DEFAULT_MEMORY_SHARE = 0.90+# vLLM checks free memory after creating its CUDA context, which nvidia-smi+# has not seen yet: a GPU needs this much free beyond vLLM's share.+START_MARGIN_MIB = 1024# A file downloads under <name><PARTIAL_SUFFIX> until it is verified.PARTIAL_SUFFIX = ".partial"GPU_QUERY = "nvidia-smi --query-gpu=uuid,compute_cap,memory.total --format=csv,noheader,nounits"+# The installer's fresh-install query: GPU_QUERY plus each GPU's free memory.+GPU_FREE_QUERY = "nvidia-smi --query-gpu=uuid,compute_cap,memory.total,memory.free --format=csv,noheader,nounits"_HF_REPO = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*")_HEX40 = re.compile(r"[0-9a-f]{40}")@@ -85,6 +96,8 @@ class Gpu(NamedTuple):uuid: strcompute_capability: strmemory_mib: int+ # nvidia-smi memory.free when the caller passed it; None means unknown.+ free_mib: int | None = Noneclass Candidate(NamedTuple):@@ -306,24 +319,51 @@ def require_valid(model: dict[str, Any]) -> dict[str, Any]:def parse_gpu(text: str) -> Gpu:- """One row of :data:`GPU_QUERY` output: ``UUID, COMPUTE_CAP, MEMORY_MIB``."""+ """One row of :data:`GPU_QUERY` output, or of :data:`GPU_FREE_QUERY` (``, FREE_MIB``).++ A free field that is not a number ([N/A]) leaves that GPU's free memory+ unknown, as a three-field row does.+ """parts = [part.strip() for part in str(text).split(",")]- if len(parts) == 3:- uuid, capability, memory = parts+ if len(parts) in (3, 4):+ uuid, capability, memory = parts[:3]size = _GPU_MEMORY.fullmatch(memory)+ free = _GPU_MEMORY.fullmatch(parts[3]) if len(parts) == 4 else Noneif _GPU_UUID.fullmatch(uuid) and _COMPUTE_CAPABILITY.fullmatch(capability) and size:- return Gpu(uuid, capability, int(size.group(1)))- raise ValueError(f"{text!r} is not UUID,COMPUTE_CAP,MEMORY_MIB ({GPU_QUERY})")+ return Gpu(uuid, capability, int(size.group(1)), int(free.group(1)) if free else None)+ raise ValueError(f"{text!r} is not UUID,COMPUTE_CAP,MEMORY_MIB[,FREE_MIB] ({GPU_FREE_QUERY})")-def gpu_fits(model: dict[str, Any], gpu: Gpu) -> bool:+def memory_share(model: dict[str, Any]) -> float:+ """The share of its GPU a vLLM entry preallocates: its ``gpu_memory_utilization``, else 0.90."""+ return float(model.get("gpu_memory_utilization") or DEFAULT_MEMORY_SHARE)+++def claim_mib(model: dict[str, Any], gpu: Gpu) -> int:+ """The memory vLLM claims on ``gpu`` at start: its share of the GPU's total."""+ return math.ceil(memory_share(model) * gpu.memory_mib)+++def free_needed_mib(model: dict[str, Any], gpu: Gpu) -> int:+ """The free memory ``gpu`` needs for vLLM to start: its claim plus its CUDA context."""+ return claim_mib(model, gpu) + START_MARGIN_MIB+++def gpu_hardware_fits(model: dict[str, Any], gpu: Gpu) -> bool:return gpu.compute_capability in model["compute_capability"] and gpu.memory_mib >= model["min_gpu_mib"]+def gpu_fits(model: dict[str, Any], gpu: Gpu) -> bool:+ """The GPU suits ``model`` and, when its free memory is known, has what vLLM needs to start free."""+ return gpu_hardware_fits(model, gpu) and (gpu.free_mib is None or gpu.free_mib >= free_needed_mib(model, gpu))++def best_gpu(model: dict[str, Any], gpus: Iterable[Gpu]) -> Gpu | None:- """The largest GPU ``model`` fits on (the first listed on a tie), or None."""+ """The largest GPU ``model`` fits on, then the one with the most free memory (the first listed on a tie), or None."""fitting = [gpu for gpu in gpus if gpu_fits(model, gpu)]- return max(fitting, key=lambda gpu: gpu.memory_mib) if fitting else None+ if not fitting:+ return None+ return max(fitting, key=lambda gpu: (gpu.memory_mib, gpu.memory_mib if gpu.free_mib is None else gpu.free_mib))def requirement_text(model: dict[str, Any]) -> str:diff --git a/ods/installers/lib/detection.sh b/ods/installers/lib/detection.shindex bd6b7e546..d77d3d42c 100755--- a/ods/installers/lib/detection.sh+++ b/ods/installers/lib/detection.sh@@ -11,7 +11,7 @@# ENABLE_RAG, ENABLE_HERMES (all used by fix_nvidia_secure_boot),# log/warn/ai/ai_ok/ai_warn/ai_bad helpers# Provides: detect_gpu(), load_capability_profile(), ods_is_wsl_host(),-# ods_windows_host_port_in_use(),+# ods_windows_host_port_in_use(), ods_gpu_memory_in_use_report(),# normalize_profile_tier(), tier_rank(), load_backend_contract(),# fix_nvidia_secure_boot(), MIN_DRIVER_VERSION# Side-effect var on Jetson: JETSON_L4T_RELEASE (e.g. "R36.4.0")@@ -25,6 +25,22 @@# Safe env loading (no eval) for script output KEY="value" lines[[ -f "${SCRIPT_DIR:-}/lib/safe-env.sh" ]] && . "${SCRIPT_DIR}/lib/safe-env.sh"+# ods_gpu_memory_in_use_report+# Reads `nvidia-smi --query-gpu=index,memory.total,memory.used+# --format=csv,noheader,nounits` rows on stdin and prints one line per GPU+# where programs already hold more than max(4 GiB, 10% of the card), such as+# another model server: "GPU <index>: <used> of <total> GiB in use". A desktop+# session stays under that floor; rows without numbers ([N/A] on+# unified-memory GPUs) are skipped.+ods_gpu_memory_in_use_report() {+ awk -F', *' '+ $2 ~ /^[0-9]+$/ && $3 ~ /^[0-9]+$/ && $2 > 0 {+ floor = $2 * 0.10+ if (floor < 4096) floor = 4096+ if ($3 + 0 > floor) printf "GPU %s: %.0f of %.0f GiB in use\n", $1, $3 / 1024, $2 / 1024+ }'+}+# WSL2 forwards Docker-published ports through the Windows host. A port can# therefore look free to lsof/ss inside Linux while Docker Desktop still# refuses the bind because a native Windows process already owns it. Query thediff --git a/ods/installers/lib/model-selector.sh b/ods/installers/lib/model-selector.shindex 6316094cc..66210d619 100644--- a/ods/installers/lib/model-selector.sh+++ b/ods/installers/lib/model-selector.sh@@ -61,10 +61,13 @@ ods_run_catalog_selector() {# ods_run_vllm_selector PYTHON [GPU_ROW...]# The vLLM engine's pick: one GPU_ROW per line of `nvidia-smi-# --query-gpu=uuid,compute_cap,memory.total --format=csv,noheader,nounits`.-# Chooses among the catalog's vLLM entries only, never a GGUF, and prints-# shell assignments with the VLLM_* keys. Exit 2: no vLLM model fits. The fit-# comes from the GPU rows alone, so the llama.cpp memory inputs are not passed.+# --query-gpu=uuid,compute_cap,memory.total[,memory.free]+# --format=csv,noheader,nounits`. Chooses among the catalog's vLLM entries+# only, never a GGUF, and prints shell assignments with the VLLM_* keys. Exit+# 2: no vLLM model fits; exit 4: one fits the hardware, but other programs+# hold the free memory vLLM needs at start (rows with memory.free only). The+# fit comes from the GPU rows alone, so the llama.cpp memory inputs are not+# passed.ods_run_vllm_selector() {local python="$1" rowshiftdiff --git a/ods/installers/phases/02-detection.sh b/ods/installers/phases/02-detection.shindex 49c5dfa49..25b832d93 100755--- a/ods/installers/phases/02-detection.sh+++ b/ods/installers/phases/02-detection.sh@@ -589,14 +589,17 @@ fi# path below unchanged, without querying the GPUs for vLLM. --llm-backend vllm# and a rerun of a vLLM installation (_phase02_vllm_kept) stop when vLLM# cannot run or no vLLM model fits; --llm-backend llama-server never considers-# vLLM. On the vLLM path every selector failure other than "nothing fits"-# (exit 2) stops the install: there is no tier-map fallback. The pick+# vLLM. A fresh install also reads each GPU's free memory: when other programs+# hold what vLLM needs at start (exit 4), it keeps llama.cpp and says why, and+# --llm-backend vllm stops. On the vLLM path every other selector failure+# than "nothing fits" (exit 2) stops the install: there is no tier-map+# fallback. The pick# (VLLM_SELECTOR_ENV) is loaded after the tier map, which it replaces; on a# rerun of a vLLM installation it is the recommendation, and the installed# vLLM model stays active (_phase02_keep_vllm_model)._phase02_choose_llm_backend() {local requested="${LLM_BACKEND_REQUESTED:-}" external="${EXTERNAL_LLM_URL:-}"- local existing=false retained="" required=false wsl=false blocker python rows status=0+ local existing=false retained="" required=false wsl=false blocker python rows query status=0local env_text error_text error_filelocal -a gpu_rows=()VLLM_SELECTOR_ENV=""@@ -639,7 +642,12 @@ _phase02_choose_llm_backend() {log "Model server: llama.cpp (Python is unavailable for the vLLM model catalog)"return 0fi- if ! rows="$(nvidia-smi --query-gpu=uuid,compute_cap,memory.total --format=csv,noheader,nounits 2>>"$LOG_FILE")"; then+ # vLLM refuses to start unless its share of the GPU is free. A fresh+ # install's GPUs hold no ODS engine yet, so their free memory is what vLLM+ # can claim; on a rerun this installation's own engine may hold it.+ query="uuid,compute_cap,memory.total"+ [[ "$existing" == true ]] || query+=",memory.free"+ if ! rows="$(nvidia-smi --query-gpu="$query" --format=csv,noheader,nounits 2>>"$LOG_FILE")"; then[[ "$required" != true ]] || _phase02_vllm_refused "nvidia-smi could not list this host's GPUs (details in $LOG_FILE)"log "Model server: llama.cpp (nvidia-smi could not list the GPUs for the vLLM engine)"return 0@@ -659,6 +667,12 @@ _phase02_choose_llm_backend() {[[ "$required" != true ]] || _phase02_vllm_refused "${error_text#error: }"log "Model server: llama.cpp (${error_text#error: })";;+ 4)+ [[ "$required" != true ]] || _phase02_vllm_refused "${error_text#error: }"+ ai_warn "ODS uses llama.cpp, not the vLLM engine: ${error_text#error: }."+ ai " Once that memory is free, rerun the installer with --llm-backend vllm to use the vLLM engine."+ log "Model server: llama.cpp (${error_text#error: })"+ ;;*)error "The vLLM model selector stopped (exit $status): ${error_text:-no details}. The ODS files may be incomplete: download ODS again and rerun the installer. Need help? Ask on the ODS Discord: ${ODS_HELP_DISCORD_URL:-https://discord.gg/4ntNp9MAwC}"exit 1diff --git a/ods/installers/phases/04-requirements.sh b/ods/installers/phases/04-requirements.shindex a719e4af2..ad0ac4047 100755--- a/ods/installers/phases/04-requirements.sh+++ b/ods/installers/phases/04-requirements.sh@@ -9,8 +9,9 @@# GPU_VRAM, GPU_NAME, GPU_COUNT, INTERACTIVE, DRY_RUN,# PREFLIGHT_REPORT_FILE, CAP_PLATFORM_ID, CAP_COMPOSE_OVERLAYS,# ENABLE_VOICE, ENABLE_WORKFLOWS, ENABLE_RAG, ENABLE_QDRANT,-# INSTALL_DIR, external_llm_env_value(),-# tier_rank(), chapter(), ai_ok(), ai_bad(), ai_warn(), log(), warn()+# INSTALL_DIR, EXTERNAL_LLM_URL, ODS_GATEWAY_ONLY, external_llm_env_value(),+# ods_gpu_memory_in_use_report() (installers/lib/detection.sh),+# tier_rank(), chapter(), ai(), ai_ok(), ai_bad(), ai_warn(), log(), warn()# Provides: REQUIREMENTS_MET, TIER_RANK, WHISPER_PORT (only when the# generated default 9000 moves; phase 06 persists it)#@@ -160,6 +161,30 @@ if [[ -z "${EXTERNAL_LLM_URL:-}" ]] && ! ods_native_llm_requested \fifi+# A fresh install sizes its model and services for each GPU's whole memory.+# Say so when other programs already hold much of it, before anything+# downloads. A rerun's own model holds memory, and an external, host-native+# or cloud model runs nothing on these GPUs, so only a fresh local install+# checks.+if [[ "${GPU_BACKEND:-}" == "nvidia" && ! -f "$INSTALL_DIR/.env" && -z "${EXTERNAL_LLM_URL:-}" \+ && "${ODS_GATEWAY_ONLY:-false}" != true && "${TIER:-}" != "CLOUD" ]] \+ && ! ods_native_llm_requested && command -v nvidia-smi >/dev/null 2>&1 \+ && declare -F ods_gpu_memory_in_use_report >/dev/null 2>&1; then+ # A warning only: report what nvidia-smi printed even when it then fails.+ _gpu_memory_rows="$(nvidia-smi --query-gpu=index,memory.total,memory.used \+ --format=csv,noheader,nounits 2>>"$LOG_FILE")" || log "nvidia-smi exited $? while listing GPU memory"+ _gpu_memory_in_use="$(printf '%s\n' "$_gpu_memory_rows" | ods_gpu_memory_in_use_report)"+ if [[ -n "$_gpu_memory_in_use" ]]; then+ ai_warn "Other programs already use GPU memory:"+ while IFS= read -r _gpu_line; do+ ai " $_gpu_line"+ done <<< "$_gpu_memory_in_use"+ ai " ODS sizes its model and services for each GPU's whole memory: stop those programs before ODS starts, or its model may not load."+ log "GPU memory in use before install: ${_gpu_memory_in_use//$'\n'/; }"+ fi+ unset _gpu_memory_rows _gpu_memory_in_use _gpu_line+fi+# Port conflict detection with process details# Warn-once guard for missing port-check tools_port_check_warned=falsediff --git a/ods/scripts/select-model.py b/ods/scripts/select-model.pyindex 8851bc315..aa80c5427 100644--- a/ods/scripts/select-model.py+++ b/ods/scripts/select-model.py@@ -50,11 +50,17 @@ from model_selection import ( # noqa: E402from model_selection import pixel_agent_ready as _pixel_agent_ready # noqa: E402from model_selection import usable_memory_gb as _usable_memory_gb # noqa: E402from model_vllm import ( # noqa: E402+ GPU_FREE_QUERY,GPU_QUERY,+ START_MARGIN_MIB,Candidate as VllmCandidate,Gpu,+ free_needed_mib,+ gpu_fits,+ gpu_hardware_fits,is_vllm_entry,manifest_bytes,+ memory_share,model_dir_name,parse_gpu,rank_entries,@@ -79,6 +85,9 @@ VLLM_POLICY = "vllm-gpu-fit-v1"VLLM_SOURCE = "catalog_vllm_pre_download"EXIT_NO_FIT = 2EXIT_CHECK_FIT_FAILED = 3+# --llm-runtime vllm: an entry fits the GPUs, but other programs hold the+# memory vLLM claims at start (--gpu rows with free memory).+EXIT_GPU_BUSY = 4def normalize_profile(value: str | None) -> str:@@ -497,13 +506,36 @@ def _vllm_no_fit(pool: list[dict[str, Any]], gpus: list[Gpu], args: argparse.Namif not gpus:return f"--llm-runtime vllm needs the host's NVIDIA GPUs: pass each one as --gpu ({GPU_QUERY})"seen = ", ".join(- f"{gpu.uuid} (compute capability {gpu.compute_capability}, {gpu.memory_mib} MiB)" for gpu in gpus+ f"{gpu.uuid} (compute capability {gpu.compute_capability}, {gpu.memory_mib} MiB"+ f"{'' if gpu.free_mib is None else f', {gpu.free_mib} MiB free'})" for gpu in gpus)floor = f" at the {args.min_context} context floor" if args.require_min_context and args.min_context else ""needs = "; ".join(f"{model['id']} needs {requirement_text(model)}" for model in pool)return f"no vLLM catalog model fits this host{floor}. GPUs: {seen}. {needs}"+def _vllm_busy(candidate: VllmCandidate, gpus: list[Gpu]) -> str:+ """Why the best entry for the GPUs' hardware cannot start: their memory is in use."""+ model = candidate.model+ freest = max((gpu for gpu in gpus if gpu_hardware_fits(model, gpu)),+ key=lambda gpu: gpu.free_mib or 0)+ return (+ f"other programs use the GPU memory the vLLM engine needs: {model['name']} needs "+ f"{free_needed_mib(model, freest)} MiB free when it starts ({memory_share(model):.2f} of the GPU "+ f"plus {START_MARGIN_MIB} MiB for its CUDA context), and the freest GPU it fits, "+ f"{freest.uuid}, has {freest.free_mib} MiB free"+ )+++def _vllm_busy_note(hardware_pick: VllmCandidate, gpus: list[Gpu]) -> str:+ """Names the entry the GPUs' hardware alone would get when its GPU's memory is in use."""+ gpu = next(gpu for gpu in gpus if gpu.uuid == hardware_pick.gpu.uuid)+ if gpu_fits(hardware_pick.model, gpu):+ return ""+ return (f" {hardware_pick.model['name']} would run on GPU {gpu.uuid}, but other programs use "+ f"the memory it needs there.")++def _vllm_payload(ranked: list[VllmCandidate], reason: str, min_context: int, host_arch: str) -> dict[str, Any]:selected = ranked[0]model, gpu = selected.model, selected.gpu@@ -513,7 +545,7 @@ def _vllm_payload(ranked: list[VllmCandidate], reason: str, min_context: int, ho"confidence": "high","runtime": "vllm","host_arch": normalize_host_arch(host_arch),- "gpu": gpu._asdict(),+ "gpu": {key: value for key, value in gpu._asdict().items() if value is not None},"memory_capacity_gb": round(gpu.memory_mib / 1024.0, 1),"memory_label": "GPU memory","memory_class": "discrete",@@ -590,10 +622,17 @@ def _vllm_main(args: argparse.Namespace) -> int:min_context = max(int(args.min_context or 0), 0)ranked = rank_entries(pool, gpus, min_context=min_context, require_min_context=args.require_min_context,installable_only=args.installable_only)+ # The same ranking with free memory unknown: what the GPUs' hardware alone gets.+ by_hardware = rank_entries(pool, [gpu._replace(free_mib=None) for gpu in gpus], min_context=min_context,+ require_min_context=args.require_min_context,+ installable_only=args.installable_only)if not ranked:+ if by_hardware:+ print(f"error: {_vllm_busy(by_hardware[0], gpus)}", file=sys.stderr)+ return EXIT_GPU_BUSYprint(f"error: {_vllm_no_fit(pool, gpus, args)}", file=sys.stderr)return EXIT_NO_FIT- reason = _vllm_reason(ranked[0], min_context)+ reason = _vllm_reason(ranked[0], min_context) + _vllm_busy_note(by_hardware[0], gpus)if not args.env:print(json.dumps(_vllm_payload(ranked, reason, min_context, args.host_arch), indent=2))return 0@@ -645,12 +684,16 @@ def main(argv: list[str] | None = None) -> int:"--llm-runtime", choices=("llama-server", "vllm"), default="llama-server",help="The model server (.env LLM_BACKEND). llama-server (default) picks a GGUF ""and never a vLLM entry; vllm picks only vLLM entries, for one --gpu, and "- "exits 2 when none fits. With vllm the fit comes from --gpu alone: --vram-mb, "- "--ram-gb, --memory-type, --tier, --profile and --max-size-mb are not read.",+ "exits 2 when none fits, or 4 when one fits the GPUs but other programs "+ "hold the memory vLLM claims at start. With vllm the fit comes from --gpu "+ "alone: --vram-mb, --ram-gb, --memory-type, --tier, --profile and "+ "--max-size-mb are not read.",)parser.add_argument(- "--gpu", action="append", default=[], metavar="UUID,COMPUTE_CAP,MEMORY_MIB",- help=f"One NVIDIA GPU, a row of `{GPU_QUERY}`; repeat per GPU. "+ "--gpu", action="append", default=[], metavar="UUID,COMPUTE_CAP,MEMORY_MIB[,FREE_MIB]",+ help=f"One NVIDIA GPU, a row of `{GPU_QUERY}` or of `{GPU_FREE_QUERY}`; repeat "+ "per GPU. With the free memory, a GPU fits only while vLLM can start there: "+ f"its share of the GPU's total memory plus {START_MARGIN_MIB} MiB free. ""Read only with --llm-runtime vllm.",)parser.add_argument("--env", action="store_true", help="print shell assignments")diff --git a/ods/tests/ci-suite.txt b/ods/tests/ci-suite.txtindex 423c885fd..1b10310ad 100644--- a/ods/tests/ci-suite.txt+++ b/ods/tests/ci-suite.txt@@ -64,6 +64,7 @@ tests/test-comfyui-cuda-cache-persisted.shtests/test-llama-cuda-cache-persisted.shtests/test-vllm-engine-service.shtests/test-vllm-installer-route.sh+tests/test-gpu-memory-in-use.shtests/test-vllm-installer-phases.shtests/test-cli-vllm-engine.shtests/test-doctor-vllm-engine.shdiff --git a/ods/tests/test-gpu-memory-in-use.sh b/ods/tests/test-gpu-memory-in-use.shnew file mode 100755index 000000000..5716c92e2--- /dev/null+++ b/ods/tests/test-gpu-memory-in-use.sh@@ -0,0 +1,112 @@+#!/usr/bin/env bash+# The requirements check (phase 04) warns a fresh local NVIDIA install when+# other programs already hold much of a GPU's memory, before anything+# downloads:+# - installers/lib/detection.sh ods_gpu_memory_in_use_report names each GPU+# where more than max(4 GiB, 10% of the card) is in use, and nothing for a+# desktop session or a unified-memory GPU that reports [N/A];+# - phase 04's block prints the warning on a fresh install, even when+# nvidia-smi fails after printing its rows, and stays quiet on a rerun+# (its own model holds memory), for an external, gateway-only,+# host-native or cloud model (nothing runs on these GPUs), on other+# backends and when nvidia-smi prints nothing.+# Needs no GPU.+set -euo pipefail++ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"+PHASE04="$ROOT_DIR/installers/phases/04-requirements.sh"+PASS=0+fail() { printf '[FAIL] %s\n' "$*" >&2; exit 1; }+pass() { printf '[PASS] %s\n' "$*"; PASS=$((PASS + 1)); }++tmp="$(mktemp -d)"+trap 'rm -rf "$tmp"' EXIT++# shellcheck source=../installers/lib/detection.sh+SCRIPT_DIR="$ROOT_DIR" source "$ROOT_DIR/installers/lib/detection.sh"++report() { printf '%b' "$1" | ods_gpu_memory_in_use_report; }++# tower2 (2026-10-27): another model server held about 82 GiB of each card.+[[ "$(report '0, 97887, 83973\n1, 97887, 83953\n')" == $'GPU 0: 82 of 96 GiB in use\nGPU 1: 82 of 96 GiB in use' ]] \+ || fail "busy GPUs must be named: $(report '0, 97887, 83973\n1, 97887, 83953\n')"+# A desktop session, and the 4 GiB floor on a 24 GB card.+[[ -z "$(report '0, 24564, 1200\n')" && -z "$(report '0, 24564, 4096\n')" \+ && "$(report '0, 24564, 4097\n')" == 'GPU 0: 4 of 24 GiB in use' ]] \+ || fail "a 24 GB card is busy only above 4 GiB in use"+# 10% of a 96 GB card is the floor there.+[[ -z "$(report '3, 97887, 9788\n')" && "$(report '3, 97887, 9789\n')" == 'GPU 3: 10 of 96 GiB in use' ]] \+ || fail "a 96 GB card is busy only above 10% in use"+[[ -z "$(report '0, [N/A], [N/A]\n')" && -z "$(report '0, 0, 0\n')" && -z "$(report '')" ]] \+ || fail "rows without memory numbers must be skipped"+pass "the report names GPUs with more than max(4 GiB, 10%) in use and skips desktops and [N/A] rows"++block="$(sed -n '/^# A fresh install sizes its model and services for each GPU.s whole memory\./,/^fi$/p' "$PHASE04")"+[[ -n "$block" ]] || fail "phase 04's GPU memory warning block was not found"++fakebin="$tmp/bin"+mkdir -p "$fakebin"+cat > "$fakebin/nvidia-smi" <<'EOF'+#!/usr/bin/env bash+printf '%s\n' "$*" >> "$FAKE_CALLS"+printf '%b' "$FAKE_ROWS"+exit "${FAKE_STATUS:-0}"+EOF+chmod +x "$fakebin/nvidia-smi"++# run NAME [statements...]: phase 04's block with the given inputs.+run() {+ local name="$1" dir="$tmp/$1" rc=0+ shift+ mkdir -p "$dir/install"+ : > "$dir/calls"+ {+ printf '%s\n' 'set -euo pipefail'+ printf 'SCRIPT_DIR=%q\nINSTALL_DIR=%q\nLOG_FILE=%q\n' "$ROOT_DIR" "$dir/install" "$dir/install.log"+ printf 'export PATH=%q FAKE_CALLS=%q\n' "$fakebin:$PATH" "$dir/calls"+ printf '%s\n' 'source "$SCRIPT_DIR/installers/lib/detection.sh"'+ printf '%s\n' 'ai() { printf "AI %s\n" "$*"; }' 'ai_warn() { printf "WARN %s\n" "$*"; }' \+ 'log() { printf "LOG %s\n" "$*" >> "$LOG_FILE"; }' \+ 'ods_native_llm_requested() { [[ -n "${NATIVE_LLM_BASE_URL:-}" ]]; }'+ printf '%s\n' 'GPU_BACKEND=nvidia TIER=NV_ULTRA EXTERNAL_LLM_URL="" ODS_GATEWAY_ONLY=false NATIVE_LLM_BASE_URL=""' \+ "export FAKE_ROWS='0, 97887, 83973\n1, 97887, 1200\n'"+ printf '%s\n' "$@" "$block" 'echo "DONE"'+ } > "$dir/run.bash"+ bash "$dir/run.bash" > "$dir/out" 2> "$dir/err" || rc=$?+ [[ "$rc" == 0 ]] || fail "$name: the block failed (rc=$rc): $(cat "$dir/err")"+}+warned() {+ [[ "$(cat "$tmp/$1/out")" == "WARN Other programs already use GPU memory:+AI GPU 0: 82 of 96 GiB in use+AI ODS sizes its model and services for each GPU's whole memory: stop those programs before ODS starts, or its model may not load.+DONE" ]]+}+quiet() { [[ "$(cat "$tmp/$1/out")" == DONE ]]; }++run fresh+warned fresh || fail "a fresh install must name the busy GPU and what to do: $(cat "$tmp/fresh/out")"+grep -qx -- '--query-gpu=index,memory.total,memory.used --format=csv,noheader,nounits' "$tmp/fresh/calls" \+ || fail "the warning must read index, total and used memory: $(cat "$tmp/fresh/calls")"+grep -qx 'LOG GPU memory in use before install: GPU 0: 82 of 96 GiB in use' "$tmp/fresh/install.log" \+ || fail "the warning must reach the install log"+run partial 'export FAKE_STATUS=15'+warned partial && grep -qx 'LOG nvidia-smi exited 15 while listing GPU memory' "$tmp/partial/install.log" \+ || fail "rows printed before an nvidia-smi failure must still be reported: $(cat "$tmp/partial/out")"+pass "a fresh local NVIDIA install names the busy GPUs before anything downloads, even when nvidia-smi then fails"++run idle "export FAKE_ROWS='0, 97887, 600\n'"+quiet idle || fail "idle GPUs must not warn: $(cat "$tmp/idle/out")"+run smi-empty "export FAKE_ROWS='' FAKE_STATUS=9"+quiet smi-empty || fail "an nvidia-smi that prints nothing leaves nothing to report: $(cat "$tmp/smi-empty/out")"+run rerun 'printf "LLM_BACKEND=vllm\n" > "$INSTALL_DIR/.env"'+quiet rerun && [[ ! -s "$tmp/rerun/calls" ]] \+ || fail "a rerun must not judge GPUs its own model holds: $(cat "$tmp/rerun/out" "$tmp/rerun/calls")"+for route in 'EXTERNAL_LLM_URL=http://127.0.0.1:11434' 'ODS_GATEWAY_ONLY=true' \+ 'NATIVE_LLM_BASE_URL=http://localhost:8080' 'TIER=CLOUD' 'GPU_BACKEND=amd'; do+ name="route-${route%%=*}"+ run "$name" "$route"+ quiet "$name" && [[ ! -s "$tmp/$name/calls" ]] || fail "$route runs no model on these GPUs, so it must not warn"+done+pass "idle GPUs, an empty nvidia-smi, a rerun, external, gateway-only, host-native, cloud and AMD stay quiet"++printf '\nResult: %s passed, 0 failed\n' "$PASS"diff --git a/ods/tests/test-vllm-installer-route.sh b/ods/tests/test-vllm-installer-route.shindex 8b767cbd3..7241e776c 100644--- a/ods/tests/test-vllm-installer-route.sh+++ b/ods/tests/test-vllm-installer-route.sh@@ -159,7 +159,12 @@ cat > "$fakebin/nvidia-smi" <<'EOF'#!/usr/bin/env bashprintf '%s\n' "$*" >> "$FAKE_NVIDIA_SMI_CALLS"[[ "${FAKE_NVIDIA_SMI_FAIL:-false}" != true ]] || { echo "NVIDIA-SMI has failed" >&2; exit 9; }-printf '%b' "$FAKE_GPU_ROWS"+# A row's fourth field is memory.free: print it only when it is queried.+if [[ "$*" == *memory.free* ]]; then+ printf '%b' "$FAKE_GPU_ROWS"+else+ printf '%b' "$FAKE_GPU_ROWS" | sed 's/^\([^,]*,[^,]*,[^,]*\),.*$/\1/'+fiEOF# Python for phase 02: scripts/preserve-active-model.py answers from the# FAKE_PRESERVE_* settings (its vLLM contract lives on the runtime side);@@ -203,7 +208,9 @@ source "$SCRIPT_DIR/installers/lib/llm-backend.sh"ods_model_selector_python() { printf '%s' "$FAKE_PYTHON_SHIM"; }ods_native_llm_requested() { [[ -n "${NATIVE_LLM_BASE_URL:-}" ]]; }log() { printf 'LOG %s\n' "$*" >> "$LOG_FILE"; }+ai() { printf 'AI %s\n' "$*"; }ai_ok() { printf 'OK %s\n' "$*"; }+ai_warn() { printf 'WARN %s\n' "$*"; }error() { printf 'ERROR %s\n' "$*" >&2; }ods_is_wsl_host() { [[ "${FAKE_WSL:-false}" == true ]]; }uname() { printf '%s\n' "${FAKE_KERNEL:-Linux}"; }@@ -305,8 +312,9 @@ want_size="${want_size%$'\r'}" # a Windows Python ends the line with CRLF|| fail "the vLLM pick must replace the tier map's GGUF: $(cat "$tmp/dgx-auto/out")"[[ "$(value dgx-auto VLLM_IMAGE)" == "$pinned" ]] || fail "VLLM_IMAGE must be the locked pin"[[ "$(value dgx-auto VLLM_ARGS)" == *'--max-model-len 131072'* ]] || fail "VLLM_ARGS must carry the catalog arguments"-[[ "$(calls dgx-auto)" == 1 ]] && grep -qx -- '--query-gpu=uuid,compute_cap,memory.total --format=csv,noheader,nounits' "$tmp/dgx-auto/calls" \- || fail "the decision must read each GPU's uuid, compute capability and memory once"+[[ "$(calls dgx-auto)" == 1 ]] \+ && grep -qx -- '--query-gpu=uuid,compute_cap,memory.total,memory.free --format=csv,noheader,nounits' "$tmp/dgx-auto/calls" \+ || fail "a fresh install's decision must read each GPU's uuid, compute capability, total and free memory once"grep -qx "OK Model server: vLLM engine, serving $want_served on GPU GPU-aaaaaaaa-gb300" "$tmp/dgx-auto/out" \|| fail "the choice must be shown: $(cat "$tmp/dgx-auto/out")"pass "a fresh DGX Station install chooses vLLM on the GB300 with the catalog's top vLLM pick ($want_served)"@@ -349,6 +357,32 @@ p6k_share="${p6k_share%$'\r'}" # a Windows Python ends the line with CRLF|| fail "one RTX PRO 6000 must choose vLLM with its entry's memory share: $(cat "$tmp/pro6000-auto/out" "$tmp/pro6000-auto/err")"pass "a lone RTX PRO 6000 chooses vLLM ($p6k_model) at its entry's $p6k_share memory share"+# tower2 (2026-10-27): RTX PRO 6000s while another model server holds about+# 84 GB of each. vLLM refuses to start unless its share (half the card,+# 48944 MiB) is free beside its CUDA context (1 GiB), so a fresh install asks+# for free memory too and keeps llama.cpp, saying why, instead of downloading+# a vLLM model that cannot start.+BUSY_ROWS='GPU-ffffffff-rtx6000, 12.0, 97887, 13278\nGPU-ffffffff-rtx6001, 12.0, 97887, 13298\n'+decide pro6000-busy "export FAKE_GPU_ROWS='$BUSY_ROWS'"+expect_llama pro6000-busy+grep -q -- '--query-gpu=uuid,compute_cap,memory.total,memory.free ' "$tmp/pro6000-busy/calls" \+ || fail "a fresh install must ask nvidia-smi for free memory: $(cat "$tmp/pro6000-busy/calls")"+grep -qF 'WARN ODS uses llama.cpp, not the vLLM engine: other programs use the GPU memory the vLLM engine needs: ' \+ "$tmp/pro6000-busy/out" \+ && grep -qF 'needs 49968 MiB free when it starts (0.50 of the GPU plus 1024 MiB for its CUDA context), and the freest GPU it fits, GPU-ffffffff-rtx6001, has 13298 MiB free.' \+ "$tmp/pro6000-busy/out" \+ && grep -qF 'AI Once that memory is free, rerun the installer with --llm-backend vllm' "$tmp/pro6000-busy/out" \+ || fail "a busy GPU must be shown as the reason, with how to get vLLM later: $(cat "$tmp/pro6000-busy/out")"+grep -q 'LOG Model server: llama.cpp (other programs use the GPU memory the vLLM engine needs' "$tmp/pro6000-busy/install.log" \+ || fail "the busy-GPU reason must reach the install log"+decide pro6000-one-free "export FAKE_GPU_ROWS='GPU-ffffffff-rtx6000, 12.0, 97887, 13278\nGPU-ffffffff-rtx6001, 12.0, 97887, 96900\n'"+[[ "$(rc_of pro6000-one-free)" == 0 && "$(value pro6000-one-free BACKEND)" == vllm \+ && "$(value pro6000-one-free VLLM_GPU_UUID)" == GPU-ffffffff-rtx6001 ]] \+ || fail "vLLM must take the GPU whose memory is free: $(cat "$tmp/pro6000-one-free/out" "$tmp/pro6000-one-free/err")"+decide pro6000-busy-forced "export FAKE_GPU_ROWS='$BUSY_ROWS'" 'LLM_BACKEND_REQUESTED=vllm'+expect_refused pro6000-busy-forced 'other programs use the GPU memory the vLLM engine needs'+pass "a fresh install gives vLLM only a GPU with its share free: busy GPUs keep llama.cpp and say why; --llm-backend vllm stops"+decide rtx-forced "export FAKE_GPU_ROWS='$RTX_ROWS'" 'LLM_BACKEND_REQUESTED=vllm'expect_refused rtx-forced 'no vLLM catalog model fits this host'grep -qF 'needs one GPU with compute capability 10.0 or 10.3' "$tmp/rtx-forced/err" \@@ -438,6 +472,19 @@ grep -qF "LOG Kept this installation's vLLM model across installer rerun: glm-5."$tmp/rerun-vllm/install.log" || fail "keeping the vLLM model must reach the install log"pass "a rerun of a vLLM install keeps its model, folder, GPU and memory share; the fresh pick stays the recommendation"+# The installation's own engine holds its share of the GPU, so a rerun asks+# for total memory only and keeps vLLM however little of the GPU is free.+grep -q -- '--query-gpu=uuid,compute_cap,memory.total --format' "$tmp/rerun-vllm/calls" \+ && ! grep -q 'memory.free' "$tmp/rerun-vllm/calls" \+ || fail "a rerun must not judge its GPUs by free memory: $(cat "$tmp/rerun-vllm/calls")"+name=rerun-vllm-busy; mkdir -p "$tmp/$name/install"; vllm_install+decide rerun-vllm-busy "export FAKE_PRESERVE_OUT='$GLM_CONTRACT'" \+ "export FAKE_GPU_ROWS='GPU-aaaaaaaa-gb300, 10.3, 256703, 30000\n'"+[[ "$(rc_of rerun-vllm-busy)" == 0 && "$(value rerun-vllm-busy BACKEND)" == vllm \+ && "$(value rerun-vllm-busy VLLM_GPU_UUID)" == GPU-aaaaaaaa-gb300 ]] \+ || fail "a rerun whose own engine holds the GPU must keep vLLM: $(cat "$tmp/rerun-vllm-busy/out" "$tmp/rerun-vllm-busy/err")"+pass "a rerun of a vLLM install is never judged by the GPU memory its own engine holds"+# The kept model never takes the fresh pick's memory share: kept without a# share of its own, it stays on compose's default even where the fresh pick# (the RTX PRO 6000 entry here) sets one.diff --git a/ods/tests/test_select_model_vllm.py b/ods/tests/test_select_model_vllm.pyindex a711b5913..37e25b3bd 100644--- a/ods/tests/test_select_model_vllm.py+++ b/ods/tests/test_select_model_vllm.py@@ -224,6 +224,99 @@ def test_the_largest_fitting_gpu_is_the_one_vllm_owns():assert env_values(out)["VLLM_GPU_UUID"] == "GPU-test-b300"+# A fresh install adds nvidia-smi memory.free as a fourth field. vLLM refuses+# to start unless its share of the card is free once its CUDA context exists:+# half an RTX PRO 6000 is ceil(0.5 * 97887) = 48944 MiB, plus START_MARGIN_MIB+# (1024) makes 49968 MiB free.+def test_a_gpu_without_room_for_vllm_is_not_given_to_vllm():+ # tower2 (2026-10-27): another model server held about 84 GB of each card.+ rows = ("GPU-test-busy-0, 12.0, 97887, 13278", "GPU-test-busy-1, 12.0, 97887, 13298")+ status, out, err = select(*VLLM, *_gpu_args(*rows), "--env")++ assert (status, out) == (4, "")+ assert err == (+ "error: other programs use the GPU memory the vLLM engine needs: Qwen 3.6 35B A3B NVFP4 (vLLM) "+ "needs 49968 MiB free when it starts (0.50 of the GPU plus 1024 MiB for its CUDA context), "+ "and the freest GPU it fits, GPU-test-busy-1, has 13298 MiB free\n"+ )+++def test_vllm_takes_a_gpu_with_room_to_start_and_says_why():+ status, out, err = select(*VLLM, *_gpu_args("GPU-test-busy, 12.0, 97887, 13278",+ "GPU-test-exact, 12.0, 97887, 49968"), "--env")+ assert status == 0, err+ env = env_values(out)+ assert env["VLLM_GPU_UUID"] == "GPU-test-exact"+ assert env["MODEL_RECOMMENDATION_REASON"].endswith(+ " Qwen 3.6 35B A3B NVFP4 (vLLM) would run on GPU GPU-test-busy, but other programs use the memory "+ "it needs there.")++ status, _, err = select(*VLLM, *_gpu_args("GPU-test-short, 12.0, 97887, 49967"), "--env")+ assert status == 4 and "has 49967 MiB free" in err+++def test_a_busy_gb300_gives_vllm_the_idle_rtx_pro_6000_and_says_why():+ # A DGX Station whose GB300 runs another model server: the RTX PRO 6000's+ # own entry runs instead of the GB300's top pick, and the reason says so.+ best_gb300, _ = _best_fitting("10.3", 256703)+ status, out, err = select(*VLLM, *_gpu_args(RTX_PRO_6000 + ", 97000", GB300 + ", 30000"), "--env")++ assert status == 0, err+ env = env_values(out)+ assert (env["VLLM_GPU_UUID"], env["LLM_MODEL"]) == ("GPU-test-rtx-pro-6000", "qwen3.6-35b-a3b-nvfp4")+ assert (f" {best_gb300['name']} would run on GPU GPU-test-gb300, but other programs use the memory it "+ f"needs there.") in env["MODEL_RECOMMENDATION_REASON"]++ # Idle, the GB300 keeps its top pick and the reason no note.+ status, out, err = select(*VLLM, *_gpu_args(RTX_PRO_6000 + ", 97000", GB300 + ", 255000"), "--env")+ assert status == 0, err+ env = env_values(out)+ assert (env["VLLM_GPU_UUID"], env["LLM_MODEL"]) == ("GPU-test-gb300", best_gb300["llm_model_name"])+ assert "other programs" not in env["MODEL_RECOMMENDATION_REASON"]++ status, out, err = select(*VLLM, *_gpu_args(RTX_PRO_6000 + ", 97000"))+ assert status == 0, err+ assert json.loads(out)["gpu"] == {"uuid": "GPU-test-rtx-pro-6000", "compute_capability": "12.0",+ "memory_mib": 97887, "free_mib": 97000}+++def test_equal_gpus_go_to_the_one_with_more_memory_free():+ rows = ("GPU-test-desktop, 12.0, 97887, 95000", "GPU-test-idle, 12.0, 97887, 97000")+ status, out, err = select(*VLLM, *_gpu_args(*rows), "--env")+ assert status == 0, err+ assert env_values(out)["VLLM_GPU_UUID"] == "GPU-test-idle"+++def test_free_memory_never_lets_a_gpu_fit_that_its_hardware_does_not():+ # A free H200 still lacks a compute capability any entry names: no fit, not busy.+ status, out, err = select(*VLLM, *_gpu_args(H200 + ", 143000"), "--env")+ assert (status, out) == (2, "")+ assert "GPU-test-h200 (compute capability 9.0, 143771 MiB, 143000 MiB free)" in err+++def test_an_unreadable_free_field_leaves_that_gpu_unknown():+ # [N/A] on one GPU must not take vLLM from the whole host.+ status, out, err = select(*VLLM, *_gpu_args(RTX_PRO_6000 + ", [N/A]", GB300 + ", 250000"), "--env")+ assert status == 0, err+ assert env_values(out)["VLLM_GPU_UUID"] == "GPU-test-gb300"++ status, out, err = select(*VLLM, *_gpu_args(RTX_PRO_6000 + ", [N/A]"), "--env")+ assert status == 0, err+ assert env_values(out)["VLLM_GPU_UUID"] == "GPU-test-rtx-pro-6000"++ # The other fields stay required.+ status, out, err = select(*VLLM, *_gpu_args("GPU-test-x, 12.0, [N/A], 1000"), "--env")+ assert (status, out) == (2, "")+ assert "is not UUID,COMPUTE_CAP,MEMORY_MIB[,FREE_MIB]" in err+++def test_vllms_default_share_is_the_memory_estimates():+ selector = _selector()+ assert selector.memory_share({}) == selector.vllm_memory_share({}) == 0.9+ assert selector.memory_share({"gpu_memory_utilization": 0.5}) \+ == selector.vllm_memory_share({"gpu_memory_utilization": 0.5}) == 0.5++def test_json_payload_names_the_gpu_folder_and_estimate():status, out, err = select(*VLLM, *_gpu_args(GB300))