Skip to content

armgit / ODS

Public Imported from GitHub
0

ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.

fix(installer): give vLLM only a GPU with its share free on a fresh install (#8306)

Mike Bradleybffd6064ba3b
diff --git a/ods/extensions/services/dashboard-api/model_vllm.py b/ods/extensions/services/dashboard-api/model_vllm.py
index 2d62bcb1b..2df9bd800 100644
--- a/ods/extensions/services/dashboard-api/model_vllm.py
+++ b/ods/extensions/services/dashboard-api/model_vllm.py
@@ -12,7 +12,10 @@ the vLLM engine (``.env`` ``LLM_BACKEND=vllm``), not llama.cpp. They carry no
``context_length`` (equal to ``--max-model-len``) and ``vllm_args``;
* ``min_gpu_mib`` and ``compute_capability``: the one GPU vLLM runs on needs
at least this much memory (nvidia-smi ``memory.total``) and one of these
- compute capabilities;
+ compute capabilities. When a GPU's free memory is known (a fresh install
+ passes nvidia-smi ``memory.free``), it fits only if vLLM's share of its
+ total memory is free, plus START_MARGIN_MIB for the CUDA context vLLM
+ creates before it checks: vLLM refuses to start with less;
* optional ``gpu_memory_utilization``: the share of that GPU vLLM
preallocates (``.env`` VLLM_GPU_MEMORY_UTILIZATION, the vllm-server
service's ``--gpu-memory-utilization``; vLLM's 0.90 when absent). An entry
@@ -43,6 +46,7 @@ model_memory.py.
from __future__ import annotations
+import math
import re
import shlex
from typing import Any, Iterable, NamedTuple
@@ -59,9 +63,16 @@ LLAMA_MEMORY_CLASSES = ("discrete", "unified", "cpu")
COMPOSE_FLAGS = frozenset({"--model", "--served-model-name", "--host", "--port", "--gpu-memory-utilization"})
# The range an entry's gpu_memory_utilization may take.
MIN_MEMORY_SHARE, MAX_MEMORY_SHARE = 0.10, 0.95
+# vLLM's own --gpu-memory-utilization default, for an entry that sets none.
+DEFAULT_MEMORY_SHARE = 0.90
+# vLLM checks free memory after creating its CUDA context, which nvidia-smi
+# has not seen yet: a GPU needs this much free beyond vLLM's share.
+START_MARGIN_MIB = 1024
# A file downloads under <name><PARTIAL_SUFFIX> until it is verified.
PARTIAL_SUFFIX = ".partial"
GPU_QUERY = "nvidia-smi --query-gpu=uuid,compute_cap,memory.total --format=csv,noheader,nounits"
+# The installer's fresh-install query: GPU_QUERY plus each GPU's free memory.
+GPU_FREE_QUERY = "nvidia-smi --query-gpu=uuid,compute_cap,memory.total,memory.free --format=csv,noheader,nounits"
_HF_REPO = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*")
_HEX40 = re.compile(r"[0-9a-f]{40}")
@@ -85,6 +96,8 @@ class Gpu(NamedTuple):
uuid: str
compute_capability: str
memory_mib: int
+ # nvidia-smi memory.free when the caller passed it; None means unknown.
+ free_mib: int | None = None
class Candidate(NamedTuple):
@@ -306,24 +319,51 @@ def require_valid(model: dict[str, Any]) -> dict[str, Any]:
def parse_gpu(text: str) -> Gpu:
- """One row of :data:`GPU_QUERY` output: ``UUID, COMPUTE_CAP, MEMORY_MIB``."""
+ """One row of :data:`GPU_QUERY` output, or of :data:`GPU_FREE_QUERY` (``, FREE_MIB``).
+
+ A free field that is not a number ([N/A]) leaves that GPU's free memory
+ unknown, as a three-field row does.
+ """
parts = [part.strip() for part in str(text).split(",")]
- if len(parts) == 3:
- uuid, capability, memory = parts
+ if len(parts) in (3, 4):
+ uuid, capability, memory = parts[:3]
size = _GPU_MEMORY.fullmatch(memory)
+ free = _GPU_MEMORY.fullmatch(parts[3]) if len(parts) == 4 else None
if _GPU_UUID.fullmatch(uuid) and _COMPUTE_CAPABILITY.fullmatch(capability) and size:
- return Gpu(uuid, capability, int(size.group(1)))
- raise ValueError(f"{text!r} is not UUID,COMPUTE_CAP,MEMORY_MIB ({GPU_QUERY})")
+ return Gpu(uuid, capability, int(size.group(1)), int(free.group(1)) if free else None)
+ raise ValueError(f"{text!r} is not UUID,COMPUTE_CAP,MEMORY_MIB[,FREE_MIB] ({GPU_FREE_QUERY})")
-def gpu_fits(model: dict[str, Any], gpu: Gpu) -> bool:
+def memory_share(model: dict[str, Any]) -> float:
+ """The share of its GPU a vLLM entry preallocates: its ``gpu_memory_utilization``, else 0.90."""
+ return float(model.get("gpu_memory_utilization") or DEFAULT_MEMORY_SHARE)
+
+
+def claim_mib(model: dict[str, Any], gpu: Gpu) -> int:
+ """The memory vLLM claims on ``gpu`` at start: its share of the GPU's total."""
+ return math.ceil(memory_share(model) * gpu.memory_mib)
+
+
+def free_needed_mib(model: dict[str, Any], gpu: Gpu) -> int:
+ """The free memory ``gpu`` needs for vLLM to start: its claim plus its CUDA context."""
+ return claim_mib(model, gpu) + START_MARGIN_MIB
+
+
+def gpu_hardware_fits(model: dict[str, Any], gpu: Gpu) -> bool:
return gpu.compute_capability in model["compute_capability"] and gpu.memory_mib >= model["min_gpu_mib"]
+def gpu_fits(model: dict[str, Any], gpu: Gpu) -> bool:
+ """The GPU suits ``model`` and, when its free memory is known, has what vLLM needs to start free."""
+ return gpu_hardware_fits(model, gpu) and (gpu.free_mib is None or gpu.free_mib >= free_needed_mib(model, gpu))
+
+
def best_gpu(model: dict[str, Any], gpus: Iterable[Gpu]) -> Gpu | None:
- """The largest GPU ``model`` fits on (the first listed on a tie), or None."""
+ """The largest GPU ``model`` fits on, then the one with the most free memory (the first listed on a tie), or None."""
fitting = [gpu for gpu in gpus if gpu_fits(model, gpu)]
- return max(fitting, key=lambda gpu: gpu.memory_mib) if fitting else None
+ if not fitting:
+ return None
+ return max(fitting, key=lambda gpu: (gpu.memory_mib, gpu.memory_mib if gpu.free_mib is None else gpu.free_mib))
def requirement_text(model: dict[str, Any]) -> str:
diff --git a/ods/installers/lib/detection.sh b/ods/installers/lib/detection.sh
index bd6b7e546..d77d3d42c 100755
--- a/ods/installers/lib/detection.sh
+++ b/ods/installers/lib/detection.sh
@@ -11,7 +11,7 @@
# ENABLE_RAG, ENABLE_HERMES (all used by fix_nvidia_secure_boot),
# log/warn/ai/ai_ok/ai_warn/ai_bad helpers
# Provides: detect_gpu(), load_capability_profile(), ods_is_wsl_host(),
-# ods_windows_host_port_in_use(),
+# ods_windows_host_port_in_use(), ods_gpu_memory_in_use_report(),
# normalize_profile_tier(), tier_rank(), load_backend_contract(),
# fix_nvidia_secure_boot(), MIN_DRIVER_VERSION
# Side-effect var on Jetson: JETSON_L4T_RELEASE (e.g. "R36.4.0")
@@ -25,6 +25,22 @@
# Safe env loading (no eval) for script output KEY="value" lines
[[ -f "${SCRIPT_DIR:-}/lib/safe-env.sh" ]] && . "${SCRIPT_DIR}/lib/safe-env.sh"
+# ods_gpu_memory_in_use_report
+# Reads `nvidia-smi --query-gpu=index,memory.total,memory.used
+# --format=csv,noheader,nounits` rows on stdin and prints one line per GPU
+# where programs already hold more than max(4 GiB, 10% of the card), such as
+# another model server: "GPU <index>: <used> of <total> GiB in use". A desktop
+# session stays under that floor; rows without numbers ([N/A] on
+# unified-memory GPUs) are skipped.
+ods_gpu_memory_in_use_report() {
+ awk -F', *' '
+ $2 ~ /^[0-9]+$/ && $3 ~ /^[0-9]+$/ && $2 > 0 {
+ floor = $2 * 0.10
+ if (floor < 4096) floor = 4096
+ if ($3 + 0 > floor) printf "GPU %s: %.0f of %.0f GiB in use\n", $1, $3 / 1024, $2 / 1024
+ }'
+}
+
# WSL2 forwards Docker-published ports through the Windows host. A port can
# therefore look free to lsof/ss inside Linux while Docker Desktop still
# refuses the bind because a native Windows process already owns it. Query the
diff --git a/ods/installers/lib/model-selector.sh b/ods/installers/lib/model-selector.sh
index 6316094cc..66210d619 100644
--- a/ods/installers/lib/model-selector.sh
+++ b/ods/installers/lib/model-selector.sh
@@ -61,10 +61,13 @@ ods_run_catalog_selector() {
# ods_run_vllm_selector PYTHON [GPU_ROW...]
# The vLLM engine's pick: one GPU_ROW per line of `nvidia-smi
-# --query-gpu=uuid,compute_cap,memory.total --format=csv,noheader,nounits`.
-# Chooses among the catalog's vLLM entries only, never a GGUF, and prints
-# shell assignments with the VLLM_* keys. Exit 2: no vLLM model fits. The fit
-# comes from the GPU rows alone, so the llama.cpp memory inputs are not passed.
+# --query-gpu=uuid,compute_cap,memory.total[,memory.free]
+# --format=csv,noheader,nounits`. Chooses among the catalog's vLLM entries
+# only, never a GGUF, and prints shell assignments with the VLLM_* keys. Exit
+# 2: no vLLM model fits; exit 4: one fits the hardware, but other programs
+# hold the free memory vLLM needs at start (rows with memory.free only). The
+# fit comes from the GPU rows alone, so the llama.cpp memory inputs are not
+# passed.
ods_run_vllm_selector() {
local python="$1" row
shift
diff --git a/ods/installers/phases/02-detection.sh b/ods/installers/phases/02-detection.sh
index 49c5dfa49..25b832d93 100755
--- a/ods/installers/phases/02-detection.sh
+++ b/ods/installers/phases/02-detection.sh
@@ -589,14 +589,17 @@ fi
# path below unchanged, without querying the GPUs for vLLM. --llm-backend vllm
# and a rerun of a vLLM installation (_phase02_vllm_kept) stop when vLLM
# cannot run or no vLLM model fits; --llm-backend llama-server never considers
-# vLLM. On the vLLM path every selector failure other than "nothing fits"
-# (exit 2) stops the install: there is no tier-map fallback. The pick
+# vLLM. A fresh install also reads each GPU's free memory: when other programs
+# hold what vLLM needs at start (exit 4), it keeps llama.cpp and says why, and
+# --llm-backend vllm stops. On the vLLM path every other selector failure
+# than "nothing fits" (exit 2) stops the install: there is no tier-map
+# fallback. The pick
# (VLLM_SELECTOR_ENV) is loaded after the tier map, which it replaces; on a
# rerun of a vLLM installation it is the recommendation, and the installed
# vLLM model stays active (_phase02_keep_vllm_model).
_phase02_choose_llm_backend() {
local requested="${LLM_BACKEND_REQUESTED:-}" external="${EXTERNAL_LLM_URL:-}"
- local existing=false retained="" required=false wsl=false blocker python rows status=0
+ local existing=false retained="" required=false wsl=false blocker python rows query status=0
local env_text error_text error_file
local -a gpu_rows=()
VLLM_SELECTOR_ENV=""
@@ -639,7 +642,12 @@ _phase02_choose_llm_backend() {
log "Model server: llama.cpp (Python is unavailable for the vLLM model catalog)"
return 0
fi
- if ! rows="$(nvidia-smi --query-gpu=uuid,compute_cap,memory.total --format=csv,noheader,nounits 2>>"$LOG_FILE")"; then
+ # vLLM refuses to start unless its share of the GPU is free. A fresh
+ # install's GPUs hold no ODS engine yet, so their free memory is what vLLM
+ # can claim; on a rerun this installation's own engine may hold it.
+ query="uuid,compute_cap,memory.total"
+ [[ "$existing" == true ]] || query+=",memory.free"
+ if ! rows="$(nvidia-smi --query-gpu="$query" --format=csv,noheader,nounits 2>>"$LOG_FILE")"; then
[[ "$required" != true ]] || _phase02_vllm_refused "nvidia-smi could not list this host's GPUs (details in $LOG_FILE)"
log "Model server: llama.cpp (nvidia-smi could not list the GPUs for the vLLM engine)"
return 0
@@ -659,6 +667,12 @@ _phase02_choose_llm_backend() {
[[ "$required" != true ]] || _phase02_vllm_refused "${error_text#error: }"
log "Model server: llama.cpp (${error_text#error: })"
;;
+ 4)
+ [[ "$required" != true ]] || _phase02_vllm_refused "${error_text#error: }"
+ ai_warn "ODS uses llama.cpp, not the vLLM engine: ${error_text#error: }."
+ ai " Once that memory is free, rerun the installer with --llm-backend vllm to use the vLLM engine."
+ log "Model server: llama.cpp (${error_text#error: })"
+ ;;
*)
error "The vLLM model selector stopped (exit $status): ${error_text:-no details}. The ODS files may be incomplete: download ODS again and rerun the installer. Need help? Ask on the ODS Discord: ${ODS_HELP_DISCORD_URL:-https://discord.gg/4ntNp9MAwC}"
exit 1
diff --git a/ods/installers/phases/04-requirements.sh b/ods/installers/phases/04-requirements.sh
index a719e4af2..ad0ac4047 100755
--- a/ods/installers/phases/04-requirements.sh
+++ b/ods/installers/phases/04-requirements.sh
@@ -9,8 +9,9 @@
# GPU_VRAM, GPU_NAME, GPU_COUNT, INTERACTIVE, DRY_RUN,
# PREFLIGHT_REPORT_FILE, CAP_PLATFORM_ID, CAP_COMPOSE_OVERLAYS,
# ENABLE_VOICE, ENABLE_WORKFLOWS, ENABLE_RAG, ENABLE_QDRANT,
-# INSTALL_DIR, external_llm_env_value(),
-# tier_rank(), chapter(), ai_ok(), ai_bad(), ai_warn(), log(), warn()
+# INSTALL_DIR, EXTERNAL_LLM_URL, ODS_GATEWAY_ONLY, external_llm_env_value(),
+# ods_gpu_memory_in_use_report() (installers/lib/detection.sh),
+# tier_rank(), chapter(), ai(), ai_ok(), ai_bad(), ai_warn(), log(), warn()
# Provides: REQUIREMENTS_MET, TIER_RANK, WHISPER_PORT (only when the
# generated default 9000 moves; phase 06 persists it)
#
@@ -160,6 +161,30 @@ if [[ -z "${EXTERNAL_LLM_URL:-}" ]] && ! ods_native_llm_requested \
fi
fi
+# A fresh install sizes its model and services for each GPU's whole memory.
+# Say so when other programs already hold much of it, before anything
+# downloads. A rerun's own model holds memory, and an external, host-native
+# or cloud model runs nothing on these GPUs, so only a fresh local install
+# checks.
+if [[ "${GPU_BACKEND:-}" == "nvidia" && ! -f "$INSTALL_DIR/.env" && -z "${EXTERNAL_LLM_URL:-}" \
+ && "${ODS_GATEWAY_ONLY:-false}" != true && "${TIER:-}" != "CLOUD" ]] \
+ && ! ods_native_llm_requested && command -v nvidia-smi >/dev/null 2>&1 \
+ && declare -F ods_gpu_memory_in_use_report >/dev/null 2>&1; then
+ # A warning only: report what nvidia-smi printed even when it then fails.
+ _gpu_memory_rows="$(nvidia-smi --query-gpu=index,memory.total,memory.used \
+ --format=csv,noheader,nounits 2>>"$LOG_FILE")" || log "nvidia-smi exited $? while listing GPU memory"
+ _gpu_memory_in_use="$(printf '%s\n' "$_gpu_memory_rows" | ods_gpu_memory_in_use_report)"
+ if [[ -n "$_gpu_memory_in_use" ]]; then
+ ai_warn "Other programs already use GPU memory:"
+ while IFS= read -r _gpu_line; do
+ ai " $_gpu_line"
+ done <<< "$_gpu_memory_in_use"
+ ai " ODS sizes its model and services for each GPU's whole memory: stop those programs before ODS starts, or its model may not load."
+ log "GPU memory in use before install: ${_gpu_memory_in_use//$'\n'/; }"
+ fi
+ unset _gpu_memory_rows _gpu_memory_in_use _gpu_line
+fi
+
# Port conflict detection with process details
# Warn-once guard for missing port-check tools
_port_check_warned=false
diff --git a/ods/scripts/select-model.py b/ods/scripts/select-model.py
index 8851bc315..aa80c5427 100644
--- a/ods/scripts/select-model.py
+++ b/ods/scripts/select-model.py
@@ -50,11 +50,17 @@ from model_selection import ( # noqa: E402
from model_selection import pixel_agent_ready as _pixel_agent_ready # noqa: E402
from model_selection import usable_memory_gb as _usable_memory_gb # noqa: E402
from model_vllm import ( # noqa: E402
+ GPU_FREE_QUERY,
GPU_QUERY,
+ START_MARGIN_MIB,
Candidate as VllmCandidate,
Gpu,
+ free_needed_mib,
+ gpu_fits,
+ gpu_hardware_fits,
is_vllm_entry,
manifest_bytes,
+ memory_share,
model_dir_name,
parse_gpu,
rank_entries,
@@ -79,6 +85,9 @@ VLLM_POLICY = "vllm-gpu-fit-v1"
VLLM_SOURCE = "catalog_vllm_pre_download"
EXIT_NO_FIT = 2
EXIT_CHECK_FIT_FAILED = 3
+# --llm-runtime vllm: an entry fits the GPUs, but other programs hold the
+# memory vLLM claims at start (--gpu rows with free memory).
+EXIT_GPU_BUSY = 4
def normalize_profile(value: str | None) -> str:
@@ -497,13 +506,36 @@ def _vllm_no_fit(pool: list[dict[str, Any]], gpus: list[Gpu], args: argparse.Nam
if not gpus:
return f"--llm-runtime vllm needs the host's NVIDIA GPUs: pass each one as --gpu ({GPU_QUERY})"
seen = ", ".join(
- f"{gpu.uuid} (compute capability {gpu.compute_capability}, {gpu.memory_mib} MiB)" for gpu in gpus
+ f"{gpu.uuid} (compute capability {gpu.compute_capability}, {gpu.memory_mib} MiB"
+ f"{'' if gpu.free_mib is None else f', {gpu.free_mib} MiB free'})" for gpu in gpus
)
floor = f" at the {args.min_context} context floor" if args.require_min_context and args.min_context else ""
needs = "; ".join(f"{model['id']} needs {requirement_text(model)}" for model in pool)
return f"no vLLM catalog model fits this host{floor}. GPUs: {seen}. {needs}"
+def _vllm_busy(candidate: VllmCandidate, gpus: list[Gpu]) -> str:
+ """Why the best entry for the GPUs' hardware cannot start: their memory is in use."""
+ model = candidate.model
+ freest = max((gpu for gpu in gpus if gpu_hardware_fits(model, gpu)),
+ key=lambda gpu: gpu.free_mib or 0)
+ return (
+ f"other programs use the GPU memory the vLLM engine needs: {model['name']} needs "
+ f"{free_needed_mib(model, freest)} MiB free when it starts ({memory_share(model):.2f} of the GPU "
+ f"plus {START_MARGIN_MIB} MiB for its CUDA context), and the freest GPU it fits, "
+ f"{freest.uuid}, has {freest.free_mib} MiB free"
+ )
+
+
+def _vllm_busy_note(hardware_pick: VllmCandidate, gpus: list[Gpu]) -> str:
+ """Names the entry the GPUs' hardware alone would get when its GPU's memory is in use."""
+ gpu = next(gpu for gpu in gpus if gpu.uuid == hardware_pick.gpu.uuid)
+ if gpu_fits(hardware_pick.model, gpu):
+ return ""
+ return (f" {hardware_pick.model['name']} would run on GPU {gpu.uuid}, but other programs use "
+ f"the memory it needs there.")
+
+
def _vllm_payload(ranked: list[VllmCandidate], reason: str, min_context: int, host_arch: str) -> dict[str, Any]:
selected = ranked[0]
model, gpu = selected.model, selected.gpu
@@ -513,7 +545,7 @@ def _vllm_payload(ranked: list[VllmCandidate], reason: str, min_context: int, ho
"confidence": "high",
"runtime": "vllm",
"host_arch": normalize_host_arch(host_arch),
- "gpu": gpu._asdict(),
+ "gpu": {key: value for key, value in gpu._asdict().items() if value is not None},
"memory_capacity_gb": round(gpu.memory_mib / 1024.0, 1),
"memory_label": "GPU memory",
"memory_class": "discrete",
@@ -590,10 +622,17 @@ def _vllm_main(args: argparse.Namespace) -> int:
min_context = max(int(args.min_context or 0), 0)
ranked = rank_entries(pool, gpus, min_context=min_context, require_min_context=args.require_min_context,
installable_only=args.installable_only)
+ # The same ranking with free memory unknown: what the GPUs' hardware alone gets.
+ by_hardware = rank_entries(pool, [gpu._replace(free_mib=None) for gpu in gpus], min_context=min_context,
+ require_min_context=args.require_min_context,
+ installable_only=args.installable_only)
if not ranked:
+ if by_hardware:
+ print(f"error: {_vllm_busy(by_hardware[0], gpus)}", file=sys.stderr)
+ return EXIT_GPU_BUSY
print(f"error: {_vllm_no_fit(pool, gpus, args)}", file=sys.stderr)
return EXIT_NO_FIT
- reason = _vllm_reason(ranked[0], min_context)
+ reason = _vllm_reason(ranked[0], min_context) + _vllm_busy_note(by_hardware[0], gpus)
if not args.env:
print(json.dumps(_vllm_payload(ranked, reason, min_context, args.host_arch), indent=2))
return 0
@@ -645,12 +684,16 @@ def main(argv: list[str] | None = None) -> int:
"--llm-runtime", choices=("llama-server", "vllm"), default="llama-server",
help="The model server (.env LLM_BACKEND). llama-server (default) picks a GGUF "
"and never a vLLM entry; vllm picks only vLLM entries, for one --gpu, and "
- "exits 2 when none fits. With vllm the fit comes from --gpu alone: --vram-mb, "
- "--ram-gb, --memory-type, --tier, --profile and --max-size-mb are not read.",
+ "exits 2 when none fits, or 4 when one fits the GPUs but other programs "
+ "hold the memory vLLM claims at start. With vllm the fit comes from --gpu "
+ "alone: --vram-mb, --ram-gb, --memory-type, --tier, --profile and "
+ "--max-size-mb are not read.",
)
parser.add_argument(
- "--gpu", action="append", default=[], metavar="UUID,COMPUTE_CAP,MEMORY_MIB",
- help=f"One NVIDIA GPU, a row of `{GPU_QUERY}`; repeat per GPU. "
+ "--gpu", action="append", default=[], metavar="UUID,COMPUTE_CAP,MEMORY_MIB[,FREE_MIB]",
+ help=f"One NVIDIA GPU, a row of `{GPU_QUERY}` or of `{GPU_FREE_QUERY}`; repeat "
+ "per GPU. With the free memory, a GPU fits only while vLLM can start there: "
+ f"its share of the GPU's total memory plus {START_MARGIN_MIB} MiB free. "
"Read only with --llm-runtime vllm.",
)
parser.add_argument("--env", action="store_true", help="print shell assignments")
diff --git a/ods/tests/ci-suite.txt b/ods/tests/ci-suite.txt
index 423c885fd..1b10310ad 100644
--- a/ods/tests/ci-suite.txt
+++ b/ods/tests/ci-suite.txt
@@ -64,6 +64,7 @@ tests/test-comfyui-cuda-cache-persisted.sh
tests/test-llama-cuda-cache-persisted.sh
tests/test-vllm-engine-service.sh
tests/test-vllm-installer-route.sh
+tests/test-gpu-memory-in-use.sh
tests/test-vllm-installer-phases.sh
tests/test-cli-vllm-engine.sh
tests/test-doctor-vllm-engine.sh
diff --git a/ods/tests/test-gpu-memory-in-use.sh b/ods/tests/test-gpu-memory-in-use.sh
new file mode 100755
index 000000000..5716c92e2
--- /dev/null
+++ b/ods/tests/test-gpu-memory-in-use.sh
@@ -0,0 +1,112 @@
+#!/usr/bin/env bash
+# The requirements check (phase 04) warns a fresh local NVIDIA install when
+# other programs already hold much of a GPU's memory, before anything
+# downloads:
+# - installers/lib/detection.sh ods_gpu_memory_in_use_report names each GPU
+# where more than max(4 GiB, 10% of the card) is in use, and nothing for a
+# desktop session or a unified-memory GPU that reports [N/A];
+# - phase 04's block prints the warning on a fresh install, even when
+# nvidia-smi fails after printing its rows, and stays quiet on a rerun
+# (its own model holds memory), for an external, gateway-only,
+# host-native or cloud model (nothing runs on these GPUs), on other
+# backends and when nvidia-smi prints nothing.
+# Needs no GPU.
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+PHASE04="$ROOT_DIR/installers/phases/04-requirements.sh"
+PASS=0
+fail() { printf '[FAIL] %s\n' "$*" >&2; exit 1; }
+pass() { printf '[PASS] %s\n' "$*"; PASS=$((PASS + 1)); }
+
+tmp="$(mktemp -d)"
+trap 'rm -rf "$tmp"' EXIT
+
+# shellcheck source=../installers/lib/detection.sh
+SCRIPT_DIR="$ROOT_DIR" source "$ROOT_DIR/installers/lib/detection.sh"
+
+report() { printf '%b' "$1" | ods_gpu_memory_in_use_report; }
+
+# tower2 (2026-10-27): another model server held about 82 GiB of each card.
+[[ "$(report '0, 97887, 83973\n1, 97887, 83953\n')" == $'GPU 0: 82 of 96 GiB in use\nGPU 1: 82 of 96 GiB in use' ]] \
+ || fail "busy GPUs must be named: $(report '0, 97887, 83973\n1, 97887, 83953\n')"
+# A desktop session, and the 4 GiB floor on a 24 GB card.
+[[ -z "$(report '0, 24564, 1200\n')" && -z "$(report '0, 24564, 4096\n')" \
+ && "$(report '0, 24564, 4097\n')" == 'GPU 0: 4 of 24 GiB in use' ]] \
+ || fail "a 24 GB card is busy only above 4 GiB in use"
+# 10% of a 96 GB card is the floor there.
+[[ -z "$(report '3, 97887, 9788\n')" && "$(report '3, 97887, 9789\n')" == 'GPU 3: 10 of 96 GiB in use' ]] \
+ || fail "a 96 GB card is busy only above 10% in use"
+[[ -z "$(report '0, [N/A], [N/A]\n')" && -z "$(report '0, 0, 0\n')" && -z "$(report '')" ]] \
+ || fail "rows without memory numbers must be skipped"
+pass "the report names GPUs with more than max(4 GiB, 10%) in use and skips desktops and [N/A] rows"
+
+block="$(sed -n '/^# A fresh install sizes its model and services for each GPU.s whole memory\./,/^fi$/p' "$PHASE04")"
+[[ -n "$block" ]] || fail "phase 04's GPU memory warning block was not found"
+
+fakebin="$tmp/bin"
+mkdir -p "$fakebin"
+cat > "$fakebin/nvidia-smi" <<'EOF'
+#!/usr/bin/env bash
+printf '%s\n' "$*" >> "$FAKE_CALLS"
+printf '%b' "$FAKE_ROWS"
+exit "${FAKE_STATUS:-0}"
+EOF
+chmod +x "$fakebin/nvidia-smi"
+
+# run NAME [statements...]: phase 04's block with the given inputs.
+run() {
+ local name="$1" dir="$tmp/$1" rc=0
+ shift
+ mkdir -p "$dir/install"
+ : > "$dir/calls"
+ {
+ printf '%s\n' 'set -euo pipefail'
+ printf 'SCRIPT_DIR=%q\nINSTALL_DIR=%q\nLOG_FILE=%q\n' "$ROOT_DIR" "$dir/install" "$dir/install.log"
+ printf 'export PATH=%q FAKE_CALLS=%q\n' "$fakebin:$PATH" "$dir/calls"
+ printf '%s\n' 'source "$SCRIPT_DIR/installers/lib/detection.sh"'
+ printf '%s\n' 'ai() { printf "AI %s\n" "$*"; }' 'ai_warn() { printf "WARN %s\n" "$*"; }' \
+ 'log() { printf "LOG %s\n" "$*" >> "$LOG_FILE"; }' \
+ 'ods_native_llm_requested() { [[ -n "${NATIVE_LLM_BASE_URL:-}" ]]; }'
+ printf '%s\n' 'GPU_BACKEND=nvidia TIER=NV_ULTRA EXTERNAL_LLM_URL="" ODS_GATEWAY_ONLY=false NATIVE_LLM_BASE_URL=""' \
+ "export FAKE_ROWS='0, 97887, 83973\n1, 97887, 1200\n'"
+ printf '%s\n' "$@" "$block" 'echo "DONE"'
+ } > "$dir/run.bash"
+ bash "$dir/run.bash" > "$dir/out" 2> "$dir/err" || rc=$?
+ [[ "$rc" == 0 ]] || fail "$name: the block failed (rc=$rc): $(cat "$dir/err")"
+}
+warned() {
+ [[ "$(cat "$tmp/$1/out")" == "WARN Other programs already use GPU memory:
+AI GPU 0: 82 of 96 GiB in use
+AI ODS sizes its model and services for each GPU's whole memory: stop those programs before ODS starts, or its model may not load.
+DONE" ]]
+}
+quiet() { [[ "$(cat "$tmp/$1/out")" == DONE ]]; }
+
+run fresh
+warned fresh || fail "a fresh install must name the busy GPU and what to do: $(cat "$tmp/fresh/out")"
+grep -qx -- '--query-gpu=index,memory.total,memory.used --format=csv,noheader,nounits' "$tmp/fresh/calls" \
+ || fail "the warning must read index, total and used memory: $(cat "$tmp/fresh/calls")"
+grep -qx 'LOG GPU memory in use before install: GPU 0: 82 of 96 GiB in use' "$tmp/fresh/install.log" \
+ || fail "the warning must reach the install log"
+run partial 'export FAKE_STATUS=15'
+warned partial && grep -qx 'LOG nvidia-smi exited 15 while listing GPU memory' "$tmp/partial/install.log" \
+ || fail "rows printed before an nvidia-smi failure must still be reported: $(cat "$tmp/partial/out")"
+pass "a fresh local NVIDIA install names the busy GPUs before anything downloads, even when nvidia-smi then fails"
+
+run idle "export FAKE_ROWS='0, 97887, 600\n'"
+quiet idle || fail "idle GPUs must not warn: $(cat "$tmp/idle/out")"
+run smi-empty "export FAKE_ROWS='' FAKE_STATUS=9"
+quiet smi-empty || fail "an nvidia-smi that prints nothing leaves nothing to report: $(cat "$tmp/smi-empty/out")"
+run rerun 'printf "LLM_BACKEND=vllm\n" > "$INSTALL_DIR/.env"'
+quiet rerun && [[ ! -s "$tmp/rerun/calls" ]] \
+ || fail "a rerun must not judge GPUs its own model holds: $(cat "$tmp/rerun/out" "$tmp/rerun/calls")"
+for route in 'EXTERNAL_LLM_URL=http://127.0.0.1:11434' 'ODS_GATEWAY_ONLY=true' \
+ 'NATIVE_LLM_BASE_URL=http://localhost:8080' 'TIER=CLOUD' 'GPU_BACKEND=amd'; do
+ name="route-${route%%=*}"
+ run "$name" "$route"
+ quiet "$name" && [[ ! -s "$tmp/$name/calls" ]] || fail "$route runs no model on these GPUs, so it must not warn"
+done
+pass "idle GPUs, an empty nvidia-smi, a rerun, external, gateway-only, host-native, cloud and AMD stay quiet"
+
+printf '\nResult: %s passed, 0 failed\n' "$PASS"
diff --git a/ods/tests/test-vllm-installer-route.sh b/ods/tests/test-vllm-installer-route.sh
index 8b767cbd3..7241e776c 100644
--- a/ods/tests/test-vllm-installer-route.sh
+++ b/ods/tests/test-vllm-installer-route.sh
@@ -159,7 +159,12 @@ cat > "$fakebin/nvidia-smi" <<'EOF'
#!/usr/bin/env bash
printf '%s\n' "$*" >> "$FAKE_NVIDIA_SMI_CALLS"
[[ "${FAKE_NVIDIA_SMI_FAIL:-false}" != true ]] || { echo "NVIDIA-SMI has failed" >&2; exit 9; }
-printf '%b' "$FAKE_GPU_ROWS"
+# A row's fourth field is memory.free: print it only when it is queried.
+if [[ "$*" == *memory.free* ]]; then
+ printf '%b' "$FAKE_GPU_ROWS"
+else
+ printf '%b' "$FAKE_GPU_ROWS" | sed 's/^\([^,]*,[^,]*,[^,]*\),.*$/\1/'
+fi
EOF
# Python for phase 02: scripts/preserve-active-model.py answers from the
# FAKE_PRESERVE_* settings (its vLLM contract lives on the runtime side);
@@ -203,7 +208,9 @@ source "$SCRIPT_DIR/installers/lib/llm-backend.sh"
ods_model_selector_python() { printf '%s' "$FAKE_PYTHON_SHIM"; }
ods_native_llm_requested() { [[ -n "${NATIVE_LLM_BASE_URL:-}" ]]; }
log() { printf 'LOG %s\n' "$*" >> "$LOG_FILE"; }
+ai() { printf 'AI %s\n' "$*"; }
ai_ok() { printf 'OK %s\n' "$*"; }
+ai_warn() { printf 'WARN %s\n' "$*"; }
error() { printf 'ERROR %s\n' "$*" >&2; }
ods_is_wsl_host() { [[ "${FAKE_WSL:-false}" == true ]]; }
uname() { printf '%s\n' "${FAKE_KERNEL:-Linux}"; }
@@ -305,8 +312,9 @@ want_size="${want_size%$'\r'}" # a Windows Python ends the line with CRLF
|| fail "the vLLM pick must replace the tier map's GGUF: $(cat "$tmp/dgx-auto/out")"
[[ "$(value dgx-auto VLLM_IMAGE)" == "$pinned" ]] || fail "VLLM_IMAGE must be the locked pin"
[[ "$(value dgx-auto VLLM_ARGS)" == *'--max-model-len 131072'* ]] || fail "VLLM_ARGS must carry the catalog arguments"
-[[ "$(calls dgx-auto)" == 1 ]] && grep -qx -- '--query-gpu=uuid,compute_cap,memory.total --format=csv,noheader,nounits' "$tmp/dgx-auto/calls" \
- || fail "the decision must read each GPU's uuid, compute capability and memory once"
+[[ "$(calls dgx-auto)" == 1 ]] \
+ && grep -qx -- '--query-gpu=uuid,compute_cap,memory.total,memory.free --format=csv,noheader,nounits' "$tmp/dgx-auto/calls" \
+ || fail "a fresh install's decision must read each GPU's uuid, compute capability, total and free memory once"
grep -qx "OK Model server: vLLM engine, serving $want_served on GPU GPU-aaaaaaaa-gb300" "$tmp/dgx-auto/out" \
|| fail "the choice must be shown: $(cat "$tmp/dgx-auto/out")"
pass "a fresh DGX Station install chooses vLLM on the GB300 with the catalog's top vLLM pick ($want_served)"
@@ -349,6 +357,32 @@ p6k_share="${p6k_share%$'\r'}" # a Windows Python ends the line with CRLF
|| fail "one RTX PRO 6000 must choose vLLM with its entry's memory share: $(cat "$tmp/pro6000-auto/out" "$tmp/pro6000-auto/err")"
pass "a lone RTX PRO 6000 chooses vLLM ($p6k_model) at its entry's $p6k_share memory share"
+# tower2 (2026-10-27): RTX PRO 6000s while another model server holds about
+# 84 GB of each. vLLM refuses to start unless its share (half the card,
+# 48944 MiB) is free beside its CUDA context (1 GiB), so a fresh install asks
+# for free memory too and keeps llama.cpp, saying why, instead of downloading
+# a vLLM model that cannot start.
+BUSY_ROWS='GPU-ffffffff-rtx6000, 12.0, 97887, 13278\nGPU-ffffffff-rtx6001, 12.0, 97887, 13298\n'
+decide pro6000-busy "export FAKE_GPU_ROWS='$BUSY_ROWS'"
+expect_llama pro6000-busy
+grep -q -- '--query-gpu=uuid,compute_cap,memory.total,memory.free ' "$tmp/pro6000-busy/calls" \
+ || fail "a fresh install must ask nvidia-smi for free memory: $(cat "$tmp/pro6000-busy/calls")"
+grep -qF 'WARN ODS uses llama.cpp, not the vLLM engine: other programs use the GPU memory the vLLM engine needs: ' \
+ "$tmp/pro6000-busy/out" \
+ && grep -qF 'needs 49968 MiB free when it starts (0.50 of the GPU plus 1024 MiB for its CUDA context), and the freest GPU it fits, GPU-ffffffff-rtx6001, has 13298 MiB free.' \
+ "$tmp/pro6000-busy/out" \
+ && grep -qF 'AI Once that memory is free, rerun the installer with --llm-backend vllm' "$tmp/pro6000-busy/out" \
+ || fail "a busy GPU must be shown as the reason, with how to get vLLM later: $(cat "$tmp/pro6000-busy/out")"
+grep -q 'LOG Model server: llama.cpp (other programs use the GPU memory the vLLM engine needs' "$tmp/pro6000-busy/install.log" \
+ || fail "the busy-GPU reason must reach the install log"
+decide pro6000-one-free "export FAKE_GPU_ROWS='GPU-ffffffff-rtx6000, 12.0, 97887, 13278\nGPU-ffffffff-rtx6001, 12.0, 97887, 96900\n'"
+[[ "$(rc_of pro6000-one-free)" == 0 && "$(value pro6000-one-free BACKEND)" == vllm \
+ && "$(value pro6000-one-free VLLM_GPU_UUID)" == GPU-ffffffff-rtx6001 ]] \
+ || fail "vLLM must take the GPU whose memory is free: $(cat "$tmp/pro6000-one-free/out" "$tmp/pro6000-one-free/err")"
+decide pro6000-busy-forced "export FAKE_GPU_ROWS='$BUSY_ROWS'" 'LLM_BACKEND_REQUESTED=vllm'
+expect_refused pro6000-busy-forced 'other programs use the GPU memory the vLLM engine needs'
+pass "a fresh install gives vLLM only a GPU with its share free: busy GPUs keep llama.cpp and say why; --llm-backend vllm stops"
+
decide rtx-forced "export FAKE_GPU_ROWS='$RTX_ROWS'" 'LLM_BACKEND_REQUESTED=vllm'
expect_refused rtx-forced 'no vLLM catalog model fits this host'
grep -qF 'needs one GPU with compute capability 10.0 or 10.3' "$tmp/rtx-forced/err" \
@@ -438,6 +472,19 @@ grep -qF "LOG Kept this installation's vLLM model across installer rerun: glm-5.
"$tmp/rerun-vllm/install.log" || fail "keeping the vLLM model must reach the install log"
pass "a rerun of a vLLM install keeps its model, folder, GPU and memory share; the fresh pick stays the recommendation"
+# The installation's own engine holds its share of the GPU, so a rerun asks
+# for total memory only and keeps vLLM however little of the GPU is free.
+grep -q -- '--query-gpu=uuid,compute_cap,memory.total --format' "$tmp/rerun-vllm/calls" \
+ && ! grep -q 'memory.free' "$tmp/rerun-vllm/calls" \
+ || fail "a rerun must not judge its GPUs by free memory: $(cat "$tmp/rerun-vllm/calls")"
+name=rerun-vllm-busy; mkdir -p "$tmp/$name/install"; vllm_install
+decide rerun-vllm-busy "export FAKE_PRESERVE_OUT='$GLM_CONTRACT'" \
+ "export FAKE_GPU_ROWS='GPU-aaaaaaaa-gb300, 10.3, 256703, 30000\n'"
+[[ "$(rc_of rerun-vllm-busy)" == 0 && "$(value rerun-vllm-busy BACKEND)" == vllm \
+ && "$(value rerun-vllm-busy VLLM_GPU_UUID)" == GPU-aaaaaaaa-gb300 ]] \
+ || fail "a rerun whose own engine holds the GPU must keep vLLM: $(cat "$tmp/rerun-vllm-busy/out" "$tmp/rerun-vllm-busy/err")"
+pass "a rerun of a vLLM install is never judged by the GPU memory its own engine holds"
+
# The kept model never takes the fresh pick's memory share: kept without a
# share of its own, it stays on compose's default even where the fresh pick
# (the RTX PRO 6000 entry here) sets one.
diff --git a/ods/tests/test_select_model_vllm.py b/ods/tests/test_select_model_vllm.py
index a711b5913..37e25b3bd 100644
--- a/ods/tests/test_select_model_vllm.py
+++ b/ods/tests/test_select_model_vllm.py
@@ -224,6 +224,99 @@ def test_the_largest_fitting_gpu_is_the_one_vllm_owns():
assert env_values(out)["VLLM_GPU_UUID"] == "GPU-test-b300"
+# A fresh install adds nvidia-smi memory.free as a fourth field. vLLM refuses
+# to start unless its share of the card is free once its CUDA context exists:
+# half an RTX PRO 6000 is ceil(0.5 * 97887) = 48944 MiB, plus START_MARGIN_MIB
+# (1024) makes 49968 MiB free.
+def test_a_gpu_without_room_for_vllm_is_not_given_to_vllm():
+ # tower2 (2026-10-27): another model server held about 84 GB of each card.
+ rows = ("GPU-test-busy-0, 12.0, 97887, 13278", "GPU-test-busy-1, 12.0, 97887, 13298")
+ status, out, err = select(*VLLM, *_gpu_args(*rows), "--env")
+
+ assert (status, out) == (4, "")
+ assert err == (
+ "error: other programs use the GPU memory the vLLM engine needs: Qwen 3.6 35B A3B NVFP4 (vLLM) "
+ "needs 49968 MiB free when it starts (0.50 of the GPU plus 1024 MiB for its CUDA context), "
+ "and the freest GPU it fits, GPU-test-busy-1, has 13298 MiB free\n"
+ )
+
+
+def test_vllm_takes_a_gpu_with_room_to_start_and_says_why():
+ status, out, err = select(*VLLM, *_gpu_args("GPU-test-busy, 12.0, 97887, 13278",
+ "GPU-test-exact, 12.0, 97887, 49968"), "--env")
+ assert status == 0, err
+ env = env_values(out)
+ assert env["VLLM_GPU_UUID"] == "GPU-test-exact"
+ assert env["MODEL_RECOMMENDATION_REASON"].endswith(
+ " Qwen 3.6 35B A3B NVFP4 (vLLM) would run on GPU GPU-test-busy, but other programs use the memory "
+ "it needs there.")
+
+ status, _, err = select(*VLLM, *_gpu_args("GPU-test-short, 12.0, 97887, 49967"), "--env")
+ assert status == 4 and "has 49967 MiB free" in err
+
+
+def test_a_busy_gb300_gives_vllm_the_idle_rtx_pro_6000_and_says_why():
+ # A DGX Station whose GB300 runs another model server: the RTX PRO 6000's
+ # own entry runs instead of the GB300's top pick, and the reason says so.
+ best_gb300, _ = _best_fitting("10.3", 256703)
+ status, out, err = select(*VLLM, *_gpu_args(RTX_PRO_6000 + ", 97000", GB300 + ", 30000"), "--env")
+
+ assert status == 0, err
+ env = env_values(out)
+ assert (env["VLLM_GPU_UUID"], env["LLM_MODEL"]) == ("GPU-test-rtx-pro-6000", "qwen3.6-35b-a3b-nvfp4")
+ assert (f" {best_gb300['name']} would run on GPU GPU-test-gb300, but other programs use the memory it "
+ f"needs there.") in env["MODEL_RECOMMENDATION_REASON"]
+
+ # Idle, the GB300 keeps its top pick and the reason no note.
+ status, out, err = select(*VLLM, *_gpu_args(RTX_PRO_6000 + ", 97000", GB300 + ", 255000"), "--env")
+ assert status == 0, err
+ env = env_values(out)
+ assert (env["VLLM_GPU_UUID"], env["LLM_MODEL"]) == ("GPU-test-gb300", best_gb300["llm_model_name"])
+ assert "other programs" not in env["MODEL_RECOMMENDATION_REASON"]
+
+ status, out, err = select(*VLLM, *_gpu_args(RTX_PRO_6000 + ", 97000"))
+ assert status == 0, err
+ assert json.loads(out)["gpu"] == {"uuid": "GPU-test-rtx-pro-6000", "compute_capability": "12.0",
+ "memory_mib": 97887, "free_mib": 97000}
+
+
+def test_equal_gpus_go_to_the_one_with_more_memory_free():
+ rows = ("GPU-test-desktop, 12.0, 97887, 95000", "GPU-test-idle, 12.0, 97887, 97000")
+ status, out, err = select(*VLLM, *_gpu_args(*rows), "--env")
+ assert status == 0, err
+ assert env_values(out)["VLLM_GPU_UUID"] == "GPU-test-idle"
+
+
+def test_free_memory_never_lets_a_gpu_fit_that_its_hardware_does_not():
+ # A free H200 still lacks a compute capability any entry names: no fit, not busy.
+ status, out, err = select(*VLLM, *_gpu_args(H200 + ", 143000"), "--env")
+ assert (status, out) == (2, "")
+ assert "GPU-test-h200 (compute capability 9.0, 143771 MiB, 143000 MiB free)" in err
+
+
+def test_an_unreadable_free_field_leaves_that_gpu_unknown():
+ # [N/A] on one GPU must not take vLLM from the whole host.
+ status, out, err = select(*VLLM, *_gpu_args(RTX_PRO_6000 + ", [N/A]", GB300 + ", 250000"), "--env")
+ assert status == 0, err
+ assert env_values(out)["VLLM_GPU_UUID"] == "GPU-test-gb300"
+
+ status, out, err = select(*VLLM, *_gpu_args(RTX_PRO_6000 + ", [N/A]"), "--env")
+ assert status == 0, err
+ assert env_values(out)["VLLM_GPU_UUID"] == "GPU-test-rtx-pro-6000"
+
+ # The other fields stay required.
+ status, out, err = select(*VLLM, *_gpu_args("GPU-test-x, 12.0, [N/A], 1000"), "--env")
+ assert (status, out) == (2, "")
+ assert "is not UUID,COMPUTE_CAP,MEMORY_MIB[,FREE_MIB]" in err
+
+
+def test_vllms_default_share_is_the_memory_estimates():
+ selector = _selector()
+ assert selector.memory_share({}) == selector.vllm_memory_share({}) == 0.9
+ assert selector.memory_share({"gpu_memory_utilization": 0.5}) \
+ == selector.vllm_memory_share({"gpu_memory_utilization": 0.5}) == 0.5
+
+
def test_json_payload_names_the_gpu_folder_and_estimate():
status, out, err = select(*VLLM, *_gpu_args(GB300))