ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.
Python ai-agentsamdcomfyuidockerllama-cppllmlocal-ain8nnvidiaopen-webuiragself-hostedspeech-to-textstrix-halotext-to-speechworkflow-automation
feat(compose): add the vLLM engine service for LLM_BACKEND=vllm (#8290)
Mike Bradley9ed172f861bf
diff --git a/.github/workflows/validate-compose.yml b/.github/workflows/validate-compose.ymlindex b3ab09ef1..985f8f4d9 100644--- a/.github/workflows/validate-compose.yml+++ b/.github/workflows/validate-compose.yml@@ -128,6 +128,31 @@ jobs:fiexit "$failed"+ - name: Validate vLLM engine overlay stack (base + nvidia + vllm)+ env:+ WEBUI_SECRET: ci-placeholder+ SEARXNG_SECRET: ci-placeholder+ N8N_USER: [email protected]+ N8N_PASS: ci-placeholder+ LITELLM_KEY: ci-placeholder+ VLLM_MODEL_DIR: "example--Model-NVFP4@0123abcd"+ VLLM_SERVED_MODEL: "Model-NVFP4"+ VLLM_GPU_UUID: "GPU-ci-vllm"+ VLLM_ARGS: "--max-model-len 8192 --kv-cache-dtype fp8"+ working-directory: ods+ run: |+ set -e+ stack=(-f docker-compose.base.yml -f docker-compose.nvidia.yml -f docker-compose.vllm.yml)+ docker compose "${stack[@]}" config --quiet+ services="$(docker compose "${stack[@]}" config --services)"+ grep -qx vllm-server <<<"$services" || { echo "FAIL: vllm-server is not in the stack"; exit 1; }+ if grep -qx llama-server <<<"$services"; then echo "FAIL: llama-server is still enabled"; exit 1; fi+ # A missing installer value stops Compose and names the variable.+ if VLLM_GPU_UUID= docker compose "${stack[@]}" config --quiet 2>missing.err; then+ echo "FAIL: the vLLM stack accepted an empty VLLM_GPU_UUID"; exit 1+ fi+ grep -q VLLM_GPU_UUID missing.err || { cat missing.err; exit 1; }+- name: Validate resolver output for AMD multi-GPUworking-directory: odsrun: |diff --git a/ods/.env.schema.json b/ods/.env.schema.jsonindex 141526ade..58149a7f7 100644--- a/ods/.env.schema.json+++ b/ods/.env.schema.json@@ -170,9 +170,36 @@},"LLM_BACKEND": {"type": "string",- "description": "Inference backend. Current installers write llama-server (every local GPU, AMD included) or external; legacy values such as lemonade and litellm remain accepted for upgrade compatibility.",+ "description": "Inference backend. Current installers write llama-server (every local GPU, AMD included) or external; vllm runs the vLLM engine (docker-compose.vllm.yml) instead of llama-server on a big NVIDIA GPU and is opt-in. Legacy values such as lemonade and litellm remain accepted for upgrade compatibility.","default": "llama-server"},+ "VLLM_IMAGE": {+ "type": "string",+ "description": "vLLM engine image for LLM_BACKEND=vllm: the digest-pinned reference recorded in config/dependency-lock.json. The installer writes it; docker-compose.vllm.yml defaults to the same pin."+ },+ "VLLM_MODEL_DIR": {+ "type": "string",+ "description": "Folder under data/models that holds the vLLM model snapshot, e.g. nvidia--GLM-5.3-Flash-NVFP4@da920bb0. Mounted read-only at /model. Required when LLM_BACKEND=vllm."+ },+ "VLLM_SERVED_MODEL": {+ "type": "string",+ "description": "Model id vLLM serves and lists in /v1/models (--served-model-name), e.g. GLM-5.3-Flash-NVFP4. Required when LLM_BACKEND=vllm."+ },+ "VLLM_ARGS": {+ "type": "string",+ "description": "Further vLLM arguments from the model's catalog entry, e.g. --max-model-len 131072 --kv-cache-dtype fp8. Compose splits the string like a shell."+ },+ "VLLM_GPU_UUID": {+ "type": "string",+ "description": "UUID of the one NVIDIA GPU the vLLM engine owns, picked at install (the largest compute capability 10.x device). Required when LLM_BACKEND=vllm."+ },+ "VLLM_GPU_MEMORY_UTILIZATION": {+ "type": "number",+ "description": "Share of the vLLM GPU's memory that vLLM pre-allocates for weights and KV cache (--gpu-memory-utilization). Lower it to leave room for other GPU services on the same GPU.",+ "minimum": 0.01,+ "maximum": 1,+ "default": 0.9+ },"EXTERNAL_LLM_URL": {"type": "string","description": "Host-facing base URL of an installer-validated Ollama or LM Studio runtime. Configure with install.sh rather than editing this field alone.",diff --git a/ods/Makefile b/ods/Makefileindex bc5ee290f..a46f8f684 100644--- a/ods/Makefile+++ b/ods/Makefile@@ -188,6 +188,8 @@ test: ## Run unit and contract tests@echo "=== ComfyUI CUDA kernel cache is persisted ==="@bash tests/test-comfyui-cuda-cache-persisted.sh@bash tests/test-llama-cuda-cache-persisted.sh+ @echo "=== vLLM engine service and its compose selection ==="+ @bash tests/test-vllm-engine-service.sh@echo ""@echo "=== macOS ods config show secret-mask ==="@bash tests/test-macos-config-show-secret-mask.shdiff --git a/ods/config/core-service-ids.json b/ods/config/core-service-ids.jsonindex c08f3f9b5..410fd3e21 100644--- a/ods/config/core-service-ids.json+++ b/ods/config/core-service-ids.json@@ -21,5 +21,6 @@"searxng","token-spy","tts",+ "vllm-server","whisper"]diff --git a/ods/config/dependency-lock.json b/ods/config/dependency-lock.jsonindex 3de2e0e1a..bcab03c9e 100644--- a/ods/config/dependency-lock.json+++ b/ods/config/dependency-lock.json@@ -55,6 +55,16 @@"path": "config/backends/nvidia.json","value": "ghcr.io/ggml-org/llama.cpp:server-cuda13-b11429@sha256:9d6ca9a08b5e0c0a6d2560bff7d4c76bdf946776bbde6a20d9d6403594d93a9f"},+ {+ "id": "vllm.vllm-server",+ "type": "image",+ "path": "docker-compose.vllm.yml",+ "value": "vllm/vllm-openai:v0.32.0@sha256:befccf0e7e4089aa23926bb6f887a85c0d46c2a99e6de5c5ffdf5cb7957c5b34",+ "platforms": {+ "linux/amd64": "sha256:6ee133356d49ed24cee03dd0cda3d3980279a11c107ce3e752c89e76540f41c9",+ "linux/arm64": "sha256:6ff5ff0ddf3658d2c2d677dac5ca2d4726ce0156b4371166d918181b210bd4bc"+ }+ },{"id": "amd.llama-server","type": "image",@@ -458,6 +468,12 @@"default": "ghcr.io/ggml-org/llama.cpp:server-cuda-b11429@sha256:883915ea20a4b350e98f7284098f4d400c21e9b366e248d397cd42d90602a84c","reason": "NVIDIA llama.cpp image can be overridden for testing, but the shipped default is locked."},+ {+ "path": "docker-compose.vllm.yml",+ "value": "${VLLM_IMAGE:-vllm/vllm-openai:v0.32.0@sha256:befccf0e7e4089aa23926bb6f887a85c0d46c2a99e6de5c5ffdf5cb7957c5b34}",+ "default": "vllm/vllm-openai:v0.32.0@sha256:befccf0e7e4089aa23926bb6f887a85c0d46c2a99e6de5c5ffdf5cb7957c5b34",+ "reason": "The installer writes VLLM_IMAGE from the vllm.vllm-server entry; the shipped default is the same digest-pinned multi-arch image."+ },{"path": "docker-compose.cpu.yml","value": "${LLAMA_SERVER_IMAGE:-ghcr.io/ggml-org/llama.cpp:server-b9014@sha256:2e7953dfef88f302bf0683bffa7dc1f8d86ef75910380bc41126ec5b8bedaf53}",diff --git a/ods/docker-compose.vllm.yml b/ods/docker-compose.vllm.ymlnew file mode 100644index 000000000..4b7d46a92--- /dev/null+++ b/ods/docker-compose.vllm.yml@@ -0,0 +1,91 @@+# ODS - vLLM engine overlay+#+# Used when .env sets LLM_BACKEND=vllm: vLLM serves the model on one big+# NVIDIA GPU (datacenter Blackwell, compute capability 10.x) instead of the+# in-stack llama-server. As on the host-native route, llama-server is off and+# model-router stays; endpoints.json points model-router at vllm-server:8000.+#+# This file is intentionally not standalone. scripts/resolve-compose-stack.sh+# layers it right after docker-compose.nvidia.yml when LLM_BACKEND=vllm, and+# refuses it together with a host-native, external or cloud model route or a+# non-NVIDIA GPU. The installer writes the VLLM_* values from the model+# catalog; without VLLM_MODEL_DIR, VLLM_SERVED_MODEL or VLLM_GPU_UUID,+# Compose stops and names the missing one.+#+# Validate the resolved stack with:+#+# LLM_BACKEND=vllm ./scripts/resolve-compose-stack.sh --ods-mode local --gpu-backend nvidia --tier NV_ULTRA++services:+ llama-server:+ profiles:+ - local-inference+ restart: "no"++ vllm-server:+ # Digest-pinned multi-arch image (config/dependency-lock.json). The+ # installer writes the same reference to VLLM_IMAGE.+ image: ${VLLM_IMAGE:-vllm/vllm-openai:v0.32.0@sha256:befccf0e7e4089aa23926bb6f887a85c0d46c2a99e6de5c5ffdf5cb7957c5b34}+ container_name: ods-vllm-server+ restart: unless-stopped+ # Reached only on the ODS network (model-router); no host port.+ expose:+ - "8000"+ security_opt:+ - no-new-privileges:true+ shm_size: 16g+ environment:+ # The first start loads 130-200 GB of weights and compiles kernels;+ # vLLM's shorter default would give up first.+ - VLLM_ENGINE_READY_TIMEOUT_S=3600+ # The weights are local. Never reach the Hugging Face Hub.+ - HF_HUB_OFFLINE=1+ # vLLM sends anonymous usage reports to its developers unless told not to.+ - VLLM_NO_USAGE_STATS=1+ - DO_NOT_TRACK=1+ # The CUDA driver's kernel cache, kept in the mounted cache folder. In+ # the container layer it was lost on every recreate (see llama-server+ # in docker-compose.nvidia.yml); 4 GiB is the driver's maximum.+ - CUDA_CACHE_PATH=/root/.cache/nv/ComputeCache+ - CUDA_CACHE_MAXSIZE=4294967296+ volumes:+ # The model's snapshot folder under data/models, read-only.+ - ./data/models/${VLLM_MODEL_DIR:?VLLM_MODEL_DIR must name the model folder under data/models}:/model:ro,z+ # FlashInfer autotune, torch.compile and CUDA kernel caches. Without+ # this mount every recreate (ods update, a restart on a new model)+ # compiles again. Rebuilt on start, so backups skip it.+ - ./data/vllm/cache:/root/.cache:z+ # Compose splits this string like a shell, so VLLM_ARGS (the catalog+ # entry's argument string) becomes separate arguments to `vllm serve`.+ command: >-+ --model /model+ --served-model-name ${VLLM_SERVED_MODEL:?VLLM_SERVED_MODEL must name the model id vLLM serves}+ --host 0.0.0.0+ --port 8000+ --gpu-memory-utilization ${VLLM_GPU_MEMORY_UTILIZATION:-0.90}+ ${VLLM_ARGS:-}+ deploy:+ resources:+ reservations:+ devices:+ # Exactly the GPU the installer picked. Do not add a second entry+ # in another overlay: Compose appends reservation lists, and the+ # container would get an extra GPU.+ - driver: nvidia+ device_ids: ["${VLLM_GPU_UUID:?VLLM_GPU_UUID must name the GPU vLLM owns}"]+ capabilities: [gpu]+ healthcheck:+ # /health answers 200 with an empty body once the engine is ready.+ # The image has python3 because vLLM runs on it; curl may be missing.+ test: ["CMD", "python3", "-c", "import sys,urllib.request;sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8000/health',timeout=5).status==200 else 1)"]+ interval: 30s+ timeout: 10s+ retries: 5+ # As long as VLLM_ENGINE_READY_TIMEOUT_S, so Docker does not call a+ # first start unhealthy before vLLM itself gives up.+ start_period: 60m+ logging:+ driver: "json-file"+ options:+ max-size: "10m"+ max-file: "3"diff --git a/ods/lib/backup-paths.sh b/ods/lib/backup-paths.shindex 095153304..ad2f1878f 100644--- a/ods/lib/backup-paths.sh+++ b/ods/lib/backup-paths.sh@@ -60,6 +60,7 @@ ODS_BACKUP_EXCLUDED_DATA_PATHS=("data/whisper" # STT model cache"data/embeddings" # TEI model cache"data/llama-server" # CUDA kernel cache, root-owned; rebuilt on the next start+ "data/vllm" # vLLM compile/autotune cache, rebuilt on start# dashboard-api runtime files: progress, status and receipts that the# running stack rewrites, or copies of state captured elsewhere."data/extensions-library" # catalog copy; re-copied by the installerdiff --git a/ods/scripts/resolve-compose-stack.sh b/ods/scripts/resolve-compose-stack.shindex c9e9d5400..587a4aafc 100755--- a/ods/scripts/resolve-compose-stack.sh+++ b/ods/scripts/resolve-compose-stack.sh@@ -219,6 +219,12 @@ selected_stack = globals().get("_selected_stack")resolved = []primary = "docker-compose.yml"+# LLM_BACKEND=vllm: the vllm-server container (docker-compose.vllm.yml)+# serves the model on a big NVIDIA GPU. Like the host-native route it turns+# the in-stack llama-server off and keeps model-router. Any other value,+# including none, keeps today's selection.+vllm_engine = selected_stack is None and _install_env_value("LLM_BACKEND").lower() == "vllm"+# An ODS-managed host-native runtime owns inference selection, even when# hardware detection supplied a local CPU/AMD/NVIDIA profile to the installer.if selected_stack is not None:@@ -307,6 +313,41 @@ if selected_stack is None and "docker-compose.amd.yml" in resolved:sys.exit(1)resolved.insert(resolved.index("docker-compose.amd.yml") + 1, "docker-compose.amd-rocm.yml")++def _vllm_route_conflict(files):+ """Why LLM_BACKEND=vllm cannot run with this selection, or "" if it can.++ vLLM replaces only the in-stack llama-server on the NVIDIA stack, so a+ route without that server, or another GPU vendor, is refused instead of+ silently starting a different engine than .env asks for.+ """+ if host_native:+ return "NATIVE_LLM_BASE_URL is set, so a host-native llama-server serves the model"+ if external_llm:+ return "EXTERNAL_LLM_URL is set, so an external model server serves the model"+ if ods_mode == "cloud" or tier == "CLOUD":+ return "ODS runs in cloud mode, which has no local model server"+ if gpu_backend != "nvidia":+ return f"the GPU backend is {gpu_backend}; the vLLM engine runs only on NVIDIA GPUs"+ if "docker-compose.nvidia.yml" not in files:+ return "this hardware selection does not include docker-compose.nvidia.yml"+ return ""+++# The vLLM overlay comes right after the NVIDIA overlay (also when a hardware+# profile chose it), which still supplies the GPU stack.+if vllm_engine:+ vllm_conflict = _vllm_route_conflict(resolved)+ if vllm_conflict:+ print(f"ERROR: LLM_BACKEND=vllm cannot be used: {vllm_conflict}. "+ "Remove LLM_BACKEND=vllm or the conflicting setting from .env.", file=sys.stderr)+ sys.exit(1)+ if not (script_dir / "docker-compose.vllm.yml").exists():+ print("ERROR: LLM_BACKEND=vllm but docker-compose.vllm.yml is missing", file=sys.stderr)+ sys.exit(1)+ if "docker-compose.vllm.yml" not in resolved:+ resolved.insert(resolved.index("docker-compose.nvidia.yml") + 1, "docker-compose.vllm.yml")+# A generated auth file is an override, never a primary/profile input. Remove# stale cached occurrences before extension discovery so it cannot make a# partial core service appear to exist or influence overlay eligibility.@@ -1436,8 +1477,9 @@ if selected_stack is None and ext_dir.exists():# A host-native runtime is also a host process. Its overlay# profiles out ODS's in-stack llama-server, so local-mode overlays# that wait on `llama-server: service_healthy` would point at a- # disabled service and break lifecycle commands.- if ods_mode in ("local", "hybrid") and tier != "CLOUD" and gpu_backend != "apple" and not host_native and not external_llm:+ # disabled service and break lifecycle commands. The vLLM overlay+ # profiles llama-server out the same way.+ if ods_mode in ("local", "hybrid") and tier != "CLOUD" and gpu_backend != "apple" and not host_native and not external_llm and not vllm_engine:local_mode_overlay = service_dir / "compose.local.yaml"if local_mode_overlay.exists():resolved.append(str(local_mode_overlay.relative_to(script_dir)))@@ -1583,6 +1625,7 @@ if user_ext_dir.exists():and tier != "CLOUD"and not host_nativeand not external_llm+ and not vllm_engine)if (not managed_local_inference@@ -1603,11 +1646,11 @@ if user_ext_dir.exists():# service_healthy` inside compose.local.yaml overlays can never be# satisfied and deadlocks the stack. The real LLM-ready gate on macOS# is the `llama-server-ready` sidecar defined in the macOS overlay.- # A host-native runtime likewise disables ODS's in-stack- # llama-server, so user-local overlays must not add local- # llama-server health dependencies. Mirrors the same guard in- # the built-in loop above (PR #1004).- if ods_mode in ("local", "hybrid") and tier != "CLOUD" and gpu_backend != "apple" and not host_native and not external_llm:+ # A host-native runtime (or the vLLM engine) likewise disables+ # ODS's in-stack llama-server, so user-local overlays must not+ # add local llama-server health dependencies. Mirrors the same+ # guard in the built-in loop above (PR #1004).+ if ods_mode in ("local", "hybrid") and tier != "CLOUD" and gpu_backend != "apple" and not host_native and not external_llm and not vllm_engine:local_mode_overlay = service_dir / "compose.local.yaml"if local_mode_overlay.exists():# Same content scan as compose.yaml/gpu overlay above —diff --git a/ods/tests/ci-suite.txt b/ods/tests/ci-suite.txtindex 7f9e30789..2a86cc4bb 100644--- a/ods/tests/ci-suite.txt+++ b/ods/tests/ci-suite.txt@@ -62,6 +62,7 @@ tests/test-cli-update-verification.shtests/test-comfyui-user-dir-persisted.shtests/test-comfyui-cuda-cache-persisted.shtests/test-llama-cuda-cache-persisted.sh+tests/test-vllm-engine-service.shtests/test_comfyui_extension_contract.pytests/test_extension_assets.pytests/test-compose-failure-evidence.shdiff --git a/ods/tests/test-vllm-engine-service.sh b/ods/tests/test-vllm-engine-service.shnew file mode 100755index 000000000..e21e89281--- /dev/null+++ b/ods/tests/test-vllm-engine-service.sh@@ -0,0 +1,343 @@+#!/usr/bin/env bash+# LLM_BACKEND=vllm runs the model in the vllm-server container+# (docker-compose.vllm.yml) instead of the in-stack llama-server, on the one+# NVIDIA GPU the installer picked. This pins the service contract the other+# vLLM pieces build on (container, port, mounts, GPU, healthcheck, command,+# image pin, .env keys, backup exclusion) and the resolver selection: the+# overlay follows docker-compose.nvidia.yml only for LLM_BACKEND=vllm, every+# route it cannot replace is refused, and every other selection is unchanged.+# Needs no Docker daemon.+set -euo pipefail++ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"+OVERLAY="${ODS_VLLM_COMPOSE_UNDER_TEST:-$ROOT_DIR/docker-compose.vllm.yml}"+RESOLVER="${ODS_RESOLVER_UNDER_TEST:-$ROOT_DIR/scripts/resolve-compose-stack.sh}"+PYTHON_CMD="${ODS_PYTHON_CMD:-python3}"++fail() { echo "[FAIL] $*" >&2; exit 1; }+pass() { echo "[PASS] $*"; }++for f in "$OVERLAY" "$RESOLVER" "$ROOT_DIR/docker-compose.base.yml" "$ROOT_DIR/docker-compose.nvidia.yml"; do+ [[ -f "$f" ]] || fail "missing $f"+done++# 1. The overlay's service contract, read as Compose will read it.+"$PYTHON_CMD" - "$OVERLAY" "$ROOT_DIR/docker-compose.base.yml" "$ROOT_DIR/config/dependency-lock.json" \+ "$ROOT_DIR/.env.schema.json" "$ROOT_DIR/config/core-service-ids.json" <<'PY' || fail "docker-compose.vllm.yml contract (see above)"+import http.server+import json+import os+import re+import shlex+import subprocess+import sys+import threading++import yaml++overlay_path, base_path, lock_path, schema_path, core_ids_path = sys.argv[1:6]+text = open(overlay_path, encoding="utf-8").read()+overlay = yaml.safe_load(text)+base = yaml.safe_load(open(base_path, encoding="utf-8"))+lock = json.load(open(lock_path, encoding="utf-8"))+schema = json.load(open(schema_path, encoding="utf-8"))["properties"]+core_ids = json.load(open(core_ids_path, encoding="utf-8"))+errors = []+++def check(condition, message):+ if not condition:+ errors.append(message)+++services = overlay.get("services") or {}+check(set(services) == {"llama-server", "vllm-server"},+ f"the overlay may only switch llama-server off and add vllm-server, got {sorted(services)}")+check(services.get("llama-server") == {"profiles": ["local-inference"], "restart": "no"},+ f"llama-server must be profiled out and not restarted: {services.get('llama-server')}")+vllm = services.get("vllm-server") or {}++# Image: the digest pin in config/dependency-lock.json, overridable by VLLM_IMAGE.+entry = next((e for e in lock["entries"] if e.get("id") == "vllm.vllm-server"), {})+pinned = str(entry.get("value", ""))+check(entry.get("path") == "docker-compose.vllm.yml"+ and re.fullmatch(r"vllm/vllm-openai:v[0-9][^@\s]*@sha256:[0-9a-f]{64}", pinned),+ f"dependency-lock.json needs a digest-pinned vllm.vllm-server entry, got {entry}")+check(vllm.get("image") == "${VLLM_IMAGE:-" + pinned + "}",+ f"vllm-server must run ${{VLLM_IMAGE}} defaulting to the locked pin, got {vllm.get('image')!r}")+allowed = [i for i in lock.get("allow_variable_refs", []) if i.get("path") == "docker-compose.vllm.yml"]+check(len(allowed) == 1 and allowed[0].get("value") == vllm.get("image") and allowed[0].get("default") == pinned,+ "allow_variable_refs must document the VLLM_IMAGE reference with the locked default")++check(vllm.get("container_name") == "ods-vllm-server", "the container must be ods-vllm-server")+check(vllm.get("restart") == "unless-stopped", "vllm-server must restart unless stopped")+check("profiles" not in vllm, "vllm-server must start whenever the overlay is selected")+check("ports" not in vllm, "vllm-server must not publish a host port")+check(vllm.get("security_opt") == ["no-new-privileges:true"], "vllm-server must set no-new-privileges")+check(str(vllm.get("shm_size")) == "16g", "vllm-server needs a 16g /dev/shm")+check(vllm.get("logging") == base.get("x-logging"), "vllm-server must use the stack's default logging policy")+llama = base["services"]["llama-server"]+check(sorted(vllm.get("networks") or ["default"]) == sorted(llama.get("networks") or ["default"]),+ "vllm-server must join the networks llama-server uses")++env = vllm.get("environment") or []+env = dict(item.split("=", 1) for item in env) if isinstance(env, list) else {k: str(v) for k, v in env.items()}+for key, value in {"VLLM_ENGINE_READY_TIMEOUT_S": "3600", "HF_HUB_OFFLINE": "1",+ "VLLM_NO_USAGE_STATS": "1", "DO_NOT_TRACK": "1"}.items():+ check(env.get(key) == value, f"vllm-server must set {key}={value}")+check(env.get("CUDA_CACHE_PATH", "").startswith("/root/.cache/"),+ "the CUDA kernel cache must live in the mounted cache folder")++# Mounts: one model folder read-only, the compile/autotune cache writable.+mounts = {}+for spec in vllm.get("volumes") or []:+ match = re.fullmatch(r"(?P<src>.+):(?P<dst>/[^:]*)(?::(?P<opts>[^:]+))?", str(spec))+ check(match is not None, f"unexpected volume {spec!r}")+ if match:+ mounts[match["dst"]] = (match["src"], set(filter(None, (match["opts"] or "").split(","))))+check(set(mounts) == {"/model", "/root/.cache"}, f"vllm-server must mount /model and /root/.cache only, got {sorted(mounts)}")+source, options = mounts.get("/model", ("", set()))+check(re.fullmatch(r"\./data/models/\$\{VLLM_MODEL_DIR:\?[^}]+\}", source),+ f"/model must be the data/models folder a required VLLM_MODEL_DIR names, got {source!r}")+check({"ro", "z"} <= options, "/model must be read-only with the SELinux z label")+source, options = mounts.get("/root/.cache", ("", set()))+check(source == "./data/vllm/cache" and "z" in options and "ro" not in options,+ "/root/.cache must be ./data/vllm/cache, writable, with the SELinux z label")++# GPU: exactly the device VLLM_GPU_UUID names.+devices = (((vllm.get("deploy") or {}).get("resources") or {}).get("reservations") or {}).get("devices")+device = devices[0] if isinstance(devices, list) and len(devices) == 1 else {}+check(device.get("driver") == "nvidia" and device.get("capabilities") == ["gpu"] and "count" not in device,+ f"vllm-server must reserve one NVIDIA GPU by id, got {devices}")+ids = device.get("device_ids")+check(isinstance(ids, list) and len(ids) == 1 and re.fullmatch(r"\$\{VLLM_GPU_UUID:\?[^}]+\}", str(ids[0])),+ f"the reservation must name only a required VLLM_GPU_UUID, got {ids}")++# Healthcheck: no curl, a long first-start grace period.+health = vllm.get("healthcheck") or {}+test = health.get("test") or []+check(test[:3] == ["CMD", "python3", "-c"] and len(test) == 4, f"the healthcheck must run python3 -c, got {test}")+check(not any(tool in " ".join(map(str, test)) for tool in ("curl", "wget")), "the healthcheck must not need curl or wget")+++def seconds(value):+ match = re.fullmatch(r"(?:(\d+)h)?(?:(\d+)m)?(?:(\d+)s)?", str(value))+ return None if not match or not any(match.groups()) else sum(+ int(part or 0) * unit for part, unit in zip(match.groups(), (3600, 60, 1)))+++start_period, interval = seconds(health.get("start_period")), seconds(health.get("interval"))+check(start_period is not None and start_period >= 1800, "the healthcheck start_period must allow at least 30 minutes")+check(interval is not None and interval <= 60 and seconds(health.get("timeout")) is not None,+ "the healthcheck needs an interval of at most a minute and a timeout")+check(isinstance(health.get("retries"), int) and health["retries"] >= 3, "the healthcheck needs at least 3 retries")++# Run the probe itself: vLLM's ready /health is an empty 200; 503 and a+# refused connection are unhealthy.+probe = str(test[3]) if len(test) == 4 else ""+check("http://127.0.0.1:8000/health" in probe, "the probe must ask http://127.0.0.1:8000/health")+status = {"code": 200}+++class Health(http.server.BaseHTTPRequestHandler):+ def do_GET(self):+ self.send_response(status["code"] if self.path == "/health" else 404)+ self.send_header("Content-Length", "0")+ self.end_headers()++ def log_message(self, *args):+ pass+++server = http.server.HTTPServer(("127.0.0.1", 0), Health)+threading.Thread(target=server.serve_forever, daemon=True).start()+port = server.server_address[1]+plain_env = {k: v for k, v in os.environ.items() if "proxy" not in k.lower()}+++def probe_rc():+ code = probe.replace("127.0.0.1:8000", f"127.0.0.1:{port}")+ return subprocess.run([sys.executable, "-c", code], env=plain_env, capture_output=True, timeout=30).returncode+++if probe:+ check(probe_rc() == 0, "the probe must accept vLLM's empty 200 /health")+ status["code"] = 503+ check(probe_rc() != 0, "the probe must fail on a 503 /health")+ server.shutdown()+ server.server_close()+ check(probe_rc() != 0, "the probe must fail when nothing listens on the port")++# Command: the string form Compose splits like a shell, in the contract order.+command = vllm.get("command")+check(isinstance(command, str), "vllm-server's command must be the string form")+VAR = re.compile(r"\$\{(?P<name>[A-Za-z_][A-Za-z0-9_]*)(?:(?P<op>:-|:\?)(?P<arg>[^}]*))?\}")+++def interpolate(value, values):+ def replace(match):+ found = values.get(match["name"], "")+ if not found and match["op"] == ":?":+ raise KeyError(match["name"])+ return found or (match["arg"] if match["op"] == ":-" else "")+ return VAR.sub(replace, value)+++if isinstance(command, str):+ args = "--max-model-len 131072 --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":3}'"+ argv = shlex.split(interpolate(command, {"VLLM_SERVED_MODEL": "GLM-5.3-Flash-NVFP4", "VLLM_ARGS": args}))+ check(argv == ["--model", "/model", "--served-model-name", "GLM-5.3-Flash-NVFP4", "--host", "0.0.0.0",+ "--port", "8000", "--gpu-memory-utilization", "0.90", "--max-model-len", "131072",+ "--speculative-config", '{"method":"mtp","num_speculative_tokens":3}'],+ f"vllm-server's command breaks the contract: {argv}")+ argv = shlex.split(interpolate(command, {"VLLM_SERVED_MODEL": "m", "VLLM_GPU_MEMORY_UTILIZATION": "0.5"}))+ check(argv[-2:] == ["--gpu-memory-utilization", "0.5"], f"VLLM_GPU_MEMORY_UTILIZATION is not applied: {argv}")+ try:+ interpolate(command, {})+ errors.append("the command must require VLLM_SERVED_MODEL")+ except KeyError:+ pass++# Every variable is a documented .env key; the GPU, folder and model id are required.+names = set(re.findall(r"\$\{([A-Za-z_][A-Za-z0-9_]*)", text))+check(names == {"VLLM_IMAGE", "VLLM_MODEL_DIR", "VLLM_SERVED_MODEL", "VLLM_ARGS", "VLLM_GPU_UUID",+ "VLLM_GPU_MEMORY_UTILIZATION"}, f"unexpected variables in the overlay: {sorted(names)}")+check(set(re.findall(r"\$\{([A-Za-z_][A-Za-z0-9_]*):\?", text)) == {"VLLM_MODEL_DIR", "VLLM_SERVED_MODEL", "VLLM_GPU_UUID"},+ "VLLM_MODEL_DIR, VLLM_SERVED_MODEL and VLLM_GPU_UUID must stop Compose when unset")+undocumented = sorted(name for name in names if name not in schema)+check(not undocumented, f".env.schema.json does not document {undocumented}")+utilization = schema.get("VLLM_GPU_MEMORY_UTILIZATION", {})+check(utilization.get("type") == "number" and utilization.get("maximum") == 1 and utilization.get("default") == 0.9,+ "VLLM_GPU_MEMORY_UTILIZATION must be a number at most 1, default 0.9")+backend = schema.get("LLM_BACKEND", {})+check("vllm" in backend["enum"] if "enum" in backend else "vllm" in backend.get("description", ""),+ "LLM_BACKEND in .env.schema.json must list vllm")+check("vllm-server" in core_ids, "config/core-service-ids.json must protect vllm-server from shadowing extensions")++for error in errors:+ print(f" - {error}")+sys.exit(1 if errors else 0)+PY+pass "docker-compose.vllm.yml runs one pinned vLLM on the installer's GPU, mounts, probe and command as contracted"++# 2. The cache is a deliberate backup decision.+# shellcheck source=/dev/null+. "$ROOT_DIR/lib/backup-paths.sh"+[[ " ${ODS_BACKUP_EXCLUDED_DATA_PATHS[*]} " == *" data/vllm "* ]] \+ || fail "lib/backup-paths.sh must exclude data/vllm (rebuilt compile cache) from backups"+[[ " ${ODS_USER_DATA_PATHS[*]} " != *" data/vllm "* ]] || fail "data/vllm is a cache, not user data"+pass "backups skip the vLLM compile cache"++# 3. Resolver selection. Each run starts from a clean model route; ROUTE_ENV+# adds the selector a case needs.+TMP_DIR="$(mktemp -d)"+trap 'rm -rf "$TMP_DIR"' EXIT+ERR_FILE="$TMP_DIR/resolver.err"+ROUTE_ENV=()+FILES=""+# resolve LLM_BACKEND|<unset> RESOLVER_ARGS... -> FILES, one Compose file per line+resolve() {+ local backend="$1" out rc=0+ shift+ local -a route=(-u LLM_BACKEND -u NATIVE_LLM_BASE_URL -u EXTERNAL_LLM_URL -u ODS_EXTERNAL_LLM_SELECTED+ -u AMD_INFERENCE_BACKEND -u ODS_SKIP_GPU_OVERLAYS -u ODS_SKIP_GPU_OVERLAYS_FOR)+ [[ "$backend" == "<unset>" ]] || route+=("LLM_BACKEND=$backend")+ out="$(env "${route[@]}" ${ROUTE_ENV[@]+"${ROUTE_ENV[@]}"} "$RESOLVER" "$@" 2>"$ERR_FILE")" || rc=$?+ FILES="$(printf '%s\n' "$out" | tr '\\' '/' | awk '{ for (i = 1; i <= NF; i++) if ($i != "-f") print $i }')"+ return "$rc"+}+has_file() { grep -qxF -- "$1" <<<"$FILES"; }+first_files() { head -n "$1" <<<"$FILES" | paste -sd, -; }+without() { awk -v drop="$1" 'index($0, drop) == 0' <<<"$2"; }++real=(--script-dir "$ROOT_DIR" --tier NV_ULTRA --gpu-backend nvidia --ods-mode local)+resolve "<unset>" "${real[@]}" || fail "default NVIDIA resolution failed: $(cat "$ERR_FILE")"+default_files="$FILES"+if has_file docker-compose.vllm.yml; then fail "the default NVIDIA stack selected the vLLM overlay"; fi+has_file extensions/services/litellm/compose.local.yaml \+ || fail "the default NVIDIA stack lost LiteLLM's llama-server health overlay"+resolve llama-server "${real[@]}" || fail "LLM_BACKEND=llama-server resolution failed: $(cat "$ERR_FILE")"+[[ "$FILES" == "$default_files" ]] || fail "LLM_BACKEND=llama-server changed the default selection"+pass "unset and llama-server LLM_BACKEND keep today's NVIDIA stack"++resolve vllm "${real[@]}" || fail "LLM_BACKEND=vllm resolution failed: $(cat "$ERR_FILE")"+[[ "$(first_files 3)" == "docker-compose.base.yml,docker-compose.nvidia.yml,docker-compose.vllm.yml" ]] \+ || fail "the vLLM overlay must follow the NVIDIA overlay: $(first_files 3)"+if grep -qF compose.local.yaml <<<"$FILES"; then+ fail "the vLLM stack kept a local-mode overlay that waits on the disabled llama-server"+fi+[[ "$(without docker-compose.vllm.yml "$FILES")" == "$(without /compose.local.yaml "$default_files")" ]] \+ || fail "LLM_BACKEND=vllm must only add the overlay and drop the llama-server health overlays"+pass "LLM_BACKEND=vllm layers base + nvidia + vllm and drops the llama-server waits"++resolve vllm "${real[@]}" --gpu-count 2 || fail "multi-GPU vLLM resolution failed: $(cat "$ERR_FILE")"+[[ "$(first_files 4)" == "docker-compose.base.yml,docker-compose.nvidia.yml,docker-compose.vllm.yml,docker-compose.multigpu-nvidia.yml" ]] \+ || fail "multi-GPU vLLM stack order: $(first_files 4)"+resolve vllm "${real[@]}" --profile-overlays docker-compose.base.yml,docker-compose.nvidia.yml \+ || fail "installer-profile vLLM resolution failed: $(cat "$ERR_FILE")"+[[ "$(first_files 3)" == "docker-compose.base.yml,docker-compose.nvidia.yml,docker-compose.vllm.yml" ]] \+ || fail "an installer hardware profile lost the vLLM overlay: $(first_files 3)"+pass "the vLLM overlay follows the NVIDIA overlay on multi-GPU and installer-profile stacks"++# refuse REASON RESOLVER_ARGS...: LLM_BACKEND=vllm must stop with REASON and no stack.+refuse() {+ local reason="$1"+ shift+ if resolve vllm "$@"; then+ fail "LLM_BACKEND=vllm was accepted with $* ${ROUTE_ENV[*]-}"+ fi+ [[ -z "$FILES" ]] || fail "a refused vLLM selection still printed a stack"+ grep -qF "LLM_BACKEND=vllm cannot be used" "$ERR_FILE" && grep -qF -- "$reason" "$ERR_FILE" \+ || fail "refusal for $* ${ROUTE_ENV[*]-} does not say '$reason': $(cat "$ERR_FILE")"+}+ROUTE_ENV=(NATIVE_LLM_BASE_URL=http://localhost:8080); refuse NATIVE_LLM_BASE_URL "${real[@]}"+ROUTE_ENV=(EXTERNAL_LLM_URL=http://127.0.0.1:11434); refuse EXTERNAL_LLM_URL "${real[@]}"+ROUTE_ENV=(ODS_EXTERNAL_LLM_SELECTED=true); refuse EXTERNAL_LLM_URL "${real[@]}"+ROUTE_ENV=()+refuse "cloud mode" --script-dir "$ROOT_DIR" --tier NV_ULTRA --gpu-backend nvidia --ods-mode cloud+refuse "cloud mode" --script-dir "$ROOT_DIR" --tier CLOUD --gpu-backend nvidia --ods-mode local+for backend in amd cpu apple intel; do+ refuse "GPU backend is $backend" --script-dir "$ROOT_DIR" --tier 1 --gpu-backend "$backend" --ods-mode local+done+refuse "does not include docker-compose.nvidia.yml" --script-dir "$ROOT_DIR" --tier AP_PRO --gpu-backend nvidia --ods-mode local+pass "LLM_BACKEND=vllm is refused with host-native, external and cloud routes and non-NVIDIA GPUs"++# 4. An installed .env selects it too, and user-extension llama-server waits+# are dropped with it. A minimal install tree keeps this independent of the+# shipped extensions.+tree="$TMP_DIR/install"+mkdir -p "$tree/lib" "$tree/data/user-extensions/probe-ext"+cp "$ROOT_DIR/docker-compose.base.yml" "$ROOT_DIR/docker-compose.nvidia.yml" "$tree/"+cp "$OVERLAY" "$tree/docker-compose.vllm.yml"+cp "$ROOT_DIR/lib/python-cmd.sh" "$tree/lib/"+printf '%s\n' 'schema_version: ods.services.v1' 'service:' ' id: probe-ext' ' name: Probe' \+ ' compose_file: compose.yaml' > "$tree/data/user-extensions/probe-ext/manifest.yaml"+printf '%s\n' 'services:' ' probe-ext:' ' image: busybox:1.36.1' \+ > "$tree/data/user-extensions/probe-ext/compose.yaml"+printf '%s\n' 'services:' ' probe-ext:' ' depends_on:' ' llama-server:' \+ ' condition: service_healthy' > "$tree/data/user-extensions/probe-ext/compose.local.yaml"+installed=(--script-dir "$tree" --tier NV_ULTRA --gpu-backend nvidia --ods-mode local)++printf 'LLM_BACKEND=llama-server\n' > "$tree/.env"+resolve "<unset>" "${installed[@]}" || fail "installed llama-server resolution failed: $(cat "$ERR_FILE")"+if has_file docker-compose.vllm.yml; then fail "an installed LLM_BACKEND=llama-server selected the vLLM overlay"; fi+has_file data/user-extensions/probe-ext/compose.local.yaml || fail "the fixture's local-mode overlay was not selected"++printf 'LLM_BACKEND="vllm"\n' > "$tree/.env"+resolve "<unset>" "${installed[@]}" || fail "installed vLLM resolution failed: $(cat "$ERR_FILE")"+[[ "$(first_files 3)" == "docker-compose.base.yml,docker-compose.nvidia.yml,docker-compose.vllm.yml" ]] \+ || fail "an installed LLM_BACKEND=vllm did not select the overlay: $(first_files 3)"+if has_file data/user-extensions/probe-ext/compose.local.yaml; then+ fail "the vLLM stack kept a user extension's wait on the disabled llama-server"+fi+has_file data/user-extensions/probe-ext/compose.yaml || fail "the vLLM stack dropped the user extension itself"++rm "$tree/docker-compose.vllm.yml"+if resolve "<unset>" "${installed[@]}"; then+ fail "LLM_BACKEND=vllm resolved without docker-compose.vllm.yml"+fi+grep -qF "docker-compose.vllm.yml is missing" "$ERR_FILE" \+ || fail "a missing vLLM overlay is not named: $(cat "$ERR_FILE")"+pass "an installed .env selects the overlay, drops extension llama-server waits, and a missing overlay is named"++echo "[PASS] vLLM engine service"