ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.
Python ai-agentsamdcomfyuidockerllama-cppllmlocal-ain8nnvidiaopen-webuiragself-hostedspeech-to-textstrix-halotext-to-speechworkflow-automation
fix(nvidia): limit llama-server to its assigned GPUs with CUDA_VISIBLE_DEVICES (#8305)
Mike Bradley5b3b2ddd8254
diff --git a/ods/bin/ods-host-agent.py b/ods/bin/ods-host-agent.pyindex 0b5130c6c..56bf42113 100755--- a/ods/bin/ods-host-agent.py+++ b/ods/bin/ods-host-agent.py@@ -21623,6 +21623,9 @@ def _llama_recreate_argv(visible_devices = str(env.get("LLAMA_SERVER_GPU_UUIDS") or "").strip()if visible_devices:replacement_env["NVIDIA_VISIBLE_DEVICES"] = visible_devices+ # As in docker-compose.multigpu-nvidia.yml: only CUDA's own+ # variable reliably limits llama-server to these GPUs.+ replacement_env["CUDA_VISIBLE_DEVICES"] = visible_deviceselif str(env.get("GPU_BACKEND") or "").lower() == "amd":for key in ("ROCR_VISIBLE_DEVICES",diff --git a/ods/docker-compose.multigpu-nvidia.yml b/ods/docker-compose.multigpu-nvidia.ymlindex 8214c95e6..c9d7761bf 100644--- a/ods/docker-compose.multigpu-nvidia.yml+++ b/ods/docker-compose.multigpu-nvidia.yml@@ -2,6 +2,14 @@ services:llama-server:environment:NVIDIA_VISIBLE_DEVICES: "${LLAMA_SERVER_GPU_UUIDS:?LLAMA_SERVER_GPU_UUIDS must be set for the NVIDIA multi-GPU overlay}"+ # NVIDIA_VISIBLE_DEVICES alone does not limit llama-server: the base+ # overlay reserves `count: all`, Docker then appends its own+ # NVIDIA_VISIBLE_DEVICES=all, and CDI ignores the variable (all four+ # GPUs stayed visible on a 4x RTX PRO 6000 host, 2026-10-27). CUDA+ # honors CUDA_VISIBLE_DEVICES, in the listed order, so llama.cpp sees+ # exactly the GPUs the assignment gave it and LLAMA_ARG_TENSOR_SPLIT+ # applies to them.+ CUDA_VISIBLE_DEVICES: "${LLAMA_SERVER_GPU_UUIDS:?LLAMA_SERVER_GPU_UUIDS must be set for the NVIDIA multi-GPU overlay}"LLAMA_ARG_SPLIT_MODE: "${LLAMA_ARG_SPLIT_MODE:-none}"LLAMA_ARG_TENSOR_SPLIT: "${LLAMA_ARG_TENSOR_SPLIT:-}"deploy:diff --git a/ods/extensions/services/dashboard-api/tests/test_model_activate.py b/ods/extensions/services/dashboard-api/tests/test_model_activate.pyindex dba5fc73c..1e09ecd62 100644--- a/ods/extensions/services/dashboard-api/tests/test_model_activate.py+++ b/ods/extensions/services/dashboard-api/tests/test_model_activate.py@@ -2162,6 +2162,8 @@ class TestRecreateLlamaServerFromInspect:"--runtime", "nvidia",]assert "NVIDIA_VISIBLE_DEVICES=GPU-ti-0,GPU-1080,GPU-ti-2" in argv+ # With --gpus all only CUDA's own variable limits llama.cpp to these GPUs.+ assert "CUDA_VISIBLE_DEVICES=GPU-ti-0,GPU-1080,GPU-ti-2" in argvassert "LLAMA_ARG_SPLIT_MODE=layer" in argvassert "LLAMA_ARG_TENSOR_SPLIT=" in argvassert "LLAMA_ARG_TENSOR_SPLIT=0.5789,0.4211" not in argvdiff --git a/ods/tests/test-multigpu-nvidia-visibility.sh b/ods/tests/test-multigpu-nvidia-visibility.shindex 635624d0d..7ac502591 100755--- a/ods/tests/test-multigpu-nvidia-visibility.sh+++ b/ods/tests/test-multigpu-nvidia-visibility.sh@@ -25,7 +25,9 @@ figrep -q 'LLAMA_SERVER_GPU_UUIDS must be set' "$tmp_dir/missing.err" \|| { cat "$tmp_dir/missing.err" >&2; exit 1; }+# NVIDIA_VISIBLE_DEVICES alone is overridden by the base overlay's `count: all`+# reservation; CUDA_VISIBLE_DEVICES is what actually limits llama.cpp.WEBUI_SECRET=test LLAMA_SERVER_GPU_UUIDS=GPU-a,GPU-b "${compose[@]}" \- | python3 -c 'import json, sys; config=json.load(sys.stdin); visible=config["services"]["llama-server"]["environment"]["NVIDIA_VISIBLE_DEVICES"]; assert visible == "GPU-a,GPU-b", visible'+ | python3 -c 'import json, sys; env=json.load(sys.stdin)["services"]["llama-server"]["environment"]; assert env["NVIDIA_VISIBLE_DEVICES"] == "GPU-a,GPU-b", env; assert env["CUDA_VISIBLE_DEVICES"] == "GPU-a,GPU-b", env'echo "test-multigpu-nvidia-visibility: ok"