Skip to content

armgit / ODS

Public Imported from GitHub
0

ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.

fix(nvidia): limit llama-server to its assigned GPUs with CUDA_VISIBLE_DEVICES (#8305)

Mike Bradley5b3b2ddd8254
diff --git a/ods/bin/ods-host-agent.py b/ods/bin/ods-host-agent.py
index 0b5130c6c..56bf42113 100755
--- a/ods/bin/ods-host-agent.py
+++ b/ods/bin/ods-host-agent.py
@@ -21623,6 +21623,9 @@ def _llama_recreate_argv(
visible_devices = str(env.get("LLAMA_SERVER_GPU_UUIDS") or "").strip()
if visible_devices:
replacement_env["NVIDIA_VISIBLE_DEVICES"] = visible_devices
+ # As in docker-compose.multigpu-nvidia.yml: only CUDA's own
+ # variable reliably limits llama-server to these GPUs.
+ replacement_env["CUDA_VISIBLE_DEVICES"] = visible_devices
elif str(env.get("GPU_BACKEND") or "").lower() == "amd":
for key in (
"ROCR_VISIBLE_DEVICES",
diff --git a/ods/docker-compose.multigpu-nvidia.yml b/ods/docker-compose.multigpu-nvidia.yml
index 8214c95e6..c9d7761bf 100644
--- a/ods/docker-compose.multigpu-nvidia.yml
+++ b/ods/docker-compose.multigpu-nvidia.yml
@@ -2,6 +2,14 @@ services:
llama-server:
environment:
NVIDIA_VISIBLE_DEVICES: "${LLAMA_SERVER_GPU_UUIDS:?LLAMA_SERVER_GPU_UUIDS must be set for the NVIDIA multi-GPU overlay}"
+ # NVIDIA_VISIBLE_DEVICES alone does not limit llama-server: the base
+ # overlay reserves `count: all`, Docker then appends its own
+ # NVIDIA_VISIBLE_DEVICES=all, and CDI ignores the variable (all four
+ # GPUs stayed visible on a 4x RTX PRO 6000 host, 2026-10-27). CUDA
+ # honors CUDA_VISIBLE_DEVICES, in the listed order, so llama.cpp sees
+ # exactly the GPUs the assignment gave it and LLAMA_ARG_TENSOR_SPLIT
+ # applies to them.
+ CUDA_VISIBLE_DEVICES: "${LLAMA_SERVER_GPU_UUIDS:?LLAMA_SERVER_GPU_UUIDS must be set for the NVIDIA multi-GPU overlay}"
LLAMA_ARG_SPLIT_MODE: "${LLAMA_ARG_SPLIT_MODE:-none}"
LLAMA_ARG_TENSOR_SPLIT: "${LLAMA_ARG_TENSOR_SPLIT:-}"
deploy:
diff --git a/ods/extensions/services/dashboard-api/tests/test_model_activate.py b/ods/extensions/services/dashboard-api/tests/test_model_activate.py
index dba5fc73c..1e09ecd62 100644
--- a/ods/extensions/services/dashboard-api/tests/test_model_activate.py
+++ b/ods/extensions/services/dashboard-api/tests/test_model_activate.py
@@ -2162,6 +2162,8 @@ class TestRecreateLlamaServerFromInspect:
"--runtime", "nvidia",
]
assert "NVIDIA_VISIBLE_DEVICES=GPU-ti-0,GPU-1080,GPU-ti-2" in argv
+ # With --gpus all only CUDA's own variable limits llama.cpp to these GPUs.
+ assert "CUDA_VISIBLE_DEVICES=GPU-ti-0,GPU-1080,GPU-ti-2" in argv
assert "LLAMA_ARG_SPLIT_MODE=layer" in argv
assert "LLAMA_ARG_TENSOR_SPLIT=" in argv
assert "LLAMA_ARG_TENSOR_SPLIT=0.5789,0.4211" not in argv
diff --git a/ods/tests/test-multigpu-nvidia-visibility.sh b/ods/tests/test-multigpu-nvidia-visibility.sh
index 635624d0d..7ac502591 100755
--- a/ods/tests/test-multigpu-nvidia-visibility.sh
+++ b/ods/tests/test-multigpu-nvidia-visibility.sh
@@ -25,7 +25,9 @@ fi
grep -q 'LLAMA_SERVER_GPU_UUIDS must be set' "$tmp_dir/missing.err" \
|| { cat "$tmp_dir/missing.err" >&2; exit 1; }
+# NVIDIA_VISIBLE_DEVICES alone is overridden by the base overlay's `count: all`
+# reservation; CUDA_VISIBLE_DEVICES is what actually limits llama.cpp.
WEBUI_SECRET=test LLAMA_SERVER_GPU_UUIDS=GPU-a,GPU-b "${compose[@]}" \
- | python3 -c 'import json, sys; config=json.load(sys.stdin); visible=config["services"]["llama-server"]["environment"]["NVIDIA_VISIBLE_DEVICES"]; assert visible == "GPU-a,GPU-b", visible'
+ | python3 -c 'import json, sys; env=json.load(sys.stdin)["services"]["llama-server"]["environment"]; assert env["NVIDIA_VISIBLE_DEVICES"] == "GPU-a,GPU-b", env; assert env["CUDA_VISIBLE_DEVICES"] == "GPU-a,GPU-b", env'
echo "test-multigpu-nvidia-visibility: ok"