Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: resolve UUID-form CUDA_VISIBLE_DEVICES to physical GPU indices (#8917)

Darshan Poudel5169487183df
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 97bcdcb82f0..d7432ec4c1e 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -10687,7 +10687,14 @@ class LlamaCppBackend:
try:
return [int(x.strip()) for x in cvd.split(",") if x.strip()]
except ValueError:
+ pass
+ # A resolvable CUDA UUID mask, else the GPU auto-pick escapes the mask onto hidden cards (#8873).
+ try:
+ from utils.hardware.hardware import _get_parent_visible_gpu_spec
+ spec = _get_parent_visible_gpu_spec()
+ except Exception:
return None
+ return spec["numeric_ids"] if spec.get("raw") == cvd.strip() else None
@staticmethod
def _visibility_mask_is_unmappable() -> bool:
@@ -12445,7 +12452,15 @@ class LlamaCppBackend:
try:
return set(int(x.strip()) for x in raw.split(",") if x.strip())
except ValueError:
- return None
+ if (
+ env_name != "CUDA_VISIBLE_DEVICES"
+ or os.environ.get("CUDA_DEVICE_ORDER") != "PCI_BUS_ID"
+ ):
+ return None
+ from utils.hardware import nvidia
+
+ ids = nvidia.resolve_uuid_mask(raw.strip())
+ return set(ids) if ids is not None else None
@staticmethod
def _vulkan_pin_args(gpu_indices: Optional[Iterable[int]]) -> list[str]:
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index 5261e4dd389..ce5fdf58790 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -194,7 +194,7 @@ class LoadRequest(BaseModel):
"supported on XPU, and physical IDs are unsupported when the parent "
"visibility mask uses "
"non-numeric or subdevice entries, including CUDA_VISIBLE_DEVICES "
- "with UUID/MIG entries and ZE_AFFINITY_MASK with subdevice tokens "
+ "with MIG or unresolvable UUID entries and ZE_AFFINITY_MASK with subdevice tokens "
"(for example '0.0,0.1') or FLAT-hierarchy tile handles. For GGUF "
"models the fitter may pin the smallest subset of this pool that fits."
),
diff --git a/studio/backend/models/training.py b/studio/backend/models/training.py
index 099bd698af7..8be9b5f79aa 100644
--- a/studio/backend/models/training.py
+++ b/studio/backend/models/training.py
@@ -605,8 +605,8 @@ class TrainingStartRequest(BaseModel):
"Physical GPU indices to use, for example [0, 1]. Omit or pass "
"[] to use automatic selection. Explicit gpu_ids are unsupported "
"when the parent visibility mask uses non-numeric or subdevice "
- "entries -- this includes CUDA_VISIBLE_DEVICES with UUID/MIG "
- "entries on NVIDIA, and ZE_AFFINITY_MASK with subdevice tokens "
+ "entries -- this includes CUDA_VISIBLE_DEVICES with MIG or "
+ "unresolvable UUID entries on NVIDIA, and ZE_AFFINITY_MASK with subdevice tokens "
"(e.g. '0.0,0.1') or FLAT-hierarchy (default) tile handles on "
"Intel XPU."
),
diff --git a/studio/backend/tests/test_gpu_selection.py b/studio/backend/tests/test_gpu_selection.py
index 3b50cba6555..b08232e535a 100644
--- a/studio/backend/tests/test_gpu_selection.py
+++ b/studio/backend/tests/test_gpu_selection.py
@@ -33,6 +33,7 @@ from utils.hardware import (
prepare_gpu_selection,
resolve_requested_gpu_ids,
)
+from utils.hardware import nvidia
import utils.hardware.hardware as _hw_module
_BACKEND_ROOT = Path(__file__).resolve().parent.parent
@@ -106,6 +107,7 @@ class _GpuCacheResetMixin:
def tearDown(self):
_hw_module._physical_gpu_count = None
_hw_module._visible_gpu_count = None
+ nvidia._uuid_mask_cache.clear()
class TestResolveRequestedGpuIds(_GpuCacheResetMixin, unittest.TestCase):
@@ -129,6 +131,102 @@ class TestResolveRequestedGpuIds(_GpuCacheResetMixin, unittest.TestCase):
):
self.assertEqual(get_parent_visible_gpu_ids(), [])
+ def _uuid_mask_ids(
+ self,
+ mask,
+ smi_stdout,
+ env = None,
+ returncode = 0,
+ probe = get_parent_visible_gpu_ids,
+ ):
+ with (
+ patch.dict(
+ os.environ,
+ {"CUDA_VISIBLE_DEVICES": mask, "CUDA_DEVICE_ORDER": "PCI_BUS_ID", **(env or {})},
+ clear = True,
+ ),
+ patch("utils.hardware.hardware.get_physical_gpu_count", return_value = 2),
+ patch("utils.hardware.nvidia.subprocess.run") as mock_run,
+ ):
+ mock_run.return_value = SimpleNamespace(returncode = returncode, stdout = smi_stdout)
+ return probe(), mock_run.call_count
+
+ def test_uuid_mask_resolves_to_physical_ids_in_mask_order(self):
+ smi = "0, GPU-d18a14b7-70a4\n1, GPU-2f902962-578c\n"
+ ids, _ = self._uuid_mask_ids("GPU-2f902962-578c,GPU-d18a14b7", smi)
+ self.assertEqual(ids, [1, 0])
+
+ def test_llama_backend_inherits_the_resolved_uuid_mask(self):
+ from core.inference.llama_cpp import LlamaCppBackend
+
+ smi = "0, GPU-d18a14b7-70a4\n1, GPU-2f902962-578c\n"
+ mask = "GPU-2f902962-578c,GPU-d18a14b7"
+ with patch.object(LlamaCppBackend, "_torch_is_rocm", return_value = False):
+ ids, _ = self._uuid_mask_ids(
+ mask, smi, probe = LlamaCppBackend._resolve_visible_physical_ids
+ )
+ self.assertEqual(ids, [1, 0])
+ allowed, _ = self._uuid_mask_ids(
+ mask,
+ smi,
+ probe = lambda: LlamaCppBackend._visible_devices_mask("CUDA_VISIBLE_DEVICES"),
+ )
+ self.assertEqual(allowed, {0, 1})
+ ids, _ = self._uuid_mask_ids(
+ "GPU-ffff", smi, probe = LlamaCppBackend._resolve_visible_physical_ids
+ )
+ self.assertIsNone(ids)
+
+ def test_failed_uuid_resolution_is_not_retried_every_call(self):
+ _, first = self._uuid_mask_ids("GPU-aaa1", "", returncode = 9)
+ _, again = self._uuid_mask_ids("GPU-aaa1", "", returncode = 9)
+ self.assertEqual((first, again), (1, 0))
+ from utils.hardware import gpu_query
+
+ gpu_query.invalidate_static("test")
+ _, after_invalidate = self._uuid_mask_ids("GPU-aaa1", "", returncode = 9)
+ self.assertEqual(after_invalidate, 1)
+
+ def test_cross_vendor_inventory_follows_uuid_mask_order(self):
+ inventory = {
+ "devices": [
+ {"index": 0, "name": "GPU Zero", "memory_total_gb": 24.0},
+ {"index": 1, "name": "GPU One", "memory_total_gb": 24.0},
+ ]
+ }
+ with (
+ patch.dict(
+ os.environ,
+ {"CUDA_VISIBLE_DEVICES": "GPU-b,GPU-a", "CUDA_DEVICE_ORDER": "PCI_BUS_ID"},
+ clear = True,
+ ),
+ patch(
+ "core.inference.llama_cpp.LlamaCppBackend._visible_devices_mask",
+ return_value = {0, 1},
+ ),
+ patch("utils.hardware.nvidia.resolve_uuid_mask", return_value = [1, 0]),
+ patch("utils.hardware.nvidia.get_physical_gpu_inventory", return_value = inventory),
+ patch(
+ "utils.hardware.nvidia.get_visible_gpu_utilization", return_value = {"devices": []}
+ ),
+ ):
+ devices = _hw_module._nvidia_inference_devices()
+ self.assertEqual([d["index"] for d in devices], [1, 0])
+
+ def test_unresolvable_uuid_mask_stays_unresolved(self):
+ smi = "0, GPU-aaa1\n1, GPU-aaa2\n"
+ for mask in ("GPU-aaa", "GPU-bbb", "MIG-aaa1", "0,GPU-aaa1", "GPU-aaa1,GPU-aaa1"):
+ with self.subTest(mask = mask):
+ self.assertEqual(self._uuid_mask_ids(mask, smi)[0], [])
+ self.assertEqual(self._uuid_mask_ids("GPU-aaa1", smi, returncode = 9)[0], [])
+
+ def test_uuid_mask_not_resolved_off_pci_bus_order_or_on_rocm(self):
+ smi = "0, GPU-aaa1\n"
+ ids, calls = self._uuid_mask_ids("GPU-aaa1", smi, {"CUDA_DEVICE_ORDER": "FASTEST_FIRST"})
+ self.assertEqual((ids, calls), ([], 0))
+ with patch("utils.hardware.hardware.IS_ROCM", True):
+ self.assertEqual(self._uuid_mask_ids("GPU-aaa1", smi), ([], 0))
+
def test_invalid_requests_raise_clear_value_errors(self):
cases = [
([1, 1], "duplicate GPU IDs"),
diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.py
index 6e775351bb9..dad36aa8a25 100644
--- a/studio/backend/utils/hardware/hardware.py
+++ b/studio/backend/utils/hardware/hardware.py
@@ -4690,6 +4690,16 @@ def _get_parent_visible_gpu_spec() -> Dict[str, Any]:
try:
numeric_ids = [int(value) for value in tokens]
except ValueError:
+ # nvidia-smi indices are PCI order, so they only name the same cards a numeric mask written back to a child would under PCI_BUS_ID (#8873).
+ if not _is_rocm_spec and os.environ.get("CUDA_DEVICE_ORDER") == "PCI_BUS_ID":
+ from . import nvidia
+ resolved_ids = nvidia.resolve_uuid_mask(cuda_visible)
+ if resolved_ids is not None:
+ return {
+ "raw": cuda_visible,
+ "numeric_ids": resolved_ids,
+ "supports_explicit_gpu_ids": True,
+ }
return {
"raw": cuda_visible,
"numeric_ids": None,
@@ -6028,8 +6038,13 @@ def _nvidia_inference_devices() -> list[Dict[str, Any]]:
return []
if allowed is not None:
# visible_ordinal is the child's numbering, which follows the mask's order.
- order = [int(x) for x in os.environ["CUDA_VISIBLE_DEVICES"].split(",") if x.strip()]
- rows.sort(key = lambda row: order.index(row["index"]))
+ try:
+ order = [int(x) for x in os.environ["CUDA_VISIBLE_DEVICES"].split(",") if x.strip()]
+ except ValueError:
+ order = nvidia.resolve_uuid_mask(os.environ["CUDA_VISIBLE_DEVICES"].strip()) or []
+ rows.sort(
+ key = lambda row: order.index(row["index"]) if row["index"] in order else len(order)
+ )
usage = nvidia.get_visible_gpu_utilization([row["index"] for row in rows])
usage_by_index = {d.get("index"): d for d in usage.get("devices") or []}
devices = []
diff --git a/studio/backend/utils/hardware/nvidia.py b/studio/backend/utils/hardware/nvidia.py
index 654a7f3fb7b..c3b6696b71c 100644
--- a/studio/backend/utils/hardware/nvidia.py
+++ b/studio/backend/utils/hardware/nvidia.py
@@ -6,6 +6,7 @@ import platform
import shutil
import subprocess
import threading
+import time
from typing import Any, Optional
from loggers import get_logger
@@ -75,6 +76,53 @@ def _uuid_visible_ordinal_map(
return visible_ordinals
+_UUID_MASK_TTL_S = 30.0
+_uuid_mask_cache: dict[tuple[str, int], tuple[float, Optional[list[int]]]] = {}
+
+
+def resolve_uuid_mask(parent_cuda_visible_devices: str) -> Optional[list[int]]:
+ """Physical nvidia-smi indices for a full-GPU UUID mask, in mask order; None if any token is unresolvable."""
+ # gpu_query never caches a failure, and the parent GPU spec is read on every poll and load:
+ # without this a hung nvidia-smi costs its full timeout per call.
+ # Keyed on the static generation so invalidate_static() (eGPU hotplug, driver reset) drops it too.
+ key = (parent_cuda_visible_devices, gpu_query._static_gen)
+ hit = _uuid_mask_cache.get(key)
+ if hit is not None and time.monotonic() - hit[0] < _UUID_MASK_TTL_S:
+ return None if hit[1] is None else list(hit[1])
+ ids = _query_uuid_mask(parent_cuda_visible_devices)
+ _uuid_mask_cache.clear()
+ _uuid_mask_cache[key] = (time.monotonic(), ids)
+ return None if ids is None else list(ids)
+
+
+def _query_uuid_mask(parent_cuda_visible_devices: str) -> Optional[list[int]]:
+ try:
+ result = gpu_query.run_nvidia_smi(
+ ["nvidia-smi", "--query-gpu=index,uuid", "--format=csv,noheader"],
+ capture_output = True,
+ text = True,
+ encoding = "utf-8",
+ errors = "replace",
+ timeout = 5,
+ env = child_env_without_native_path_secret(),
+ **_windows_hidden_subprocess_kwargs(),
+ )
+ except (OSError, subprocess.TimeoutExpired) as e:
+ logger.warning("nvidia-smi query failed while resolving a UUID mask: %s", e)
+ return None
+ if result.returncode != 0:
+ return None
+ gpu_rows = []
+ for line in result.stdout.strip().splitlines():
+ parts = [p.strip() for p in line.split(",")]
+ if len(parts) == 2 and parts[0].isdecimal():
+ gpu_rows.append((int(parts[0]), parts[1]))
+ visible_ordinals = _uuid_visible_ordinal_map(parent_cuda_visible_devices, gpu_rows)
+ if visible_ordinals is None:
+ return None
+ return sorted(visible_ordinals, key = visible_ordinals.__getitem__)
+
+
def get_physical_gpu_count() -> Optional[int]:
"""Return physical GPU count via nvidia-smi, or None on failure."""
try: