Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: resolve UUID-form CUDA_VISIBLE_DEVICES to physical GPU indices (#8917)
Darshan Poudel5169487183df
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 97bcdcb82f0..d7432ec4c1e 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -10687,7 +10687,14 @@ class LlamaCppBackend:try:return [int(x.strip()) for x in cvd.split(",") if x.strip()]except ValueError:+ pass+ # A resolvable CUDA UUID mask, else the GPU auto-pick escapes the mask onto hidden cards (#8873).+ try:+ from utils.hardware.hardware import _get_parent_visible_gpu_spec+ spec = _get_parent_visible_gpu_spec()+ except Exception:return None+ return spec["numeric_ids"] if spec.get("raw") == cvd.strip() else None@staticmethoddef _visibility_mask_is_unmappable() -> bool:@@ -12445,7 +12452,15 @@ class LlamaCppBackend:try:return set(int(x.strip()) for x in raw.split(",") if x.strip())except ValueError:- return None+ if (+ env_name != "CUDA_VISIBLE_DEVICES"+ or os.environ.get("CUDA_DEVICE_ORDER") != "PCI_BUS_ID"+ ):+ return None+ from utils.hardware import nvidia++ ids = nvidia.resolve_uuid_mask(raw.strip())+ return set(ids) if ids is not None else None@staticmethoddef _vulkan_pin_args(gpu_indices: Optional[Iterable[int]]) -> list[str]:diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.pyindex 5261e4dd389..ce5fdf58790 100644--- a/studio/backend/models/inference.py+++ b/studio/backend/models/inference.py@@ -194,7 +194,7 @@ class LoadRequest(BaseModel):"supported on XPU, and physical IDs are unsupported when the parent ""visibility mask uses ""non-numeric or subdevice entries, including CUDA_VISIBLE_DEVICES "- "with UUID/MIG entries and ZE_AFFINITY_MASK with subdevice tokens "+ "with MIG or unresolvable UUID entries and ZE_AFFINITY_MASK with subdevice tokens ""(for example '0.0,0.1') or FLAT-hierarchy tile handles. For GGUF ""models the fitter may pin the smallest subset of this pool that fits."),diff --git a/studio/backend/models/training.py b/studio/backend/models/training.pyindex 099bd698af7..8be9b5f79aa 100644--- a/studio/backend/models/training.py+++ b/studio/backend/models/training.py@@ -605,8 +605,8 @@ class TrainingStartRequest(BaseModel):"Physical GPU indices to use, for example [0, 1]. Omit or pass ""[] to use automatic selection. Explicit gpu_ids are unsupported ""when the parent visibility mask uses non-numeric or subdevice "- "entries -- this includes CUDA_VISIBLE_DEVICES with UUID/MIG "- "entries on NVIDIA, and ZE_AFFINITY_MASK with subdevice tokens "+ "entries -- this includes CUDA_VISIBLE_DEVICES with MIG or "+ "unresolvable UUID entries on NVIDIA, and ZE_AFFINITY_MASK with subdevice tokens ""(e.g. '0.0,0.1') or FLAT-hierarchy (default) tile handles on ""Intel XPU."),diff --git a/studio/backend/tests/test_gpu_selection.py b/studio/backend/tests/test_gpu_selection.pyindex 3b50cba6555..b08232e535a 100644--- a/studio/backend/tests/test_gpu_selection.py+++ b/studio/backend/tests/test_gpu_selection.py@@ -33,6 +33,7 @@ from utils.hardware import (prepare_gpu_selection,resolve_requested_gpu_ids,)+from utils.hardware import nvidiaimport utils.hardware.hardware as _hw_module_BACKEND_ROOT = Path(__file__).resolve().parent.parent@@ -106,6 +107,7 @@ class _GpuCacheResetMixin:def tearDown(self):_hw_module._physical_gpu_count = None_hw_module._visible_gpu_count = None+ nvidia._uuid_mask_cache.clear()class TestResolveRequestedGpuIds(_GpuCacheResetMixin, unittest.TestCase):@@ -129,6 +131,102 @@ class TestResolveRequestedGpuIds(_GpuCacheResetMixin, unittest.TestCase):):self.assertEqual(get_parent_visible_gpu_ids(), [])+ def _uuid_mask_ids(+ self,+ mask,+ smi_stdout,+ env = None,+ returncode = 0,+ probe = get_parent_visible_gpu_ids,+ ):+ with (+ patch.dict(+ os.environ,+ {"CUDA_VISIBLE_DEVICES": mask, "CUDA_DEVICE_ORDER": "PCI_BUS_ID", **(env or {})},+ clear = True,+ ),+ patch("utils.hardware.hardware.get_physical_gpu_count", return_value = 2),+ patch("utils.hardware.nvidia.subprocess.run") as mock_run,+ ):+ mock_run.return_value = SimpleNamespace(returncode = returncode, stdout = smi_stdout)+ return probe(), mock_run.call_count++ def test_uuid_mask_resolves_to_physical_ids_in_mask_order(self):+ smi = "0, GPU-d18a14b7-70a4\n1, GPU-2f902962-578c\n"+ ids, _ = self._uuid_mask_ids("GPU-2f902962-578c,GPU-d18a14b7", smi)+ self.assertEqual(ids, [1, 0])++ def test_llama_backend_inherits_the_resolved_uuid_mask(self):+ from core.inference.llama_cpp import LlamaCppBackend++ smi = "0, GPU-d18a14b7-70a4\n1, GPU-2f902962-578c\n"+ mask = "GPU-2f902962-578c,GPU-d18a14b7"+ with patch.object(LlamaCppBackend, "_torch_is_rocm", return_value = False):+ ids, _ = self._uuid_mask_ids(+ mask, smi, probe = LlamaCppBackend._resolve_visible_physical_ids+ )+ self.assertEqual(ids, [1, 0])+ allowed, _ = self._uuid_mask_ids(+ mask,+ smi,+ probe = lambda: LlamaCppBackend._visible_devices_mask("CUDA_VISIBLE_DEVICES"),+ )+ self.assertEqual(allowed, {0, 1})+ ids, _ = self._uuid_mask_ids(+ "GPU-ffff", smi, probe = LlamaCppBackend._resolve_visible_physical_ids+ )+ self.assertIsNone(ids)++ def test_failed_uuid_resolution_is_not_retried_every_call(self):+ _, first = self._uuid_mask_ids("GPU-aaa1", "", returncode = 9)+ _, again = self._uuid_mask_ids("GPU-aaa1", "", returncode = 9)+ self.assertEqual((first, again), (1, 0))+ from utils.hardware import gpu_query++ gpu_query.invalidate_static("test")+ _, after_invalidate = self._uuid_mask_ids("GPU-aaa1", "", returncode = 9)+ self.assertEqual(after_invalidate, 1)++ def test_cross_vendor_inventory_follows_uuid_mask_order(self):+ inventory = {+ "devices": [+ {"index": 0, "name": "GPU Zero", "memory_total_gb": 24.0},+ {"index": 1, "name": "GPU One", "memory_total_gb": 24.0},+ ]+ }+ with (+ patch.dict(+ os.environ,+ {"CUDA_VISIBLE_DEVICES": "GPU-b,GPU-a", "CUDA_DEVICE_ORDER": "PCI_BUS_ID"},+ clear = True,+ ),+ patch(+ "core.inference.llama_cpp.LlamaCppBackend._visible_devices_mask",+ return_value = {0, 1},+ ),+ patch("utils.hardware.nvidia.resolve_uuid_mask", return_value = [1, 0]),+ patch("utils.hardware.nvidia.get_physical_gpu_inventory", return_value = inventory),+ patch(+ "utils.hardware.nvidia.get_visible_gpu_utilization", return_value = {"devices": []}+ ),+ ):+ devices = _hw_module._nvidia_inference_devices()+ self.assertEqual([d["index"] for d in devices], [1, 0])++ def test_unresolvable_uuid_mask_stays_unresolved(self):+ smi = "0, GPU-aaa1\n1, GPU-aaa2\n"+ for mask in ("GPU-aaa", "GPU-bbb", "MIG-aaa1", "0,GPU-aaa1", "GPU-aaa1,GPU-aaa1"):+ with self.subTest(mask = mask):+ self.assertEqual(self._uuid_mask_ids(mask, smi)[0], [])+ self.assertEqual(self._uuid_mask_ids("GPU-aaa1", smi, returncode = 9)[0], [])++ def test_uuid_mask_not_resolved_off_pci_bus_order_or_on_rocm(self):+ smi = "0, GPU-aaa1\n"+ ids, calls = self._uuid_mask_ids("GPU-aaa1", smi, {"CUDA_DEVICE_ORDER": "FASTEST_FIRST"})+ self.assertEqual((ids, calls), ([], 0))+ with patch("utils.hardware.hardware.IS_ROCM", True):+ self.assertEqual(self._uuid_mask_ids("GPU-aaa1", smi), ([], 0))+def test_invalid_requests_raise_clear_value_errors(self):cases = [([1, 1], "duplicate GPU IDs"),diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.pyindex 6e775351bb9..dad36aa8a25 100644--- a/studio/backend/utils/hardware/hardware.py+++ b/studio/backend/utils/hardware/hardware.py@@ -4690,6 +4690,16 @@ def _get_parent_visible_gpu_spec() -> Dict[str, Any]:try:numeric_ids = [int(value) for value in tokens]except ValueError:+ # nvidia-smi indices are PCI order, so they only name the same cards a numeric mask written back to a child would under PCI_BUS_ID (#8873).+ if not _is_rocm_spec and os.environ.get("CUDA_DEVICE_ORDER") == "PCI_BUS_ID":+ from . import nvidia+ resolved_ids = nvidia.resolve_uuid_mask(cuda_visible)+ if resolved_ids is not None:+ return {+ "raw": cuda_visible,+ "numeric_ids": resolved_ids,+ "supports_explicit_gpu_ids": True,+ }return {"raw": cuda_visible,"numeric_ids": None,@@ -6028,8 +6038,13 @@ def _nvidia_inference_devices() -> list[Dict[str, Any]]:return []if allowed is not None:# visible_ordinal is the child's numbering, which follows the mask's order.- order = [int(x) for x in os.environ["CUDA_VISIBLE_DEVICES"].split(",") if x.strip()]- rows.sort(key = lambda row: order.index(row["index"]))+ try:+ order = [int(x) for x in os.environ["CUDA_VISIBLE_DEVICES"].split(",") if x.strip()]+ except ValueError:+ order = nvidia.resolve_uuid_mask(os.environ["CUDA_VISIBLE_DEVICES"].strip()) or []+ rows.sort(+ key = lambda row: order.index(row["index"]) if row["index"] in order else len(order)+ )usage = nvidia.get_visible_gpu_utilization([row["index"] for row in rows])usage_by_index = {d.get("index"): d for d in usage.get("devices") or []}devices = []diff --git a/studio/backend/utils/hardware/nvidia.py b/studio/backend/utils/hardware/nvidia.pyindex 654a7f3fb7b..c3b6696b71c 100644--- a/studio/backend/utils/hardware/nvidia.py+++ b/studio/backend/utils/hardware/nvidia.py@@ -6,6 +6,7 @@ import platformimport shutilimport subprocessimport threading+import timefrom typing import Any, Optionalfrom loggers import get_logger@@ -75,6 +76,53 @@ def _uuid_visible_ordinal_map(return visible_ordinals+_UUID_MASK_TTL_S = 30.0+_uuid_mask_cache: dict[tuple[str, int], tuple[float, Optional[list[int]]]] = {}+++def resolve_uuid_mask(parent_cuda_visible_devices: str) -> Optional[list[int]]:+ """Physical nvidia-smi indices for a full-GPU UUID mask, in mask order; None if any token is unresolvable."""+ # gpu_query never caches a failure, and the parent GPU spec is read on every poll and load:+ # without this a hung nvidia-smi costs its full timeout per call.+ # Keyed on the static generation so invalidate_static() (eGPU hotplug, driver reset) drops it too.+ key = (parent_cuda_visible_devices, gpu_query._static_gen)+ hit = _uuid_mask_cache.get(key)+ if hit is not None and time.monotonic() - hit[0] < _UUID_MASK_TTL_S:+ return None if hit[1] is None else list(hit[1])+ ids = _query_uuid_mask(parent_cuda_visible_devices)+ _uuid_mask_cache.clear()+ _uuid_mask_cache[key] = (time.monotonic(), ids)+ return None if ids is None else list(ids)+++def _query_uuid_mask(parent_cuda_visible_devices: str) -> Optional[list[int]]:+ try:+ result = gpu_query.run_nvidia_smi(+ ["nvidia-smi", "--query-gpu=index,uuid", "--format=csv,noheader"],+ capture_output = True,+ text = True,+ encoding = "utf-8",+ errors = "replace",+ timeout = 5,+ env = child_env_without_native_path_secret(),+ **_windows_hidden_subprocess_kwargs(),+ )+ except (OSError, subprocess.TimeoutExpired) as e:+ logger.warning("nvidia-smi query failed while resolving a UUID mask: %s", e)+ return None+ if result.returncode != 0:+ return None+ gpu_rows = []+ for line in result.stdout.strip().splitlines():+ parts = [p.strip() for p in line.split(",")]+ if len(parts) == 2 and parts[0].isdecimal():+ gpu_rows.append((int(parts[0]), parts[1]))+ visible_ordinals = _uuid_visible_ordinal_map(parent_cuda_visible_devices, gpu_rows)+ if visible_ordinals is None:+ return None+ return sorted(visible_ordinals, key = visible_ordinals.__getitem__)++def get_physical_gpu_count() -> Optional[int]:"""Return physical GPU count via nvidia-smi, or None on failure."""try: