Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: read GPU metadata from the CUDA driver API, not torch.cuda.get_device_properties (#17120)
Daniel Han2f8467f454f4
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 81d725f0630..d9bbbeca6a0 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -12487,6 +12487,11 @@ class LlamaCppBackend:"""True if the integrated-CUDA answer is cached for this mask; the preflight never probes, since the budget is already taken and both arms fill the cache."""return LlamaCppBackend._integrated_cuda_mask_key() in LlamaCppBackend._INTEGRATED_CUDA_IDS+ @staticmethod+ def _cuda_device_info():+ from utils.hardware import nvidia+ return nvidia.cuda_device_info_or_torch()+@staticmethoddef _integrated_cuda_gpu_ids() -> set[int]:"""PHYSICAL ids of visible integrated CUDA GPUs; ROCm is left to _rocm_unified_memory_gpu_ids."""@@ -12503,29 +12508,19 @@ class LlamaCppBackend:return set()if not (hasattr(torch, "cuda") and torch.cuda.is_available()):return set()+ info = LlamaCppBackend._cuda_device_info()+ if info is None:+ return set()# Same ordinal-to-physical map as the ROCm twin, so masked CUDA_VISIBLE_DEVICES=2 hosts stay correct.physical_ids = LlamaCppBackend._resolve_visible_physical_ids()- integrated: set[int] = set()- # An incomplete pass is returned but not cached, so a failed query is retried.- complete = True- for ordinal in range(torch.cuda.device_count()):- try:- props = torch.cuda.get_device_properties(ordinal)- except Exception:- complete = False- continue- # Both attribute spellings: torch renamed `integrated`.- if not (- getattr(props, "is_integrated", False) or getattr(props, "integrated", False)- ):- continue- integrated.add(- physical_ids[ordinal]- if physical_ids is not None and ordinal < len(physical_ids)- else ordinal- )- # Cached for the preflight only if the pass reached every device; a torch that raised says nothing.- if complete:+ integrated = {+ physical_ids[ordinal]+ if physical_ids is not None and ordinal < len(physical_ids)+ else ordinal+ for ordinal, device in enumerate(info)+ if device is not None and device[1]+ }+ if all(device is not None for device in info):LlamaCppBackend._INTEGRATED_CUDA_IDS[LlamaCppBackend._integrated_cuda_mask_key()] = integrated@@ -12538,7 +12533,7 @@ class LlamaCppBackend:@staticmethoddef _integrated_cuda_pool_total_mib() -> dict[int, int]:- """Pool MiB per integrated CUDA device, from props.total_memory: what it can actually allocate."""+ """Pool MiB per integrated CUDA device, from the driver's total memory: what it can actually allocate."""key = LlamaCppBackend._integrated_cuda_mask_key()cached = LlamaCppBackend._INTEGRATED_CUDA_POOL_MIB.get(key)if cached is not None:@@ -12547,24 +12542,21 @@ class LlamaCppBackend:if not integrated:return {}try:- import torch-+ info = LlamaCppBackend._cuda_device_info()+ if info is None:+ return {}physical_ids = LlamaCppBackend._resolve_visible_physical_ids()totals: dict[int, int] = {}- complete = True- for ordinal in range(torch.cuda.device_count()):+ complete = all(device is not None for device in info)+ for ordinal, device in enumerate(info):idx = (physical_ids[ordinal]if physical_ids is not None and ordinal < len(physical_ids)else ordinal)- if idx not in integrated:+ if device is None or idx not in integrated:continue- try:- props = torch.cuda.get_device_properties(ordinal)- totals[idx] = int(props.total_memory) // (1024 * 1024)- except Exception:- complete = False+ totals[idx] = int(device[2]) // (1024 * 1024)if complete:LlamaCppBackend._INTEGRATED_CUDA_POOL_MIB[key] = totalsreturn totals@@ -12728,17 +12720,16 @@ class LlamaCppBackend:return False # ROCm reuses torch.cuda.*; not a CUDA partif not (hasattr(torch, "cuda") and torch.cuda.is_available()):return False- count = torch.cuda.device_count()+ info = LlamaCppBackend._cuda_device_info() or []# Mirror _get_gpu_free_memory; unset or unparsable CUDA_VISIBLE_DEVICES keeps physical id == ordinal.physical_ids = LlamaCppBackend._resolve_visible_physical_ids()names_by_id: dict[int, str] = {}- for ordinal in range(count):- try:- name = (torch.cuda.get_device_properties(ordinal).name or "").lower()- except Exception:+ for ordinal, device in enumerate(info):+ if device is None:continue+ name = (device[3] or "").lower()pid = (physical_ids[ordinal]if physical_ids is not None and ordinal < len(physical_ids)diff --git a/studio/backend/tests/conftest.py b/studio/backend/tests/conftest.pyindex b7dec809c27..e4c1b68f4a6 100644--- a/studio/backend/tests/conftest.py+++ b/studio/backend/tests/conftest.py@@ -104,6 +104,134 @@ def _no_vae_structural_verdict_cache(monkeypatch):monkeypatch.setenv("UNSLOTH_VAE_FUSED_STRUCTURAL_CACHE", "0")+def _cuda_device_info_from_the_test_torch():+ """The devices the test's fake torch describes; none for the real torch (never this host's driver)."""+ torch = sys.modules.get("torch")+ cuda = getattr(torch, "cuda", None)+ props = getattr(cuda, "get_device_properties", None)+ if props is None or str(getattr(props, "__module__", "") or "").startswith("torch"):+ return []+ try:+ count = int(cuda.device_count())+ except Exception:+ return None+ devices = []+ for ordinal in range(count):+ try:+ p = props(ordinal)+ except Exception:+ devices.append(None)+ continue+ integrated = bool(getattr(p, "is_integrated", False) or getattr(p, "integrated", False))+ devices.append(+ (+ f"0000:{ordinal + 1:02x}:00.0",+ integrated,+ int(getattr(p, "total_memory", 0) or 0),+ str(getattr(p, "name", "") or ""),+ )+ )+ return devices+++def _c_cuda_mask():+ """CUDA_VISIBLE_DEVICES as cuInit reads it: the C environment, which os.environ writes reach."""+ import ctypes++ getenv = ctypes.CDLL(None).getenv+ getenv.restype, getenv.argtypes = ctypes.c_char_p, [ctypes.c_char_p]+ raw = getenv(b"CUDA_VISIBLE_DEVICES")+ return None if raw is None else raw.decode(errors = "replace").strip()+++# A run that hid every GPU must never open a context: a CPU test that rewrites the mask and reaches a real CUDA+# init would open one on each card it names (xdist workers held 600-900 MiB on six GPUs), so that init fails it.+_SESSION_HID_GPUS = os.name != "nt" and _c_cuda_mask() in ("", "-1")+_cuda_init_state = {"cpu_test": False, "attempts": []}+++def _refuse_cuda_init_once_unhidden(torch):+ c = getattr(torch, "_C", None)+ real = getattr(c, "_cuda_init", None)+ if real is None or getattr(real, "_studio_cpu_test_check", False):+ return++ def _cuda_init(*a, **k):+ mask = _c_cuda_mask()+ if _cuda_init_state["cpu_test"] and mask not in ("", "-1"):+ import traceback+ _cuda_init_state["attempts"].append(+ (mask, "".join(traceback.format_stack(limit = 12)[:-1]))+ )+ raise RuntimeError("a CPU test initialised CUDA after unhiding the GPUs (refused)")+ return real(*a, **k)++ _cuda_init._studio_cpu_test_check = True+ c._cuda_init = _cuda_init+++class _AfterTorchImport:+ def find_spec(+ self,+ name,+ path = None,+ target = None,+ ):+ if name != "torch" or "torch" in sys.modules:+ return None+ sys.meta_path.remove(self)+ spec = _ilu.find_spec("torch")+ if spec is None or spec.loader is None:+ return spec+ exec_module = spec.loader.exec_module++ def _exec(module):+ exec_module(module)+ _refuse_cuda_init_once_unhidden(module)++ spec.loader.exec_module = _exec+ return spec+++if _SESSION_HID_GPUS:+ if "torch" in sys.modules:+ _refuse_cuda_init_once_unhidden(sys.modules["torch"])+ else:+ sys.meta_path.insert(0, _AfterTorchImport())++[email protected](autouse = True)+def _no_cuda_context_from_a_cpu_test(request):+ if not _SESSION_HID_GPUS or request.node.get_closest_marker("gpu") is not None:+ yield+ return+ _cuda_init_state["cpu_test"], _cuda_init_state["attempts"] = True, []+ try:+ yield+ finally:+ _cuda_init_state["cpu_test"] = False+ if _cuda_init_state["attempts"]:+ mask, stack = _cuda_init_state["attempts"][0]+ pytest.fail(+ f"this CPU test initialised CUDA with CUDA_VISIBLE_DEVICES={mask!r} after the run had hidden "+ f"every GPU; its next device call opens a context on each card it names. Fake the device query (torch.cuda, "+ f"utils.hardware.nvidia.get_cuda_device_info) or mark the test gpu. First call:\n{stack}",+ pytrace = False,+ )++[email protected](autouse = True)+def _no_real_cuda_device_query(monkeypatch):+ try:+ from utils.hardware import nvidia+ except Exception:+ return+ monkeypatch.setattr(+ nvidia, "get_cuda_device_info", _cuda_device_info_from_the_test_torch, raising = False+ )+ monkeypatch.setattr(nvidia, "_CUDA_DEVICE_INFO_CACHE", {}, raising = False)++@pytest.hookimpl(trylast = True)def pytest_collection_modifyitems(config, items):"""After xdist tags grouped tests; see tests/_shared/xdist_group_check.py."""diff --git a/studio/backend/tests/test_gpu_metadata_no_cuda_context.py b/studio/backend/tests/test_gpu_metadata_no_cuda_context.pynew file mode 100644index 00000000000..6b2f75f79bc--- /dev/null+++ b/studio/backend/tests/test_gpu_metadata_no_cuda_context.py@@ -0,0 +1,299 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++from __future__ import annotations++import os+import subprocess+import sys+import types++import pytest++from core.inference.llama_cpp import LlamaCppBackend+from utils.hardware import nvidia++GIB = 1 << 30+# The real query, captured at import: the autouse conftest fixture replaces it for every test.+_REAL_GET_CUDA_DEVICE_INFO = getattr(nvidia, "get_cuda_device_info", None)+++class _CountingTorch(types.ModuleType):+ def __init__(self, count):+ super().__init__("torch")+ self.calls = []+ self.version = types.SimpleNamespace(hip = None)++ def _counted(name, value = None):+ def _call(*a, **k):+ self.calls.append(name)+ if name == "get_device_properties":+ return types.SimpleNamespace(+ name = "NVIDIA B200", total_memory = 180 * GIB, is_integrated = 0+ )+ return value++ return _call++ self.cuda = types.SimpleNamespace(+ is_available = lambda: True,+ device_count = lambda: count,+ get_device_properties = _counted("get_device_properties"),+ _lazy_init = _counted("_lazy_init"),+ init = _counted("init"),+ )+++def host(monkeypatch):+ monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)+ for cache in ("_INTEGRATED_CUDA_IDS", "_INTEGRATED_CUDA_POOL_MIB"):+ monkeypatch.setattr(LlamaCppBackend, cache, {}, raising = False)+ torch = _CountingTorch(7)+ monkeypatch.setitem(sys.modules, "torch", torch)+ queries = []+ devices = [(f"0000:{i + 1:02x}:00.0", False, 180 * GIB, "NVIDIA B200") for i in range(6)]+ devices.append(("0000:0f:00.0", True, 120 * GIB, "NVIDIA GB10"))++ def _child():+ queries.append(1)+ return list(devices)++ monkeypatch.setattr(nvidia, "get_cuda_device_info", _REAL_GET_CUDA_DEVICE_INFO, raising = False)+ monkeypatch.setattr(nvidia, "_query_cuda_device_info", _child, raising = False)+ return types.SimpleNamespace(torch = torch, queries = queries)+++def test_metadata_lookups_never_initialise_torch_cuda(host):+ answers = (+ LlamaCppBackend._integrated_cuda_gpu_ids(),+ LlamaCppBackend._integrated_cuda_pool_total_mib(),+ LlamaCppBackend._integrated_cuda_unified_memory([0]),+ LlamaCppBackend._integrated_cuda_selection_is_all_shared([6]),+ LlamaCppBackend._all_selected_gpus_match(LlamaCppBackend._DATACENTER_GPU_RE, [0, 1]),+ )+ assert host.torch.calls == []+ assert answers == ({6}, {6: 120 * 1024}, False, True, True)+ assert len(host.queries) == 1+++def _torch_cannot_answer_either(host, count):+ def _raise(ordinal):+ raise RuntimeError("no properties")++ host.torch.cuda.device_count = lambda: count+ host.torch.cuda.get_device_properties = _raise+++def test_an_ordinal_that_did_not_answer_is_used_but_not_settled(host, monkeypatch):+ monkeypatch.setattr(+ nvidia,+ "get_cuda_device_info",+ lambda: [("0000:01:00.0", True, 120 * GIB, "NVIDIA GB10"), None],+ )+ _torch_cannot_answer_either(host, 2)+ assert LlamaCppBackend._integrated_cuda_gpu_ids() == {0}+ assert LlamaCppBackend._integrated_cuda_probe_is_free() is False+++def test_an_unknown_answer_neither_side_knows_is_not_cached(host, monkeypatch):+ monkeypatch.setattr(nvidia, "get_cuda_device_info", lambda: None)+ _torch_cannot_answer_either(host, 2)+ assert LlamaCppBackend._integrated_cuda_gpu_ids() == set()+ assert LlamaCppBackend._integrated_cuda_probe_is_free() is False+++def _torch_with_an_integrated_card(host):+ def _props(ordinal):+ host.torch.calls.append("get_device_properties")+ if ordinal == 1:+ return types.SimpleNamespace(+ name = "NVIDIA GB10", total_memory = 120 * GIB, is_integrated = 1+ )+ return types.SimpleNamespace(name = "NVIDIA B200", total_memory = 180 * GIB, is_integrated = 0)++ host.torch.cuda.device_count = lambda: 2+ host.torch.cuda.get_device_properties = _props+++ "driver_answer", [None, [], [("0000:01:00.0", False, 180 * GIB, "NVIDIA B200"), None]]+)+def test_a_failed_driver_probe_never_hides_a_card_torch_sees(host, monkeypatch, driver_answer):+ monkeypatch.setattr(nvidia, "get_cuda_device_info", lambda: driver_answer)+ _torch_with_an_integrated_card(host)+ assert LlamaCppBackend._integrated_cuda_gpu_ids() == {1}+ assert LlamaCppBackend._integrated_cuda_pool_total_mib() == {1: 120 * 1024}+ assert host.torch.calls.count("get_device_properties") >= 1+++def _child(+ monkeypatch,+ returncode,+ stdout = "",+):+ seen = {}++ def _run(cmd, **kw):+ seen["cmd"] = cmd+ return subprocess.CompletedProcess(cmd, returncode, stdout, "")++ monkeypatch.setattr(nvidia.subprocess, "run", _run)+ return seen+++ "returncode, stdout, expected",+ [+ (+ 0,+ "0000:3b:00.0 0 85899345920 NVIDIA H100 80GB HBM3\n0000:00:00.0 1 128000000000 NVIDIA GB10\n",+ [+ ("0000:3b:00.0", False, 85899345920, "NVIDIA H100 80GB HBM3"),+ ("0000:00:00.0", True, 128000000000, "NVIDIA GB10"),+ ],+ ),+ (+ 0,+ "0000:3b:00.0 0 85899345920 NVIDIA H100\nERR\n",+ [("0000:3b:00.0", False, 85899345920, "NVIDIA H100"), None],+ ),+ (0, "- 1 128000000000 NVIDIA GB10\n", [("", True, 128000000000, "NVIDIA GB10")]),+ (0, "", []),+ (2, "", []),+ (3, "", None),+ (0, "garbage\n", None),+ ],+)+def test_the_driver_child_answer_is_parsed(monkeypatch, returncode, stdout, expected):+ monkeypatch.setattr(nvidia, "get_cuda_device_info", _REAL_GET_CUDA_DEVICE_INFO)+ seen = _child(monkeypatch, returncode, stdout)+ assert nvidia.get_cuda_device_info() == expected+ assert seen["cmd"][1:] == ["-c", nvidia._CUDA_DEVICE_INFO_SCRIPT]+++def test_a_child_that_times_out_is_unknown(monkeypatch):+ monkeypatch.setattr(nvidia, "get_cuda_device_info", _REAL_GET_CUDA_DEVICE_INFO)++ def _run(cmd, **kw):+ raise subprocess.TimeoutExpired(cmd, kw.get("timeout"))++ monkeypatch.setattr(nvidia.subprocess, "run", _run)+ assert nvidia.get_cuda_device_info() is None+++def test_an_unknown_answer_is_not_asked_again_for_a_while(monkeypatch):+ monkeypatch.setattr(nvidia, "get_cuda_device_info", _REAL_GET_CUDA_DEVICE_INFO)+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0")+ clock = [1000.0]+ monkeypatch.setattr(nvidia.time, "monotonic", lambda: clock[0])+ answers = iter(+ [+ (3, ""),+ (0, "0000:01:00.0 0 8589934592 NVIDIA T4\nERR\n"),+ (0, "0000:01:00.0 0 8589934592 NVIDIA T4\n"),+ ]+ )+ runs = []++ def _run(cmd, **kw):+ runs.append(1)+ rc, out = next(answers)+ return subprocess.CompletedProcess(cmd, rc, out, "")++ monkeypatch.setattr(nvidia.subprocess, "run", _run)+ assert nvidia.get_cuda_device_info() is None+ assert nvidia.get_cuda_device_info() is None and len(runs) == 1+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "1")+ assert nvidia.get_cuda_device_info() == [("0000:01:00.0", False, 8589934592, "NVIDIA T4"), None]+ assert len(runs) == 2+ clock[0] += nvidia._CUDA_DEVICE_INFO_RETRY_S + 1+ assert nvidia.get_cuda_device_info() == [("0000:01:00.0", False, 8589934592, "NVIDIA T4")]+ assert len(runs) == 3+ clock[0] += 10 * nvidia._CUDA_DEVICE_INFO_RETRY_S+ assert nvidia.get_cuda_device_info() == [("0000:01:00.0", False, 8589934592, "NVIDIA T4")]+ assert len(runs) == 3+++def test_the_real_query_answers_no_device_under_an_empty_mask(monkeypatch):+ monkeypatch.setattr(nvidia, "get_cuda_device_info", _REAL_GET_CUDA_DEVICE_INFO)+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "")+ assert nvidia.get_cuda_device_info() == []+++def test_the_driver_child_runs_and_finds_no_device_under_an_empty_mask():+ env = dict(os.environ, CUDA_VISIBLE_DEVICES = "")+ r = subprocess.run(+ [sys.executable, "-c", nvidia._CUDA_DEVICE_INFO_SCRIPT],+ capture_output = True,+ text = True,+ timeout = 60,+ env = env,+ )+ assert r.returncode in (0, 2), r.stderr+ assert r.stdout.strip() == ""+++def test_an_ordinal_that_did_not_answer_leaves_the_pool_unsettled(host, monkeypatch):+ monkeypatch.setattr(+ nvidia,+ "get_cuda_device_info",+ lambda: [("0000:01:00.0", True, 120 * GIB, "NVIDIA GB10"), None],+ )+ _torch_cannot_answer_either(host, 2)+ assert LlamaCppBackend._integrated_cuda_pool_total_mib() == {0: 120 * 1024}+ assert LlamaCppBackend._INTEGRATED_CUDA_POOL_MIB == {}+++def test_the_inventory_poll_reads_nvidia_cards_from_the_driver(host, monkeypatch):+ from utils.hardware import hardware as hw++ monkeypatch.setattr(hw, "IS_ROCM", False)+ monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA)+ inventory = hw._torch_get_device_inventory([4, 6])+ assert host.torch.calls == []+ assert [+ (d["index"], d["visible_ordinal"], d["name"], d["total_gb"], d["_cuda_integrated"])+ for d in inventory+ ] == [+ (4, 0, "NVIDIA B200", 180.0, False),+ (6, 1, "NVIDIA B200", 180.0, False),+ ]+++def test_the_device_list_keeps_the_names_that_answered(host, monkeypatch, capsys):+ from utils.hardware import hardware as hw++ monkeypatch.setattr(+ nvidia,+ "get_cuda_device_info",+ lambda: [None, ("0000:02:00.0", False, 180 * GIB, "NVIDIA B200")],+ )+ _torch_cannot_answer_either(host, 2)+ hw._print_cuda_device_list(False)+ out = capsys.readouterr().out+ assert "[0] <unavailable>" in out and "[1] NVIDIA B200" in out+ with pytest.raises(RuntimeError, match = "device 0"):+ hw._cuda_device0_name()+++def test_hardware_redetection_asks_the_driver_again(monkeypatch):+ from utils.hardware import gpu_query++ monkeypatch.setattr(nvidia, "get_cuda_device_info", _REAL_GET_CUDA_DEVICE_INFO)+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0")+ answers = iter(+ ["0000:01:00.0 0 8589934592 NVIDIA T4\n", "0000:02:00.0 0 25769803776 NVIDIA L4\n"]+ )+ runs = []++ def _run(cmd, **kw):+ runs.append(1)+ return subprocess.CompletedProcess(cmd, 0, next(answers), "")++ monkeypatch.setattr(nvidia.subprocess, "run", _run)+ assert nvidia.get_cuda_device_info()[0][3] == "NVIDIA T4"+ assert nvidia.get_cuda_device_info()[0][3] == "NVIDIA T4" and len(runs) == 1+ gpu_query.invalidate_static("test: card replaced")+ assert nvidia.get_cuda_device_info() == [("0000:02:00.0", False, 25769803776, "NVIDIA L4")]+ assert len(runs) == 2diff --git a/studio/backend/tests/test_gpu_selection.py b/studio/backend/tests/test_gpu_selection.pyindex f86cc7afdb7..11ab20acd3b 100644--- a/studio/backend/tests/test_gpu_selection.py+++ b/studio/backend/tests/test_gpu_selection.py@@ -456,6 +456,8 @@ class TestVisibleGpuUtilization(_GpuCacheResetMixin, unittest.TestCase):with (patch.dict(os.environ, {"CUDA_VISIBLE_DEVICES": "1,3"}, clear = True),patch("utils.hardware.hardware.get_device", return_value = DeviceType.CUDA),+ # torch's view would be this host's real cards+ patch("utils.hardware.hardware._torch_get_device_module", return_value = (None, None)),patch("utils.hardware.nvidia.subprocess.run") as mock_run,):mock_run.return_value = SimpleNamespace(@@ -485,6 +487,8 @@ class TestVisibleGpuUtilization(_GpuCacheResetMixin, unittest.TestCase):with (patch.dict(os.environ, {"CUDA_VISIBLE_DEVICES": "1,3"}, clear = True),patch("utils.hardware.hardware.get_device", return_value = DeviceType.CUDA),+ # torch's view would be this host's real cards+ patch("utils.hardware.hardware._torch_get_device_module", return_value = (None, None)),patch("utils.hardware.nvidia.subprocess.run") as mock_run,):mock_run.return_value = SimpleNamespace(diff --git a/studio/backend/tests/test_llama_cpp_placement.py b/studio/backend/tests/test_llama_cpp_placement.pyindex 88b5eddb68b..4d408be13fa 100644--- a/studio/backend/tests/test_llama_cpp_placement.py+++ b/studio/backend/tests/test_llama_cpp_placement.py@@ -6205,6 +6205,10 @@ def _mixed_host_launch(monkeypatch.setattr(LlamaCppBackend, "_rocm_unified_memory_gpu_ids", staticmethod(lambda: set(rocm)))+ # The carve-out query reads torch's devices; a GPU-less host answers None (unknown) and so does this one.+ monkeypatch.setattr(+ LlamaCppBackend, "_rocm_selected_pool_mib", staticmethod(lambda gpu_indices = None: None)+ )monkeypatch.setattr(LlamaCppBackend,"_INTEGRATED_CUDA_IDS",diff --git a/studio/backend/tests/test_media_auto_gpu_ordinary_host.py b/studio/backend/tests/test_media_auto_gpu_ordinary_host.pyindex a207037f6d9..3ddc10b4597 100644--- a/studio/backend/tests/test_media_auto_gpu_ordinary_host.py+++ b/studio/backend/tests/test_media_auto_gpu_ordinary_host.py@@ -19,6 +19,10 @@ from core.inference import diffusion_device as ddfrom utils.hardware import hardware as hwfrom utils.hardware.hardware import DeviceType+from utils.hardware import nvidia as _nvidia++_REAL_GET_CUDA_DEVICE_INFO = getattr(_nvidia, "get_cuda_device_info", None)+def _dev(index, total, used):return {"index": index, "vram_total_gb": total, "vram_used_gb": used}@@ -530,9 +534,16 @@ def test_fastest_first_mask_with_an_integrated_card_keeps_torch(monkeypatch):@pytest.mark.parametrize("child_out, expected",[- ("0000:3b:00.0 0\n0000:1a:00.0 0\n", [1, 0]),- ("0000:3b:00.0 0\n0000:1a:00.0 1\n", []), # integrated card mapped+ (+ "0000:3b:00.0 0 85899345920 NVIDIA H100\n0000:1a:00.0 0 85899345920 NVIDIA H100\n",+ [1, 0],+ ),+ (+ "0000:3b:00.0 0 85899345920 NVIDIA H100\n0000:1a:00.0 1 128000000000 NVIDIA GB10\n",+ [],+ ), # integrated card mapped("0000:3b:00.0\n", None), # no attribute: unreadable+ ("- 0 85899345920 NVIDIA H100\n", None), # the driver gave no bus id: nothing to join on],)def test_cuda_ordinal_rows_parse_bus_ids_and_integrated(monkeypatch, child_out, expected):@@ -540,6 +551,8 @@ def test_cuda_ordinal_rows_parse_bus_ids_and_integrated(monkeypatch, child_out,from utils.hardware import nvidia+ # The real driver query on a faked child (conftest replaces it with the test torch's devices).+ monkeypatch.setattr(nvidia, "get_cuda_device_info", _REAL_GET_CUDA_DEVICE_INFO)child = types.SimpleNamespace(returncode = 0, stdout = child_out)smi = types.SimpleNamespace(returncode = 0, stdout = "0, 00000000:1A:00.0\n1, 00000000:3B:00.0\n")monkeypatch.setattr(subprocess, "run", lambda *a, **k: child)diff --git a/studio/backend/tests/test_rocm_multi_gpu_vram_system_wide.py b/studio/backend/tests/test_rocm_multi_gpu_vram_system_wide.pyindex 08b5817b227..5512a8b0bc8 100644--- a/studio/backend/tests/test_rocm_multi_gpu_vram_system_wide.py+++ b/studio/backend/tests/test_rocm_multi_gpu_vram_system_wide.py@@ -433,6 +433,8 @@ def test_visible_utilization_rocm_fallback_overlays(monkeypatch):{"index": 1, "visible_ordinal": 1, "used_gb": 0.01, "total_gb": 8.0},],)+ # Names come from torch's device list, which on this host would be its real cards; CI has no torch.+ monkeypatch.setattr(hw, "_torch_get_device_inventory", lambda ids: [])overlaid = []monkeypatch.setattr(hw, "_overlay_system_wide_vram", lambda devices: overlaid.append(len(devices))diff --git a/studio/backend/tests/test_utils.py b/studio/backend/tests/test_utils.pyindex b993f4ffeb3..210cd18d9de 100644--- a/studio/backend/tests/test_utils.py+++ b/studio/backend/tests/test_utils.py@@ -91,7 +91,8 @@ class TestGetDevice:@needs_torchdef test_device0_probe_failure_is_logged_with_allocator_config(self, monkeypatch):- # failed CUDA initialization can crash later, so log its cause at error rather than debug+ # a failed device-0 probe can mean a broken CUDA setup that crashes later, so log it at error;+ # both probes fail, so the NVIDIA (driver query) and ROCm (get_device_properties) branches are coveredmonkeypatch.delenv("PYTORCH_CUDA_ALLOC_CONF", raising = False)monkeypatch.delenv("PYTORCH_HIP_ALLOC_CONF", raising = False)monkeypatch.setenv("PYTORCH_ALLOC_CONF", "expandable_segments:maybe")@@ -102,6 +103,7 @@ class TestGetDevice:patch("utils.hardware.hardware._has_torch", return_value = True),patch("torch.cuda.is_available", return_value = True),patch("torch.cuda.device_count", return_value = 1),+ patch("utils.hardware.nvidia.get_cuda_device_info", side_effect = error),patch("torch.cuda.get_device_properties", side_effect = error),patch.object(_hw_module, "logger") as logger,):diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.pyindex e190122271b..3b3e60e3232 100644--- a/studio/backend/utils/hardware/hardware.py+++ b/studio/backend/utils/hardware/hardware.py@@ -1529,6 +1529,22 @@ def overturn_the_mlx_verdict(epoch: Optional[int] = None) -> bool:return DEVICE is not None and DETECTION_COMPLETE.is_set() and not CHAT_ONLY+def _cuda_device_names() -> list:+ from utils.hardware import nvidia++ info = nvidia.cuda_device_info_or_torch()+ if not info:+ raise RuntimeError("the CUDA driver did not list the devices")+ return [device[3] if device is not None else None for device in info]+++def _cuda_device0_name() -> str:+ name = _cuda_device_names()[0]+ if name is None:+ raise RuntimeError("CUDA device 0 did not answer the driver query")+ return name++def _print_cuda_device_list(is_rocm: bool) -> None:"""List every visible CUDA/ROCm GPU with its index at startup: the "Hardware detected" banner names only device 0, hiding the other cards. Indices are CUDA ordinals, matching `nvidia-smi -L` when no mask is set. CUDA_DEVICE_ORDER governs only CUDA, so it is shown for CUDA but not ROCm. Purely informational and never raises."""try:@@ -1543,9 +1559,19 @@ def _print_cuda_device_list(is_rocm: bool) -> None:order = os.environ.get("CUDA_DEVICE_ORDER", "default")header = f"CUDA devices ({count}, CUDA_DEVICE_ORDER={order}):"lines = [header]+ names = None+ if not is_rocm:+ try:+ names = _cuda_device_names()+ except Exception as e:+ logger.debug("CUDA device name probe failed: %s", e)+ names = []for i in range(count):try:- name = torch.cuda.get_device_properties(i).name+ if names is not None:+ name = names[i] if i < len(names) and names[i] is not None else "<unavailable>"+ else:+ name = torch.cuda.get_device_properties(i).nameexcept Exception as e:logger.debug("CUDA device %d property probe failed: %s", i, e)name = "<unavailable>"@@ -1748,8 +1774,14 @@ def _detect_hardware_locked() -> DeviceType:if not cuda_unavailable:DEVICE = DeviceType.CUDACHAT_ONLY = False+ # AMD SDK wheels omit torch.version.hip; ROCm label uses __version__; DEVICE stays CUDA+ _hip_ver = getattr(torch.version, "hip", None)+ _is_rocm = _hip_ver is not None or "rocm" in torch.__version__.lower()try:- device_name = torch.cuda.get_device_properties(0).name+ # Not get_device_properties for NVIDIA: it initialises CUDA in the server.+ device_name = (+ torch.cuda.get_device_properties(0).name if _is_rocm else _cuda_device0_name()+ )except Exception as e:# failed CUDA init can make later PyTorch calls segfault, so log the causefrom utils.allocator_conf import ALLOCATOR_CONF_ENV_VARS@@ -1764,9 +1796,7 @@ def _detect_hardware_locked() -> DeviceType:)device_name = "<unavailable>"- # AMD SDK wheels omit torch.version.hip; ROCm label uses __version__; DEVICE stays CUDA- _hip_ver = getattr(torch.version, "hip", None)- if _hip_ver is not None or "rocm" in torch.__version__.lower():+ if _is_rocm:IS_ROCM = True_configure_rocm_miopen(torch)_hip_label = _hip_ver or torch.__version__@@ -3115,11 +3145,43 @@ def _cuda_props_are_integrated(props: Any, backend: Optional[str] = "cuda") -> bdef _torch_get_device_inventory(device_indices: list[int]) -> list[Dict[str, Any]]:- """Per-GPU name and total VRAM only, without creating a driver context: get_device_properties is answered from the driver's device list, while mem_get_info attaches a primary context worth ~612 MiB the process never gives back, and a telemetry poll must not be what pins that, so callers wanting live occupancy go to _torch_get_per_device_info instead. props.total_memory equals mem_get_info's total everywhere except a ROCm APU (see _rocm_props_total_is_carve_out), and used_gb is always None ("telemetry unavailable")."""+ """Per-GPU name and total VRAM only, without creating a driver context: NVIDIA from the driver API (nvidia.cuda_device_info_or_torch), ROCm / XPU from get_device_properties, while mem_get_info attaches a primary context worth ~612 MiB the process never gives back, and a telemetry poll must not be what pins that, so callers wanting live occupancy go to _torch_get_per_device_info instead. props.total_memory equals mem_get_info's total everywhere except a ROCm APU (see _rocm_props_total_is_carve_out), and used_gb is always None ("telemetry unavailable")."""mod, backend = _torch_get_device_module()if mod is None:return []+ torch_mod = sys.modules.get("torch")+ if (+ backend == "cuda"+ and mod is getattr(torch_mod, "cuda", None)+ and not IS_ROCM+ and not getattr(getattr(torch_mod, "version", None), "hip", None)+ ):+ from utils.hardware import nvidia++ info = nvidia.cuda_device_info_or_torch(torch_mod)+ nvidia_devices = []+ for ordinal, phys_idx in enumerate(device_indices):+ device = info[ordinal] if info is not None and ordinal < len(info) else None+ if device is None:+ logger.debug("driver inventory probe got no answer for ordinal %d", ordinal)+ continue+ nvidia_devices.append(+ {+ "index": phys_idx,+ "visible_ordinal": ordinal,+ "name": device[3],+ "total_gb": round(device[2] / (1024**3), 2),+ "used_gb": None,+ "shared_memory": False,+ "shared_memory_host_backed_gb": None,+ "_rocm_known_unified": False,+ "_rocm_gfx": "",+ "_cuda_integrated": bool(device[1]),+ }+ )+ return nvidia_devices+devices = []for ordinal, phys_idx in enumerate(device_indices):try:diff --git a/studio/backend/utils/hardware/nvidia.py b/studio/backend/utils/hardware/nvidia.pyindex 9f3a21121b7..00bbf5fa045 100644--- a/studio/backend/utils/hardware/nvidia.py+++ b/studio/backend/utils/hardware/nvidia.py@@ -672,30 +672,6 @@ def get_card_identities() -> dict[int, tuple[str, ...]]:return identities-# Driver API only: cuInit enumerates devices under the inherited CUDA_VISIBLE_DEVICES / CUDA_DEVICE_ORDER-# without creating a context, and runs in a child so the Studio process never initialises CUDA.-_CUDA_BUS_ID_SCRIPT = r"""-import ctypes, sys-lib = ctypes.WinDLL("nvcuda.dll") if sys.platform == "win32" else ctypes.CDLL("libcuda.so.1")-if lib.cuInit(0) != 0:- sys.exit(3)-count = ctypes.c_int(0)-if lib.cuDeviceGetCount(ctypes.byref(count)) != 0:- sys.exit(4)-for ordinal in range(count.value):- dev = ctypes.c_int(0)- buf = ctypes.create_string_buffer(64)- integrated = ctypes.c_int(0)- if (- lib.cuDeviceGet(ctypes.byref(dev), ordinal) != 0- or lib.cuDeviceGetPCIBusId(buf, 64, dev) != 0- or lib.cuDeviceGetAttribute(ctypes.byref(integrated), 18, dev) != 0 # CU_DEVICE_ATTRIBUTE_INTEGRATED- ):- sys.exit(5)- print(buf.value.decode(), integrated.value)-"""--def _normalize_bus_id(bus_id: str) -> Optional[tuple[int, str]]:"""'00000000:3B:00.0' (nvidia-smi) and '0000:3b:00.0' (driver API) -> (domain, 'BUS:DEV.FN')."""domain, sep, rest = bus_id.strip().partition(":")@@ -711,25 +687,14 @@ def get_cuda_ordinal_smi_rows() -> Optional[list[int]]:"""nvidia-smi row index for each CUDA ordinal of this process's environment, matched by PCI bus id.Establishes the ordinal mapping under any CUDA_DEVICE_ORDER / numeric mask without opening a CUDA- context here: the ordinals come from a short child on the driver API, the rows from nvidia-smi.+ context here: the ordinals come from the driver query (get_cuda_device_info), the rows from nvidia-smi.None when either side is unreadable or the two do not join; [] when an integrated card is among them,whose nvidia-smi memory is a carve-out rather than its shared pool."""- import sys-- from utils.child_stdio import utf8_child_env-+ devices = get_cuda_device_info()+ if not devices or any(device is None for device in devices):+ return Nonetry:- child = subprocess.run(- [sys.executable, "-c", _CUDA_BUS_ID_SCRIPT],- capture_output = True,- text = True,- encoding = "utf-8",- errors = "replace",- timeout = 15,- env = utf8_child_env(child_env_without_native_path_secret()),- **_windows_hidden_subprocess_kwargs(),- )smi = gpu_query.run_nvidia_smi([_nvidia_smi_executable(), "--query-gpu=index,pci.bus_id", "--format=csv,noheader"],capture_output = True,@@ -742,7 +707,7 @@ def get_cuda_ordinal_smi_rows() -> Optional[list[int]]:)except (OSError, subprocess.TimeoutExpired):return None- if child.returncode != 0 or smi.returncode != 0:+ if smi.returncode != 0:return Nonerow_by_bus: dict[tuple[int, str], int] = {}for line in smi.stdout.strip().splitlines():@@ -752,16 +717,149 @@ def get_cuda_ordinal_smi_rows() -> Optional[list[int]]:return Nonerow_by_bus[key] = int(index)rows = []- for line in child.stdout.strip().splitlines():- bus_id, _, integrated = line.strip().partition(" ")+ for bus_id, integrated, _total, _name in devices:key = _normalize_bus_id(bus_id)- if key not in row_by_bus or integrated not in ("0", "1"):+ if key not in row_by_bus:return None- if integrated == "1":+ if integrated:# The row is a carve-out or N/A; only torch reports the shared pool. Known, so cacheable.return []rows.append(row_by_bus[key])- return rows or None+ return rows+++# Exit 2 (no NVIDIA driver) and no output (no device) are definite answers.+_CUDA_DEVICE_INFO_SCRIPT = r"""+import ctypes, sys+try:+ lib = ctypes.WinDLL("nvcuda.dll") if sys.platform == "win32" else ctypes.CDLL("libcuda.so.1")+except OSError:+ sys.exit(2)+rc = lib.cuInit(0)+if rc == 100: # CUDA_ERROR_NO_DEVICE+ sys.exit(0)+if rc != 0:+ sys.exit(3)+count = ctypes.c_int(0)+if lib.cuDeviceGetCount(ctypes.byref(count)) != 0:+ sys.exit(4)+total_mem = lib.cuDeviceTotalMem_v2+total_mem.argtypes = [ctypes.POINTER(ctypes.c_size_t), ctypes.c_int]+for ordinal in range(count.value):+ dev = ctypes.c_int(0)+ bus = ctypes.create_string_buffer(64)+ name = ctypes.create_string_buffer(256)+ integrated = ctypes.c_int(0)+ total = ctypes.c_size_t(0)+ if (+ lib.cuDeviceGet(ctypes.byref(dev), ordinal) != 0+ or lib.cuDeviceGetAttribute(ctypes.byref(integrated), 18, dev) != 0 # CU_DEVICE_ATTRIBUTE_INTEGRATED+ or total_mem(ctypes.byref(total), dev) != 0+ or lib.cuDeviceGetName(name, 256, dev) != 0+ ):+ print("ERR") # this ordinal did not answer; the others still count+ continue+ # The bus id is informational: a driver that cannot report it (some virtualised ones) still answers.+ bus_id = bus.value.decode() if lib.cuDeviceGetPCIBusId(bus, 64, dev) == 0 and bus.value else "-"+ print(bus_id, integrated.value, total.value, name.value.decode(errors = "replace"))+"""+++_CUDA_DEVICE_INFO_RETRY_S = 60.0+_CUDA_DEVICE_INFO_CACHE: dict[tuple, tuple[Optional[float], Any]] = {}+++def get_cuda_device_info() -> Optional[list[Optional[tuple[str, bool, int, str]]]]:+ """(bus id or "", integrated, total bytes, name) per CUDA ordinal (None: no answer); [] no device, None unknown.+ A driver-API child, never get_device_properties: that initialises CUDA here, then device calls pin contexts."""+ # Per mask and inventory generation: hardware re-detection (gpu_query.invalidate_static) asks again.+ key = (+ os.environ.get("CUDA_VISIBLE_DEVICES"),+ os.environ.get("CUDA_DEVICE_ORDER"),+ gpu_query.static_generation(),+ )+ hit = _CUDA_DEVICE_INFO_CACHE.get(key)+ if hit is not None and (+ hit[0] is None or time.monotonic() - hit[0] < _CUDA_DEVICE_INFO_RETRY_S+ ):+ return None if hit[1] is None else list(hit[1])+ info = _query_cuda_device_info()+ complete = info is not None and all(device is not None for device in info)+ _CUDA_DEVICE_INFO_CACHE[key] = (None if complete else time.monotonic(), info)+ return None if info is None else list(info)+++def cuda_device_info_or_torch(+ torch_mod: Any = None,+) -> Optional[list[Optional[tuple[str, bool, int, str]]]]:+ """get_cuda_device_info() with unanswered ordinals filled from get_device_properties: detection only widens."""+ import sys++ info = get_cuda_device_info()+ torch = sys.modules.get("torch") if torch_mod is None else torch_mod+ cuda = getattr(torch, "cuda", None)+ try:+ count = int(cuda.device_count()) if cuda is not None else None+ except Exception:+ count = None+ if count is None or (+ info is not None and len(info) == count and all(d is not None for d in info)+ ):+ return info+ out = list(info) if info is not None and len(info) == count else [None] * count+ for ordinal, device in enumerate(out):+ if device is not None:+ continue+ try:+ props = cuda.get_device_properties(ordinal)+ except Exception:+ continue+ integrated = bool(+ getattr(props, "is_integrated", False) or getattr(props, "integrated", False)+ )+ out[ordinal] = ("", integrated, int(props.total_memory), str(props.name or ""))+ return out if info is not None or any(d is not None for d in out) else None+++def _query_cuda_device_info() -> Optional[list[Optional[tuple[str, bool, int, str]]]]:+ import sys++ from utils.child_stdio import utf8_child_env++ try:+ child = subprocess.run(+ [sys.executable, "-c", _CUDA_DEVICE_INFO_SCRIPT],+ capture_output = True,+ text = True,+ encoding = "utf-8",+ errors = "replace",+ timeout = 15,+ env = utf8_child_env(child_env_without_native_path_secret()),+ **_windows_hidden_subprocess_kwargs(),+ )+ except (OSError, subprocess.TimeoutExpired):+ return None+ if child.returncode == 2:+ return []+ if child.returncode != 0:+ return None+ devices: list[Optional[tuple[str, bool, int, str]]] = []+ for line in child.stdout.strip().splitlines():+ if line.strip() == "ERR":+ devices.append(None)+ continue+ parts = line.strip().split(" ", 3)+ if len(parts) < 3 or parts[1] not in ("0", "1") or not parts[2].isdigit():+ return None+ devices.append(+ (+ "" if parts[0] == "-" else parts[0],+ parts[1] == "1",+ int(parts[2]),+ parts[3] if len(parts) > 3 else "",+ )+ )+ return devicesdef get_pci_bus_ids() -> dict[int, tuple[int, str]]: