Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: read GPU metadata from the CUDA driver API, not torch.cuda.get_device_properties (#17120)

Daniel Han2f8467f454f4
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 81d725f0630..d9bbbeca6a0 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -12487,6 +12487,11 @@ class LlamaCppBackend:
"""True if the integrated-CUDA answer is cached for this mask; the preflight never probes, since the budget is already taken and both arms fill the cache."""
return LlamaCppBackend._integrated_cuda_mask_key() in LlamaCppBackend._INTEGRATED_CUDA_IDS
+ @staticmethod
+ def _cuda_device_info():
+ from utils.hardware import nvidia
+ return nvidia.cuda_device_info_or_torch()
+
@staticmethod
def _integrated_cuda_gpu_ids() -> set[int]:
"""PHYSICAL ids of visible integrated CUDA GPUs; ROCm is left to _rocm_unified_memory_gpu_ids."""
@@ -12503,29 +12508,19 @@ class LlamaCppBackend:
return set()
if not (hasattr(torch, "cuda") and torch.cuda.is_available()):
return set()
+ info = LlamaCppBackend._cuda_device_info()
+ if info is None:
+ return set()
# Same ordinal-to-physical map as the ROCm twin, so masked CUDA_VISIBLE_DEVICES=2 hosts stay correct.
physical_ids = LlamaCppBackend._resolve_visible_physical_ids()
- integrated: set[int] = set()
- # An incomplete pass is returned but not cached, so a failed query is retried.
- complete = True
- for ordinal in range(torch.cuda.device_count()):
- try:
- props = torch.cuda.get_device_properties(ordinal)
- except Exception:
- complete = False
- continue
- # Both attribute spellings: torch renamed `integrated`.
- if not (
- getattr(props, "is_integrated", False) or getattr(props, "integrated", False)
- ):
- continue
- integrated.add(
- physical_ids[ordinal]
- if physical_ids is not None and ordinal < len(physical_ids)
- else ordinal
- )
- # Cached for the preflight only if the pass reached every device; a torch that raised says nothing.
- if complete:
+ integrated = {
+ physical_ids[ordinal]
+ if physical_ids is not None and ordinal < len(physical_ids)
+ else ordinal
+ for ordinal, device in enumerate(info)
+ if device is not None and device[1]
+ }
+ if all(device is not None for device in info):
LlamaCppBackend._INTEGRATED_CUDA_IDS[
LlamaCppBackend._integrated_cuda_mask_key()
] = integrated
@@ -12538,7 +12533,7 @@ class LlamaCppBackend:
@staticmethod
def _integrated_cuda_pool_total_mib() -> dict[int, int]:
- """Pool MiB per integrated CUDA device, from props.total_memory: what it can actually allocate."""
+ """Pool MiB per integrated CUDA device, from the driver's total memory: what it can actually allocate."""
key = LlamaCppBackend._integrated_cuda_mask_key()
cached = LlamaCppBackend._INTEGRATED_CUDA_POOL_MIB.get(key)
if cached is not None:
@@ -12547,24 +12542,21 @@ class LlamaCppBackend:
if not integrated:
return {}
try:
- import torch
-
+ info = LlamaCppBackend._cuda_device_info()
+ if info is None:
+ return {}
physical_ids = LlamaCppBackend._resolve_visible_physical_ids()
totals: dict[int, int] = {}
- complete = True
- for ordinal in range(torch.cuda.device_count()):
+ complete = all(device is not None for device in info)
+ for ordinal, device in enumerate(info):
idx = (
physical_ids[ordinal]
if physical_ids is not None and ordinal < len(physical_ids)
else ordinal
)
- if idx not in integrated:
+ if device is None or idx not in integrated:
continue
- try:
- props = torch.cuda.get_device_properties(ordinal)
- totals[idx] = int(props.total_memory) // (1024 * 1024)
- except Exception:
- complete = False
+ totals[idx] = int(device[2]) // (1024 * 1024)
if complete:
LlamaCppBackend._INTEGRATED_CUDA_POOL_MIB[key] = totals
return totals
@@ -12728,17 +12720,16 @@ class LlamaCppBackend:
return False # ROCm reuses torch.cuda.*; not a CUDA part
if not (hasattr(torch, "cuda") and torch.cuda.is_available()):
return False
- count = torch.cuda.device_count()
+ info = LlamaCppBackend._cuda_device_info() or []
# Mirror _get_gpu_free_memory; unset or unparsable CUDA_VISIBLE_DEVICES keeps physical id == ordinal.
physical_ids = LlamaCppBackend._resolve_visible_physical_ids()
names_by_id: dict[int, str] = {}
- for ordinal in range(count):
- try:
- name = (torch.cuda.get_device_properties(ordinal).name or "").lower()
- except Exception:
+ for ordinal, device in enumerate(info):
+ if device is None:
continue
+ name = (device[3] or "").lower()
pid = (
physical_ids[ordinal]
if physical_ids is not None and ordinal < len(physical_ids)
diff --git a/studio/backend/tests/conftest.py b/studio/backend/tests/conftest.py
index b7dec809c27..e4c1b68f4a6 100644
--- a/studio/backend/tests/conftest.py
+++ b/studio/backend/tests/conftest.py
@@ -104,6 +104,134 @@ def _no_vae_structural_verdict_cache(monkeypatch):
monkeypatch.setenv("UNSLOTH_VAE_FUSED_STRUCTURAL_CACHE", "0")
+def _cuda_device_info_from_the_test_torch():
+ """The devices the test's fake torch describes; none for the real torch (never this host's driver)."""
+ torch = sys.modules.get("torch")
+ cuda = getattr(torch, "cuda", None)
+ props = getattr(cuda, "get_device_properties", None)
+ if props is None or str(getattr(props, "__module__", "") or "").startswith("torch"):
+ return []
+ try:
+ count = int(cuda.device_count())
+ except Exception:
+ return None
+ devices = []
+ for ordinal in range(count):
+ try:
+ p = props(ordinal)
+ except Exception:
+ devices.append(None)
+ continue
+ integrated = bool(getattr(p, "is_integrated", False) or getattr(p, "integrated", False))
+ devices.append(
+ (
+ f"0000:{ordinal + 1:02x}:00.0",
+ integrated,
+ int(getattr(p, "total_memory", 0) or 0),
+ str(getattr(p, "name", "") or ""),
+ )
+ )
+ return devices
+
+
+def _c_cuda_mask():
+ """CUDA_VISIBLE_DEVICES as cuInit reads it: the C environment, which os.environ writes reach."""
+ import ctypes
+
+ getenv = ctypes.CDLL(None).getenv
+ getenv.restype, getenv.argtypes = ctypes.c_char_p, [ctypes.c_char_p]
+ raw = getenv(b"CUDA_VISIBLE_DEVICES")
+ return None if raw is None else raw.decode(errors = "replace").strip()
+
+
+# A run that hid every GPU must never open a context: a CPU test that rewrites the mask and reaches a real CUDA
+# init would open one on each card it names (xdist workers held 600-900 MiB on six GPUs), so that init fails it.
+_SESSION_HID_GPUS = os.name != "nt" and _c_cuda_mask() in ("", "-1")
+_cuda_init_state = {"cpu_test": False, "attempts": []}
+
+
+def _refuse_cuda_init_once_unhidden(torch):
+ c = getattr(torch, "_C", None)
+ real = getattr(c, "_cuda_init", None)
+ if real is None or getattr(real, "_studio_cpu_test_check", False):
+ return
+
+ def _cuda_init(*a, **k):
+ mask = _c_cuda_mask()
+ if _cuda_init_state["cpu_test"] and mask not in ("", "-1"):
+ import traceback
+ _cuda_init_state["attempts"].append(
+ (mask, "".join(traceback.format_stack(limit = 12)[:-1]))
+ )
+ raise RuntimeError("a CPU test initialised CUDA after unhiding the GPUs (refused)")
+ return real(*a, **k)
+
+ _cuda_init._studio_cpu_test_check = True
+ c._cuda_init = _cuda_init
+
+
+class _AfterTorchImport:
+ def find_spec(
+ self,
+ name,
+ path = None,
+ target = None,
+ ):
+ if name != "torch" or "torch" in sys.modules:
+ return None
+ sys.meta_path.remove(self)
+ spec = _ilu.find_spec("torch")
+ if spec is None or spec.loader is None:
+ return spec
+ exec_module = spec.loader.exec_module
+
+ def _exec(module):
+ exec_module(module)
+ _refuse_cuda_init_once_unhidden(module)
+
+ spec.loader.exec_module = _exec
+ return spec
+
+
+if _SESSION_HID_GPUS:
+ if "torch" in sys.modules:
+ _refuse_cuda_init_once_unhidden(sys.modules["torch"])
+ else:
+ sys.meta_path.insert(0, _AfterTorchImport())
+
+
[email protected](autouse = True)
+def _no_cuda_context_from_a_cpu_test(request):
+ if not _SESSION_HID_GPUS or request.node.get_closest_marker("gpu") is not None:
+ yield
+ return
+ _cuda_init_state["cpu_test"], _cuda_init_state["attempts"] = True, []
+ try:
+ yield
+ finally:
+ _cuda_init_state["cpu_test"] = False
+ if _cuda_init_state["attempts"]:
+ mask, stack = _cuda_init_state["attempts"][0]
+ pytest.fail(
+ f"this CPU test initialised CUDA with CUDA_VISIBLE_DEVICES={mask!r} after the run had hidden "
+ f"every GPU; its next device call opens a context on each card it names. Fake the device query (torch.cuda, "
+ f"utils.hardware.nvidia.get_cuda_device_info) or mark the test gpu. First call:\n{stack}",
+ pytrace = False,
+ )
+
+
[email protected](autouse = True)
+def _no_real_cuda_device_query(monkeypatch):
+ try:
+ from utils.hardware import nvidia
+ except Exception:
+ return
+ monkeypatch.setattr(
+ nvidia, "get_cuda_device_info", _cuda_device_info_from_the_test_torch, raising = False
+ )
+ monkeypatch.setattr(nvidia, "_CUDA_DEVICE_INFO_CACHE", {}, raising = False)
+
+
@pytest.hookimpl(trylast = True)
def pytest_collection_modifyitems(config, items):
"""After xdist tags grouped tests; see tests/_shared/xdist_group_check.py."""
diff --git a/studio/backend/tests/test_gpu_metadata_no_cuda_context.py b/studio/backend/tests/test_gpu_metadata_no_cuda_context.py
new file mode 100644
index 00000000000..6b2f75f79bc
--- /dev/null
+++ b/studio/backend/tests/test_gpu_metadata_no_cuda_context.py
@@ -0,0 +1,299 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+import types
+
+import pytest
+
+from core.inference.llama_cpp import LlamaCppBackend
+from utils.hardware import nvidia
+
+GIB = 1 << 30
+# The real query, captured at import: the autouse conftest fixture replaces it for every test.
+_REAL_GET_CUDA_DEVICE_INFO = getattr(nvidia, "get_cuda_device_info", None)
+
+
+class _CountingTorch(types.ModuleType):
+ def __init__(self, count):
+ super().__init__("torch")
+ self.calls = []
+ self.version = types.SimpleNamespace(hip = None)
+
+ def _counted(name, value = None):
+ def _call(*a, **k):
+ self.calls.append(name)
+ if name == "get_device_properties":
+ return types.SimpleNamespace(
+ name = "NVIDIA B200", total_memory = 180 * GIB, is_integrated = 0
+ )
+ return value
+
+ return _call
+
+ self.cuda = types.SimpleNamespace(
+ is_available = lambda: True,
+ device_count = lambda: count,
+ get_device_properties = _counted("get_device_properties"),
+ _lazy_init = _counted("_lazy_init"),
+ init = _counted("init"),
+ )
+
+
+def host(monkeypatch):
+ monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)
+ for cache in ("_INTEGRATED_CUDA_IDS", "_INTEGRATED_CUDA_POOL_MIB"):
+ monkeypatch.setattr(LlamaCppBackend, cache, {}, raising = False)
+ torch = _CountingTorch(7)
+ monkeypatch.setitem(sys.modules, "torch", torch)
+ queries = []
+ devices = [(f"0000:{i + 1:02x}:00.0", False, 180 * GIB, "NVIDIA B200") for i in range(6)]
+ devices.append(("0000:0f:00.0", True, 120 * GIB, "NVIDIA GB10"))
+
+ def _child():
+ queries.append(1)
+ return list(devices)
+
+ monkeypatch.setattr(nvidia, "get_cuda_device_info", _REAL_GET_CUDA_DEVICE_INFO, raising = False)
+ monkeypatch.setattr(nvidia, "_query_cuda_device_info", _child, raising = False)
+ return types.SimpleNamespace(torch = torch, queries = queries)
+
+
+def test_metadata_lookups_never_initialise_torch_cuda(host):
+ answers = (
+ LlamaCppBackend._integrated_cuda_gpu_ids(),
+ LlamaCppBackend._integrated_cuda_pool_total_mib(),
+ LlamaCppBackend._integrated_cuda_unified_memory([0]),
+ LlamaCppBackend._integrated_cuda_selection_is_all_shared([6]),
+ LlamaCppBackend._all_selected_gpus_match(LlamaCppBackend._DATACENTER_GPU_RE, [0, 1]),
+ )
+ assert host.torch.calls == []
+ assert answers == ({6}, {6: 120 * 1024}, False, True, True)
+ assert len(host.queries) == 1
+
+
+def _torch_cannot_answer_either(host, count):
+ def _raise(ordinal):
+ raise RuntimeError("no properties")
+
+ host.torch.cuda.device_count = lambda: count
+ host.torch.cuda.get_device_properties = _raise
+
+
+def test_an_ordinal_that_did_not_answer_is_used_but_not_settled(host, monkeypatch):
+ monkeypatch.setattr(
+ nvidia,
+ "get_cuda_device_info",
+ lambda: [("0000:01:00.0", True, 120 * GIB, "NVIDIA GB10"), None],
+ )
+ _torch_cannot_answer_either(host, 2)
+ assert LlamaCppBackend._integrated_cuda_gpu_ids() == {0}
+ assert LlamaCppBackend._integrated_cuda_probe_is_free() is False
+
+
+def test_an_unknown_answer_neither_side_knows_is_not_cached(host, monkeypatch):
+ monkeypatch.setattr(nvidia, "get_cuda_device_info", lambda: None)
+ _torch_cannot_answer_either(host, 2)
+ assert LlamaCppBackend._integrated_cuda_gpu_ids() == set()
+ assert LlamaCppBackend._integrated_cuda_probe_is_free() is False
+
+
+def _torch_with_an_integrated_card(host):
+ def _props(ordinal):
+ host.torch.calls.append("get_device_properties")
+ if ordinal == 1:
+ return types.SimpleNamespace(
+ name = "NVIDIA GB10", total_memory = 120 * GIB, is_integrated = 1
+ )
+ return types.SimpleNamespace(name = "NVIDIA B200", total_memory = 180 * GIB, is_integrated = 0)
+
+ host.torch.cuda.device_count = lambda: 2
+ host.torch.cuda.get_device_properties = _props
+
+
+ "driver_answer", [None, [], [("0000:01:00.0", False, 180 * GIB, "NVIDIA B200"), None]]
+)
+def test_a_failed_driver_probe_never_hides_a_card_torch_sees(host, monkeypatch, driver_answer):
+ monkeypatch.setattr(nvidia, "get_cuda_device_info", lambda: driver_answer)
+ _torch_with_an_integrated_card(host)
+ assert LlamaCppBackend._integrated_cuda_gpu_ids() == {1}
+ assert LlamaCppBackend._integrated_cuda_pool_total_mib() == {1: 120 * 1024}
+ assert host.torch.calls.count("get_device_properties") >= 1
+
+
+def _child(
+ monkeypatch,
+ returncode,
+ stdout = "",
+):
+ seen = {}
+
+ def _run(cmd, **kw):
+ seen["cmd"] = cmd
+ return subprocess.CompletedProcess(cmd, returncode, stdout, "")
+
+ monkeypatch.setattr(nvidia.subprocess, "run", _run)
+ return seen
+
+
+ "returncode, stdout, expected",
+ [
+ (
+ 0,
+ "0000:3b:00.0 0 85899345920 NVIDIA H100 80GB HBM3\n0000:00:00.0 1 128000000000 NVIDIA GB10\n",
+ [
+ ("0000:3b:00.0", False, 85899345920, "NVIDIA H100 80GB HBM3"),
+ ("0000:00:00.0", True, 128000000000, "NVIDIA GB10"),
+ ],
+ ),
+ (
+ 0,
+ "0000:3b:00.0 0 85899345920 NVIDIA H100\nERR\n",
+ [("0000:3b:00.0", False, 85899345920, "NVIDIA H100"), None],
+ ),
+ (0, "- 1 128000000000 NVIDIA GB10\n", [("", True, 128000000000, "NVIDIA GB10")]),
+ (0, "", []),
+ (2, "", []),
+ (3, "", None),
+ (0, "garbage\n", None),
+ ],
+)
+def test_the_driver_child_answer_is_parsed(monkeypatch, returncode, stdout, expected):
+ monkeypatch.setattr(nvidia, "get_cuda_device_info", _REAL_GET_CUDA_DEVICE_INFO)
+ seen = _child(monkeypatch, returncode, stdout)
+ assert nvidia.get_cuda_device_info() == expected
+ assert seen["cmd"][1:] == ["-c", nvidia._CUDA_DEVICE_INFO_SCRIPT]
+
+
+def test_a_child_that_times_out_is_unknown(monkeypatch):
+ monkeypatch.setattr(nvidia, "get_cuda_device_info", _REAL_GET_CUDA_DEVICE_INFO)
+
+ def _run(cmd, **kw):
+ raise subprocess.TimeoutExpired(cmd, kw.get("timeout"))
+
+ monkeypatch.setattr(nvidia.subprocess, "run", _run)
+ assert nvidia.get_cuda_device_info() is None
+
+
+def test_an_unknown_answer_is_not_asked_again_for_a_while(monkeypatch):
+ monkeypatch.setattr(nvidia, "get_cuda_device_info", _REAL_GET_CUDA_DEVICE_INFO)
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0")
+ clock = [1000.0]
+ monkeypatch.setattr(nvidia.time, "monotonic", lambda: clock[0])
+ answers = iter(
+ [
+ (3, ""),
+ (0, "0000:01:00.0 0 8589934592 NVIDIA T4\nERR\n"),
+ (0, "0000:01:00.0 0 8589934592 NVIDIA T4\n"),
+ ]
+ )
+ runs = []
+
+ def _run(cmd, **kw):
+ runs.append(1)
+ rc, out = next(answers)
+ return subprocess.CompletedProcess(cmd, rc, out, "")
+
+ monkeypatch.setattr(nvidia.subprocess, "run", _run)
+ assert nvidia.get_cuda_device_info() is None
+ assert nvidia.get_cuda_device_info() is None and len(runs) == 1
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "1")
+ assert nvidia.get_cuda_device_info() == [("0000:01:00.0", False, 8589934592, "NVIDIA T4"), None]
+ assert len(runs) == 2
+ clock[0] += nvidia._CUDA_DEVICE_INFO_RETRY_S + 1
+ assert nvidia.get_cuda_device_info() == [("0000:01:00.0", False, 8589934592, "NVIDIA T4")]
+ assert len(runs) == 3
+ clock[0] += 10 * nvidia._CUDA_DEVICE_INFO_RETRY_S
+ assert nvidia.get_cuda_device_info() == [("0000:01:00.0", False, 8589934592, "NVIDIA T4")]
+ assert len(runs) == 3
+
+
+def test_the_real_query_answers_no_device_under_an_empty_mask(monkeypatch):
+ monkeypatch.setattr(nvidia, "get_cuda_device_info", _REAL_GET_CUDA_DEVICE_INFO)
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "")
+ assert nvidia.get_cuda_device_info() == []
+
+
+def test_the_driver_child_runs_and_finds_no_device_under_an_empty_mask():
+ env = dict(os.environ, CUDA_VISIBLE_DEVICES = "")
+ r = subprocess.run(
+ [sys.executable, "-c", nvidia._CUDA_DEVICE_INFO_SCRIPT],
+ capture_output = True,
+ text = True,
+ timeout = 60,
+ env = env,
+ )
+ assert r.returncode in (0, 2), r.stderr
+ assert r.stdout.strip() == ""
+
+
+def test_an_ordinal_that_did_not_answer_leaves_the_pool_unsettled(host, monkeypatch):
+ monkeypatch.setattr(
+ nvidia,
+ "get_cuda_device_info",
+ lambda: [("0000:01:00.0", True, 120 * GIB, "NVIDIA GB10"), None],
+ )
+ _torch_cannot_answer_either(host, 2)
+ assert LlamaCppBackend._integrated_cuda_pool_total_mib() == {0: 120 * 1024}
+ assert LlamaCppBackend._INTEGRATED_CUDA_POOL_MIB == {}
+
+
+def test_the_inventory_poll_reads_nvidia_cards_from_the_driver(host, monkeypatch):
+ from utils.hardware import hardware as hw
+
+ monkeypatch.setattr(hw, "IS_ROCM", False)
+ monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA)
+ inventory = hw._torch_get_device_inventory([4, 6])
+ assert host.torch.calls == []
+ assert [
+ (d["index"], d["visible_ordinal"], d["name"], d["total_gb"], d["_cuda_integrated"])
+ for d in inventory
+ ] == [
+ (4, 0, "NVIDIA B200", 180.0, False),
+ (6, 1, "NVIDIA B200", 180.0, False),
+ ]
+
+
+def test_the_device_list_keeps_the_names_that_answered(host, monkeypatch, capsys):
+ from utils.hardware import hardware as hw
+
+ monkeypatch.setattr(
+ nvidia,
+ "get_cuda_device_info",
+ lambda: [None, ("0000:02:00.0", False, 180 * GIB, "NVIDIA B200")],
+ )
+ _torch_cannot_answer_either(host, 2)
+ hw._print_cuda_device_list(False)
+ out = capsys.readouterr().out
+ assert "[0] <unavailable>" in out and "[1] NVIDIA B200" in out
+ with pytest.raises(RuntimeError, match = "device 0"):
+ hw._cuda_device0_name()
+
+
+def test_hardware_redetection_asks_the_driver_again(monkeypatch):
+ from utils.hardware import gpu_query
+
+ monkeypatch.setattr(nvidia, "get_cuda_device_info", _REAL_GET_CUDA_DEVICE_INFO)
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0")
+ answers = iter(
+ ["0000:01:00.0 0 8589934592 NVIDIA T4\n", "0000:02:00.0 0 25769803776 NVIDIA L4\n"]
+ )
+ runs = []
+
+ def _run(cmd, **kw):
+ runs.append(1)
+ return subprocess.CompletedProcess(cmd, 0, next(answers), "")
+
+ monkeypatch.setattr(nvidia.subprocess, "run", _run)
+ assert nvidia.get_cuda_device_info()[0][3] == "NVIDIA T4"
+ assert nvidia.get_cuda_device_info()[0][3] == "NVIDIA T4" and len(runs) == 1
+ gpu_query.invalidate_static("test: card replaced")
+ assert nvidia.get_cuda_device_info() == [("0000:02:00.0", False, 25769803776, "NVIDIA L4")]
+ assert len(runs) == 2
diff --git a/studio/backend/tests/test_gpu_selection.py b/studio/backend/tests/test_gpu_selection.py
index f86cc7afdb7..11ab20acd3b 100644
--- a/studio/backend/tests/test_gpu_selection.py
+++ b/studio/backend/tests/test_gpu_selection.py
@@ -456,6 +456,8 @@ class TestVisibleGpuUtilization(_GpuCacheResetMixin, unittest.TestCase):
with (
patch.dict(os.environ, {"CUDA_VISIBLE_DEVICES": "1,3"}, clear = True),
patch("utils.hardware.hardware.get_device", return_value = DeviceType.CUDA),
+ # torch's view would be this host's real cards
+ patch("utils.hardware.hardware._torch_get_device_module", return_value = (None, None)),
patch("utils.hardware.nvidia.subprocess.run") as mock_run,
):
mock_run.return_value = SimpleNamespace(
@@ -485,6 +487,8 @@ class TestVisibleGpuUtilization(_GpuCacheResetMixin, unittest.TestCase):
with (
patch.dict(os.environ, {"CUDA_VISIBLE_DEVICES": "1,3"}, clear = True),
patch("utils.hardware.hardware.get_device", return_value = DeviceType.CUDA),
+ # torch's view would be this host's real cards
+ patch("utils.hardware.hardware._torch_get_device_module", return_value = (None, None)),
patch("utils.hardware.nvidia.subprocess.run") as mock_run,
):
mock_run.return_value = SimpleNamespace(
diff --git a/studio/backend/tests/test_llama_cpp_placement.py b/studio/backend/tests/test_llama_cpp_placement.py
index 88b5eddb68b..4d408be13fa 100644
--- a/studio/backend/tests/test_llama_cpp_placement.py
+++ b/studio/backend/tests/test_llama_cpp_placement.py
@@ -6205,6 +6205,10 @@ def _mixed_host_launch(
monkeypatch.setattr(
LlamaCppBackend, "_rocm_unified_memory_gpu_ids", staticmethod(lambda: set(rocm))
)
+ # The carve-out query reads torch's devices; a GPU-less host answers None (unknown) and so does this one.
+ monkeypatch.setattr(
+ LlamaCppBackend, "_rocm_selected_pool_mib", staticmethod(lambda gpu_indices = None: None)
+ )
monkeypatch.setattr(
LlamaCppBackend,
"_INTEGRATED_CUDA_IDS",
diff --git a/studio/backend/tests/test_media_auto_gpu_ordinary_host.py b/studio/backend/tests/test_media_auto_gpu_ordinary_host.py
index a207037f6d9..3ddc10b4597 100644
--- a/studio/backend/tests/test_media_auto_gpu_ordinary_host.py
+++ b/studio/backend/tests/test_media_auto_gpu_ordinary_host.py
@@ -19,6 +19,10 @@ from core.inference import diffusion_device as dd
from utils.hardware import hardware as hw
from utils.hardware.hardware import DeviceType
+from utils.hardware import nvidia as _nvidia
+
+_REAL_GET_CUDA_DEVICE_INFO = getattr(_nvidia, "get_cuda_device_info", None)
+
def _dev(index, total, used):
return {"index": index, "vram_total_gb": total, "vram_used_gb": used}
@@ -530,9 +534,16 @@ def test_fastest_first_mask_with_an_integrated_card_keeps_torch(monkeypatch):
@pytest.mark.parametrize(
"child_out, expected",
[
- ("0000:3b:00.0 0\n0000:1a:00.0 0\n", [1, 0]),
- ("0000:3b:00.0 0\n0000:1a:00.0 1\n", []), # integrated card mapped
+ (
+ "0000:3b:00.0 0 85899345920 NVIDIA H100\n0000:1a:00.0 0 85899345920 NVIDIA H100\n",
+ [1, 0],
+ ),
+ (
+ "0000:3b:00.0 0 85899345920 NVIDIA H100\n0000:1a:00.0 1 128000000000 NVIDIA GB10\n",
+ [],
+ ), # integrated card mapped
("0000:3b:00.0\n", None), # no attribute: unreadable
+ ("- 0 85899345920 NVIDIA H100\n", None), # the driver gave no bus id: nothing to join on
],
)
def test_cuda_ordinal_rows_parse_bus_ids_and_integrated(monkeypatch, child_out, expected):
@@ -540,6 +551,8 @@ def test_cuda_ordinal_rows_parse_bus_ids_and_integrated(monkeypatch, child_out,
from utils.hardware import nvidia
+ # The real driver query on a faked child (conftest replaces it with the test torch's devices).
+ monkeypatch.setattr(nvidia, "get_cuda_device_info", _REAL_GET_CUDA_DEVICE_INFO)
child = types.SimpleNamespace(returncode = 0, stdout = child_out)
smi = types.SimpleNamespace(returncode = 0, stdout = "0, 00000000:1A:00.0\n1, 00000000:3B:00.0\n")
monkeypatch.setattr(subprocess, "run", lambda *a, **k: child)
diff --git a/studio/backend/tests/test_rocm_multi_gpu_vram_system_wide.py b/studio/backend/tests/test_rocm_multi_gpu_vram_system_wide.py
index 08b5817b227..5512a8b0bc8 100644
--- a/studio/backend/tests/test_rocm_multi_gpu_vram_system_wide.py
+++ b/studio/backend/tests/test_rocm_multi_gpu_vram_system_wide.py
@@ -433,6 +433,8 @@ def test_visible_utilization_rocm_fallback_overlays(monkeypatch):
{"index": 1, "visible_ordinal": 1, "used_gb": 0.01, "total_gb": 8.0},
],
)
+ # Names come from torch's device list, which on this host would be its real cards; CI has no torch.
+ monkeypatch.setattr(hw, "_torch_get_device_inventory", lambda ids: [])
overlaid = []
monkeypatch.setattr(
hw, "_overlay_system_wide_vram", lambda devices: overlaid.append(len(devices))
diff --git a/studio/backend/tests/test_utils.py b/studio/backend/tests/test_utils.py
index b993f4ffeb3..210cd18d9de 100644
--- a/studio/backend/tests/test_utils.py
+++ b/studio/backend/tests/test_utils.py
@@ -91,7 +91,8 @@ class TestGetDevice:
@needs_torch
def test_device0_probe_failure_is_logged_with_allocator_config(self, monkeypatch):
- # failed CUDA initialization can crash later, so log its cause at error rather than debug
+ # a failed device-0 probe can mean a broken CUDA setup that crashes later, so log it at error;
+ # both probes fail, so the NVIDIA (driver query) and ROCm (get_device_properties) branches are covered
monkeypatch.delenv("PYTORCH_CUDA_ALLOC_CONF", raising = False)
monkeypatch.delenv("PYTORCH_HIP_ALLOC_CONF", raising = False)
monkeypatch.setenv("PYTORCH_ALLOC_CONF", "expandable_segments:maybe")
@@ -102,6 +103,7 @@ class TestGetDevice:
patch("utils.hardware.hardware._has_torch", return_value = True),
patch("torch.cuda.is_available", return_value = True),
patch("torch.cuda.device_count", return_value = 1),
+ patch("utils.hardware.nvidia.get_cuda_device_info", side_effect = error),
patch("torch.cuda.get_device_properties", side_effect = error),
patch.object(_hw_module, "logger") as logger,
):
diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.py
index e190122271b..3b3e60e3232 100644
--- a/studio/backend/utils/hardware/hardware.py
+++ b/studio/backend/utils/hardware/hardware.py
@@ -1529,6 +1529,22 @@ def overturn_the_mlx_verdict(epoch: Optional[int] = None) -> bool:
return DEVICE is not None and DETECTION_COMPLETE.is_set() and not CHAT_ONLY
+def _cuda_device_names() -> list:
+ from utils.hardware import nvidia
+
+ info = nvidia.cuda_device_info_or_torch()
+ if not info:
+ raise RuntimeError("the CUDA driver did not list the devices")
+ return [device[3] if device is not None else None for device in info]
+
+
+def _cuda_device0_name() -> str:
+ name = _cuda_device_names()[0]
+ if name is None:
+ raise RuntimeError("CUDA device 0 did not answer the driver query")
+ return name
+
+
def _print_cuda_device_list(is_rocm: bool) -> None:
"""List every visible CUDA/ROCm GPU with its index at startup: the "Hardware detected" banner names only device 0, hiding the other cards. Indices are CUDA ordinals, matching `nvidia-smi -L` when no mask is set. CUDA_DEVICE_ORDER governs only CUDA, so it is shown for CUDA but not ROCm. Purely informational and never raises."""
try:
@@ -1543,9 +1559,19 @@ def _print_cuda_device_list(is_rocm: bool) -> None:
order = os.environ.get("CUDA_DEVICE_ORDER", "default")
header = f"CUDA devices ({count}, CUDA_DEVICE_ORDER={order}):"
lines = [header]
+ names = None
+ if not is_rocm:
+ try:
+ names = _cuda_device_names()
+ except Exception as e:
+ logger.debug("CUDA device name probe failed: %s", e)
+ names = []
for i in range(count):
try:
- name = torch.cuda.get_device_properties(i).name
+ if names is not None:
+ name = names[i] if i < len(names) and names[i] is not None else "<unavailable>"
+ else:
+ name = torch.cuda.get_device_properties(i).name
except Exception as e:
logger.debug("CUDA device %d property probe failed: %s", i, e)
name = "<unavailable>"
@@ -1748,8 +1774,14 @@ def _detect_hardware_locked() -> DeviceType:
if not cuda_unavailable:
DEVICE = DeviceType.CUDA
CHAT_ONLY = False
+ # AMD SDK wheels omit torch.version.hip; ROCm label uses __version__; DEVICE stays CUDA
+ _hip_ver = getattr(torch.version, "hip", None)
+ _is_rocm = _hip_ver is not None or "rocm" in torch.__version__.lower()
try:
- device_name = torch.cuda.get_device_properties(0).name
+ # Not get_device_properties for NVIDIA: it initialises CUDA in the server.
+ device_name = (
+ torch.cuda.get_device_properties(0).name if _is_rocm else _cuda_device0_name()
+ )
except Exception as e:
# failed CUDA init can make later PyTorch calls segfault, so log the cause
from utils.allocator_conf import ALLOCATOR_CONF_ENV_VARS
@@ -1764,9 +1796,7 @@ def _detect_hardware_locked() -> DeviceType:
)
device_name = "<unavailable>"
- # AMD SDK wheels omit torch.version.hip; ROCm label uses __version__; DEVICE stays CUDA
- _hip_ver = getattr(torch.version, "hip", None)
- if _hip_ver is not None or "rocm" in torch.__version__.lower():
+ if _is_rocm:
IS_ROCM = True
_configure_rocm_miopen(torch)
_hip_label = _hip_ver or torch.__version__
@@ -3115,11 +3145,43 @@ def _cuda_props_are_integrated(props: Any, backend: Optional[str] = "cuda") -> b
def _torch_get_device_inventory(device_indices: list[int]) -> list[Dict[str, Any]]:
- """Per-GPU name and total VRAM only, without creating a driver context: get_device_properties is answered from the driver's device list, while mem_get_info attaches a primary context worth ~612 MiB the process never gives back, and a telemetry poll must not be what pins that, so callers wanting live occupancy go to _torch_get_per_device_info instead. props.total_memory equals mem_get_info's total everywhere except a ROCm APU (see _rocm_props_total_is_carve_out), and used_gb is always None ("telemetry unavailable")."""
+ """Per-GPU name and total VRAM only, without creating a driver context: NVIDIA from the driver API (nvidia.cuda_device_info_or_torch), ROCm / XPU from get_device_properties, while mem_get_info attaches a primary context worth ~612 MiB the process never gives back, and a telemetry poll must not be what pins that, so callers wanting live occupancy go to _torch_get_per_device_info instead. props.total_memory equals mem_get_info's total everywhere except a ROCm APU (see _rocm_props_total_is_carve_out), and used_gb is always None ("telemetry unavailable")."""
mod, backend = _torch_get_device_module()
if mod is None:
return []
+ torch_mod = sys.modules.get("torch")
+ if (
+ backend == "cuda"
+ and mod is getattr(torch_mod, "cuda", None)
+ and not IS_ROCM
+ and not getattr(getattr(torch_mod, "version", None), "hip", None)
+ ):
+ from utils.hardware import nvidia
+
+ info = nvidia.cuda_device_info_or_torch(torch_mod)
+ nvidia_devices = []
+ for ordinal, phys_idx in enumerate(device_indices):
+ device = info[ordinal] if info is not None and ordinal < len(info) else None
+ if device is None:
+ logger.debug("driver inventory probe got no answer for ordinal %d", ordinal)
+ continue
+ nvidia_devices.append(
+ {
+ "index": phys_idx,
+ "visible_ordinal": ordinal,
+ "name": device[3],
+ "total_gb": round(device[2] / (1024**3), 2),
+ "used_gb": None,
+ "shared_memory": False,
+ "shared_memory_host_backed_gb": None,
+ "_rocm_known_unified": False,
+ "_rocm_gfx": "",
+ "_cuda_integrated": bool(device[1]),
+ }
+ )
+ return nvidia_devices
+
devices = []
for ordinal, phys_idx in enumerate(device_indices):
try:
diff --git a/studio/backend/utils/hardware/nvidia.py b/studio/backend/utils/hardware/nvidia.py
index 9f3a21121b7..00bbf5fa045 100644
--- a/studio/backend/utils/hardware/nvidia.py
+++ b/studio/backend/utils/hardware/nvidia.py
@@ -672,30 +672,6 @@ def get_card_identities() -> dict[int, tuple[str, ...]]:
return identities
-# Driver API only: cuInit enumerates devices under the inherited CUDA_VISIBLE_DEVICES / CUDA_DEVICE_ORDER
-# without creating a context, and runs in a child so the Studio process never initialises CUDA.
-_CUDA_BUS_ID_SCRIPT = r"""
-import ctypes, sys
-lib = ctypes.WinDLL("nvcuda.dll") if sys.platform == "win32" else ctypes.CDLL("libcuda.so.1")
-if lib.cuInit(0) != 0:
- sys.exit(3)
-count = ctypes.c_int(0)
-if lib.cuDeviceGetCount(ctypes.byref(count)) != 0:
- sys.exit(4)
-for ordinal in range(count.value):
- dev = ctypes.c_int(0)
- buf = ctypes.create_string_buffer(64)
- integrated = ctypes.c_int(0)
- if (
- lib.cuDeviceGet(ctypes.byref(dev), ordinal) != 0
- or lib.cuDeviceGetPCIBusId(buf, 64, dev) != 0
- or lib.cuDeviceGetAttribute(ctypes.byref(integrated), 18, dev) != 0 # CU_DEVICE_ATTRIBUTE_INTEGRATED
- ):
- sys.exit(5)
- print(buf.value.decode(), integrated.value)
-"""
-
-
def _normalize_bus_id(bus_id: str) -> Optional[tuple[int, str]]:
"""'00000000:3B:00.0' (nvidia-smi) and '0000:3b:00.0' (driver API) -> (domain, 'BUS:DEV.FN')."""
domain, sep, rest = bus_id.strip().partition(":")
@@ -711,25 +687,14 @@ def get_cuda_ordinal_smi_rows() -> Optional[list[int]]:
"""nvidia-smi row index for each CUDA ordinal of this process's environment, matched by PCI bus id.
Establishes the ordinal mapping under any CUDA_DEVICE_ORDER / numeric mask without opening a CUDA
- context here: the ordinals come from a short child on the driver API, the rows from nvidia-smi.
+ context here: the ordinals come from the driver query (get_cuda_device_info), the rows from nvidia-smi.
None when either side is unreadable or the two do not join; [] when an integrated card is among them,
whose nvidia-smi memory is a carve-out rather than its shared pool.
"""
- import sys
-
- from utils.child_stdio import utf8_child_env
-
+ devices = get_cuda_device_info()
+ if not devices or any(device is None for device in devices):
+ return None
try:
- child = subprocess.run(
- [sys.executable, "-c", _CUDA_BUS_ID_SCRIPT],
- capture_output = True,
- text = True,
- encoding = "utf-8",
- errors = "replace",
- timeout = 15,
- env = utf8_child_env(child_env_without_native_path_secret()),
- **_windows_hidden_subprocess_kwargs(),
- )
smi = gpu_query.run_nvidia_smi(
[_nvidia_smi_executable(), "--query-gpu=index,pci.bus_id", "--format=csv,noheader"],
capture_output = True,
@@ -742,7 +707,7 @@ def get_cuda_ordinal_smi_rows() -> Optional[list[int]]:
)
except (OSError, subprocess.TimeoutExpired):
return None
- if child.returncode != 0 or smi.returncode != 0:
+ if smi.returncode != 0:
return None
row_by_bus: dict[tuple[int, str], int] = {}
for line in smi.stdout.strip().splitlines():
@@ -752,16 +717,149 @@ def get_cuda_ordinal_smi_rows() -> Optional[list[int]]:
return None
row_by_bus[key] = int(index)
rows = []
- for line in child.stdout.strip().splitlines():
- bus_id, _, integrated = line.strip().partition(" ")
+ for bus_id, integrated, _total, _name in devices:
key = _normalize_bus_id(bus_id)
- if key not in row_by_bus or integrated not in ("0", "1"):
+ if key not in row_by_bus:
return None
- if integrated == "1":
+ if integrated:
# The row is a carve-out or N/A; only torch reports the shared pool. Known, so cacheable.
return []
rows.append(row_by_bus[key])
- return rows or None
+ return rows
+
+
+# Exit 2 (no NVIDIA driver) and no output (no device) are definite answers.
+_CUDA_DEVICE_INFO_SCRIPT = r"""
+import ctypes, sys
+try:
+ lib = ctypes.WinDLL("nvcuda.dll") if sys.platform == "win32" else ctypes.CDLL("libcuda.so.1")
+except OSError:
+ sys.exit(2)
+rc = lib.cuInit(0)
+if rc == 100: # CUDA_ERROR_NO_DEVICE
+ sys.exit(0)
+if rc != 0:
+ sys.exit(3)
+count = ctypes.c_int(0)
+if lib.cuDeviceGetCount(ctypes.byref(count)) != 0:
+ sys.exit(4)
+total_mem = lib.cuDeviceTotalMem_v2
+total_mem.argtypes = [ctypes.POINTER(ctypes.c_size_t), ctypes.c_int]
+for ordinal in range(count.value):
+ dev = ctypes.c_int(0)
+ bus = ctypes.create_string_buffer(64)
+ name = ctypes.create_string_buffer(256)
+ integrated = ctypes.c_int(0)
+ total = ctypes.c_size_t(0)
+ if (
+ lib.cuDeviceGet(ctypes.byref(dev), ordinal) != 0
+ or lib.cuDeviceGetAttribute(ctypes.byref(integrated), 18, dev) != 0 # CU_DEVICE_ATTRIBUTE_INTEGRATED
+ or total_mem(ctypes.byref(total), dev) != 0
+ or lib.cuDeviceGetName(name, 256, dev) != 0
+ ):
+ print("ERR") # this ordinal did not answer; the others still count
+ continue
+ # The bus id is informational: a driver that cannot report it (some virtualised ones) still answers.
+ bus_id = bus.value.decode() if lib.cuDeviceGetPCIBusId(bus, 64, dev) == 0 and bus.value else "-"
+ print(bus_id, integrated.value, total.value, name.value.decode(errors = "replace"))
+"""
+
+
+_CUDA_DEVICE_INFO_RETRY_S = 60.0
+_CUDA_DEVICE_INFO_CACHE: dict[tuple, tuple[Optional[float], Any]] = {}
+
+
+def get_cuda_device_info() -> Optional[list[Optional[tuple[str, bool, int, str]]]]:
+ """(bus id or "", integrated, total bytes, name) per CUDA ordinal (None: no answer); [] no device, None unknown.
+ A driver-API child, never get_device_properties: that initialises CUDA here, then device calls pin contexts."""
+ # Per mask and inventory generation: hardware re-detection (gpu_query.invalidate_static) asks again.
+ key = (
+ os.environ.get("CUDA_VISIBLE_DEVICES"),
+ os.environ.get("CUDA_DEVICE_ORDER"),
+ gpu_query.static_generation(),
+ )
+ hit = _CUDA_DEVICE_INFO_CACHE.get(key)
+ if hit is not None and (
+ hit[0] is None or time.monotonic() - hit[0] < _CUDA_DEVICE_INFO_RETRY_S
+ ):
+ return None if hit[1] is None else list(hit[1])
+ info = _query_cuda_device_info()
+ complete = info is not None and all(device is not None for device in info)
+ _CUDA_DEVICE_INFO_CACHE[key] = (None if complete else time.monotonic(), info)
+ return None if info is None else list(info)
+
+
+def cuda_device_info_or_torch(
+ torch_mod: Any = None,
+) -> Optional[list[Optional[tuple[str, bool, int, str]]]]:
+ """get_cuda_device_info() with unanswered ordinals filled from get_device_properties: detection only widens."""
+ import sys
+
+ info = get_cuda_device_info()
+ torch = sys.modules.get("torch") if torch_mod is None else torch_mod
+ cuda = getattr(torch, "cuda", None)
+ try:
+ count = int(cuda.device_count()) if cuda is not None else None
+ except Exception:
+ count = None
+ if count is None or (
+ info is not None and len(info) == count and all(d is not None for d in info)
+ ):
+ return info
+ out = list(info) if info is not None and len(info) == count else [None] * count
+ for ordinal, device in enumerate(out):
+ if device is not None:
+ continue
+ try:
+ props = cuda.get_device_properties(ordinal)
+ except Exception:
+ continue
+ integrated = bool(
+ getattr(props, "is_integrated", False) or getattr(props, "integrated", False)
+ )
+ out[ordinal] = ("", integrated, int(props.total_memory), str(props.name or ""))
+ return out if info is not None or any(d is not None for d in out) else None
+
+
+def _query_cuda_device_info() -> Optional[list[Optional[tuple[str, bool, int, str]]]]:
+ import sys
+
+ from utils.child_stdio import utf8_child_env
+
+ try:
+ child = subprocess.run(
+ [sys.executable, "-c", _CUDA_DEVICE_INFO_SCRIPT],
+ capture_output = True,
+ text = True,
+ encoding = "utf-8",
+ errors = "replace",
+ timeout = 15,
+ env = utf8_child_env(child_env_without_native_path_secret()),
+ **_windows_hidden_subprocess_kwargs(),
+ )
+ except (OSError, subprocess.TimeoutExpired):
+ return None
+ if child.returncode == 2:
+ return []
+ if child.returncode != 0:
+ return None
+ devices: list[Optional[tuple[str, bool, int, str]]] = []
+ for line in child.stdout.strip().splitlines():
+ if line.strip() == "ERR":
+ devices.append(None)
+ continue
+ parts = line.strip().split(" ", 3)
+ if len(parts) < 3 or parts[1] not in ("0", "1") or not parts[2].isdigit():
+ return None
+ devices.append(
+ (
+ "" if parts[0] == "-" else parts[0],
+ parts[1] == "1",
+ int(parts[2]),
+ parts[3] if len(parts) > 3 else "",
+ )
+ )
+ return devices
def get_pci_bus_ids() -> dict[int, tuple[int, str]]: