Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: find nvidia-smi under WSL on every query, count Core Ultra Arc iGPUs as XPU, cut GPU masks at an invalid index (#12962)

Daniel Hancf018742db86
diff --git a/studio/backend/tests/test_gpu_detect_wsl_coreultra_mask.py b/studio/backend/tests/test_gpu_detect_wsl_coreultra_mask.py
new file mode 100644
index 00000000000..4e114d11f3c
--- /dev/null
+++ b/studio/backend/tests/test_gpu_detect_wsl_coreultra_mask.py
@@ -0,0 +1,186 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""WSL nvidia-smi off PATH on every query, Core Ultra iGPU PCI ids, and masks cut at an invalid index; all faked."""
+
+from __future__ import annotations
+
+import subprocess
+
+import pytest
+
+import utils.hardware.hardware as hw
+from utils.hardware import nvidia
+
+_UUID = "GPU-11111111-2222-3333-4444-555555555555"
+
+
+def _fake_smi(where, calls):
+ """run_nvidia_smi that only answers when spawned as one of ``where``, the way a missing binary raises."""
+
+ def run(argv, **_kwargs):
+ calls.append(argv[0])
+ if argv[0] not in where:
+ raise FileNotFoundError(argv[0])
+ args = " ".join(argv[1:])
+ if args == "-L":
+ out = "GPU 0: NVIDIA GeForce RTX 4090 (UUID: %s)\n" % _UUID
+ elif "index,uuid" in args and "utilization" not in args:
+ out = "0, %s\n" % _UUID
+ elif args.startswith("--query-gpu=index,"):
+ out = "0, 37, 51, 1024, 24564, 80.5, 450.0\n"
+ else:
+ out = "37, 51, 1024, 24564, 80.5, 450.0\n"
+ return subprocess.CompletedProcess(argv, 0, stdout = out, stderr = "")
+
+ return run
+
+
+def _host(monkeypatch, *, system, which, wsl_file):
+ monkeypatch.setattr(nvidia.platform, "system", lambda: system)
+ monkeypatch.setattr(nvidia.shutil, "which", lambda _name: which)
+ monkeypatch.setattr(
+ nvidia.os.path, "isfile", lambda p: wsl_file and p == nvidia._WSL_NVIDIA_SMI
+ )
+ nvidia._uuid_mask_cache.clear()
+
+
+def _queries():
+ return {
+ "count": nvidia.get_physical_gpu_count(),
+ "primary": nvidia.get_primary_gpu_utilization().get("available"),
+ "visible": len(nvidia.get_visible_gpu_utilization([0], "0")["devices"]),
+ "uuid": nvidia._query_uuid_mask(_UUID),
+ }
+
+
+def test_wsl_nvidia_smi_off_path_answers_every_query(monkeypatch):
+ calls = []
+ _host(monkeypatch, system = "Linux", which = None, wsl_file = True)
+ monkeypatch.setattr(
+ nvidia.gpu_query, "run_nvidia_smi", _fake_smi({nvidia._WSL_NVIDIA_SMI}, calls)
+ )
+ assert _queries() == {"count": 1, "primary": True, "visible": 1, "uuid": [0]}
+ assert set(calls) == {nvidia._WSL_NVIDIA_SMI}
+
+
+def test_nvidia_smi_on_path_is_still_the_one_spawned(monkeypatch):
+ calls = []
+ _host(monkeypatch, system = "Linux", which = "/usr/bin/nvidia-smi", wsl_file = True)
+ # Both spellings reach the same binary through PATH.
+ monkeypatch.setattr(
+ nvidia.gpu_query, "run_nvidia_smi", _fake_smi({"nvidia-smi", "/usr/bin/nvidia-smi"}, calls)
+ )
+ assert _queries() == {"count": 1, "primary": True, "visible": 1, "uuid": [0]}
+
+
[email protected]("system", ["Linux", "Windows"])
+def test_no_nvidia_smi_anywhere_still_reports_unavailable(monkeypatch, system):
+ calls = []
+ _host(monkeypatch, system = system, which = None, wsl_file = False)
+ monkeypatch.delenv("ProgramFiles", raising = False)
+ monkeypatch.delenv("SystemRoot", raising = False)
+ monkeypatch.setattr(nvidia.gpu_query, "run_nvidia_smi", _fake_smi(set(), calls))
+ assert _queries() == {"count": None, "primary": False, "visible": 0, "uuid": None}
+ assert set(calls) == {"nvidia-smi"}
+
+
+ ("device_id", "xpu_class"),
+ [
+ ("0x7d55", True), # Core Ultra (Meteor Lake-H) Arc Graphics
+ ("0x7d51", True), # Core Ultra 200H (Arrow Lake-H) Arc Graphics
+ ("0x64a0", True), # Core Ultra 200V (Lunar Lake) Arc 140V
+ ("0xb080", True), # Core Ultra Series 3 (Panther Lake) Arc B390
+ ("0x56a0", True),
+ ("0xe20b", True),
+ ("0x7d45", False),
+ ("0x7d67", False),
+ ("0x9a49", False),
+ ("0x46a6", False),
+ ("0x7d60", False), # Meteor Lake-M, absent from Intel compute-runtime
+ ("0x7dd5", False), # Meteor Lake "Intel Graphics", not Arc
+ ("0x7dd1", False), # Arrow Lake-H "Intel Graphics", not Arc
+ ("0x6420", False), # Lunar Lake "Intel Graphics", not Arc
+ ("0xb0a0", False), # Panther Lake "Intel Graphics", not Arc
+ ],
+)
+def test_core_ultra_arc_igpus_are_xpu_class(tmp_path, device_id, xpu_class):
+ (tmp_path / "device").write_text(device_id + "\n", encoding = "utf-8")
+ assert hw._intel_pci_device_is_xpu_class(str(tmp_path)) is xpu_class
+
+
+def test_a_core_ultra_record_establishes_a_mismatch(monkeypatch, tmp_path):
+ monkeypatch.setattr(hw, "_expected_xpu_flavor_was_chosen", lambda: False)
+ monkeypatch.setattr(hw, "_torch_reports_an_xpu_runtime", lambda: False)
+ monkeypatch.setattr(hw, "_vendors_masked_off", lambda: set())
+ (tmp_path / "device").write_text("0x64a0\n", encoding = "utf-8")
+ lnl = [
+ {
+ "vendor": "intel",
+ "name": None,
+ "index": 0,
+ "xpu_class": hw._intel_pci_device_is_xpu_class(str(tmp_path)),
+ }
+ ]
+ assert hw._devices_that_can_establish_a_mismatch(lnl) == lnl
+
+
+ ("mask", "numeric_ids"),
+ [
+ ("0,2,-1,1", [0, 2]), # NVIDIA's documented example
+ ("-1,0", []),
+ ("1,0,abc,2", [1, 0]),
+ ("0,abc", [0]),
+ ("0,1gpu2,2", [0, 1, 2]), # strtoul prefix, as torch parses it
+ ("1,0,1", []), # a repeated ordinal empties the set
+ ("0,,1", [0]), # an empty index ends the list
+ ("1,,1", [1]),
+ ("0,1,", [0, 1]),
+ (",0", []), # a leading empty index exposes nothing
+ (",", []),
+ ("-1", []),
+ ("", []),
+ ("1,0", [1, 0]),
+ ("2", [2]),
+ (_UUID, None),
+ ("0," + _UUID, None),
+ ("abc", None),
+ ],
+)
+def test_a_cuda_mask_stops_at_the_first_invalid_index(monkeypatch, mask, numeric_ids):
+ monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA)
+ monkeypatch.setattr(hw, "IS_ROCM", False)
+ for var in ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_DEVICE_ORDER"):
+ monkeypatch.delenv(var, raising = False)
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", mask)
+ spec = hw._get_parent_visible_gpu_spec()
+ assert spec["numeric_ids"] == numeric_ids
+ assert spec["supports_explicit_gpu_ids"] is (numeric_ids is not None)
+
+
+ ("env", "numeric_ids"),
+ [
+ ({"HIP_VISIBLE_DEVICES": "0,0,1"}, [0, 1]), # clr skips a repeat
+ ({"ROCR_VISIBLE_DEVICES": "0,0,1"}, [0]), # ROCr stops at one
+ ({"HIP_VISIBLE_DEVICES": "0,1gpu2,2"}, [0]), # clr takes only a plain index
+ ({"HIP_VISIBLE_DEVICES": "1,0"}, [1, 0]),
+ ({"HIP_VISIBLE_DEVICES": "0,-1,1"}, [0]),
+ ],
+)
+def test_a_rocm_mask_keeps_the_runtime_repeat_rule(monkeypatch, env, numeric_ids):
+ monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA)
+ monkeypatch.setattr(hw, "IS_ROCM", True)
+ monkeypatch.setattr(hw.sys, "platform", "linux")
+ for var in (
+ "HIP_VISIBLE_DEVICES",
+ "ROCR_VISIBLE_DEVICES",
+ "CUDA_VISIBLE_DEVICES",
+ "CUDA_DEVICE_ORDER",
+ ):
+ monkeypatch.delenv(var, raising = False)
+ for var, value in env.items():
+ monkeypatch.setenv(var, value)
+ assert hw._get_parent_visible_gpu_spec()["numeric_ids"] == numeric_ids
diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.py
index a50d184a805..e067a2eb2f1 100644
--- a/studio/backend/utils/hardware/hardware.py
+++ b/studio/backend/utils/hardware/hardware.py
@@ -459,6 +459,8 @@ def _adapter_name_is_live(name: Optional[str], live_names: list[str]) -> bool:
# XPU-capable Intel PCI IDs (pciids.h: DG2/ATS-M, PVC, BMG); an allowlist since DG1 Iris Xe MAX is discrete but unsupported.
_INTEL_XPU_PCI_ID_RANGES = ((0x5690, 0x56C2), (0x0B69, 0x0BE5), (0xE200, 0xE2FF))
+# Core Ultra iGPUs with Arc Graphics, which PyTorch XPU lists: the ids Intel compute-runtime (devices_base.inl) names Arc (MTL-H, ARL-H, LNL, PTL).
+_INTEL_XPU_PCI_IDS = frozenset((0x7D55, 0x7D51, 0x64A0, 0xB080, 0xB081, 0xB082, 0xB083))
def _intel_pci_device_is_xpu_class(device_dir: str) -> Optional[bool]:
@@ -468,7 +470,9 @@ def _intel_pci_device_is_xpu_class(device_dir: str) -> Optional[bool]:
device_id = int(fh.read().strip(), 16)
except (OSError, ValueError):
return None
- return any(lo <= device_id <= hi for lo, hi in _INTEL_XPU_PCI_ID_RANGES)
+ return device_id in _INTEL_XPU_PCI_IDS or any(
+ lo <= device_id <= hi for lo, hi in _INTEL_XPU_PCI_ID_RANGES
+ )
def _linux_drm_sysfs_records(*, distinguish_failure: bool = False) -> "list[Dict[str, Any]] | None":
@@ -4670,6 +4674,7 @@ def _get_parent_visible_gpu_spec() -> Dict[str, Any]:
"CUDA_VISIBLE_DEVICES" not in os.environ
and ("HIP_VISIBLE_DEVICES" in os.environ or "ROCR_VISIBLE_DEVICES" in os.environ)
)
+ rocr_mask = False
if _is_rocm_spec:
hip_vis = os.environ.get("HIP_VISIBLE_DEVICES")
# ROCR_VISIBLE_DEVICES is Linux-only: Windows HIP has no ROCr layer, so a stray ROCR var there masks nothing and must not be read as the ordinal->physical mapping.
@@ -4678,6 +4683,7 @@ def _get_parent_visible_gpu_spec() -> Dict[str, Any]:
cuda_visible = hip_vis
elif rocr_vis is not None:
cuda_visible = rocr_vis
+ rocr_mask = True
if cuda_visible is None:
cuda_visible = os.environ.get("CUDA_VISIBLE_DEVICES")
@@ -4696,10 +4702,28 @@ def _get_parent_visible_gpu_spec() -> Dict[str, Any]:
"supports_explicit_gpu_ids": True,
}
- tokens = [value.strip() for value in cuda_visible.split(",") if value.strip()]
- try:
- numeric_ids = [int(value) for value in tokens]
- except ValueError:
+ # Each runtime's own rule: a negative, empty or non-numeric index ends the list ("0,2,-1,1" exposes 0
+ # and 2). CUDA reads strtoul-style ("1gpu2" is 1) and empties the set on a repeat (torch's
+ # _parse_visible_devices); HIP takes only a plain index and skips a repeat, ROCr stops at one.
+ numeric_ids = []
+ for value in (token.strip() for token in cuda_visible.split(",")):
+ prefix = re.fullmatch(r"-?\d+", value) if _is_rocm_spec else re.match(r"[+-]?\d+", value)
+ if prefix is None:
+ # A UUID/MIG id, or a mask not starting with a number, keeps the UUID path below.
+ if value and (not numeric_ids or value.upper().startswith(("GPU-", "MIG-"))):
+ numeric_ids = None
+ break
+ gpu_id = int(prefix.group())
+ if gpu_id < 0:
+ break
+ if gpu_id in numeric_ids:
+ if _is_rocm_spec and not rocr_mask:
+ continue
+ if not _is_rocm_spec:
+ numeric_ids = []
+ break
+ numeric_ids.append(gpu_id)
+ if numeric_ids is None:
# nvidia-smi indices are PCI order, so they only name the same cards a numeric mask written back to a child would under PCI_BUS_ID (#8873).
if not _is_rocm_spec and os.environ.get("CUDA_DEVICE_ORDER") == "PCI_BUS_ID":
from . import nvidia
diff --git a/studio/backend/utils/hardware/nvidia.py b/studio/backend/utils/hardware/nvidia.py
index c3b6696b71c..71cbe00c217 100644
--- a/studio/backend/utils/hardware/nvidia.py
+++ b/studio/backend/utils/hardware/nvidia.py
@@ -98,7 +98,7 @@ def resolve_uuid_mask(parent_cuda_visible_devices: str) -> Optional[list[int]]:
def _query_uuid_mask(parent_cuda_visible_devices: str) -> Optional[list[int]]:
try:
result = gpu_query.run_nvidia_smi(
- ["nvidia-smi", "--query-gpu=index,uuid", "--format=csv,noheader"],
+ [_nvidia_smi_executable(), "--query-gpu=index,uuid", "--format=csv,noheader"],
capture_output = True,
text = True,
encoding = "utf-8",
@@ -127,7 +127,7 @@ def get_physical_gpu_count() -> Optional[int]:
"""Return physical GPU count via nvidia-smi, or None on failure."""
try:
result = gpu_query.run_nvidia_smi(
- ["nvidia-smi", "-L"],
+ [_nvidia_smi_executable(), "-L"],
capture_output = True,
text = True,
encoding = "utf-8",
@@ -151,7 +151,7 @@ def get_primary_gpu_utilization() -> dict[str, Any]:
try:
result = gpu_query.run_nvidia_smi(
[
- "nvidia-smi",
+ _nvidia_smi_executable(),
"--query-gpu=utilization.gpu,temperature.gpu,"
"memory.used,memory.total,power.draw,power.limit",
"--format=csv,noheader,nounits",
@@ -200,7 +200,7 @@ def get_visible_gpu_utilization(
try:
result = gpu_query.run_nvidia_smi(
[
- "nvidia-smi",
+ _nvidia_smi_executable(),
f"--query-gpu={query_fields}",
"--format=csv,noheader,nounits",
],