Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep a working GPU when one probe fails, and name the card behind no_gpu (#11944)
Daniel Hand2804cc84d95
diff --git a/studio/backend/tests/test_gpu_probe_robustness.py b/studio/backend/tests/test_gpu_probe_robustness.pynew file mode 100644index 00000000000..ec73305d325--- /dev/null+++ b/studio/backend/tests/test_gpu_probe_robustness.py@@ -0,0 +1,163 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""A working GPU survives a failing probe or a missing name (XPU, Linux Arc, WSL nvidia-smi); all faked."""++from __future__ import annotations++import sys+import types+from types import SimpleNamespace++import pytest++import utils.hardware.hardware as hw+from utils.hardware import nvidia+++def _boom(*_a, **_k):+ raise RuntimeError("probe failed")+++def _fake_torch(+ *,+ cuda = False,+ xpu = False,+ cuda_raises = False,+ xpu_raises = False,+ xpu_name_raises = False,+):+ torch = types.ModuleType("torch")+ torch.__version__ = "2.11.0+xpu" if xpu else "2.11.0+cu128"+ torch.version = SimpleNamespace(hip = None, cuda = "12.8", xpu = None)+ torch.cuda = SimpleNamespace(+ is_available = _boom if cuda_raises else (lambda: cuda),+ device_count = lambda: 1 if cuda else 0,+ get_device_properties = lambda _i: SimpleNamespace(name = "NVIDIA RTX A4000"),+ )+ torch.xpu = SimpleNamespace(+ is_available = _boom if xpu_raises else (lambda: xpu),+ get_device_name = _boom if xpu_name_raises else (lambda _i: "Intel(R) Arc(TM) B580"),+ )+ return torch+++def _detect(+ monkeypatch,+ torch,+ *,+ force_xpu = False,+):+ monkeypatch.setitem(sys.modules, "torch", torch)+ monkeypatch.setattr(hw, "TORCH_IMPORT_ERROR", None)+ monkeypatch.setattr(hw, "is_apple_silicon", lambda: False)+ monkeypatch.setattr(hw, "_mismatch_verdict_for_this_host", lambda *_a: (None, None))+ monkeypatch.setattr(hw.platform, "system", lambda: "Linux")+ for var in ("UNSLOTH_FORCE_XPU", "ZE_AFFINITY_MASK", "CUDA_VISIBLE_DEVICES"):+ monkeypatch.delenv(var, raising = False)+ if force_xpu:+ monkeypatch.setenv("UNSLOTH_FORCE_XPU", "1")+ for name, value in (+ ("DEVICE", None),+ ("CHAT_ONLY", True),+ ("CHAT_ONLY_REASON", None),+ ("CHAT_ONLY_DETAIL", None),+ ):+ monkeypatch.setattr(hw, name, value)+ with hw._DETECT_LOCK:+ return hw._detect_hardware_locked()+++def test_a_raising_cuda_probe_falls_through_to_a_working_xpu(monkeypatch):+ assert _detect(monkeypatch, _fake_torch(cuda_raises = True, xpu = True)) == hw.DeviceType.XPU+ assert hw.CHAT_ONLY is False+++def test_an_xpu_whose_name_cannot_be_read_is_still_an_xpu(monkeypatch):+ assert _detect(monkeypatch, _fake_torch(xpu = True, xpu_name_raises = True)) == hw.DeviceType.XPU+ assert hw.CHAT_ONLY is False+++def test_a_forced_xpu_whose_name_cannot_be_read_is_still_an_xpu(monkeypatch):+ torch = _fake_torch(xpu = True, xpu_name_raises = True)+ assert _detect(monkeypatch, torch, force_xpu = True) == hw.DeviceType.XPU+++def test_a_raising_xpu_probe_is_a_measured_cpu_host_not_a_detection_failure(monkeypatch):+ assert _detect(monkeypatch, _fake_torch(xpu_raises = True)) == hw.DeviceType.CPU+ assert hw.CHAT_ONLY_REASON == "no_gpu"+++def test_the_winner_is_unchanged_when_every_probe_succeeds(monkeypatch):+ assert _detect(monkeypatch, _fake_torch(cuda = True, xpu = True)) == hw.DeviceType.CUDA+ assert _detect(monkeypatch, _fake_torch(xpu = True)) == hw.DeviceType.XPU+++ ("device_id", "xpu_class"),+ [+ ("0x56a0", True), # Arc A770 (DG2)+ ("0x56c0", True), # Data Center GPU Flex 170 (ATS-M)+ ("0x0bd5", True), # Data Center GPU Max 1550 (PVC)+ ("0xe20b", True), # Arc B580 (BMG)+ ("0x4905", False), # Iris Xe MAX (DG1): discrete, but no XPU wheel supports it+ ("0x46a6", False), # Alder Lake iGPU+ ("garbage", None),+ ],+)+def test_the_pci_device_id_tells_an_xpu_card_from_other_intel_graphics(+ tmp_path, device_id, xpu_class+):+ (tmp_path / "device").write_text(device_id + "\n", encoding = "utf-8")+ assert hw._intel_pci_device_is_xpu_class(str(tmp_path)) is xpu_class+++def test_an_unreadable_pci_device_id_is_unknown(tmp_path):+ assert hw._intel_pci_device_is_xpu_class(str(tmp_path / "missing")) is None+++def test_an_xpu_class_intel_record_establishes_a_mismatch(monkeypatch):+ monkeypatch.setattr(hw, "_expected_xpu_flavor_was_chosen", lambda: False)+ monkeypatch.setattr(hw, "_torch_reports_an_xpu_runtime", lambda: False)+ monkeypatch.setattr(hw, "_vendors_masked_off", lambda: set())+ arc = [{"vendor": "intel", "name": None, "index": 0, "xpu_class": True}]+ assert hw._devices_that_can_establish_a_mismatch(arc) == arc+ # Controls: an iGPU or DG1 host's correct CPU install must not be flagged.+ for xpu_class in (False, None):+ igpu = [{"vendor": "intel", "name": None, "index": 0, "xpu_class": xpu_class}]+ assert hw._devices_that_can_establish_a_mismatch(igpu) == []+++ ("system", "vendors", "pinned"),+ [+ ("Linux", {"intel"}, True),+ ("Windows", {"intel"}, False), # install.ps1 autodetects Arc, so Repair works there+ ("Linux", {"nvidia"}, False),+ ("Linux", {"intel", "nvidia"}, False),+ ],+)+def test_a_linux_intel_host_is_told_the_pin_instead_of_a_repair_that_reinstalls_cpu(+ monkeypatch, system, vendors, pinned+):+ monkeypatch.setattr(hw.platform, "system", lambda: system)+ monkeypatch.setattr(hw, "CHAT_ONLY_MISMATCH_VENDORS", frozenset(vendors))+ message = hw._gpu_present_but_unusable_message("training", ("torch_cpu_build", "2.11.0+cpu"))+ assert ("UNSLOTH_TORCH_INDEX_FAMILY=xpu" in message) is pinned+ assert ("Repair installation" in message) is not pinned++[email protected]("present", [True, False])+def test_wsl_nvidia_smi_is_found_off_path(monkeypatch, present):+ monkeypatch.setattr(nvidia.platform, "system", lambda: "Linux")+ monkeypatch.setattr(nvidia.shutil, "which", lambda _name: None)+ monkeypatch.setattr(nvidia.os.path, "isfile", lambda p: present and p == nvidia._WSL_NVIDIA_SMI)+ expected = nvidia._WSL_NVIDIA_SMI if present else "nvidia-smi"+ assert nvidia._nvidia_smi_executable() == expected+++def test_path_still_wins_over_the_wsl_location(monkeypatch):+ monkeypatch.setattr(nvidia.platform, "system", lambda: "Linux")+ monkeypatch.setattr(nvidia.shutil, "which", lambda _name: "/usr/bin/nvidia-smi")+ monkeypatch.setattr(nvidia.os.path, "isfile", lambda _p: True)+ assert nvidia._nvidia_smi_executable() == "/usr/bin/nvidia-smi"diff --git a/studio/backend/tests/test_torch_cpu_build_on_nvidia_host.py b/studio/backend/tests/test_torch_cpu_build_on_nvidia_host.pyindex 5316dfba8a1..bce4ae3d102 100644--- a/studio/backend/tests/test_torch_cpu_build_on_nvidia_host.py+++ b/studio/backend/tests/test_torch_cpu_build_on_nvidia_host.py@@ -567,6 +567,8 @@ def test_nvidia_smi_is_resolved_from_the_standard_windows_locations(monkeypatch)def test_path_resolution_is_a_no_op_off_windows_and_when_path_has_it(monkeypatch):monkeypatch.setattr(nvidia.platform, "system", lambda: "Linux")monkeypatch.setattr(nvidia.shutil, "which", lambda _name: None)+ # Not a WSL host: the /usr/lib/wsl/lib fallback has its own tests.+ monkeypatch.setattr(nvidia.os.path, "isfile", lambda _p: False)assert nvidia._nvidia_smi_executable() == "nvidia-smi"monkeypatch.setattr(nvidia.platform, "system", lambda: "Windows")diff --git a/studio/backend/utils/hardware/amd.py b/studio/backend/utils/hardware/amd.pyindex 716d6c24786..e41134f6b47 100644--- a/studio/backend/utils/hardware/amd.py+++ b/studio/backend/utils/hardware/amd.py@@ -1382,7 +1382,7 @@ def _groups_that_own(paths: list) -> tuple:"""How to open ``paths``, read from the nodes themselves."render,video" is not always the right pair, and sometimes no group is the answer at- all, so the seven buckets returned each carry a different repair:+ all, so the eight buckets returned each carry a different repair:``joinable`` membership WOULD open it, so ``usermod -a -G`` is the fix.``unnamed`` GIDs with no entry in the group database, the container casediff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.pyindex 3bedb9243b7..f25895dc695 100644--- a/studio/backend/utils/hardware/hardware.py+++ b/studio/backend/utils/hardware/hardware.py@@ -454,6 +454,20 @@ def _adapter_name_is_live(name: Optional[str], live_names: list[str]) -> bool:)+# XPU-capable Intel PCI IDs (pciids.h: DG2/ATS-M, PVC, BMG); an allowlist since DG1 Iris Xe MAX is discrete but unsupported.+_INTEL_XPU_PCI_ID_RANGES = ((0x5690, 0x56C2), (0x0B69, 0x0BE5), (0xE200, 0xE2FF))+++def _intel_pci_device_is_xpu_class(device_dir: str) -> Optional[bool]:+ """Whether the card's PCI device ID is an XPU-capable family; None if unreadable."""+ try:+ with open(os.path.join(device_dir, "device"), encoding = "utf-8") as fh:+ device_id = int(fh.read().strip(), 16)+ except (OSError, ValueError):+ return None+ return any(lo <= device_id <= hi for lo, hi in _INTEL_XPU_PCI_ID_RANGES)++def _linux_drm_sysfs_records(*, distinguish_failure: bool = False) -> "list[Dict[str, Any]] | None":"""Every AMD (0x1002) or Intel (0x8086) card the DRM drivers have bound, from /sys/class/drm. amdgpu's mem_info_vram_total is a byte count; Intel publishes no equivalent on the discrete path, so an Arc card is reported with unknown capacity rather than left out. Only cardN is walked, since connector entries and render nodes would double-count. No name is reported: the kernel publishes none, and borrowing an amd-smi row whose ordering is not guaranteed would attach the wrong one. Never raises, and returns None when the walk could not be trusted, so the inventory marks the vendors unanswered rather than publishing "no cards" for a TTL."""root = "/sys/class/drm"@@ -488,15 +502,16 @@ def _linux_drm_sysfs_records(*, distinguish_failure: bool = False) -> "list[Dicttotal_gb = round(total_bytes / 1024**3, 2) if total_bytes > 0 else Noneexcept (OSError, ValueError):pass- records.append(- {- "vendor": vendors[vendor],- "index": len(records),- "name": None,- "memory_total_gb": total_gb,- "source": "sysfs-drm",- }- )+ record = {+ "vendor": vendors[vendor],+ "index": len(records),+ "name": None,+ "memory_total_gb": total_gb,+ "source": "sysfs-drm",+ }+ if vendors[vendor] == "intel":+ record["xpu_class"] = _intel_pci_device_is_xpu_class(device)+ records.append(record)if unreadable and distinguish_failure:# One card that could not be read makes the whole walk a partial answer, and a partial answer published as a complete one drops a card for a TTL.return None@@ -846,7 +861,12 @@ def _devices_that_can_establish_a_mismatch(devices: list[Dict[str, Any]]) -> lisif device.get("vendor") != "intel":keep.append(device)continue- if xpu_expected or _XPU_ADAPTER_NAME_RE.search(str(device.get("name") or "")):+ # Nameless Linux record: PCI device ID stands in for setup.ps1's Arc / Data Center name rule.+ if (+ xpu_expected+ or _XPU_ADAPTER_NAME_RE.search(str(device.get("name") or ""))+ or device.get("xpu_class") is True+ ):keep.append(device)return keep@@ -1347,6 +1367,15 @@ def ensure_hardware_detected(epoch: Optional[int] = None) -> DeviceType:return DEVICE+def _xpu_device_name_or_placeholder(torch) -> str:+ """A failing name probe must not demote an already-found XPU to CPU + detection_failed."""+ try:+ return torch.xpu.get_device_name(0)+ except Exception as e:+ logger.debug("XPU device 0 name probe failed: %s", e)+ return "<unavailable>"++def _detect_hardware_locked() -> DeviceType:"""detect_hardware() body. Call only with _DETECT_LOCK held."""global DEVICE, CHAT_ONLY, CHAT_ONLY_REASON, CHAT_ONLY_DETAIL, IS_ROCM@@ -1387,7 +1416,7 @@ def _detect_hardware_locked() -> DeviceType:DEVICE = DeviceType.XPUCHAT_ONLY = FalseCHAT_ONLY_REASON = None- device_name = torch.xpu.get_device_name(0)+ device_name = _xpu_device_name_or_placeholder(torch)if force_xpu and not ze_mask:reason = "UNSLOTH_FORCE_XPU=1"elif force_xpu:@@ -1397,7 +1426,8 @@ def _detect_hardware_locked() -> DeviceType:print(f"Hardware detected: XPU -- {device_name} ({reason})")return DEVICE- if torch.cuda.is_available():+ # Reuse the guarded answer: a raising second is_available() would skip the XPU branch below.+ if not cuda_unavailable:DEVICE = DeviceType.CUDACHAT_ONLY = Falsetry:@@ -1419,10 +1449,15 @@ def _detect_hardware_locked() -> DeviceType:if torch_ok:import torch- if hasattr(torch, "xpu") and torch.xpu.is_available():+ try:+ xpu_ok = hasattr(torch, "xpu") and torch.xpu.is_available()+ except Exception as e:+ logger.debug("XPU availability probe failed: %s", e)+ xpu_ok = False+ if xpu_ok:DEVICE = DeviceType.XPUCHAT_ONLY = False- device_name = torch.xpu.get_device_name(0)+ device_name = _xpu_device_name_or_placeholder(torch)print(f"Hardware detected: XPU — {device_name}")return DEVICE@@ -1600,6 +1635,16 @@ def current_chat_only_verdict() -> tuple[Optional[str], Optional[str]]:_WHEEL_LABEL_OTHER_VENDOR_RE = re.compile(r"\+[a-z]*(?:cu\d|xpu)")+def _intel_xpu_pin_hint(vendors: "set[str]") -> str:+ """Linux installers install XPU only when asked, so a plain re-run reinstalls CPU; Windows autodetects Arc."""+ if vendors != {"intel"} or platform.system() != "Linux":+ return ""+ return (+ "On Linux the Unsloth installer installs the Intel XPU build only when asked: re-run "+ "it with UNSLOTH_TORCH_INDEX_FAMILY=xpu set."+ )++def _gpu_present_but_unusable_message(feature: str, verdict: Optional[tuple[Optional[str], Optional[str]]] = None) -> Optional[str]:@@ -1652,11 +1697,13 @@ def _gpu_present_but_unusable_message(return f"This host has a GPU, but {feature} cannot use it. {node_hint}"# Both routes, always. The repair row exists only in the desktop app and only for a backend it manages, so a browser-hosted Studio, or a desktop attached to a server someone started from a terminal, was being sent to a control that is not on the page.if reason == "torch_cpu_build":+ repair = _intel_xpu_pin_hint(vendors) or (+ "Reinstall the GPU build: use Repair installation in Settings in the desktop app, "+ "or re-run the Unsloth installer."+ )return (f"This host has a GPU, but the installed PyTorch is a CPU-only build{installed}, "- f"so {feature} cannot use it. Reinstall the GPU build: use Repair installation "- f"in Settings in the desktop app, or re-run the Unsloth installer."- + (f" {node_hint}" if node_hint else "")+ f"so {feature} cannot use it. {repair}" + (f" {node_hint}" if node_hint else ""))return (f"This host has a GPU, but the installed PyTorch{installed} cannot initialise it, so "diff --git a/studio/backend/utils/hardware/nvidia.py b/studio/backend/utils/hardware/nvidia.pyindex 26cb01c7eb6..1a68a2ab632 100644--- a/studio/backend/utils/hardware/nvidia.py+++ b/studio/backend/utils/hardware/nvidia.py@@ -238,12 +238,18 @@ def get_visible_gpu_utilization(}+_WSL_NVIDIA_SMI = "/usr/lib/wsl/lib/nvidia-smi"++def _nvidia_smi_executable() -> str:"""The nvidia-smi to run, resolving the standard Windows locations off PATH. A driver install can leave nvidia-smi.exe in the NVSMI directory or the driver store without putting either on PATH, and a bare "nvidia-smi" then raises FileNotFoundError, leaving the physical inventory empty on exactly the host this inventory exists for: real GPUs, a PyTorch that cannot see them. Same two locations setup.ps1 falls back to. Returns the bare name when nothing better is found, so the caller's existing OSError handling still applies."""found = shutil.which("nvidia-smi")if found:return foundif platform.system() != "Windows":+ # WSL: often off PATH (secure_path strips it), and no /proc/driver/nvidia fallback.+ if platform.system() == "Linux" and os.path.isfile(_WSL_NVIDIA_SMI):+ return _WSL_NVIDIA_SMIreturn "nvidia-smi"for base, tail in ((os.environ.get("ProgramFiles"), r"NVIDIA Corporation\NVSMI\nvidia-smi.exe"),