Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: pick up a GPU whose driver loads after Studio starts (#13232)
Daniel Hanc0f64cd3db9e
diff --git a/studio/backend/tests/test_physical_gpu_count_refresh.py b/studio/backend/tests/test_physical_gpu_count_refresh.pynew file mode 100644index 00000000000..b807917d806--- /dev/null+++ b/studio/backend/tests/test_physical_gpu_count_refresh.py@@ -0,0 +1,199 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""A GPU whose driver loads after Studio starts must still appear, and a later probe must never hide one (#9510)."""++from __future__ import annotations++from types import SimpleNamespace++import pytest++import utils.hardware.hardware as hw+from utils.hardware import nvidia+++def smi(monkeypatch):+ monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)+ monkeypatch.delenv("HIP_VISIBLE_DEVICES", raising = False)+ monkeypatch.delenv("ROCR_VISIBLE_DEVICES", raising = False)+ monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA)+ monkeypatch.setattr(hw, "IS_ROCM", False)+ monkeypatch.setattr(hw, "_physical_gpu_count", None)+ monkeypatch.setattr(hw, "_physical_gpu_count_from_smi", False)+ monkeypatch.setattr(hw, "_physical_gpu_count_checked_at", None, raising = False)+ monkeypatch.setattr(hw, "_physical_gpu_count_refreshing", False, raising = False)+ # The re-probe runs off-thread in production; inline here so each assertion sees its result.+ monkeypatch.setattr(+ hw,+ "_schedule_single_flight_refresh",+ lambda flag, refresh_lock, work_lock, run, *a: run(),+ )+ monkeypatch.setattr(hw, "_torch_get_physical_gpu_count", lambda: 1)+ clock = {"now": 1000.0}+ monkeypatch.setattr(hw.time, "monotonic", lambda: clock["now"])+ answer = {"count": 1, "calls": 0}++ def count(gpu_rows_only = False):+ answer["calls"] += 1+ return answer["count"]++ monkeypatch.setattr(nvidia, "get_physical_gpu_count", count)++ def advance(count, seconds = getattr(hw, "_PHYSICAL_GPU_COUNT_TTL_SECONDS", 60.0)):+ answer["count"] = count+ clock["now"] += seconds++ advance.answer = answer+ return advance+++def test_a_late_second_gpu_becomes_selectable(smi):+ assert hw.get_parent_visible_gpu_ids() == [0]+ smi(2)+ assert hw.get_physical_gpu_count() == 2+ assert hw.get_parent_visible_gpu_ids() == [0, 1]+ assert hw.resolve_requested_gpu_ids([1]) == [1]+++def test_the_count_is_not_reprobed_inside_the_ttl(smi):+ assert hw.get_physical_gpu_count() == 1+ smi(2, seconds = 1.0)+ assert hw.get_physical_gpu_count() == 1+ assert smi.answer["calls"] == 1++[email protected]("later", [None, 1, 0])+def test_a_failed_or_lower_probe_never_hides_a_gpu(smi, later):+ smi(2, seconds = 0.0)+ assert hw.get_physical_gpu_count() == 2+ smi(later)+ assert hw.get_physical_gpu_count() == 2+ assert hw.get_parent_visible_gpu_ids() == [0, 1]+++def test_an_smi_that_answers_later_replaces_the_torch_fallback(smi):+ smi(None, seconds = 0.0)+ assert hw.get_physical_gpu_count() == 1+ assert hw._physical_gpu_count_from_smi is False+ smi(2)+ assert hw.get_physical_gpu_count() == 2+ assert hw._physical_gpu_count_from_smi is True+++def test_non_cuda_counts_are_never_reprobed(smi, monkeypatch):+ monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.MLX)+ assert hw.get_physical_gpu_count() == 1+ smi(2)+ assert hw.get_physical_gpu_count() == 1+ assert smi.answer["calls"] == 0+++def test_rocm_counts_are_never_reprobed(smi, monkeypatch):+ from utils.hardware import amd++ calls = []+ monkeypatch.setattr(hw, "IS_ROCM", True)+ monkeypatch.setattr(amd, "get_physical_gpu_count", lambda: calls.append(1) or 1)+ assert hw.get_physical_gpu_count() == 1+ smi(2)+ assert hw.get_physical_gpu_count() == 1+ assert len(calls) == 1+++def test_mig_child_rows_are_not_new_gpus(monkeypatch):+ rows = {"out": "GPU 0: NVIDIA A100 (UUID: GPU-a)\n"}+ monkeypatch.setattr(+ nvidia.gpu_query,+ "run_nvidia_smi",+ lambda argv, **kw: SimpleNamespace(returncode = 0, stdout = rows["out"]),+ )+ assert nvidia.get_physical_gpu_count(gpu_rows_only = True) == 1+ rows["out"] += (+ " MIG 1g.5gb Device 0: (UUID: MIG-a)\n MIG 1g.5gb Device 1: (UUID: MIG-b)\n"+ )+ assert nvidia.get_physical_gpu_count(gpu_rows_only = True) == 1+ rows["out"] += "GPU 1: NVIDIA A100 (UUID: GPU-b)\n"+ assert nvidia.get_physical_gpu_count(gpu_rows_only = True) == 2+++def test_the_refresh_asks_for_physical_rows_only(smi, monkeypatch):+ seen = []+ monkeypatch.setattr(+ nvidia,+ "get_physical_gpu_count",+ lambda gpu_rows_only = False: seen.append(gpu_rows_only) or 1,+ )+ hw.get_physical_gpu_count()+ smi(1)+ hw.get_physical_gpu_count()+ assert seen == [False, True]+++def test_the_ttl_starts_after_the_probe_returns(smi, monkeypatch):+ def slow_probe(gpu_rows_only = False):+ smi(1, seconds = 5.0)+ return 1++ monkeypatch.setattr(nvidia, "get_physical_gpu_count", slow_probe)+ hw.get_physical_gpu_count()+ # The probe took 5 s; an expiry measured from before it would re-probe 5 s early.+ assert hw.time.monotonic() - hw._physical_gpu_count_checked_at == 0.0+++def test_the_memory_table_covers_every_parent_visible_gpu(smi, monkeypatch):+ smi(2, seconds = 0.0)+ monkeypatch.setattr(hw, "get_visible_gpu_count", lambda: 1)+ monkeypatch.setattr(+ hw, "estimate_fp16_model_size_bytes", lambda *a, **k: (8 * (1024**3), "config")+ )+ monkeypatch.setattr(+ hw, "_resolve_model_identifier_for_gpu_estimate", lambda *a, **k: "unsloth/test"+ )+ monkeypatch.setattr(+ hw,+ "_load_config_for_gpu_estimate",+ lambda *a, **k: SimpleNamespace(+ hidden_size = 4096,+ num_hidden_layers = 32,+ num_attention_heads = 32,+ num_key_value_heads = 8,+ intermediate_size = 14336,+ vocab_size = 128256,+ tie_word_embeddings = False,+ ),+ )+ _, metadata = hw.estimate_required_model_memory_gb(+ "unsloth/test", training_type = "LoRA/QLoRA", load_in_4bit = True+ )+ assert metadata["estimation_mode"] == "detailed"+ assert "min_per_gpu_2" in metadata["vram_breakdown"]+++def test_the_reprobe_never_blocks_the_caller(smi, monkeypatch):+ import threading++ monkeypatch.undo()+ release = threading.Event()+ monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA)+ monkeypatch.setattr(hw, "IS_ROCM", False)+ monkeypatch.setattr(hw, "_physical_gpu_count", 1)+ monkeypatch.setattr(hw, "_physical_gpu_count_from_smi", True)+ monkeypatch.setattr(hw, "_physical_gpu_count_refreshing", False)+ monkeypatch.setattr(hw, "_physical_gpu_count_checked_at", hw.time.monotonic() - 3600.0)++ def wedged_driver(gpu_rows_only = False):+ release.wait(10)+ return 2++ monkeypatch.setattr(nvidia, "get_physical_gpu_count", wedged_driver)+ started = hw.time.monotonic()+ assert hw.get_physical_gpu_count() == 1+ assert hw.get_physical_gpu_count() == 1+ assert hw.time.monotonic() - started < 1.0+ release.set()+ deadline = hw.time.monotonic() + 10+ while hw._physical_gpu_count != 2 and hw.time.monotonic() < deadline:+ hw.time.sleep(0.01)+ assert hw.get_physical_gpu_count() == 2diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.pyindex c54c589052a..0aac8073f8a 100644--- a/studio/backend/utils/hardware/hardware.py+++ b/studio/backend/utils/hardware/hardware.py@@ -4761,6 +4761,12 @@ def get_visible_gpu_utilization() -> Dict[str, Any]:_physical_gpu_count: Optional[int] = None# Whether the cached count came from the SMI (physical) or the torch fallback (visibility-filtered). Only the former can answer "is this host single-GPU"._physical_gpu_count_from_smi: bool = False+# NVIDIA counts are re-read once a minute so a GPU whose driver loads after Studio starts still appears (#9510); `nvidia-smi -L` is already TTL cached by gpu_query. None = never re-read (ROCm: amd-smi is uncached and slow; XPU, MLX, CPU).+_PHYSICAL_GPU_COUNT_TTL_SECONDS = 60.0+_physical_gpu_count_checked_at: Optional[float] = None+_physical_gpu_count_refresh_lock = threading.Lock()+_physical_gpu_count_work_lock = threading.Lock()+_physical_gpu_count_refreshing = False_visible_gpu_count: Optional[int] = None@@ -5656,7 +5662,8 @@ def estimate_required_model_memory_gb(metadata["estimation_mode"] = "detailed"metadata["attention_implementation"] = vram_config.attention_implementationmetadata["vram_breakdown"] = breakdown.to_gb_dict()- max_gpus = max(1, get_visible_gpu_count())+ # The parent-visible ids can outgrow the visible count when a GPU appears after startup; auto-selection ranks every id and reads this table.+ max_gpus = max(1, get_visible_gpu_count(), len(get_parent_visible_gpu_ids()))for n_gpus in range(1, max_gpus + 1):metadata["vram_breakdown"][f"min_per_gpu_{n_gpus}"] = round(breakdown.min_gpu_vram(n_gpus) / (1024**3), 3@@ -6118,27 +6125,69 @@ def prepare_gpu_selection(return selected_gpu_ids, metadata+def _nvidia_physical_gpu_row_count() -> Optional[int]:+ # Physical rows only: MIG instances created after startup add indented `nvidia-smi -L` rows, not GPUs.+ try:+ from . import nvidia+ return nvidia.get_physical_gpu_count(gpu_rows_only = True)+ except Exception:+ return None+++def _run_physical_gpu_count_probe() -> None:+ """Keep a re-read NVIDIA count only if it grew: a failed probe or a lower count keeps the old one, so detection never narrows."""+ global _physical_gpu_count, _physical_gpu_count_from_smi, _physical_gpu_count_checked_at+ count = _nvidia_physical_gpu_row_count()+ # Stamped after the probe, so the next expiry also outlives gpu_query's own cache entry for it.+ _physical_gpu_count_checked_at = time.monotonic()+ previous = _physical_gpu_count+ if count is None or previous is None or count <= previous:+ return+ logger.info("Physical GPU count grew from %d to %d; using the new GPUs", previous, count)+ _physical_gpu_count = count+ _physical_gpu_count_from_smi = True+++def _refresh_physical_gpu_count() -> None:+ """Re-probe once per TTL off the caller's thread: async routes reach this, and `nvidia-smi` can take seconds on a busy driver. The caller keeps the cached count; growth shows on the next call."""+ checked_at = _physical_gpu_count_checked_at+ if checked_at is None or time.monotonic() - checked_at < _PHYSICAL_GPU_COUNT_TTL_SECONDS:+ return+ _schedule_single_flight_refresh(+ "_physical_gpu_count_refreshing",+ _physical_gpu_count_refresh_lock,+ _physical_gpu_count_work_lock,+ _run_physical_gpu_count_probe,+ "gpu-count-refresh",+ "GPU count",+ )++def get_physical_gpu_count() -> int:- """Number of physical GPUs on the machine, from `nvidia-smi -L` (unaffected by CUDA_VISIBLE_DEVICES) with a torch fallback for AMD ROCm and Intel XPU. Cached after the first call."""- global _physical_gpu_count, _physical_gpu_count_from_smi+ """Number of physical GPUs on the machine, from `nvidia-smi -L` (unaffected by CUDA_VISIBLE_DEVICES) with a torch fallback for AMD ROCm and Intel XPU. Cached; on NVIDIA re-read once a minute and only ever grows."""+ global _physical_gpu_count, _physical_gpu_count_from_smi, _physical_gpu_count_checked_atif _physical_gpu_count is not None:+ _refresh_physical_gpu_count()return _physical_gpu_countdevice = get_device()if device == DeviceType.CUDA:+ count = Nonetry:if IS_ROCM:from . import amd as _smi_modelse:from . import nvidia as _smi_modcount = _smi_mod.get_physical_gpu_count()- if count is not None:- _physical_gpu_count = count- _physical_gpu_count_from_smi = True- return _physical_gpu_countexcept Exception:pass+ if not IS_ROCM:+ _physical_gpu_count_checked_at = time.monotonic()+ if count is not None:+ _physical_gpu_count = count+ _physical_gpu_count_from_smi = True+ return _physical_gpu_count# SMI unavailable: fall back to torch.count = _torch_get_physical_gpu_count()_physical_gpu_count = count if count is not None else 1diff --git a/studio/backend/utils/hardware/nvidia.py b/studio/backend/utils/hardware/nvidia.pyindex 71cbe00c217..5839a0606cb 100644--- a/studio/backend/utils/hardware/nvidia.py+++ b/studio/backend/utils/hardware/nvidia.py@@ -123,8 +123,8 @@ def _query_uuid_mask(parent_cuda_visible_devices: str) -> Optional[list[int]]:return sorted(visible_ordinals, key = visible_ordinals.__getitem__)-def get_physical_gpu_count() -> Optional[int]:- """Return physical GPU count via nvidia-smi, or None on failure."""+def get_physical_gpu_count(gpu_rows_only: bool = False) -> Optional[int]:+ """Return physical GPU count via nvidia-smi, or None on failure. ``gpu_rows_only`` skips the indented MIG child rows."""try:result = gpu_query.run_nvidia_smi([_nvidia_smi_executable(), "-L"],@@ -137,7 +137,10 @@ def get_physical_gpu_count() -> Optional[int]:**_windows_hidden_subprocess_kwargs(),)if result.returncode == 0 and result.stdout.strip():- return len(result.stdout.strip().splitlines())+ lines = result.stdout.strip().splitlines()+ if gpu_rows_only:+ return sum(1 for line in lines if line.startswith("GPU "))+ return len(lines)logger.warning("nvidia-smi -L returned code %d; caller should fall back to torch",result.returncode,