Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: pick up a GPU whose driver loads after Studio starts (#13232)

Daniel Hanc0f64cd3db9e
diff --git a/studio/backend/tests/test_physical_gpu_count_refresh.py b/studio/backend/tests/test_physical_gpu_count_refresh.py
new file mode 100644
index 00000000000..b807917d806
--- /dev/null
+++ b/studio/backend/tests/test_physical_gpu_count_refresh.py
@@ -0,0 +1,199 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""A GPU whose driver loads after Studio starts must still appear, and a later probe must never hide one (#9510)."""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+
+import utils.hardware.hardware as hw
+from utils.hardware import nvidia
+
+
+def smi(monkeypatch):
+ monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)
+ monkeypatch.delenv("HIP_VISIBLE_DEVICES", raising = False)
+ monkeypatch.delenv("ROCR_VISIBLE_DEVICES", raising = False)
+ monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA)
+ monkeypatch.setattr(hw, "IS_ROCM", False)
+ monkeypatch.setattr(hw, "_physical_gpu_count", None)
+ monkeypatch.setattr(hw, "_physical_gpu_count_from_smi", False)
+ monkeypatch.setattr(hw, "_physical_gpu_count_checked_at", None, raising = False)
+ monkeypatch.setattr(hw, "_physical_gpu_count_refreshing", False, raising = False)
+ # The re-probe runs off-thread in production; inline here so each assertion sees its result.
+ monkeypatch.setattr(
+ hw,
+ "_schedule_single_flight_refresh",
+ lambda flag, refresh_lock, work_lock, run, *a: run(),
+ )
+ monkeypatch.setattr(hw, "_torch_get_physical_gpu_count", lambda: 1)
+ clock = {"now": 1000.0}
+ monkeypatch.setattr(hw.time, "monotonic", lambda: clock["now"])
+ answer = {"count": 1, "calls": 0}
+
+ def count(gpu_rows_only = False):
+ answer["calls"] += 1
+ return answer["count"]
+
+ monkeypatch.setattr(nvidia, "get_physical_gpu_count", count)
+
+ def advance(count, seconds = getattr(hw, "_PHYSICAL_GPU_COUNT_TTL_SECONDS", 60.0)):
+ answer["count"] = count
+ clock["now"] += seconds
+
+ advance.answer = answer
+ return advance
+
+
+def test_a_late_second_gpu_becomes_selectable(smi):
+ assert hw.get_parent_visible_gpu_ids() == [0]
+ smi(2)
+ assert hw.get_physical_gpu_count() == 2
+ assert hw.get_parent_visible_gpu_ids() == [0, 1]
+ assert hw.resolve_requested_gpu_ids([1]) == [1]
+
+
+def test_the_count_is_not_reprobed_inside_the_ttl(smi):
+ assert hw.get_physical_gpu_count() == 1
+ smi(2, seconds = 1.0)
+ assert hw.get_physical_gpu_count() == 1
+ assert smi.answer["calls"] == 1
+
+
[email protected]("later", [None, 1, 0])
+def test_a_failed_or_lower_probe_never_hides_a_gpu(smi, later):
+ smi(2, seconds = 0.0)
+ assert hw.get_physical_gpu_count() == 2
+ smi(later)
+ assert hw.get_physical_gpu_count() == 2
+ assert hw.get_parent_visible_gpu_ids() == [0, 1]
+
+
+def test_an_smi_that_answers_later_replaces_the_torch_fallback(smi):
+ smi(None, seconds = 0.0)
+ assert hw.get_physical_gpu_count() == 1
+ assert hw._physical_gpu_count_from_smi is False
+ smi(2)
+ assert hw.get_physical_gpu_count() == 2
+ assert hw._physical_gpu_count_from_smi is True
+
+
+def test_non_cuda_counts_are_never_reprobed(smi, monkeypatch):
+ monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.MLX)
+ assert hw.get_physical_gpu_count() == 1
+ smi(2)
+ assert hw.get_physical_gpu_count() == 1
+ assert smi.answer["calls"] == 0
+
+
+def test_rocm_counts_are_never_reprobed(smi, monkeypatch):
+ from utils.hardware import amd
+
+ calls = []
+ monkeypatch.setattr(hw, "IS_ROCM", True)
+ monkeypatch.setattr(amd, "get_physical_gpu_count", lambda: calls.append(1) or 1)
+ assert hw.get_physical_gpu_count() == 1
+ smi(2)
+ assert hw.get_physical_gpu_count() == 1
+ assert len(calls) == 1
+
+
+def test_mig_child_rows_are_not_new_gpus(monkeypatch):
+ rows = {"out": "GPU 0: NVIDIA A100 (UUID: GPU-a)\n"}
+ monkeypatch.setattr(
+ nvidia.gpu_query,
+ "run_nvidia_smi",
+ lambda argv, **kw: SimpleNamespace(returncode = 0, stdout = rows["out"]),
+ )
+ assert nvidia.get_physical_gpu_count(gpu_rows_only = True) == 1
+ rows["out"] += (
+ " MIG 1g.5gb Device 0: (UUID: MIG-a)\n MIG 1g.5gb Device 1: (UUID: MIG-b)\n"
+ )
+ assert nvidia.get_physical_gpu_count(gpu_rows_only = True) == 1
+ rows["out"] += "GPU 1: NVIDIA A100 (UUID: GPU-b)\n"
+ assert nvidia.get_physical_gpu_count(gpu_rows_only = True) == 2
+
+
+def test_the_refresh_asks_for_physical_rows_only(smi, monkeypatch):
+ seen = []
+ monkeypatch.setattr(
+ nvidia,
+ "get_physical_gpu_count",
+ lambda gpu_rows_only = False: seen.append(gpu_rows_only) or 1,
+ )
+ hw.get_physical_gpu_count()
+ smi(1)
+ hw.get_physical_gpu_count()
+ assert seen == [False, True]
+
+
+def test_the_ttl_starts_after_the_probe_returns(smi, monkeypatch):
+ def slow_probe(gpu_rows_only = False):
+ smi(1, seconds = 5.0)
+ return 1
+
+ monkeypatch.setattr(nvidia, "get_physical_gpu_count", slow_probe)
+ hw.get_physical_gpu_count()
+ # The probe took 5 s; an expiry measured from before it would re-probe 5 s early.
+ assert hw.time.monotonic() - hw._physical_gpu_count_checked_at == 0.0
+
+
+def test_the_memory_table_covers_every_parent_visible_gpu(smi, monkeypatch):
+ smi(2, seconds = 0.0)
+ monkeypatch.setattr(hw, "get_visible_gpu_count", lambda: 1)
+ monkeypatch.setattr(
+ hw, "estimate_fp16_model_size_bytes", lambda *a, **k: (8 * (1024**3), "config")
+ )
+ monkeypatch.setattr(
+ hw, "_resolve_model_identifier_for_gpu_estimate", lambda *a, **k: "unsloth/test"
+ )
+ monkeypatch.setattr(
+ hw,
+ "_load_config_for_gpu_estimate",
+ lambda *a, **k: SimpleNamespace(
+ hidden_size = 4096,
+ num_hidden_layers = 32,
+ num_attention_heads = 32,
+ num_key_value_heads = 8,
+ intermediate_size = 14336,
+ vocab_size = 128256,
+ tie_word_embeddings = False,
+ ),
+ )
+ _, metadata = hw.estimate_required_model_memory_gb(
+ "unsloth/test", training_type = "LoRA/QLoRA", load_in_4bit = True
+ )
+ assert metadata["estimation_mode"] == "detailed"
+ assert "min_per_gpu_2" in metadata["vram_breakdown"]
+
+
+def test_the_reprobe_never_blocks_the_caller(smi, monkeypatch):
+ import threading
+
+ monkeypatch.undo()
+ release = threading.Event()
+ monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA)
+ monkeypatch.setattr(hw, "IS_ROCM", False)
+ monkeypatch.setattr(hw, "_physical_gpu_count", 1)
+ monkeypatch.setattr(hw, "_physical_gpu_count_from_smi", True)
+ monkeypatch.setattr(hw, "_physical_gpu_count_refreshing", False)
+ monkeypatch.setattr(hw, "_physical_gpu_count_checked_at", hw.time.monotonic() - 3600.0)
+
+ def wedged_driver(gpu_rows_only = False):
+ release.wait(10)
+ return 2
+
+ monkeypatch.setattr(nvidia, "get_physical_gpu_count", wedged_driver)
+ started = hw.time.monotonic()
+ assert hw.get_physical_gpu_count() == 1
+ assert hw.get_physical_gpu_count() == 1
+ assert hw.time.monotonic() - started < 1.0
+ release.set()
+ deadline = hw.time.monotonic() + 10
+ while hw._physical_gpu_count != 2 and hw.time.monotonic() < deadline:
+ hw.time.sleep(0.01)
+ assert hw.get_physical_gpu_count() == 2
diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.py
index c54c589052a..0aac8073f8a 100644
--- a/studio/backend/utils/hardware/hardware.py
+++ b/studio/backend/utils/hardware/hardware.py
@@ -4761,6 +4761,12 @@ def get_visible_gpu_utilization() -> Dict[str, Any]:
_physical_gpu_count: Optional[int] = None
# Whether the cached count came from the SMI (physical) or the torch fallback (visibility-filtered). Only the former can answer "is this host single-GPU".
_physical_gpu_count_from_smi: bool = False
+# NVIDIA counts are re-read once a minute so a GPU whose driver loads after Studio starts still appears (#9510); `nvidia-smi -L` is already TTL cached by gpu_query. None = never re-read (ROCm: amd-smi is uncached and slow; XPU, MLX, CPU).
+_PHYSICAL_GPU_COUNT_TTL_SECONDS = 60.0
+_physical_gpu_count_checked_at: Optional[float] = None
+_physical_gpu_count_refresh_lock = threading.Lock()
+_physical_gpu_count_work_lock = threading.Lock()
+_physical_gpu_count_refreshing = False
_visible_gpu_count: Optional[int] = None
@@ -5656,7 +5662,8 @@ def estimate_required_model_memory_gb(
metadata["estimation_mode"] = "detailed"
metadata["attention_implementation"] = vram_config.attention_implementation
metadata["vram_breakdown"] = breakdown.to_gb_dict()
- max_gpus = max(1, get_visible_gpu_count())
+ # The parent-visible ids can outgrow the visible count when a GPU appears after startup; auto-selection ranks every id and reads this table.
+ max_gpus = max(1, get_visible_gpu_count(), len(get_parent_visible_gpu_ids()))
for n_gpus in range(1, max_gpus + 1):
metadata["vram_breakdown"][f"min_per_gpu_{n_gpus}"] = round(
breakdown.min_gpu_vram(n_gpus) / (1024**3), 3
@@ -6118,27 +6125,69 @@ def prepare_gpu_selection(
return selected_gpu_ids, metadata
+def _nvidia_physical_gpu_row_count() -> Optional[int]:
+ # Physical rows only: MIG instances created after startup add indented `nvidia-smi -L` rows, not GPUs.
+ try:
+ from . import nvidia
+ return nvidia.get_physical_gpu_count(gpu_rows_only = True)
+ except Exception:
+ return None
+
+
+def _run_physical_gpu_count_probe() -> None:
+ """Keep a re-read NVIDIA count only if it grew: a failed probe or a lower count keeps the old one, so detection never narrows."""
+ global _physical_gpu_count, _physical_gpu_count_from_smi, _physical_gpu_count_checked_at
+ count = _nvidia_physical_gpu_row_count()
+ # Stamped after the probe, so the next expiry also outlives gpu_query's own cache entry for it.
+ _physical_gpu_count_checked_at = time.monotonic()
+ previous = _physical_gpu_count
+ if count is None or previous is None or count <= previous:
+ return
+ logger.info("Physical GPU count grew from %d to %d; using the new GPUs", previous, count)
+ _physical_gpu_count = count
+ _physical_gpu_count_from_smi = True
+
+
+def _refresh_physical_gpu_count() -> None:
+ """Re-probe once per TTL off the caller's thread: async routes reach this, and `nvidia-smi` can take seconds on a busy driver. The caller keeps the cached count; growth shows on the next call."""
+ checked_at = _physical_gpu_count_checked_at
+ if checked_at is None or time.monotonic() - checked_at < _PHYSICAL_GPU_COUNT_TTL_SECONDS:
+ return
+ _schedule_single_flight_refresh(
+ "_physical_gpu_count_refreshing",
+ _physical_gpu_count_refresh_lock,
+ _physical_gpu_count_work_lock,
+ _run_physical_gpu_count_probe,
+ "gpu-count-refresh",
+ "GPU count",
+ )
+
+
def get_physical_gpu_count() -> int:
- """Number of physical GPUs on the machine, from `nvidia-smi -L` (unaffected by CUDA_VISIBLE_DEVICES) with a torch fallback for AMD ROCm and Intel XPU. Cached after the first call."""
- global _physical_gpu_count, _physical_gpu_count_from_smi
+ """Number of physical GPUs on the machine, from `nvidia-smi -L` (unaffected by CUDA_VISIBLE_DEVICES) with a torch fallback for AMD ROCm and Intel XPU. Cached; on NVIDIA re-read once a minute and only ever grows."""
+ global _physical_gpu_count, _physical_gpu_count_from_smi, _physical_gpu_count_checked_at
if _physical_gpu_count is not None:
+ _refresh_physical_gpu_count()
return _physical_gpu_count
device = get_device()
if device == DeviceType.CUDA:
+ count = None
try:
if IS_ROCM:
from . import amd as _smi_mod
else:
from . import nvidia as _smi_mod
count = _smi_mod.get_physical_gpu_count()
- if count is not None:
- _physical_gpu_count = count
- _physical_gpu_count_from_smi = True
- return _physical_gpu_count
except Exception:
pass
+ if not IS_ROCM:
+ _physical_gpu_count_checked_at = time.monotonic()
+ if count is not None:
+ _physical_gpu_count = count
+ _physical_gpu_count_from_smi = True
+ return _physical_gpu_count
# SMI unavailable: fall back to torch.
count = _torch_get_physical_gpu_count()
_physical_gpu_count = count if count is not None else 1
diff --git a/studio/backend/utils/hardware/nvidia.py b/studio/backend/utils/hardware/nvidia.py
index 71cbe00c217..5839a0606cb 100644
--- a/studio/backend/utils/hardware/nvidia.py
+++ b/studio/backend/utils/hardware/nvidia.py
@@ -123,8 +123,8 @@ def _query_uuid_mask(parent_cuda_visible_devices: str) -> Optional[list[int]]:
return sorted(visible_ordinals, key = visible_ordinals.__getitem__)
-def get_physical_gpu_count() -> Optional[int]:
- """Return physical GPU count via nvidia-smi, or None on failure."""
+def get_physical_gpu_count(gpu_rows_only: bool = False) -> Optional[int]:
+ """Return physical GPU count via nvidia-smi, or None on failure. ``gpu_rows_only`` skips the indented MIG child rows."""
try:
result = gpu_query.run_nvidia_smi(
[_nvidia_smi_executable(), "-L"],
@@ -137,7 +137,10 @@ def get_physical_gpu_count() -> Optional[int]:
**_windows_hidden_subprocess_kwargs(),
)
if result.returncode == 0 and result.stdout.strip():
- return len(result.stdout.strip().splitlines())
+ lines = result.stdout.strip().splitlines()
+ if gpu_rows_only:
+ return sum(1 for line in lines if line.startswith("GPU "))
+ return len(lines)
logger.warning(
"nvidia-smi -L returned code %d; caller should fall back to torch",
result.returncode,