Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep automatic GPU placement inert on ordinary hosts (#14020)

Daniel Han2f63c766cca2
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 781d40161e1..82d5db085c5 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -98,7 +98,7 @@ from .diffusion_device import (
install_frame_pad_fix,
pin_cuda_ordinal,
placed_cuda_ordinal,
- resolve_auto_cuda_ordinal,
+ auto_cuda_ordinal_for_load,
resolve_diffusion_device_target,
resolve_selected_cuda_ordinal,
)
@@ -2988,8 +2988,10 @@ class DiffusionBackend:
apply_diffusion_device_ordinal(target)
# And put an AUTOMATIC load back on its own card. generate() runs on a pooled asyncio.to_thread worker, so a
# previous pinned model leaves that thread set to its GPU for good; with no ordinal to pin, the bare "cuda"
- # Generators below would then target that card while these weights sit on the default one.
- pin_cuda_ordinal(state.placed_ordinal)
+ # Generators below would then target that card while these weights sit on the default one. Skipped when the
+ # pin above already set that card.
+ if state.gpu_ordinal is None or state.placed_ordinal != state.gpu_ordinal:
+ pin_cuda_ordinal(state.placed_ordinal)
return target
def _resolve_gguf_path(
@@ -3827,9 +3829,10 @@ class DiffusionBackend:
# the contract says to drop. Re-ranked only when the caller did not already do it: free VRAM moves between the
# route's preflight and here, so resolving twice can approve a scheme against one card and place the weights
# on another.
- if gpu_ordinal is None:
+ auto_ordinal = gpu_ordinal is None and not gpu_ids
+ if gpu_ordinal is None and gpu_ids:
gpu_ordinal = (
- (resolve_selected_cuda_ordinal(gpu_ids) if gpu_ids else resolve_auto_cuda_ordinal())
+ resolve_selected_cuda_ordinal(gpu_ids)
if resolve_diffusion_device_target().device == "cuda"
else None
)
@@ -3847,6 +3850,9 @@ class DiffusionBackend:
and resolve_model_kind(gguf_filename, model_kind) == "gguf"
):
raise ValueError(_WHOLE_PIPELINE_GGUF_LORA_MSG)
+ if auto_ordinal:
+ # Automatic, after validation: one GPU, Apple, CPU and ROCm skip every probe here.
+ gpu_ordinal = auto_cuda_ordinal_for_load()
# Refuse an EXPLICIT precision this host can never honor BEFORE the load starts, so the route answers 409 with
# the reason instead of evicting the resident model, downloading several GB and only then failing. The
# declines that need the real footprint can only be found mid-load and surface through load-progress.
diff --git a/studio/backend/core/inference/diffusion_device.py b/studio/backend/core/inference/diffusion_device.py
index eacda51bb00..224bf740061 100644
--- a/studio/backend/core/inference/diffusion_device.py
+++ b/studio/backend/core/inference/diffusion_device.py
@@ -395,6 +395,8 @@ _PLANNED_ORDINAL: ContextVar[Any] = ContextVar("media_planned_gpu_ordinal", defa
# Free-VRAM readings this close are noise, so the lower ordinal keeps an idle uniform host on card 0.
_AUTO_FREE_TIE_BYTES = 256 * 1024**2
+# Set once any thread was pinned to a card other than 0: pooled workers may then sit elsewhere.
+_PINNED_AWAY_FROM_ZERO = False
def resolve_auto_cuda_ordinal() -> Optional[int]:
@@ -406,7 +408,8 @@ def resolve_auto_cuda_ordinal() -> Optional[int]:
single-GPU fit, ties to the lowest ordinal. Read from nvidia-smi alone (no CUDA context per card, so no
torch fallback) and keyed through the parent-visible list, so a mask keeps deciding which cards exist.
- None on one visible card (unchanged), a UUID/MIG mask (no physical ids), ROCm (an APU's free reading is the
+ None when card 0 wins (ties included) on identical cards, on one visible card (unchanged),
+ a UUID/MIG mask (no physical ids), ROCm (an APU's free reading is the
shared host pool, which would outrank a discrete card), a CUDA order other than nvidia-smi's (its rows are
then not torch's ordinals), an integrated GPU (its rows are a carve-out or N/A, and its real budget is the
shared host pool), missing nvidia-smi telemetry and any failure. Never raises. Only cards at least as
@@ -422,8 +425,12 @@ def resolve_auto_cuda_ordinal() -> Optional[int]:
if hw.IS_ROCM:
return None
visible = hw.get_parent_visible_gpu_ids()
- if len(visible) < 2 or not hw._cuda_order_matches_smi():
+ if len(visible) < 2:
return None
+ if not hw._cuda_order_matches_smi():
+ # No ranking (rows are not ordinals), but on cards that are not provably identical keep the
+ # current card explicit, so a pooled thread left on another card cannot preflight against it.
+ return None if hw._cuda_cards_identical() is True else 0
if hw._integrated_cuda_rows(visible)[0]:
return None
result = hw._smi_visible_utilization(hw._get_parent_visible_gpu_spec())
@@ -456,7 +463,75 @@ def resolve_auto_cuda_ordinal() -> Optional[int]:
if not free_by_ordinal:
return None
best = max(free_by_ordinal.values())
- return min(o for o, free in free_by_ordinal.items() if best - free <= _AUTO_FREE_TIE_BYTES)
+ winner = min(o for o, free in free_by_ordinal.items() if best - free <= _AUTO_FREE_TIE_BYTES)
+ if winner:
+ return winner
+ # Card 0 won. Identical cards (same capability and size) keep None, the unpinned path they always had,
+ # while no thread was ever pinned elsewhere. Mixed cards, or a process where a pooled thread may sit on
+ # another card, keep 0 explicit so preflight and load read the ranked card.
+ if _PINNED_AWAY_FROM_ZERO:
+ return 0
+ totals = {d.get("vram_total_gb") for d in devices if d.get("index") in visible}
+ if len({caps.get(i) for i in visible}) == 1 and len(totals) == 1:
+ # Physical rows hide MIG slice sizes, so a MIG card makes the set mixed.
+ mig = hw._nvidia_mig_gpu_indices_or_none() # None (unknown) keeps card 0 explicit
+ if mig is not None and not set(visible) & mig:
+ return None
+ return 0
+
+
+def auto_cuda_ranking_possible_cached() -> Optional[bool]:
+ """``auto_cuda_ranking_possible`` when it can be answered from state already in memory, else None.
+
+ For the event loop: a cold device detection, an uncounted unmasked host or a non-numeric mask may run
+ nvidia-smi, so those go to a worker thread instead.
+ """
+ try:
+ from utils.hardware import hardware as hw
+
+ if hw.DEVICE is None:
+ return None
+ if hw.IS_ROCM or hw.DEVICE != hw.DeviceType.CUDA:
+ return auto_cuda_ranking_possible()
+ mask = os.environ.get("CUDA_VISIBLE_DEVICES")
+ if mask is None:
+ if hw._physical_gpu_count is None:
+ return None
+ elif not all(token.strip().isdigit() for token in mask.split(",") if token.strip()):
+ return None
+ except Exception: # noqa: BLE001 -- let the worker-thread gate decide
+ return None
+ return auto_cuda_ranking_possible()
+
+
+def auto_cuda_ranking_possible() -> bool:
+ """Whether an automatic load could pick another card: NVIDIA CUDA with 2+ parent-visible GPUs.
+
+ Read from the cached device and mask only (no torch, no capability or dtype probe), so zero / one GPU,
+ ROCm, XPU, MPS and CPU keep the pre-ranking fast path. Never raises.
+ """
+ try:
+ from utils.hardware import hardware as hw
+
+ # Resolve first: a cold get_device() is what sets IS_ROCM.
+ device = hw.get_device()
+ if hw.IS_ROCM or device != hw.DeviceType.CUDA:
+ return False
+ return len(hw.get_parent_visible_gpu_ids()) >= 2
+ except Exception: # noqa: BLE001 -- unknown host keeps the current device
+ return False
+
+
+def auto_cuda_ordinal_for_load() -> Optional[int]:
+ """``resolve_auto_cuda_ordinal`` behind the cheap gate and the CUDA target check, for automatic loads."""
+ planned = _PLANNED_ORDINAL.get()
+ if planned is not _UNPLANNED:
+ return planned
+ if not auto_cuda_ranking_possible():
+ return None
+ if resolve_diffusion_device_target().device != "cuda":
+ return None
+ return resolve_auto_cuda_ordinal()
@contextmanager
@@ -505,11 +580,14 @@ def apply_diffusion_device_ordinal(target: DiffusionDeviceTarget) -> None:
def pin_cuda_ordinal(ordinal: Optional[int]) -> None:
"""``torch.cuda.set_device``, thread-local, never fatal. A no-op for None."""
+ global _PINNED_AWAY_FROM_ZERO
if ordinal is None:
return
try:
import torch
torch.cuda.set_device(ordinal)
+ if ordinal != 0:
+ _PINNED_AWAY_FROM_ZERO = True
except Exception: # noqa: BLE001 -- placement still works off torch_device; never fail a load here
pass
diff --git a/studio/backend/core/inference/media_locality.py b/studio/backend/core/inference/media_locality.py
index 0f379fe888f..62ed50952ed 100644
--- a/studio/backend/core/inference/media_locality.py
+++ b/studio/backend/core/inference/media_locality.py
@@ -275,14 +275,34 @@ def plan_gpu_ordinal() -> Optional[int]:
pinned, ordinal = planned_gpu_ordinal()
if pinned:
return ordinal
- from core.inference.diffusion_device import (
- resolve_auto_cuda_ordinal,
- resolve_diffusion_device_target,
- )
+ from core.inference.diffusion_device import auto_cuda_ordinal_for_load
+
+ return auto_cuda_ordinal_for_load()
+
- if resolve_diffusion_device_target().device != "cuda":
+def catalog_gpu_ordinal() -> Optional[int]:
+ """The one ordinal a catalog refresh plans every pick against, ranked once unless the cards are identical.
+
+ Identical visible cards pick the same precision and so the same artifacts on any of them, so the
+ refresh plans against the current device as it always did instead of re-ranking per pick.
+ """
+ pinned, ordinal = planned_gpu_ordinal()
+ if pinned:
+ return ordinal
+ from core.inference.diffusion_device import auto_cuda_ranking_possible
+
+ if not auto_cuda_ranking_possible():
return None
- return resolve_auto_cuda_ordinal()
+ try:
+ from utils.hardware import hardware as hw
+
+ # Under a CUDA order nvidia-smi does not share, mask ids are not its rows: compare every card.
+ ids = hw.get_parent_visible_gpu_ids() if hw._cuda_order_matches_smi() else None
+ if hw._cuda_cards_identical(ids) is True:
+ return None
+ except Exception: # noqa: BLE001 -- unknown identity ranks once, like mixed cards
+ pass
+ return plan_gpu_ordinal()
def _pipeline_components_present(root: Path) -> bool:
@@ -559,4 +579,5 @@ __all__ = [
"plan_gpu_ordinal",
"planned_gpu_ordinal",
"planners_for",
+ "catalog_gpu_ordinal",
]
diff --git a/studio/backend/core/inference/sd_cpp_backend.py b/studio/backend/core/inference/sd_cpp_backend.py
index 7709a14606b..08432a0e9c6 100644
--- a/studio/backend/core/inference/sd_cpp_backend.py
+++ b/studio/backend/core/inference/sd_cpp_backend.py
@@ -34,7 +34,7 @@ from typing import Any, Optional
from core.inference.diffusion_auto_policy import build_resolved_record, format_generation_for_log
from core.inference.diffusion_compat import flux2_inner_dim_for_pick
from core.inference.diffusion_device import (
- resolve_auto_cuda_ordinal,
+ auto_cuda_ordinal_for_load,
resolve_diffusion_device_target,
resolve_selected_cuda_ordinal,
)
@@ -2934,14 +2934,13 @@ class SdCppDiffusionBackend:
# but a direct caller (an MCP client, a test, a plugin) hands over gpu_ids alone, and without this the native
# engine is the one engine that would drop the pick silently. Re-ranked only when nobody has, so a
# route-resolved winner is never second-guessed against free VRAM that has moved since.
- if gpu_ordinal is None:
- # Automatic ranks the torch cards too, unless sd.cpp runs on one torch cannot see.
- if off_torch is not None or resolve_diffusion_device_target().device != "cuda":
- gpu_ordinal = None
- elif gpu_ids:
- gpu_ordinal = resolve_selected_cuda_ordinal(gpu_ids)
- else:
- gpu_ordinal = resolve_auto_cuda_ordinal()
+ auto_ordinal = gpu_ordinal is None and not gpu_ids and off_torch is None
+ if gpu_ordinal is None and gpu_ids:
+ gpu_ordinal = (
+ resolve_selected_cuda_ordinal(gpu_ids)
+ if resolve_diffusion_device_target().device == "cuda"
+ else None
+ )
if not gguf_filename:
raise ValueError(
"gguf_filename is required: the native engine loads single-file GGUF checkpoints only."
@@ -2957,6 +2956,9 @@ class SdCppDiffusionBackend:
)
if not family_sd_cpp_supported(fam):
raise ValueError(f"Family '{fam.name}' has no native sd.cpp asset mapping.")
+ if auto_ordinal:
+ # Automatic ranks the torch cards too, after validation, unless sd.cpp runs on one torch cannot see.
+ gpu_ordinal = auto_cuda_ordinal_for_load()
self._loading_family = fam
base = resolve_base_repo(fam, base_repo)
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index 5227c7f1ecd..3323cf291d0 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -100,7 +100,7 @@ from .diffusion_device import (
install_rocm_vae_bf16_decode,
pin_cuda_ordinal,
placed_cuda_ordinal,
- resolve_auto_cuda_ordinal,
+ auto_cuda_ordinal_for_load,
resolve_diffusion_device_target,
resolve_selected_cuda_ordinal,
)
@@ -3057,8 +3057,9 @@ class VideoBackend:
target = self._device_target(state.gpu_ordinal)
# And an automatic load back onto its own card, in case this thread is a shared one a previous pinned load left
# elsewhere. The image path reaches this through a pooled executor; here it costs one no-op call to keep the two
- # engines identical.
- pin_cuda_ordinal(state.placed_ordinal)
+ # engines identical. Skipped when the pin above already set that card.
+ if state.gpu_ordinal is None or state.placed_ordinal != state.gpu_ordinal:
+ pin_cuda_ordinal(state.placed_ordinal)
return target
# ── validation ───────────────────────────────────────────────────────────
@@ -3379,9 +3380,10 @@ class VideoBackend:
# on the resolved backend, since XPU / MPS / CPU ignore physical ids and would otherwise 400 a selection the
# contract drops. Re-ranked only when the caller did not already do it: free VRAM moves between the route's
# preflight and here, so resolving twice can approve a scheme against one card and place the weights on another.
- if gpu_ordinal is None:
+ auto_ordinal = gpu_ordinal is None and not gpu_ids
+ if gpu_ordinal is None and gpu_ids:
gpu_ordinal = (
- (resolve_selected_cuda_ordinal(gpu_ids) if gpu_ids else resolve_auto_cuda_ordinal())
+ resolve_selected_cuda_ordinal(gpu_ids)
if resolve_diffusion_device_target().device == "cuda"
else None
)
@@ -3396,6 +3398,9 @@ class VideoBackend:
text_encoder_quant = text_encoder_quant,
h3_task = h3_task,
)
+ if auto_ordinal:
+ # Automatic, after validation: one GPU, Apple, CPU and ROCm skip every probe here.
+ gpu_ordinal = auto_cuda_ordinal_for_load()
# Refuse an EXPLICIT precision this host can never honor BEFORE the load starts, so the route answers 409 with
# the reason instead of evicting the resident model and pulling tens of GB first. Footprint-dependent declines
# still surface via load-progress. The ROUTE makes the same call earlier still (see
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 35585ae3d6f..70f4339e378 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -37121,31 +37121,41 @@ def _validated_media_picks(catalog_at: float) -> dict:
with _MEDIA_PICK_CACHE_LOCK:
if _MEDIA_PICK_CACHE["at"] == catalog_at:
return _MEDIA_PICK_CACHE["picks"]
- from core.inference.media_locality import is_edit_only, missing_download_bytes
+ from core.inference.media_locality import (
+ _PLANNED_ORDINAL,
+ catalog_gpu_ordinal,
+ is_edit_only,
+ missing_download_bytes,
+ )
from core.inference.media_model_index import (
available_media_model_ids,
resolve_local_media_model,
)
picks: dict = {}
- for task in _MEDIA_MODEL_TASKS:
- rows = []
- for model_id in available_media_model_ids(task):
- pick = resolve_local_media_model(model_id, task = task)
- if pick is None:
- continue
- # The local catalog tags an instruction-editing checkpoint text-to-image, but it
- # ships no txt2img workflow: the switch refuses it with a 400 and a resident one
- # is refused by /v1/images/generations too.
- if task == "text-to-image" and is_edit_only(pick):
- continue
- try:
- local = missing_download_bytes(_media_owner(task), pick) == 0
- except Exception as exc: # noqa: BLE001
- logger.debug("media locality unavailable for %s: %s", model_id, exc)
- local = False
- rows.append((model_id, pick, local))
- picks[task] = rows
+ # One placement decision per refresh, never a live ranking per pick.
+ planned = _PLANNED_ORDINAL.set(catalog_gpu_ordinal())
+ try:
+ for task in _MEDIA_MODEL_TASKS:
+ rows = []
+ for model_id in available_media_model_ids(task):
+ pick = resolve_local_media_model(model_id, task = task)
+ if pick is None:
+ continue
+ # The local catalog tags an instruction-editing checkpoint text-to-image, but it
+ # ships no txt2img workflow: the switch refuses it with a 400 and a resident one
+ # is refused by /v1/images/generations too.
+ if task == "text-to-image" and is_edit_only(pick):
+ continue
+ try:
+ local = missing_download_bytes(_media_owner(task), pick) == 0
+ except Exception as exc: # noqa: BLE001
+ logger.debug("media locality unavailable for %s: %s", model_id, exc)
+ local = False
+ rows.append((model_id, pick, local))
+ picks[task] = rows
+ finally:
+ _PLANNED_ORDINAL.reset(planned)
_MEDIA_PICK_CACHE.update(at = catalog_at, picks = picks)
return picks
@@ -49458,23 +49468,32 @@ async def _selected_gpu_ordinal(gpu_ids, *, allow_ranking: bool = True) -> Optio
resolve_selected_cuda_ordinal,
)
- if not gpu_ids and not allow_ranking:
- return None
- device = await asyncio.to_thread(lambda: resolve_diffusion_device_target().device)
- if device != "cuda":
- return None
if not gpu_ids:
+ if not allow_ranking:
+ return None
# An auto-switch load uses the card its locality check planned against.
from core.inference.media_locality import planned_gpu_ordinal
pinned, ordinal = planned_gpu_ordinal()
if pinned:
return ordinal
+ # Zero / one GPU, ROCm, Apple and CPU: nothing to rank, so no device or dtype probe either.
+ possible = diffusion_device.auto_cuda_ranking_possible_cached()
+ if possible is None: # would probe: off the event loop
+ possible = await asyncio.to_thread(diffusion_device.auto_cuda_ranking_possible)
+ if not possible:
+ return None
+ device = await asyncio.to_thread(lambda: resolve_diffusion_device_target().device)
+ if device != "cuda":
+ return None
# Automatic ranks the visible cards too (#8235); the current device is PCI slot 0.
# Pinned for this request, so begin_load's own fallback reuses it (None included).
ordinal = await asyncio.to_thread(diffusion_device.resolve_auto_cuda_ordinal)
diffusion_device._PLANNED_ORDINAL.set(ordinal)
return ordinal
+ device = await asyncio.to_thread(lambda: resolve_diffusion_device_target().device)
+ if device != "cuda":
+ return None
# The keyword only when it is not the default, so the ordinary path calls the resolver with
# the same one-argument shape it always had (a monkeypatched seam still fits it).
if allow_ranking:
diff --git a/studio/backend/tests/test_diffusion_auto_gpu.py b/studio/backend/tests/test_diffusion_auto_gpu.py
index b43defe2e1a..77d09e626ee 100644
--- a/studio/backend/tests/test_diffusion_auto_gpu.py
+++ b/studio/backend/tests/test_diffusion_auto_gpu.py
@@ -36,7 +36,9 @@ def _host(
if caps is None:
caps = {d["index"]: (8, 6) for d in devices}
monkeypatch.setattr(nvidia, "get_compute_capabilities", lambda: dict(caps))
+ monkeypatch.setattr(hw, "_nvidia_mig_gpu_indices_or_none", lambda: set())
monkeypatch.setattr(hw, "IS_ROCM", rocm)
+ monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA)
monkeypatch.setattr(hw, "get_parent_visible_gpu_ids", lambda: list(visible))
monkeypatch.setattr(hw, "_cuda_order_matches_smi", lambda: order_matches)
monkeypatch.setattr(hw, "_integrated_cuda_rows", lambda ids: ({}, "index"))
@@ -57,10 +59,15 @@ def test_auto_maps_physical_ids_through_the_mask(monkeypatch):
assert dd.resolve_auto_cuda_ordinal() == 1
-def test_auto_ties_keep_the_lowest_ordinal(monkeypatch):
- # An idle uniform host stays on card 0, MiB-level noise included.
+def test_auto_ties_keep_the_current_device(monkeypatch):
+ # An idle uniform host stays on card 0, MiB-level noise included, and as None (no explicit pin).
_host(monkeypatch, [0, 1, 2], [_dev(0, 80.0, 0.6), _dev(1, 80.0, 0.5), _dev(2, 80.0, 0.6)])
- assert dd.resolve_auto_cuda_ordinal() == 0
+ assert dd.resolve_auto_cuda_ordinal() is None
+
+
+def test_auto_ties_among_other_cards_keep_the_lowest_ordinal(monkeypatch):
+ _host(monkeypatch, [0, 1, 2], [_dev(0, 80.0, 70.0), _dev(1, 80.0, 0.5), _dev(2, 80.0, 0.6)])
+ assert dd.resolve_auto_cuda_ordinal() == 1
@pytest.mark.parametrize(
@@ -87,10 +94,13 @@ def test_auto_never_raises(monkeypatch):
assert dd.resolve_auto_cuda_ordinal() is None
-def test_auto_keeps_the_current_device_when_cuda_order_is_not_smi_order(monkeypatch):
- # CUDA_DEVICE_ORDER=FASTEST_FIRST: nvidia-smi rows are not torch ordinals, so no ranking.
[email protected]("identical, expected", [(True, None), (False, 0), (None, 0)])
+def test_auto_does_not_rank_when_cuda_order_is_not_smi_order(monkeypatch, identical, expected):
+ # CUDA_DEVICE_ORDER=FASTEST_FIRST: nvidia-smi rows are not torch ordinals, so no ranking. Identical
+ # cards keep None; otherwise card 0 stays explicit so a pooled thread cannot preflight elsewhere.
_host(monkeypatch, [0, 1], [_dev(0, 8.0, 1.0), _dev(1, 24.0, 0.5)], order_matches = False)
- assert dd.resolve_auto_cuda_ordinal() is None
+ monkeypatch.setattr(hw, "_cuda_cards_identical", lambda gpu_ids = None: identical)
+ assert dd.resolve_auto_cuda_ordinal() is expected
def test_auto_never_falls_back_to_torch_telemetry(monkeypatch):
@@ -107,6 +117,7 @@ def _route(monkeypatch, device = "cuda"):
dd, "resolve_diffusion_device_target", lambda: types.SimpleNamespace(device = device)
)
monkeypatch.setattr(dd, "resolve_auto_cuda_ordinal", lambda: 1)
+ monkeypatch.setattr(dd, "auto_cuda_ranking_possible", lambda: True)
monkeypatch.setattr(dd, "resolve_selected_cuda_ordinal", lambda ids, **_k: min(ids))
return routes._selected_gpu_ordinal
@@ -187,7 +198,7 @@ def test_direct_backend_loads_rank_automatic_too(rel):
for node in ast.walk(ast.parse(src))
if isinstance(node, ast.Call) and isinstance(node.func, ast.Name)
}
- assert "resolve_auto_cuda_ordinal" in calls
+ assert "auto_cuda_ordinal_for_load" in calls
def test_auto_skips_a_freer_card_older_than_the_current_one(monkeypatch):
@@ -195,6 +206,7 @@ def test_auto_skips_a_freer_card_older_than_the_current_one(monkeypatch):
_host(
monkeypatch, [0, 1], [_dev(0, 24.0, 20.0), _dev(1, 8.0, 0.0)], caps = {0: (8, 6), 1: (6, 1)}
)
+ # Mixed cards keep card 0 explicit.
assert dd.resolve_auto_cuda_ordinal() == 0
@@ -225,6 +237,7 @@ def test_route_pins_its_automatic_decision_for_the_backend(monkeypatch):
)
answers = iter([None, 1])
monkeypatch.setattr(dd, "resolve_auto_cuda_ordinal", lambda: next(answers))
+ monkeypatch.setattr(dd, "auto_cuda_ranking_possible", lambda: True)
real = dd.__dict__["_PLANNED_ORDINAL"]
async def load():
diff --git a/studio/backend/tests/test_gpu_ids_cuda_device_order.py b/studio/backend/tests/test_gpu_ids_cuda_device_order.py
index 0a7aa5834f9..1dec77534c4 100644
--- a/studio/backend/tests/test_gpu_ids_cuda_device_order.py
+++ b/studio/backend/tests/test_gpu_ids_cuda_device_order.py
@@ -95,6 +95,8 @@ def _host(monkeypatch, physical = 2):
monkeypatch.setattr(hw, "rocm_gpu_ids_without_torch_kernels", lambda: set())
monkeypatch.setattr(hw, "get_visible_gpu_utilization", _smi_utilization)
monkeypatch.setattr(hw, "_torch_get_per_device_info", _torch_rows)
+ # No out-of-process ordinal mapping here, so the mismatched shape ranks through torch.
+ monkeypatch.setattr(hw, "_smi_rows_for_cuda_ordinals", lambda parent_ids: None)
def _auto(env):
diff --git a/studio/backend/tests/test_media_auto_gpu_ordinary_host.py b/studio/backend/tests/test_media_auto_gpu_ordinary_host.py
new file mode 100644
index 00000000000..c94a87ec323
--- /dev/null
+++ b/studio/backend/tests/test_media_auto_gpu_ordinary_host.py
@@ -0,0 +1,602 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Automatic image / video GPU placement (#8235) leaves ordinary hosts as they were.
+
+Zero / one GPU, Apple, CPU and ROCm run no device, dtype or capability probe for an
+automatic request; the current device (a tie included) stays None so generation gains
+no explicit set_device pins; identical cards under FASTEST_FIRST keep nvidia-smi
+telemetry; a /v1/models catalog refresh does not rank live on homogeneous hosts."""
+
+import asyncio
+import os
+import types
+from unittest.mock import patch
+
+import pytest
+
+from core.inference import diffusion_device as dd
+from utils.hardware import hardware as hw
+from utils.hardware.hardware import DeviceType
+
+
+def _dev(index, total, used):
+ return {"index": index, "vram_total_gb": total, "vram_used_gb": used}
+
+
+def _counting_target(
+ monkeypatch,
+ module,
+ device = "cuda",
+):
+ calls = []
+
+ def _target(**_kw):
+ calls.append(1)
+ return types.SimpleNamespace(device = device)
+
+ monkeypatch.setattr(module, "resolve_diffusion_device_target", _target)
+ return calls
+
+
+def _counting_rank(monkeypatch, answer = 1):
+ calls = []
+
+ def _rank():
+ calls.append(1)
+ return answer
+
+ monkeypatch.setattr(dd, "resolve_auto_cuda_ordinal", _rank)
+ return calls
+
+
+def _ordinary(
+ monkeypatch,
+ device = DeviceType.CUDA,
+ visible = (0,),
+ rocm = False,
+):
+ monkeypatch.setattr(hw, "IS_ROCM", rocm)
+ monkeypatch.setattr(hw, "get_device", lambda: device)
+ monkeypatch.setattr(hw, "get_parent_visible_gpu_ids", lambda: list(visible))
+
+
[email protected](autouse = True)
+def _no_earlier_pins(monkeypatch):
+ monkeypatch.setattr(dd, "_PINNED_AWAY_FROM_ZERO", False, raising = False)
+
+
+# Item 1: the current device stays None.
+
+
+ "devices",
+ [
+ [_dev(0, 80.0, 0.5), _dev(1, 80.0, 0.5)], # idle identical cards
+ [_dev(0, 80.0, 0.5), _dev(1, 80.0, 40.0)], # card 0 is the freest
+ ],
+)
+def test_auto_returns_none_when_the_current_device_wins(monkeypatch, devices):
+ _two_cards(monkeypatch, devices, {0: (8, 0), 1: (8, 0)})
+ assert dd.resolve_auto_cuda_ordinal() is None
+
+
+def test_auto_keeps_card_zero_explicit_beside_a_mig_card(monkeypatch):
+ # Identical physical rows, but card 1 is MIG: its slice size is not the row's.
+ _two_cards(
+ monkeypatch, [_dev(0, 80.0, 0.5), _dev(1, 80.0, 0.5)], {0: (8, 0), 1: (8, 0)}, mig = {1}
+ )
+ assert dd.resolve_auto_cuda_ordinal() == 0
+
+
+ "devices, caps",
+ [
+ ([_dev(0, 80.0, 0.5), _dev(1, 80.0, 40.0)], {0: (8, 0), 1: (9, 0)}), # mixed capability
+ ([_dev(0, 80.0, 0.5), _dev(1, 40.0, 30.0)], {0: (8, 0), 1: (8, 0)}), # mixed size
+ ],
+)
+def test_auto_keeps_card_zero_explicit_on_mixed_cards(monkeypatch, devices, caps):
+ # A pooled thread left on card 1 must not preflight a card-0 load against card 1.
+ _two_cards(monkeypatch, devices, caps)
+ assert dd.resolve_auto_cuda_ordinal() == 0
+
+
+def _two_cards(
+ monkeypatch,
+ devices,
+ caps,
+ mig = (),
+):
+ from utils.hardware import nvidia
+
+ monkeypatch.setattr(nvidia, "get_compute_capabilities", lambda: dict(caps))
+ monkeypatch.setattr(hw, "_nvidia_mig_gpu_indices_or_none", lambda: set(mig))
+ monkeypatch.setattr(hw, "IS_ROCM", False)
+ monkeypatch.setattr(hw, "get_parent_visible_gpu_ids", lambda: [0, 1])
+ monkeypatch.setattr(hw, "_cuda_order_matches_smi", lambda: True)
+ monkeypatch.setattr(hw, "_integrated_cuda_rows", lambda ids: ({}, "index"))
+ monkeypatch.setattr(hw, "_get_parent_visible_gpu_spec", lambda: {"numeric_ids": [0, 1]})
+ monkeypatch.setattr(
+ hw, "_smi_visible_utilization", lambda spec: {"available": True, "devices": devices}
+ )
+
+
[email protected]("gpu_ordinal, expected", [(None, [("pin", 0)]), (0, ["apply"])])
+def test_video_generation_sets_the_device_once(monkeypatch, gpu_ordinal, expected):
+ # Auto (None) keeps the single placed-device pin; an explicit 0 pins once, not twice.
+ from core.inference import video
+
+ pins = []
+ monkeypatch.setattr(video, "resolve_diffusion_device_target", lambda **_k: object())
+ monkeypatch.setattr(video, "apply_diffusion_device_ordinal", lambda t: pins.append("apply"))
+ monkeypatch.setattr(video, "pin_cuda_ordinal", lambda o: pins.append(("pin", o)))
+ state = types.SimpleNamespace(gpu_ordinal = gpu_ordinal, placed_ordinal = 0)
+ video.VideoBackend._state_device_target(video.VideoBackend.__new__(video.VideoBackend), state)
+ assert pins == expected
+
+
[email protected]("gpu_ordinal, expected", [(None, ["apply", ("pin", 0)]), (0, ["apply"])])
+def test_image_generation_sets_the_device_once(monkeypatch, gpu_ordinal, expected):
+ from core.inference import diffusion
+
+ pins = []
+ backend = diffusion.DiffusionBackend.__new__(diffusion.DiffusionBackend)
+ monkeypatch.setattr(backend, "_target_for_ordinal", lambda fam, o: object(), raising = False)
+ monkeypatch.setattr(diffusion, "apply_diffusion_device_ordinal", lambda t: pins.append("apply"))
+ monkeypatch.setattr(diffusion, "pin_cuda_ordinal", lambda o: pins.append(("pin", o)))
+ state = types.SimpleNamespace(family = None, gpu_ordinal = gpu_ordinal, placed_ordinal = 0)
+ diffusion.DiffusionBackend._state_device_target(backend, state)
+ assert pins == expected
+
+
+# Item 2: no probe on ordinary hosts, and validation first.
+
+
+ "device, visible, rocm",
+ [
+ (DeviceType.CUDA, (0,), False), # one GPU
+ (DeviceType.CUDA, (), False), # zero visible
+ (DeviceType.MLX, (), False), # Apple Silicon
+ (DeviceType.CPU, (), False),
+ (DeviceType.CUDA, (0, 1), True), # ROCm
+ ],
+)
+def test_route_auto_runs_no_device_probe_on_ordinary_hosts(monkeypatch, device, visible, rocm):
+ from routes import inference as routes
+
+ _ordinary(monkeypatch, device = device, visible = visible, rocm = rocm)
+ targets = _counting_target(monkeypatch, dd)
+ ranks = _counting_rank(monkeypatch)
+ assert asyncio.run(routes._selected_gpu_ordinal(None)) is None
+ assert (targets, ranks) == ([], [])
+
+
+def test_route_auto_still_ranks_on_a_multi_gpu_cuda_host(monkeypatch):
+ from routes import inference as routes
+
+ _ordinary(monkeypatch, visible = (0, 1))
+ targets = _counting_target(monkeypatch, dd)
+ ranks = _counting_rank(monkeypatch)
+
+ async def load():
+ return await routes._selected_gpu_ordinal(None)
+
+ assert asyncio.run(load()) == 1
+ assert (len(targets), len(ranks)) == (1, 1)
+
+
+ "module_name, cls_name", [("diffusion", "DiffusionBackend"), ("video", "VideoBackend")]
+)
+def test_direct_invalid_auto_load_is_refused_before_any_probe(monkeypatch, module_name, cls_name):
+ import importlib
+
+ module = importlib.import_module(f"core.inference.{module_name}")
+ _ordinary(monkeypatch, visible = (0, 1))
+ targets = _counting_target(monkeypatch, module)
+ dd_targets = _counting_target(monkeypatch, dd)
+ ranks = _counting_rank(monkeypatch)
+ backend = getattr(module, cls_name)()
+ with pytest.raises(ValueError):
+ backend.begin_load("nobody/not-a-diffusion-model-at-all")
+ assert (targets, dd_targets, ranks) == ([], [], [])
+
+
[email protected]("device, visible", [(DeviceType.CUDA, (0,)), (DeviceType.MLX, ())])
+def test_backend_auto_fallback_runs_no_probe_on_ordinary_hosts(monkeypatch, device, visible):
+ _ordinary(monkeypatch, device = device, visible = visible)
+ targets = _counting_target(monkeypatch, dd)
+ ranks = _counting_rank(monkeypatch)
+ assert dd.auto_cuda_ordinal_for_load() is None
+ assert (targets, ranks) == ([], [])
+
+
+# Item 3: FASTEST_FIRST numeric masks read nvidia-smi rows joined by PCI bus id, not torch.
+
+
+def _fastest_first_host(monkeypatch, rows):
+ torch_calls = []
+
+ def _torch_rows(indices):
+ torch_calls.append(list(indices))
+ return [
+ {"index": idx, "visible_ordinal": o, "total_gb": 80.0, "used_gb": 10.0 + o * 60.0}
+ for o, idx in enumerate(indices)
+ ]
+
+ monkeypatch.setattr(hw, "IS_ROCM", False)
+ monkeypatch.setattr(hw, "get_device", lambda: DeviceType.CUDA)
+ monkeypatch.setattr(hw, "get_physical_gpu_count", lambda: 2)
+ monkeypatch.setattr(hw, "_physical_gpu_count_from_smi", True)
+ monkeypatch.setattr(hw, "rocm_gpu_ids_without_torch_kernels", lambda: set())
+ # Masked SMI rows (PCI order): row 0 busy, row 1 free.
+ smi_rows = [_dev(0, 80.0, 70.0), _dev(1, 80.0, 10.0)]
+ monkeypatch.setattr(hw, "get_visible_gpu_utilization", lambda: {"devices": smi_rows})
+ monkeypatch.setattr(
+ hw,
+ "_smi_query",
+ lambda name, ids, **_k: {
+ "available": True,
+ "devices": [d for d in smi_rows if d["index"] in ids],
+ },
+ )
+ monkeypatch.setattr(hw, "_torch_get_per_device_info", _torch_rows)
+ from utils.hardware import nvidia
+
+ reads = []
+ monkeypatch.setattr(nvidia, "get_cuda_ordinal_smi_rows", lambda: reads.append(1) or rows)
+ monkeypatch.setattr(hw, "_cuda_ordinal_rows_cache", {})
+ monkeypatch.setattr(hw, "_nvidia_mig_gpu_indices_or_none", lambda: set())
+ return torch_calls, reads
+
+
+def _auto(env):
+ with patch.dict(os.environ, env, clear = True):
+ ids, _meta = hw.auto_select_gpu_ids("m", required_override_gb = 16.0)
+ return ids
+
+
+_FASTEST = {"CUDA_DEVICE_ORDER": "FASTEST_FIRST", "CUDA_VISIBLE_DEVICES": "0,1"}
+
+
[email protected]("rows, expected", [([0, 1], [1]), ([1, 0], [0])])
+def test_fastest_first_mask_ranks_smi_rows_by_bus_id(monkeypatch, rows, expected):
+ # rows = the nvidia-smi row behind each CUDA ordinal; [1, 0] is a tie order that differs from PCI.
+ torch_calls, reads = _fastest_first_host(monkeypatch, rows)
+ assert _auto(_FASTEST) == expected
+ assert _auto(_FASTEST) == expected
+ assert torch_calls == []
+ assert reads == [1] # mapping found once per environment
+
+
+def test_fastest_first_mask_without_a_mapping_keeps_torch(monkeypatch):
+ torch_calls, _reads = _fastest_first_host(monkeypatch, None)
+ assert _auto(_FASTEST) == [0]
+ assert torch_calls == [[0, 1]]
+
+
[email protected]("mig", [{1}, None]) # a mapped MIG card, or MIG discovery failing
+def test_fastest_first_mask_falls_back_to_torch_around_mig(monkeypatch, mig):
+ # nvidia-smi shows a MIG card's whole memory; torch reports the slice, so it ranks instead.
+ torch_calls, _reads = _fastest_first_host(monkeypatch, [1, 0])
+ monkeypatch.setattr(hw, "_nvidia_mig_gpu_indices_or_none", lambda: mig)
+ with patch.dict(os.environ, _FASTEST, clear = True):
+ assert hw._smi_rows_for_cuda_ordinals([0, 1]) is None
+ hw.auto_select_gpu_ids("m", required_override_gb = 8.0)
+ assert torch_calls == [[0, 1]]
+
+
+def test_fastest_first_mask_without_mig_keeps_nvidia_smi(monkeypatch):
+ torch_calls, _reads = _fastest_first_host(monkeypatch, [1, 0])
+ monkeypatch.setattr(hw, "_nvidia_mig_gpu_indices_or_none", lambda: {5}) # MIG elsewhere only
+ with patch.dict(os.environ, _FASTEST, clear = True):
+ assert hw._smi_rows_for_cuda_ordinals([0, 1]) is not None
+ assert torch_calls == []
+
+
+def test_torch_fallback_rows_skip_the_physical_mig_correction(monkeypatch):
+ # No bus-id mapping: torch rows carry mask ids and already report slice memory. Physical row 0 is a
+ # MIG card, but mask id 0 is another card, so correcting it would hide the card that has room.
+ from core.inference.llama_cpp import LlamaCppBackend
+ from utils.hardware import nvidia
+
+ torch_calls, _reads = _fastest_first_host(monkeypatch, None)
+ monkeypatch.setattr(nvidia, "get_mig_gpu_indices", lambda: {0})
+ monkeypatch.setattr(
+ LlamaCppBackend, "_get_gpu_memory_nvml", staticmethod(lambda: [(0, 1024, 5120)])
+ )
+ assert _auto(_FASTEST) == [0] # torch: mask id 0 has 70 GB free
+ assert torch_calls == [[0, 1]]
+
+
+def test_bus_ids_from_both_tools_join():
+ from utils.hardware import nvidia
+ assert nvidia._normalize_bus_id("00000000:3B:00.0") == nvidia._normalize_bus_id("0000:3b:00.0")
+ assert nvidia._normalize_bus_id("garbage") is None
+
+
+def test_card_identities_are_read_once(monkeypatch):
+ from utils.hardware import nvidia
+
+ reads = []
+ row = ("NVIDIA A100-SXM4-80GB", "81920 MiB", "8.0", "1410 MHz")
+ monkeypatch.setattr(hw, "IS_ROCM", False)
+ monkeypatch.setattr(hw, "get_physical_gpu_count", lambda: 2)
+ monkeypatch.setattr(nvidia, "get_card_identities", lambda: reads.append(1) or {0: row, 1: row})
+ monkeypatch.setattr(hw, "_nvidia_mig_gpu_indices_or_none", lambda: set())
+ monkeypatch.setattr(hw, "_cuda_card_identities", {})
+ assert hw._cuda_cards_identical() is True
+ assert hw._cuda_cards_identical([0, 1]) is True
+ assert reads == [1]
+
+
+# Item 4: catalog refreshes do not rank live on single-GPU / homogeneous hosts.
+
+
+def _catalog(monkeypatch):
+ from core.inference import media_locality, media_model_index
+ from routes import inference as routes
+
+ planned = []
+ monkeypatch.setattr(media_model_index, "available_media_model_ids", lambda task: ["a", "b"])
+ monkeypatch.setattr(
+ media_model_index,
+ "resolve_local_media_model",
+ lambda model_id, task: types.SimpleNamespace(model_id = model_id),
+ )
+ monkeypatch.setattr(media_locality, "is_edit_only", lambda pick: False)
+ monkeypatch.setattr(
+ media_locality,
+ "missing_download_bytes",
+ lambda owner, pick, hf_token = None: planned.append(media_locality.plan_gpu_ordinal()) or 0,
+ )
+ monkeypatch.setattr(routes, "_MEDIA_PICK_CACHE", {"at": None, "picks": {}})
+ monkeypatch.setattr(hw, "_cuda_order_matches_smi", lambda: True)
+ return routes, planned
+
+
[email protected]("visible, identical", [((0,), True), ((0,), None), ((0, 1), True)])
+def test_catalog_refresh_does_not_rank_on_ordinary_hosts(monkeypatch, visible, identical):
+ routes, planned = _catalog(monkeypatch)
+ _ordinary(monkeypatch, visible = visible)
+ monkeypatch.setattr(hw, "_cuda_cards_identical", lambda gpu_ids = None: identical)
+ _counting_target(monkeypatch, dd)
+ ranks = _counting_rank(monkeypatch)
+ routes._validated_media_picks(1.0)
+ assert ranks == []
+ assert planned and set(planned) == {None}
+
+
[email protected]("identical", [False, None])
+def test_catalog_refresh_ranks_once_on_mixed_or_unknown_cards(monkeypatch, identical):
+ routes, planned = _catalog(monkeypatch)
+ _ordinary(monkeypatch, visible = (0, 1))
+ monkeypatch.setattr(hw, "_cuda_cards_identical", lambda gpu_ids = None: identical)
+ _counting_target(monkeypatch, dd)
+ ranks = _counting_rank(monkeypatch, answer = 1)
+ routes._validated_media_picks(1.0)
+ assert len(ranks) == 1
+ assert set(planned) == {1}
+ from core.inference.media_locality import planned_gpu_ordinal
+
+ assert planned_gpu_ordinal() == (False, None)
+
+
+# The route's gate answers from memory, or runs on a worker thread.
+
+
+ "device, count, mask, cached",
+ [
+ (DeviceType.CUDA, 1, None, False), # counted single GPU: answered inline
+ (DeviceType.MLX, None, None, False), # Apple: answered inline
+ (None, None, None, None), # detection not run yet
+ (DeviceType.CUDA, None, None, None), # unmasked, never counted
+ (DeviceType.CUDA, 2, "GPU-abc,GPU-def", None), # UUID mask may run nvidia-smi
+ ],
+)
+def test_cached_gate_defers_anything_that_could_probe(monkeypatch, device, count, mask, cached):
+ monkeypatch.setattr(hw, "IS_ROCM", False)
+ monkeypatch.setattr(hw, "DEVICE", device)
+ monkeypatch.setattr(hw, "_physical_gpu_count", count)
+ monkeypatch.setattr(hw, "get_device", lambda: device)
+ monkeypatch.setattr(hw, "get_parent_visible_gpu_ids", lambda: list(range(count or 0)))
+ env = {} if mask is None else {"CUDA_VISIBLE_DEVICES": mask}
+ with patch.dict(os.environ, env, clear = True):
+ assert dd.auto_cuda_ranking_possible_cached() is cached
+
+
+def test_route_runs_an_uncached_gate_off_the_event_loop(monkeypatch):
+ from routes import inference as routes
+
+ monkeypatch.setattr(dd, "auto_cuda_ranking_possible_cached", lambda: None)
+ loop_thread = []
+
+ def _gate():
+ import threading
+ loop_thread.append(threading.current_thread() is threading.main_thread())
+ return False
+
+ monkeypatch.setattr(dd, "auto_cuda_ranking_possible", _gate)
+ targets = _counting_target(monkeypatch, dd)
+ assert asyncio.run(routes._selected_gpu_ordinal(None)) is None
+ assert loop_thread == [False]
+ assert targets == []
+
+
+def test_identical_rows_with_a_mig_card_are_not_identical(monkeypatch):
+ from utils.hardware import nvidia
+
+ row = ("NVIDIA A100-SXM4-80GB", "81920 MiB", "8.0", "1410 MHz")
+ monkeypatch.setattr(hw, "IS_ROCM", False)
+ monkeypatch.setattr(hw, "get_physical_gpu_count", lambda: 2)
+ monkeypatch.setattr(nvidia, "get_card_identities", lambda: {0: row, 1: row})
+ monkeypatch.setattr(hw, "_cuda_card_identities", {})
+ monkeypatch.setattr(hw, "_nvidia_mig_gpu_indices_or_none", lambda: {1})
+ assert hw._cuda_cards_identical() is False
+ assert hw._cuda_cards_identical([0]) is True
+
+
+def test_catalog_compares_every_card_when_mask_ids_are_not_smi_rows(monkeypatch):
+ # FASTEST_FIRST subset mask "0,2": those are CUDA ordinals, not nvidia-smi rows 0 and 2.
+ routes, planned = _catalog(monkeypatch)
+ _ordinary(monkeypatch, visible = (0, 2))
+ monkeypatch.setattr(hw, "_cuda_order_matches_smi", lambda: False)
+ asked = []
+ # Rows 0 and 2 match, row 1 differs: only the all-cards answer is trustworthy.
+ monkeypatch.setattr(
+ hw,
+ "_cuda_cards_identical",
+ lambda gpu_ids = None: asked.append(gpu_ids) or gpu_ids is not None,
+ )
+ _counting_target(monkeypatch, dd)
+ ranks = _counting_rank(monkeypatch, answer = 0)
+ routes._validated_media_picks(1.0)
+ assert asked == [None]
+ assert len(ranks) == 1
+
+
+ "listing",
+ [
+ types.SimpleNamespace(returncode = 1, stdout = ""), # nvidia-smi -L failed
+ types.SimpleNamespace(returncode = 0, stdout = ""), # empty output
+ "timeout",
+ ],
+)
+def test_failed_mig_discovery_is_unknown_not_identical(monkeypatch, listing):
+ import subprocess
+
+ from utils.hardware import nvidia
+
+ def _run(argv, **_k):
+ if listing == "timeout":
+ raise subprocess.TimeoutExpired(argv, 5)
+ return listing
+
+ row = ("NVIDIA A100-SXM4-80GB", "81920 MiB", "8.0", "1410 MHz")
+ monkeypatch.setattr(hw, "IS_ROCM", False)
+ monkeypatch.setattr(hw, "get_physical_gpu_count", lambda: 2)
+ monkeypatch.setattr(nvidia, "get_card_identities", lambda: {0: row, 1: row})
+ monkeypatch.setattr(hw, "_cuda_card_identities", {})
+ monkeypatch.setattr(nvidia.gpu_query, "run_nvidia_smi", _run)
+ assert hw._nvidia_mig_gpu_indices_or_none() is None
+ assert hw._cuda_cards_identical() is None
+ # The load path keeps card 0 explicit when MIG is unknown.
+ _two_cards(monkeypatch, [_dev(0, 80.0, 0.5), _dev(1, 80.0, 0.5)], {0: (8, 0), 1: (8, 0)})
+ monkeypatch.setattr(hw, "_nvidia_mig_gpu_indices_or_none", lambda: None)
+ assert dd.resolve_auto_cuda_ordinal() == 0
+
+
+def test_mig_listing_parses_slices():
+ from unittest.mock import patch as _patch
+
+ from utils.hardware import nvidia
+
+ out = "GPU 0: A100 (UUID: GPU-a)\n MIG 3g.40gb Device 0: (UUID: MIG-x)\nGPU 1: A100 (UUID: GPU-b)\n"
+ with _patch.object(
+ nvidia.gpu_query,
+ "run_nvidia_smi",
+ lambda argv, **_k: types.SimpleNamespace(returncode = 0, stdout = out),
+ ):
+ assert hw._nvidia_mig_gpu_indices_or_none() == {0}
+
+
+def test_fastest_first_mask_with_an_integrated_card_keeps_torch(monkeypatch):
+ # [] = mapping known but an integrated card is mapped: its nvidia-smi row is a carve-out.
+ torch_calls, reads = _fastest_first_host(monkeypatch, [])
+ with patch.dict(os.environ, _FASTEST, clear = True):
+ assert hw._smi_rows_for_cuda_ordinals([0, 1]) is None
+ assert hw._smi_rows_for_cuda_ordinals([0, 1]) is None
+ hw.auto_select_gpu_ids("m", required_override_gb = 8.0)
+ assert torch_calls == [[0, 1]]
+ assert reads == [1] # cached, the child is not re-run
+
+
+ "child_out, expected",
+ [
+ ("0000:3b:00.0 0\n0000:1a:00.0 0\n", [1, 0]),
+ ("0000:3b:00.0 0\n0000:1a:00.0 1\n", []), # integrated card mapped
+ ("0000:3b:00.0\n", None), # no attribute: unreadable
+ ],
+)
+def test_cuda_ordinal_rows_parse_bus_ids_and_integrated(monkeypatch, child_out, expected):
+ import subprocess
+
+ from utils.hardware import nvidia
+
+ child = types.SimpleNamespace(returncode = 0, stdout = child_out)
+ smi = types.SimpleNamespace(returncode = 0, stdout = "0, 00000000:1A:00.0\n1, 00000000:3B:00.0\n")
+ monkeypatch.setattr(subprocess, "run", lambda *a, **k: child)
+ monkeypatch.setattr(nvidia.gpu_query, "run_nvidia_smi", lambda *a, **k: smi)
+ assert nvidia.get_cuda_ordinal_smi_rows() == expected
+
+
+def test_identity_compares_every_listed_row_not_a_stale_count(monkeypatch):
+ # A torch-fallback count of 2 must not hide a third, different card nvidia-smi lists.
+ from utils.hardware import nvidia
+
+ row = ("NVIDIA A100-SXM4-80GB", "81920 MiB", "8.0", "1410 MHz")
+ other = ("NVIDIA RTX A6000", "49140 MiB", "8.6", "2100 MHz")
+ monkeypatch.setattr(hw, "IS_ROCM", False)
+ monkeypatch.setattr(hw, "get_physical_gpu_count", lambda: 2)
+ monkeypatch.setattr(nvidia, "get_card_identities", lambda: {0: row, 1: row, 2: other})
+ monkeypatch.setattr(hw, "_cuda_card_identities", {})
+ monkeypatch.setattr(hw, "_nvidia_mig_gpu_indices_or_none", lambda: set())
+ assert hw._cuda_cards_identical() is False
+ assert hw._cuda_cards_identical([0, 1]) is True
+
+
+def test_hardware_redetection_retires_the_ordinal_mapping(monkeypatch):
+ from utils.hardware import gpu_query
+
+ _torch_calls, reads = _fastest_first_host(monkeypatch, [0, 1])
+ assert _auto(_FASTEST) == [1]
+ gpu_query.invalidate_static("test: eGPU attached")
+ assert _auto(_FASTEST) == [1]
+ assert reads == [1, 1]
+
+
+def test_cold_rocm_detection_is_settled_before_the_gate_reads_it(monkeypatch):
+ _ordinary(monkeypatch, visible = (0, 1))
+ monkeypatch.setattr(hw, "IS_ROCM", False)
+
+ def cold_detect():
+ hw.IS_ROCM = True # ROCm reports as CUDA once detection has run
+ return DeviceType.CUDA
+
+ monkeypatch.setattr(hw, "get_device", cold_detect)
+ assert dd.auto_cuda_ranking_possible() is False
+
+
+def test_card_zero_stays_explicit_once_a_thread_was_pinned_elsewhere(monkeypatch):
+ import torch
+
+ _two_cards(monkeypatch, [_dev(0, 80.0, 0.5), _dev(1, 80.0, 40.0)], {0: (8, 0), 1: (8, 0)})
+ monkeypatch.setattr(torch.cuda, "set_device", lambda ordinal: None)
+ dd.pin_cuda_ordinal(0)
+ assert dd.resolve_auto_cuda_ordinal() is None # pinning card 0 leaves no worker elsewhere
+ dd.pin_cuda_ordinal(1)
+ # A pooled worker may now sit on card 1: the winner is named so preflight reads card 0.
+ assert dd.resolve_auto_cuda_ordinal() == 0
+
+
+def test_hardware_redetection_rereads_card_identities(monkeypatch):
+ from utils.hardware import gpu_query, nvidia
+
+ a100 = ("NVIDIA A100-SXM4-80GB", "81920 MiB", "8.0", "1410 MHz")
+ h100 = ("NVIDIA H100 80GB HBM3", "81559 MiB", "9.0", "1980 MHz")
+ inventory = [{0: a100, 1: a100}]
+ monkeypatch.setattr(hw, "IS_ROCM", False)
+ monkeypatch.setattr(hw, "get_physical_gpu_count", lambda: 2)
+ monkeypatch.setattr(nvidia, "get_card_identities", lambda: inventory[0])
+ monkeypatch.setattr(hw, "_nvidia_mig_gpu_indices_or_none", lambda: set())
+ monkeypatch.setattr(hw, "_cuda_card_identities", {})
+ assert hw._cuda_cards_identical() is True
+ inventory[0] = {0: a100, 1: h100} # card 1 swapped, same count
+ assert hw._cuda_cards_identical() is True # cached until re-detection
+ gpu_query.invalidate_static("test: card swapped")
+ assert hw._cuda_cards_identical() is False
diff --git a/studio/backend/tests/test_media_auto_switch.py b/studio/backend/tests/test_media_auto_switch.py
index b8b19d20b00..87a4c796fb1 100644
--- a/studio/backend/tests/test_media_auto_switch.py
+++ b/studio/backend/tests/test_media_auto_switch.py
@@ -792,6 +792,7 @@ def test_the_load_runs_on_the_card_the_locality_check_planned(
)
ranks = itertools.chain([1, 1], itertools.repeat(0))
monkeypatch.setattr(dd, "resolve_auto_cuda_ordinal", lambda: next(ranks))
+ monkeypatch.setattr(dd, "auto_cuda_ranking_possible", lambda: True)
planned: list = []
backend.download_plan = lambda model_path, **kw: planned.append(kw["gpu_ordinal"]) or {
"total_bytes": 0
diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.py
index 0bf3b2aed96..ea473d4f104 100644
--- a/studio/backend/utils/hardware/hardware.py
+++ b/studio/backend/utils/hardware/hardware.py
@@ -2054,6 +2054,136 @@ def _mask_names_cuda_ordinals(parent_visible_spec: Dict[str, Any]) -> bool:
return not _cuda_order_matches_smi()
+_cuda_card_identities: Dict[int, tuple] = {}
+_cuda_card_identities_gen: Optional[int] = None
+
+
+def _cuda_cards_identical(gpu_ids: Optional[list[int]] = None) -> Optional[bool]:
+ """Whether the NVIDIA cards ``gpu_ids`` (default: every physical card) share one identity per nvidia-smi (name, total memory, compute cap, max SM clock) and none is MIG, out of process. None when unknown. Read once and cached: a card's identity does not change, and a grown card count re-reads."""
+ global _cuda_card_identities, _cuda_card_identities_gen
+ if IS_ROCM:
+ return None
+ from . import gpu_query
+
+ count = get_physical_gpu_count()
+ wanted = list(range(count)) if gpu_ids is None else list(gpu_ids)
+ # Hardware re-detection (a swapped card) bumps the static generation.
+ if (
+ _cuda_card_identities_gen != gpu_query._static_gen
+ or not set(wanted) <= set(_cuda_card_identities)
+ or len(_cuda_card_identities) < count
+ ):
+ gen = gpu_query._static_gen
+ try:
+ from . import nvidia
+ identities = nvidia.get_card_identities()
+ except Exception:
+ return None
+ if not identities:
+ return None
+ _cuda_card_identities, _cuda_card_identities_gen = identities, gen
+ if gpu_ids is None:
+ # Every row nvidia-smi lists, not just the count: a torch-fallback count can miss cards.
+ wanted = sorted(_cuda_card_identities)
+ rows = [_cuda_card_identities.get(gpu_id) for gpu_id in wanted]
+ if not rows or any(row is None for row in rows):
+ return None
+ if len(set(rows)) != 1:
+ return False
+ # A MIG card is seen through a slice whose size the physical row does not show.
+ mig = _nvidia_mig_gpu_indices_or_none()
+ if mig is None:
+ return None
+ return not (set(wanted) & mig)
+
+
+def _nvidia_mig_gpu_indices_or_none() -> Optional[set[int]]:
+ """Physical indices of NVIDIA cards with MIG instances, from ``nvidia-smi -L``, or None when the
+ listing failed or came back empty (unknown, unlike a confirmed set with no MIG card)."""
+ try:
+ from . import nvidia
+ result = nvidia.gpu_query.run_nvidia_smi(
+ [nvidia._nvidia_smi_executable(), "-L"],
+ capture_output = True,
+ text = True,
+ encoding = "utf-8",
+ errors = "replace",
+ timeout = 5,
+ env = nvidia.child_env_without_native_path_secret(),
+ **nvidia._windows_hidden_subprocess_kwargs(),
+ )
+ except Exception:
+ return None
+ if result.returncode != 0 or not isinstance(result.stdout, str) or not result.stdout.strip():
+ return None
+ indices: set[int] = set()
+ current: Optional[int] = None
+ for line in result.stdout.splitlines():
+ if line.startswith("GPU "):
+ head = line[4:].split(":", 1)[0].strip()
+ current = int(head) if head.isdecimal() else None
+ elif current is not None and line.strip().startswith("MIG "):
+ indices.add(current)
+ return indices
+
+
+_cuda_ordinal_rows_cache: Dict[tuple, list[int]] = {}
+
+
+def _smi_rows_for_cuda_ordinals(parent_ids: list[int]) -> Optional[Dict[str, Any]]:
+ """nvidia-smi utilization keyed by the mask's ids, read from the rows the CUDA ordinals really are.
+
+ For a numeric mask under a non-PCI CUDA_DEVICE_ORDER, where mask entries are CUDA ordinals and not
+ nvidia-smi rows. The ordinal -> row join is by PCI bus id, found once per environment out of process
+ (no CUDA context here). None when it cannot be established, so the caller keeps the torch path.
+ """
+ from . import gpu_query
+
+ # The static generation moves on hardware re-detection (eGPU, driver change), retiring old mappings.
+ key = (
+ os.environ.get("CUDA_VISIBLE_DEVICES"),
+ os.environ.get("CUDA_DEVICE_ORDER"),
+ gpu_query._static_gen,
+ )
+ rows = _cuda_ordinal_rows_cache.get(key)
+ if rows is None:
+ try:
+ from . import nvidia
+ rows = nvidia.get_cuda_ordinal_smi_rows()
+ except Exception:
+ rows = None
+ if rows is None:
+ return None
+ _cuda_ordinal_rows_cache.clear()
+ _cuda_ordinal_rows_cache[key] = rows
+ if not rows: # an integrated card: keep the pool-aware torch telemetry
+ return None
+ if len(rows) != len(parent_ids) or len(set(rows)) != len(rows):
+ return None
+ result = _smi_query(
+ "get_visible_gpu_utilization", sorted(rows), parent_cuda_visible_devices = None
+ )
+ # nvidia-smi rows show a MIG card's whole memory, not its slice: with a MIG card among the rows,
+ # or MIG discovery failing, keep the slice-aware torch telemetry instead.
+ mig = _nvidia_mig_gpu_indices_or_none()
+ if mig is None or mig & set(rows):
+ return None
+ by_row = {d.get("index"): d for d in (result or {}).get("devices", [])}
+ if not all(row in by_row for row in rows):
+ return None
+ return {
+ "mig_corrected": True,
+ "devices": [
+ {
+ "index": parent_ids[ordinal],
+ "vram_total_gb": by_row[row].get("vram_total_gb"),
+ "vram_used_gb": by_row[row].get("vram_used_gb"),
+ }
+ for ordinal, row in enumerate(rows)
+ ],
+ }
+
+
def _amd_smi_ids_for_hip_ids(hip_ids: Optional[list[int]]) -> Optional[list[int]]:
"""Translate visible HIP ordinals to amd-smi physical GPU IDs."""
if hip_ids is None or not hip_ids:
@@ -6118,9 +6248,20 @@ def auto_select_gpu_ids(
metadata["selected_gpu_ids"] = parent_ids
return parent_ids, metadata
- if _mask_names_cuda_ordinals(parent_visible_spec):
- # The display keeps the CLI rows; ranking needs the cards the worker will get.
+ smi_by_ordinal = (
+ _smi_rows_for_cuda_ordinals(get_parent_visible_gpu_ids())
+ if _mask_names_cuda_ordinals(parent_visible_spec)
+ else None
+ )
+ if smi_by_ordinal is not None:
+ # nvidia-smi telemetry for the cards the ids select, joined by PCI bus id out of process.
+ utilization = smi_by_ordinal
+ elif _mask_names_cuda_ordinals(parent_visible_spec):
+ # The display keeps the CLI rows; ranking needs the cards the worker will get. Torch already
+ # reports a MIG slice's own memory, and these rows carry mask ids, not the physical rows the
+ # MIG correction keys on, so it must not run on them.
utilization = {
+ "mig_corrected": True,
"devices": [
{
"index": td["index"],
@@ -6128,12 +6269,13 @@ def auto_select_gpu_ids(
"vram_used_gb": td["used_gb"],
}
for td in _torch_get_per_device_info(get_parent_visible_gpu_ids())
- ]
+ ],
}
else:
utilization = get_visible_gpu_utilization()
devices = [d for d in utilization.get("devices", []) if d.get("index") not in _uncovered]
- devices = _with_mig_instance_memory(devices)
+ if not utilization.get("mig_corrected"):
+ devices = _with_mig_instance_memory(devices)
parent_ids = _covered(get_parent_visible_gpu_ids())
if not devices:
diff --git a/studio/backend/utils/hardware/nvidia.py b/studio/backend/utils/hardware/nvidia.py
index b91aa0e05a2..4c6676ab6fd 100644
--- a/studio/backend/utils/hardware/nvidia.py
+++ b/studio/backend/utils/hardware/nvidia.py
@@ -587,6 +587,131 @@ def get_compute_capabilities() -> dict[int, tuple[int, int]]:
return capabilities
+def get_card_identities() -> dict[int, tuple[str, ...]]:
+ """nvidia-smi row index -> (name, total memory, compute cap, max SM clock), no mask and no torch.
+
+ Equal rows pick the same precision and artifacts on either card. {} when unreadable.
+ """
+ try:
+ result = gpu_query.run_nvidia_smi(
+ [
+ _nvidia_smi_executable(),
+ "--query-gpu=index,name,memory.total,compute_cap,clocks.max.sm",
+ "--format=csv,noheader",
+ ],
+ capture_output = True,
+ text = True,
+ encoding = "utf-8",
+ errors = "replace",
+ timeout = 5,
+ env = child_env_without_native_path_secret(),
+ **_windows_hidden_subprocess_kwargs(),
+ )
+ except (OSError, subprocess.TimeoutExpired):
+ return {}
+ if result.returncode != 0:
+ return {}
+ identities: dict[int, tuple[str, ...]] = {}
+ for line in result.stdout.strip().splitlines():
+ index, *fields = (part.strip() for part in line.split(","))
+ if not index.isdigit() or len(fields) != 4:
+ return {}
+ identities[int(index)] = tuple(fields)
+ return identities
+
+
+# Driver API only: cuInit enumerates devices under the inherited CUDA_VISIBLE_DEVICES / CUDA_DEVICE_ORDER
+# without creating a context, and runs in a child so the Studio process never initialises CUDA.
+_CUDA_BUS_ID_SCRIPT = r"""
+import ctypes, sys
+lib = ctypes.WinDLL("nvcuda.dll") if sys.platform == "win32" else ctypes.CDLL("libcuda.so.1")
+if lib.cuInit(0) != 0:
+ sys.exit(3)
+count = ctypes.c_int(0)
+if lib.cuDeviceGetCount(ctypes.byref(count)) != 0:
+ sys.exit(4)
+for ordinal in range(count.value):
+ dev = ctypes.c_int(0)
+ buf = ctypes.create_string_buffer(64)
+ integrated = ctypes.c_int(0)
+ if (
+ lib.cuDeviceGet(ctypes.byref(dev), ordinal) != 0
+ or lib.cuDeviceGetPCIBusId(buf, 64, dev) != 0
+ or lib.cuDeviceGetAttribute(ctypes.byref(integrated), 18, dev) != 0 # CU_DEVICE_ATTRIBUTE_INTEGRATED
+ ):
+ sys.exit(5)
+ print(buf.value.decode(), integrated.value)
+"""
+
+
+def _normalize_bus_id(bus_id: str) -> Optional[tuple[int, str]]:
+ """'00000000:3B:00.0' (nvidia-smi) and '0000:3b:00.0' (driver API) -> (domain, 'BUS:DEV.FN')."""
+ domain, sep, rest = bus_id.strip().partition(":")
+ if not sep:
+ return None
+ try:
+ return int(domain, 16), rest.upper()
+ except ValueError:
+ return None
+
+
+def get_cuda_ordinal_smi_rows() -> Optional[list[int]]:
+ """nvidia-smi row index for each CUDA ordinal of this process's environment, matched by PCI bus id.
+
+ Establishes the ordinal mapping under any CUDA_DEVICE_ORDER / numeric mask without opening a CUDA
+ context here: the ordinals come from a short child on the driver API, the rows from nvidia-smi.
+ None when either side is unreadable or the two do not join; [] when an integrated card is among them,
+ whose nvidia-smi memory is a carve-out rather than its shared pool.
+ """
+ import sys
+
+ from utils.child_stdio import utf8_child_env
+
+ try:
+ child = subprocess.run(
+ [sys.executable, "-c", _CUDA_BUS_ID_SCRIPT],
+ capture_output = True,
+ text = True,
+ encoding = "utf-8",
+ errors = "replace",
+ timeout = 15,
+ env = utf8_child_env(child_env_without_native_path_secret()),
+ **_windows_hidden_subprocess_kwargs(),
+ )
+ smi = gpu_query.run_nvidia_smi(
+ [_nvidia_smi_executable(), "--query-gpu=index,pci.bus_id", "--format=csv,noheader"],
+ capture_output = True,
+ text = True,
+ encoding = "utf-8",
+ errors = "replace",
+ timeout = 5,
+ env = child_env_without_native_path_secret(),
+ **_windows_hidden_subprocess_kwargs(),
+ )
+ except (OSError, subprocess.TimeoutExpired):
+ return None
+ if child.returncode != 0 or smi.returncode != 0:
+ return None
+ row_by_bus: dict[tuple[int, str], int] = {}
+ for line in smi.stdout.strip().splitlines():
+ index, _, bus_id = (part.strip() for part in line.partition(","))
+ key = _normalize_bus_id(bus_id)
+ if not index.isdigit() or key is None:
+ return None
+ row_by_bus[key] = int(index)
+ rows = []
+ for line in child.stdout.strip().splitlines():
+ bus_id, _, integrated = line.strip().partition(" ")
+ key = _normalize_bus_id(bus_id)
+ if key not in row_by_bus or integrated not in ("0", "1"):
+ return None
+ if integrated == "1":
+ # The row is a carve-out or N/A; only torch reports the shared pool. Known, so cacheable.
+ return []
+ rows.append(row_by_bus[key])
+ return rows or None
+
+
def get_physical_gpu_inventory() -> dict[str, Any]:
"""Every NVIDIA GPU the driver enumerates, with no visibility mask and no torch. Display-only inventory: ``index`` is nvidia-smi's own row number, a physical id and NOT something a caller may pin, because the whole point of this probe is that PyTorch cannot open these devices. A failed probe comes back as a structured unavailable result, so this never raises out of an endpoint."""
rows = _query_gpu_inventory("get_physical_gpu_inventory")