Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep automatic GPU placement inert on ordinary hosts (#14020)
Daniel Han2f63c766cca2
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 781d40161e1..82d5db085c5 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -98,7 +98,7 @@ from .diffusion_device import (install_frame_pad_fix,pin_cuda_ordinal,placed_cuda_ordinal,- resolve_auto_cuda_ordinal,+ auto_cuda_ordinal_for_load,resolve_diffusion_device_target,resolve_selected_cuda_ordinal,)@@ -2988,8 +2988,10 @@ class DiffusionBackend:apply_diffusion_device_ordinal(target)# And put an AUTOMATIC load back on its own card. generate() runs on a pooled asyncio.to_thread worker, so a# previous pinned model leaves that thread set to its GPU for good; with no ordinal to pin, the bare "cuda"- # Generators below would then target that card while these weights sit on the default one.- pin_cuda_ordinal(state.placed_ordinal)+ # Generators below would then target that card while these weights sit on the default one. Skipped when the+ # pin above already set that card.+ if state.gpu_ordinal is None or state.placed_ordinal != state.gpu_ordinal:+ pin_cuda_ordinal(state.placed_ordinal)return targetdef _resolve_gguf_path(@@ -3827,9 +3829,10 @@ class DiffusionBackend:# the contract says to drop. Re-ranked only when the caller did not already do it: free VRAM moves between the# route's preflight and here, so resolving twice can approve a scheme against one card and place the weights# on another.- if gpu_ordinal is None:+ auto_ordinal = gpu_ordinal is None and not gpu_ids+ if gpu_ordinal is None and gpu_ids:gpu_ordinal = (- (resolve_selected_cuda_ordinal(gpu_ids) if gpu_ids else resolve_auto_cuda_ordinal())+ resolve_selected_cuda_ordinal(gpu_ids)if resolve_diffusion_device_target().device == "cuda"else None)@@ -3847,6 +3850,9 @@ class DiffusionBackend:and resolve_model_kind(gguf_filename, model_kind) == "gguf"):raise ValueError(_WHOLE_PIPELINE_GGUF_LORA_MSG)+ if auto_ordinal:+ # Automatic, after validation: one GPU, Apple, CPU and ROCm skip every probe here.+ gpu_ordinal = auto_cuda_ordinal_for_load()# Refuse an EXPLICIT precision this host can never honor BEFORE the load starts, so the route answers 409 with# the reason instead of evicting the resident model, downloading several GB and only then failing. The# declines that need the real footprint can only be found mid-load and surface through load-progress.diff --git a/studio/backend/core/inference/diffusion_device.py b/studio/backend/core/inference/diffusion_device.pyindex eacda51bb00..224bf740061 100644--- a/studio/backend/core/inference/diffusion_device.py+++ b/studio/backend/core/inference/diffusion_device.py@@ -395,6 +395,8 @@ _PLANNED_ORDINAL: ContextVar[Any] = ContextVar("media_planned_gpu_ordinal", defa# Free-VRAM readings this close are noise, so the lower ordinal keeps an idle uniform host on card 0._AUTO_FREE_TIE_BYTES = 256 * 1024**2+# Set once any thread was pinned to a card other than 0: pooled workers may then sit elsewhere.+_PINNED_AWAY_FROM_ZERO = Falsedef resolve_auto_cuda_ordinal() -> Optional[int]:@@ -406,7 +408,8 @@ def resolve_auto_cuda_ordinal() -> Optional[int]:single-GPU fit, ties to the lowest ordinal. Read from nvidia-smi alone (no CUDA context per card, so notorch fallback) and keyed through the parent-visible list, so a mask keeps deciding which cards exist.- None on one visible card (unchanged), a UUID/MIG mask (no physical ids), ROCm (an APU's free reading is the+ None when card 0 wins (ties included) on identical cards, on one visible card (unchanged),+ a UUID/MIG mask (no physical ids), ROCm (an APU's free reading is theshared host pool, which would outrank a discrete card), a CUDA order other than nvidia-smi's (its rows arethen not torch's ordinals), an integrated GPU (its rows are a carve-out or N/A, and its real budget is theshared host pool), missing nvidia-smi telemetry and any failure. Never raises. Only cards at least as@@ -422,8 +425,12 @@ def resolve_auto_cuda_ordinal() -> Optional[int]:if hw.IS_ROCM:return Nonevisible = hw.get_parent_visible_gpu_ids()- if len(visible) < 2 or not hw._cuda_order_matches_smi():+ if len(visible) < 2:return None+ if not hw._cuda_order_matches_smi():+ # No ranking (rows are not ordinals), but on cards that are not provably identical keep the+ # current card explicit, so a pooled thread left on another card cannot preflight against it.+ return None if hw._cuda_cards_identical() is True else 0if hw._integrated_cuda_rows(visible)[0]:return Noneresult = hw._smi_visible_utilization(hw._get_parent_visible_gpu_spec())@@ -456,7 +463,75 @@ def resolve_auto_cuda_ordinal() -> Optional[int]:if not free_by_ordinal:return Nonebest = max(free_by_ordinal.values())- return min(o for o, free in free_by_ordinal.items() if best - free <= _AUTO_FREE_TIE_BYTES)+ winner = min(o for o, free in free_by_ordinal.items() if best - free <= _AUTO_FREE_TIE_BYTES)+ if winner:+ return winner+ # Card 0 won. Identical cards (same capability and size) keep None, the unpinned path they always had,+ # while no thread was ever pinned elsewhere. Mixed cards, or a process where a pooled thread may sit on+ # another card, keep 0 explicit so preflight and load read the ranked card.+ if _PINNED_AWAY_FROM_ZERO:+ return 0+ totals = {d.get("vram_total_gb") for d in devices if d.get("index") in visible}+ if len({caps.get(i) for i in visible}) == 1 and len(totals) == 1:+ # Physical rows hide MIG slice sizes, so a MIG card makes the set mixed.+ mig = hw._nvidia_mig_gpu_indices_or_none() # None (unknown) keeps card 0 explicit+ if mig is not None and not set(visible) & mig:+ return None+ return 0+++def auto_cuda_ranking_possible_cached() -> Optional[bool]:+ """``auto_cuda_ranking_possible`` when it can be answered from state already in memory, else None.++ For the event loop: a cold device detection, an uncounted unmasked host or a non-numeric mask may run+ nvidia-smi, so those go to a worker thread instead.+ """+ try:+ from utils.hardware import hardware as hw++ if hw.DEVICE is None:+ return None+ if hw.IS_ROCM or hw.DEVICE != hw.DeviceType.CUDA:+ return auto_cuda_ranking_possible()+ mask = os.environ.get("CUDA_VISIBLE_DEVICES")+ if mask is None:+ if hw._physical_gpu_count is None:+ return None+ elif not all(token.strip().isdigit() for token in mask.split(",") if token.strip()):+ return None+ except Exception: # noqa: BLE001 -- let the worker-thread gate decide+ return None+ return auto_cuda_ranking_possible()+++def auto_cuda_ranking_possible() -> bool:+ """Whether an automatic load could pick another card: NVIDIA CUDA with 2+ parent-visible GPUs.++ Read from the cached device and mask only (no torch, no capability or dtype probe), so zero / one GPU,+ ROCm, XPU, MPS and CPU keep the pre-ranking fast path. Never raises.+ """+ try:+ from utils.hardware import hardware as hw++ # Resolve first: a cold get_device() is what sets IS_ROCM.+ device = hw.get_device()+ if hw.IS_ROCM or device != hw.DeviceType.CUDA:+ return False+ return len(hw.get_parent_visible_gpu_ids()) >= 2+ except Exception: # noqa: BLE001 -- unknown host keeps the current device+ return False+++def auto_cuda_ordinal_for_load() -> Optional[int]:+ """``resolve_auto_cuda_ordinal`` behind the cheap gate and the CUDA target check, for automatic loads."""+ planned = _PLANNED_ORDINAL.get()+ if planned is not _UNPLANNED:+ return planned+ if not auto_cuda_ranking_possible():+ return None+ if resolve_diffusion_device_target().device != "cuda":+ return None+ return resolve_auto_cuda_ordinal()@contextmanager@@ -505,11 +580,14 @@ def apply_diffusion_device_ordinal(target: DiffusionDeviceTarget) -> None:def pin_cuda_ordinal(ordinal: Optional[int]) -> None:"""``torch.cuda.set_device``, thread-local, never fatal. A no-op for None."""+ global _PINNED_AWAY_FROM_ZEROif ordinal is None:returntry:import torchtorch.cuda.set_device(ordinal)+ if ordinal != 0:+ _PINNED_AWAY_FROM_ZERO = Trueexcept Exception: # noqa: BLE001 -- placement still works off torch_device; never fail a load herepassdiff --git a/studio/backend/core/inference/media_locality.py b/studio/backend/core/inference/media_locality.pyindex 0f379fe888f..62ed50952ed 100644--- a/studio/backend/core/inference/media_locality.py+++ b/studio/backend/core/inference/media_locality.py@@ -275,14 +275,34 @@ def plan_gpu_ordinal() -> Optional[int]:pinned, ordinal = planned_gpu_ordinal()if pinned:return ordinal- from core.inference.diffusion_device import (- resolve_auto_cuda_ordinal,- resolve_diffusion_device_target,- )+ from core.inference.diffusion_device import auto_cuda_ordinal_for_load++ return auto_cuda_ordinal_for_load()+- if resolve_diffusion_device_target().device != "cuda":+def catalog_gpu_ordinal() -> Optional[int]:+ """The one ordinal a catalog refresh plans every pick against, ranked once unless the cards are identical.++ Identical visible cards pick the same precision and so the same artifacts on any of them, so the+ refresh plans against the current device as it always did instead of re-ranking per pick.+ """+ pinned, ordinal = planned_gpu_ordinal()+ if pinned:+ return ordinal+ from core.inference.diffusion_device import auto_cuda_ranking_possible++ if not auto_cuda_ranking_possible():return None- return resolve_auto_cuda_ordinal()+ try:+ from utils.hardware import hardware as hw++ # Under a CUDA order nvidia-smi does not share, mask ids are not its rows: compare every card.+ ids = hw.get_parent_visible_gpu_ids() if hw._cuda_order_matches_smi() else None+ if hw._cuda_cards_identical(ids) is True:+ return None+ except Exception: # noqa: BLE001 -- unknown identity ranks once, like mixed cards+ pass+ return plan_gpu_ordinal()def _pipeline_components_present(root: Path) -> bool:@@ -559,4 +579,5 @@ __all__ = ["plan_gpu_ordinal","planned_gpu_ordinal","planners_for",+ "catalog_gpu_ordinal",]diff --git a/studio/backend/core/inference/sd_cpp_backend.py b/studio/backend/core/inference/sd_cpp_backend.pyindex 7709a14606b..08432a0e9c6 100644--- a/studio/backend/core/inference/sd_cpp_backend.py+++ b/studio/backend/core/inference/sd_cpp_backend.py@@ -34,7 +34,7 @@ from typing import Any, Optionalfrom core.inference.diffusion_auto_policy import build_resolved_record, format_generation_for_logfrom core.inference.diffusion_compat import flux2_inner_dim_for_pickfrom core.inference.diffusion_device import (- resolve_auto_cuda_ordinal,+ auto_cuda_ordinal_for_load,resolve_diffusion_device_target,resolve_selected_cuda_ordinal,)@@ -2934,14 +2934,13 @@ class SdCppDiffusionBackend:# but a direct caller (an MCP client, a test, a plugin) hands over gpu_ids alone, and without this the native# engine is the one engine that would drop the pick silently. Re-ranked only when nobody has, so a# route-resolved winner is never second-guessed against free VRAM that has moved since.- if gpu_ordinal is None:- # Automatic ranks the torch cards too, unless sd.cpp runs on one torch cannot see.- if off_torch is not None or resolve_diffusion_device_target().device != "cuda":- gpu_ordinal = None- elif gpu_ids:- gpu_ordinal = resolve_selected_cuda_ordinal(gpu_ids)- else:- gpu_ordinal = resolve_auto_cuda_ordinal()+ auto_ordinal = gpu_ordinal is None and not gpu_ids and off_torch is None+ if gpu_ordinal is None and gpu_ids:+ gpu_ordinal = (+ resolve_selected_cuda_ordinal(gpu_ids)+ if resolve_diffusion_device_target().device == "cuda"+ else None+ )if not gguf_filename:raise ValueError("gguf_filename is required: the native engine loads single-file GGUF checkpoints only."@@ -2957,6 +2956,9 @@ class SdCppDiffusionBackend:)if not family_sd_cpp_supported(fam):raise ValueError(f"Family '{fam.name}' has no native sd.cpp asset mapping.")+ if auto_ordinal:+ # Automatic ranks the torch cards too, after validation, unless sd.cpp runs on one torch cannot see.+ gpu_ordinal = auto_cuda_ordinal_for_load()self._loading_family = fambase = resolve_base_repo(fam, base_repo)diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex 5227c7f1ecd..3323cf291d0 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -100,7 +100,7 @@ from .diffusion_device import (install_rocm_vae_bf16_decode,pin_cuda_ordinal,placed_cuda_ordinal,- resolve_auto_cuda_ordinal,+ auto_cuda_ordinal_for_load,resolve_diffusion_device_target,resolve_selected_cuda_ordinal,)@@ -3057,8 +3057,9 @@ class VideoBackend:target = self._device_target(state.gpu_ordinal)# And an automatic load back onto its own card, in case this thread is a shared one a previous pinned load left# elsewhere. The image path reaches this through a pooled executor; here it costs one no-op call to keep the two- # engines identical.- pin_cuda_ordinal(state.placed_ordinal)+ # engines identical. Skipped when the pin above already set that card.+ if state.gpu_ordinal is None or state.placed_ordinal != state.gpu_ordinal:+ pin_cuda_ordinal(state.placed_ordinal)return target# ── validation ───────────────────────────────────────────────────────────@@ -3379,9 +3380,10 @@ class VideoBackend:# on the resolved backend, since XPU / MPS / CPU ignore physical ids and would otherwise 400 a selection the# contract drops. Re-ranked only when the caller did not already do it: free VRAM moves between the route's# preflight and here, so resolving twice can approve a scheme against one card and place the weights on another.- if gpu_ordinal is None:+ auto_ordinal = gpu_ordinal is None and not gpu_ids+ if gpu_ordinal is None and gpu_ids:gpu_ordinal = (- (resolve_selected_cuda_ordinal(gpu_ids) if gpu_ids else resolve_auto_cuda_ordinal())+ resolve_selected_cuda_ordinal(gpu_ids)if resolve_diffusion_device_target().device == "cuda"else None)@@ -3396,6 +3398,9 @@ class VideoBackend:text_encoder_quant = text_encoder_quant,h3_task = h3_task,)+ if auto_ordinal:+ # Automatic, after validation: one GPU, Apple, CPU and ROCm skip every probe here.+ gpu_ordinal = auto_cuda_ordinal_for_load()# Refuse an EXPLICIT precision this host can never honor BEFORE the load starts, so the route answers 409 with# the reason instead of evicting the resident model and pulling tens of GB first. Footprint-dependent declines# still surface via load-progress. The ROUTE makes the same call earlier still (seediff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 35585ae3d6f..70f4339e378 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -37121,31 +37121,41 @@ def _validated_media_picks(catalog_at: float) -> dict:with _MEDIA_PICK_CACHE_LOCK:if _MEDIA_PICK_CACHE["at"] == catalog_at:return _MEDIA_PICK_CACHE["picks"]- from core.inference.media_locality import is_edit_only, missing_download_bytes+ from core.inference.media_locality import (+ _PLANNED_ORDINAL,+ catalog_gpu_ordinal,+ is_edit_only,+ missing_download_bytes,+ )from core.inference.media_model_index import (available_media_model_ids,resolve_local_media_model,)picks: dict = {}- for task in _MEDIA_MODEL_TASKS:- rows = []- for model_id in available_media_model_ids(task):- pick = resolve_local_media_model(model_id, task = task)- if pick is None:- continue- # The local catalog tags an instruction-editing checkpoint text-to-image, but it- # ships no txt2img workflow: the switch refuses it with a 400 and a resident one- # is refused by /v1/images/generations too.- if task == "text-to-image" and is_edit_only(pick):- continue- try:- local = missing_download_bytes(_media_owner(task), pick) == 0- except Exception as exc: # noqa: BLE001- logger.debug("media locality unavailable for %s: %s", model_id, exc)- local = False- rows.append((model_id, pick, local))- picks[task] = rows+ # One placement decision per refresh, never a live ranking per pick.+ planned = _PLANNED_ORDINAL.set(catalog_gpu_ordinal())+ try:+ for task in _MEDIA_MODEL_TASKS:+ rows = []+ for model_id in available_media_model_ids(task):+ pick = resolve_local_media_model(model_id, task = task)+ if pick is None:+ continue+ # The local catalog tags an instruction-editing checkpoint text-to-image, but it+ # ships no txt2img workflow: the switch refuses it with a 400 and a resident one+ # is refused by /v1/images/generations too.+ if task == "text-to-image" and is_edit_only(pick):+ continue+ try:+ local = missing_download_bytes(_media_owner(task), pick) == 0+ except Exception as exc: # noqa: BLE001+ logger.debug("media locality unavailable for %s: %s", model_id, exc)+ local = False+ rows.append((model_id, pick, local))+ picks[task] = rows+ finally:+ _PLANNED_ORDINAL.reset(planned)_MEDIA_PICK_CACHE.update(at = catalog_at, picks = picks)return picks@@ -49458,23 +49468,32 @@ async def _selected_gpu_ordinal(gpu_ids, *, allow_ranking: bool = True) -> Optioresolve_selected_cuda_ordinal,)- if not gpu_ids and not allow_ranking:- return None- device = await asyncio.to_thread(lambda: resolve_diffusion_device_target().device)- if device != "cuda":- return Noneif not gpu_ids:+ if not allow_ranking:+ return None# An auto-switch load uses the card its locality check planned against.from core.inference.media_locality import planned_gpu_ordinalpinned, ordinal = planned_gpu_ordinal()if pinned:return ordinal+ # Zero / one GPU, ROCm, Apple and CPU: nothing to rank, so no device or dtype probe either.+ possible = diffusion_device.auto_cuda_ranking_possible_cached()+ if possible is None: # would probe: off the event loop+ possible = await asyncio.to_thread(diffusion_device.auto_cuda_ranking_possible)+ if not possible:+ return None+ device = await asyncio.to_thread(lambda: resolve_diffusion_device_target().device)+ if device != "cuda":+ return None# Automatic ranks the visible cards too (#8235); the current device is PCI slot 0.# Pinned for this request, so begin_load's own fallback reuses it (None included).ordinal = await asyncio.to_thread(diffusion_device.resolve_auto_cuda_ordinal)diffusion_device._PLANNED_ORDINAL.set(ordinal)return ordinal+ device = await asyncio.to_thread(lambda: resolve_diffusion_device_target().device)+ if device != "cuda":+ return None# The keyword only when it is not the default, so the ordinary path calls the resolver with# the same one-argument shape it always had (a monkeypatched seam still fits it).if allow_ranking:diff --git a/studio/backend/tests/test_diffusion_auto_gpu.py b/studio/backend/tests/test_diffusion_auto_gpu.pyindex b43defe2e1a..77d09e626ee 100644--- a/studio/backend/tests/test_diffusion_auto_gpu.py+++ b/studio/backend/tests/test_diffusion_auto_gpu.py@@ -36,7 +36,9 @@ def _host(if caps is None:caps = {d["index"]: (8, 6) for d in devices}monkeypatch.setattr(nvidia, "get_compute_capabilities", lambda: dict(caps))+ monkeypatch.setattr(hw, "_nvidia_mig_gpu_indices_or_none", lambda: set())monkeypatch.setattr(hw, "IS_ROCM", rocm)+ monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA)monkeypatch.setattr(hw, "get_parent_visible_gpu_ids", lambda: list(visible))monkeypatch.setattr(hw, "_cuda_order_matches_smi", lambda: order_matches)monkeypatch.setattr(hw, "_integrated_cuda_rows", lambda ids: ({}, "index"))@@ -57,10 +59,15 @@ def test_auto_maps_physical_ids_through_the_mask(monkeypatch):assert dd.resolve_auto_cuda_ordinal() == 1-def test_auto_ties_keep_the_lowest_ordinal(monkeypatch):- # An idle uniform host stays on card 0, MiB-level noise included.+def test_auto_ties_keep_the_current_device(monkeypatch):+ # An idle uniform host stays on card 0, MiB-level noise included, and as None (no explicit pin)._host(monkeypatch, [0, 1, 2], [_dev(0, 80.0, 0.6), _dev(1, 80.0, 0.5), _dev(2, 80.0, 0.6)])- assert dd.resolve_auto_cuda_ordinal() == 0+ assert dd.resolve_auto_cuda_ordinal() is None+++def test_auto_ties_among_other_cards_keep_the_lowest_ordinal(monkeypatch):+ _host(monkeypatch, [0, 1, 2], [_dev(0, 80.0, 70.0), _dev(1, 80.0, 0.5), _dev(2, 80.0, 0.6)])+ assert dd.resolve_auto_cuda_ordinal() == 1@pytest.mark.parametrize(@@ -87,10 +94,13 @@ def test_auto_never_raises(monkeypatch):assert dd.resolve_auto_cuda_ordinal() is None-def test_auto_keeps_the_current_device_when_cuda_order_is_not_smi_order(monkeypatch):- # CUDA_DEVICE_ORDER=FASTEST_FIRST: nvidia-smi rows are not torch ordinals, so no ranking.[email protected]("identical, expected", [(True, None), (False, 0), (None, 0)])+def test_auto_does_not_rank_when_cuda_order_is_not_smi_order(monkeypatch, identical, expected):+ # CUDA_DEVICE_ORDER=FASTEST_FIRST: nvidia-smi rows are not torch ordinals, so no ranking. Identical+ # cards keep None; otherwise card 0 stays explicit so a pooled thread cannot preflight elsewhere._host(monkeypatch, [0, 1], [_dev(0, 8.0, 1.0), _dev(1, 24.0, 0.5)], order_matches = False)- assert dd.resolve_auto_cuda_ordinal() is None+ monkeypatch.setattr(hw, "_cuda_cards_identical", lambda gpu_ids = None: identical)+ assert dd.resolve_auto_cuda_ordinal() is expecteddef test_auto_never_falls_back_to_torch_telemetry(monkeypatch):@@ -107,6 +117,7 @@ def _route(monkeypatch, device = "cuda"):dd, "resolve_diffusion_device_target", lambda: types.SimpleNamespace(device = device))monkeypatch.setattr(dd, "resolve_auto_cuda_ordinal", lambda: 1)+ monkeypatch.setattr(dd, "auto_cuda_ranking_possible", lambda: True)monkeypatch.setattr(dd, "resolve_selected_cuda_ordinal", lambda ids, **_k: min(ids))return routes._selected_gpu_ordinal@@ -187,7 +198,7 @@ def test_direct_backend_loads_rank_automatic_too(rel):for node in ast.walk(ast.parse(src))if isinstance(node, ast.Call) and isinstance(node.func, ast.Name)}- assert "resolve_auto_cuda_ordinal" in calls+ assert "auto_cuda_ordinal_for_load" in callsdef test_auto_skips_a_freer_card_older_than_the_current_one(monkeypatch):@@ -195,6 +206,7 @@ def test_auto_skips_a_freer_card_older_than_the_current_one(monkeypatch):_host(monkeypatch, [0, 1], [_dev(0, 24.0, 20.0), _dev(1, 8.0, 0.0)], caps = {0: (8, 6), 1: (6, 1)})+ # Mixed cards keep card 0 explicit.assert dd.resolve_auto_cuda_ordinal() == 0@@ -225,6 +237,7 @@ def test_route_pins_its_automatic_decision_for_the_backend(monkeypatch):)answers = iter([None, 1])monkeypatch.setattr(dd, "resolve_auto_cuda_ordinal", lambda: next(answers))+ monkeypatch.setattr(dd, "auto_cuda_ranking_possible", lambda: True)real = dd.__dict__["_PLANNED_ORDINAL"]async def load():diff --git a/studio/backend/tests/test_gpu_ids_cuda_device_order.py b/studio/backend/tests/test_gpu_ids_cuda_device_order.pyindex 0a7aa5834f9..1dec77534c4 100644--- a/studio/backend/tests/test_gpu_ids_cuda_device_order.py+++ b/studio/backend/tests/test_gpu_ids_cuda_device_order.py@@ -95,6 +95,8 @@ def _host(monkeypatch, physical = 2):monkeypatch.setattr(hw, "rocm_gpu_ids_without_torch_kernels", lambda: set())monkeypatch.setattr(hw, "get_visible_gpu_utilization", _smi_utilization)monkeypatch.setattr(hw, "_torch_get_per_device_info", _torch_rows)+ # No out-of-process ordinal mapping here, so the mismatched shape ranks through torch.+ monkeypatch.setattr(hw, "_smi_rows_for_cuda_ordinals", lambda parent_ids: None)def _auto(env):diff --git a/studio/backend/tests/test_media_auto_gpu_ordinary_host.py b/studio/backend/tests/test_media_auto_gpu_ordinary_host.pynew file mode 100644index 00000000000..c94a87ec323--- /dev/null+++ b/studio/backend/tests/test_media_auto_gpu_ordinary_host.py@@ -0,0 +1,602 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Automatic image / video GPU placement (#8235) leaves ordinary hosts as they were.++Zero / one GPU, Apple, CPU and ROCm run no device, dtype or capability probe for an+automatic request; the current device (a tie included) stays None so generation gains+no explicit set_device pins; identical cards under FASTEST_FIRST keep nvidia-smi+telemetry; a /v1/models catalog refresh does not rank live on homogeneous hosts."""++import asyncio+import os+import types+from unittest.mock import patch++import pytest++from core.inference import diffusion_device as dd+from utils.hardware import hardware as hw+from utils.hardware.hardware import DeviceType+++def _dev(index, total, used):+ return {"index": index, "vram_total_gb": total, "vram_used_gb": used}+++def _counting_target(+ monkeypatch,+ module,+ device = "cuda",+):+ calls = []++ def _target(**_kw):+ calls.append(1)+ return types.SimpleNamespace(device = device)++ monkeypatch.setattr(module, "resolve_diffusion_device_target", _target)+ return calls+++def _counting_rank(monkeypatch, answer = 1):+ calls = []++ def _rank():+ calls.append(1)+ return answer++ monkeypatch.setattr(dd, "resolve_auto_cuda_ordinal", _rank)+ return calls+++def _ordinary(+ monkeypatch,+ device = DeviceType.CUDA,+ visible = (0,),+ rocm = False,+):+ monkeypatch.setattr(hw, "IS_ROCM", rocm)+ monkeypatch.setattr(hw, "get_device", lambda: device)+ monkeypatch.setattr(hw, "get_parent_visible_gpu_ids", lambda: list(visible))++[email protected](autouse = True)+def _no_earlier_pins(monkeypatch):+ monkeypatch.setattr(dd, "_PINNED_AWAY_FROM_ZERO", False, raising = False)+++# Item 1: the current device stays None.+++ "devices",+ [+ [_dev(0, 80.0, 0.5), _dev(1, 80.0, 0.5)], # idle identical cards+ [_dev(0, 80.0, 0.5), _dev(1, 80.0, 40.0)], # card 0 is the freest+ ],+)+def test_auto_returns_none_when_the_current_device_wins(monkeypatch, devices):+ _two_cards(monkeypatch, devices, {0: (8, 0), 1: (8, 0)})+ assert dd.resolve_auto_cuda_ordinal() is None+++def test_auto_keeps_card_zero_explicit_beside_a_mig_card(monkeypatch):+ # Identical physical rows, but card 1 is MIG: its slice size is not the row's.+ _two_cards(+ monkeypatch, [_dev(0, 80.0, 0.5), _dev(1, 80.0, 0.5)], {0: (8, 0), 1: (8, 0)}, mig = {1}+ )+ assert dd.resolve_auto_cuda_ordinal() == 0+++ "devices, caps",+ [+ ([_dev(0, 80.0, 0.5), _dev(1, 80.0, 40.0)], {0: (8, 0), 1: (9, 0)}), # mixed capability+ ([_dev(0, 80.0, 0.5), _dev(1, 40.0, 30.0)], {0: (8, 0), 1: (8, 0)}), # mixed size+ ],+)+def test_auto_keeps_card_zero_explicit_on_mixed_cards(monkeypatch, devices, caps):+ # A pooled thread left on card 1 must not preflight a card-0 load against card 1.+ _two_cards(monkeypatch, devices, caps)+ assert dd.resolve_auto_cuda_ordinal() == 0+++def _two_cards(+ monkeypatch,+ devices,+ caps,+ mig = (),+):+ from utils.hardware import nvidia++ monkeypatch.setattr(nvidia, "get_compute_capabilities", lambda: dict(caps))+ monkeypatch.setattr(hw, "_nvidia_mig_gpu_indices_or_none", lambda: set(mig))+ monkeypatch.setattr(hw, "IS_ROCM", False)+ monkeypatch.setattr(hw, "get_parent_visible_gpu_ids", lambda: [0, 1])+ monkeypatch.setattr(hw, "_cuda_order_matches_smi", lambda: True)+ monkeypatch.setattr(hw, "_integrated_cuda_rows", lambda ids: ({}, "index"))+ monkeypatch.setattr(hw, "_get_parent_visible_gpu_spec", lambda: {"numeric_ids": [0, 1]})+ monkeypatch.setattr(+ hw, "_smi_visible_utilization", lambda spec: {"available": True, "devices": devices}+ )++[email protected]("gpu_ordinal, expected", [(None, [("pin", 0)]), (0, ["apply"])])+def test_video_generation_sets_the_device_once(monkeypatch, gpu_ordinal, expected):+ # Auto (None) keeps the single placed-device pin; an explicit 0 pins once, not twice.+ from core.inference import video++ pins = []+ monkeypatch.setattr(video, "resolve_diffusion_device_target", lambda **_k: object())+ monkeypatch.setattr(video, "apply_diffusion_device_ordinal", lambda t: pins.append("apply"))+ monkeypatch.setattr(video, "pin_cuda_ordinal", lambda o: pins.append(("pin", o)))+ state = types.SimpleNamespace(gpu_ordinal = gpu_ordinal, placed_ordinal = 0)+ video.VideoBackend._state_device_target(video.VideoBackend.__new__(video.VideoBackend), state)+ assert pins == expected++[email protected]("gpu_ordinal, expected", [(None, ["apply", ("pin", 0)]), (0, ["apply"])])+def test_image_generation_sets_the_device_once(monkeypatch, gpu_ordinal, expected):+ from core.inference import diffusion++ pins = []+ backend = diffusion.DiffusionBackend.__new__(diffusion.DiffusionBackend)+ monkeypatch.setattr(backend, "_target_for_ordinal", lambda fam, o: object(), raising = False)+ monkeypatch.setattr(diffusion, "apply_diffusion_device_ordinal", lambda t: pins.append("apply"))+ monkeypatch.setattr(diffusion, "pin_cuda_ordinal", lambda o: pins.append(("pin", o)))+ state = types.SimpleNamespace(family = None, gpu_ordinal = gpu_ordinal, placed_ordinal = 0)+ diffusion.DiffusionBackend._state_device_target(backend, state)+ assert pins == expected+++# Item 2: no probe on ordinary hosts, and validation first.+++ "device, visible, rocm",+ [+ (DeviceType.CUDA, (0,), False), # one GPU+ (DeviceType.CUDA, (), False), # zero visible+ (DeviceType.MLX, (), False), # Apple Silicon+ (DeviceType.CPU, (), False),+ (DeviceType.CUDA, (0, 1), True), # ROCm+ ],+)+def test_route_auto_runs_no_device_probe_on_ordinary_hosts(monkeypatch, device, visible, rocm):+ from routes import inference as routes++ _ordinary(monkeypatch, device = device, visible = visible, rocm = rocm)+ targets = _counting_target(monkeypatch, dd)+ ranks = _counting_rank(monkeypatch)+ assert asyncio.run(routes._selected_gpu_ordinal(None)) is None+ assert (targets, ranks) == ([], [])+++def test_route_auto_still_ranks_on_a_multi_gpu_cuda_host(monkeypatch):+ from routes import inference as routes++ _ordinary(monkeypatch, visible = (0, 1))+ targets = _counting_target(monkeypatch, dd)+ ranks = _counting_rank(monkeypatch)++ async def load():+ return await routes._selected_gpu_ordinal(None)++ assert asyncio.run(load()) == 1+ assert (len(targets), len(ranks)) == (1, 1)+++ "module_name, cls_name", [("diffusion", "DiffusionBackend"), ("video", "VideoBackend")]+)+def test_direct_invalid_auto_load_is_refused_before_any_probe(monkeypatch, module_name, cls_name):+ import importlib++ module = importlib.import_module(f"core.inference.{module_name}")+ _ordinary(monkeypatch, visible = (0, 1))+ targets = _counting_target(monkeypatch, module)+ dd_targets = _counting_target(monkeypatch, dd)+ ranks = _counting_rank(monkeypatch)+ backend = getattr(module, cls_name)()+ with pytest.raises(ValueError):+ backend.begin_load("nobody/not-a-diffusion-model-at-all")+ assert (targets, dd_targets, ranks) == ([], [], [])++[email protected]("device, visible", [(DeviceType.CUDA, (0,)), (DeviceType.MLX, ())])+def test_backend_auto_fallback_runs_no_probe_on_ordinary_hosts(monkeypatch, device, visible):+ _ordinary(monkeypatch, device = device, visible = visible)+ targets = _counting_target(monkeypatch, dd)+ ranks = _counting_rank(monkeypatch)+ assert dd.auto_cuda_ordinal_for_load() is None+ assert (targets, ranks) == ([], [])+++# Item 3: FASTEST_FIRST numeric masks read nvidia-smi rows joined by PCI bus id, not torch.+++def _fastest_first_host(monkeypatch, rows):+ torch_calls = []++ def _torch_rows(indices):+ torch_calls.append(list(indices))+ return [+ {"index": idx, "visible_ordinal": o, "total_gb": 80.0, "used_gb": 10.0 + o * 60.0}+ for o, idx in enumerate(indices)+ ]++ monkeypatch.setattr(hw, "IS_ROCM", False)+ monkeypatch.setattr(hw, "get_device", lambda: DeviceType.CUDA)+ monkeypatch.setattr(hw, "get_physical_gpu_count", lambda: 2)+ monkeypatch.setattr(hw, "_physical_gpu_count_from_smi", True)+ monkeypatch.setattr(hw, "rocm_gpu_ids_without_torch_kernels", lambda: set())+ # Masked SMI rows (PCI order): row 0 busy, row 1 free.+ smi_rows = [_dev(0, 80.0, 70.0), _dev(1, 80.0, 10.0)]+ monkeypatch.setattr(hw, "get_visible_gpu_utilization", lambda: {"devices": smi_rows})+ monkeypatch.setattr(+ hw,+ "_smi_query",+ lambda name, ids, **_k: {+ "available": True,+ "devices": [d for d in smi_rows if d["index"] in ids],+ },+ )+ monkeypatch.setattr(hw, "_torch_get_per_device_info", _torch_rows)+ from utils.hardware import nvidia++ reads = []+ monkeypatch.setattr(nvidia, "get_cuda_ordinal_smi_rows", lambda: reads.append(1) or rows)+ monkeypatch.setattr(hw, "_cuda_ordinal_rows_cache", {})+ monkeypatch.setattr(hw, "_nvidia_mig_gpu_indices_or_none", lambda: set())+ return torch_calls, reads+++def _auto(env):+ with patch.dict(os.environ, env, clear = True):+ ids, _meta = hw.auto_select_gpu_ids("m", required_override_gb = 16.0)+ return ids+++_FASTEST = {"CUDA_DEVICE_ORDER": "FASTEST_FIRST", "CUDA_VISIBLE_DEVICES": "0,1"}++[email protected]("rows, expected", [([0, 1], [1]), ([1, 0], [0])])+def test_fastest_first_mask_ranks_smi_rows_by_bus_id(monkeypatch, rows, expected):+ # rows = the nvidia-smi row behind each CUDA ordinal; [1, 0] is a tie order that differs from PCI.+ torch_calls, reads = _fastest_first_host(monkeypatch, rows)+ assert _auto(_FASTEST) == expected+ assert _auto(_FASTEST) == expected+ assert torch_calls == []+ assert reads == [1] # mapping found once per environment+++def test_fastest_first_mask_without_a_mapping_keeps_torch(monkeypatch):+ torch_calls, _reads = _fastest_first_host(monkeypatch, None)+ assert _auto(_FASTEST) == [0]+ assert torch_calls == [[0, 1]]++[email protected]("mig", [{1}, None]) # a mapped MIG card, or MIG discovery failing+def test_fastest_first_mask_falls_back_to_torch_around_mig(monkeypatch, mig):+ # nvidia-smi shows a MIG card's whole memory; torch reports the slice, so it ranks instead.+ torch_calls, _reads = _fastest_first_host(monkeypatch, [1, 0])+ monkeypatch.setattr(hw, "_nvidia_mig_gpu_indices_or_none", lambda: mig)+ with patch.dict(os.environ, _FASTEST, clear = True):+ assert hw._smi_rows_for_cuda_ordinals([0, 1]) is None+ hw.auto_select_gpu_ids("m", required_override_gb = 8.0)+ assert torch_calls == [[0, 1]]+++def test_fastest_first_mask_without_mig_keeps_nvidia_smi(monkeypatch):+ torch_calls, _reads = _fastest_first_host(monkeypatch, [1, 0])+ monkeypatch.setattr(hw, "_nvidia_mig_gpu_indices_or_none", lambda: {5}) # MIG elsewhere only+ with patch.dict(os.environ, _FASTEST, clear = True):+ assert hw._smi_rows_for_cuda_ordinals([0, 1]) is not None+ assert torch_calls == []+++def test_torch_fallback_rows_skip_the_physical_mig_correction(monkeypatch):+ # No bus-id mapping: torch rows carry mask ids and already report slice memory. Physical row 0 is a+ # MIG card, but mask id 0 is another card, so correcting it would hide the card that has room.+ from core.inference.llama_cpp import LlamaCppBackend+ from utils.hardware import nvidia++ torch_calls, _reads = _fastest_first_host(monkeypatch, None)+ monkeypatch.setattr(nvidia, "get_mig_gpu_indices", lambda: {0})+ monkeypatch.setattr(+ LlamaCppBackend, "_get_gpu_memory_nvml", staticmethod(lambda: [(0, 1024, 5120)])+ )+ assert _auto(_FASTEST) == [0] # torch: mask id 0 has 70 GB free+ assert torch_calls == [[0, 1]]+++def test_bus_ids_from_both_tools_join():+ from utils.hardware import nvidia+ assert nvidia._normalize_bus_id("00000000:3B:00.0") == nvidia._normalize_bus_id("0000:3b:00.0")+ assert nvidia._normalize_bus_id("garbage") is None+++def test_card_identities_are_read_once(monkeypatch):+ from utils.hardware import nvidia++ reads = []+ row = ("NVIDIA A100-SXM4-80GB", "81920 MiB", "8.0", "1410 MHz")+ monkeypatch.setattr(hw, "IS_ROCM", False)+ monkeypatch.setattr(hw, "get_physical_gpu_count", lambda: 2)+ monkeypatch.setattr(nvidia, "get_card_identities", lambda: reads.append(1) or {0: row, 1: row})+ monkeypatch.setattr(hw, "_nvidia_mig_gpu_indices_or_none", lambda: set())+ monkeypatch.setattr(hw, "_cuda_card_identities", {})+ assert hw._cuda_cards_identical() is True+ assert hw._cuda_cards_identical([0, 1]) is True+ assert reads == [1]+++# Item 4: catalog refreshes do not rank live on single-GPU / homogeneous hosts.+++def _catalog(monkeypatch):+ from core.inference import media_locality, media_model_index+ from routes import inference as routes++ planned = []+ monkeypatch.setattr(media_model_index, "available_media_model_ids", lambda task: ["a", "b"])+ monkeypatch.setattr(+ media_model_index,+ "resolve_local_media_model",+ lambda model_id, task: types.SimpleNamespace(model_id = model_id),+ )+ monkeypatch.setattr(media_locality, "is_edit_only", lambda pick: False)+ monkeypatch.setattr(+ media_locality,+ "missing_download_bytes",+ lambda owner, pick, hf_token = None: planned.append(media_locality.plan_gpu_ordinal()) or 0,+ )+ monkeypatch.setattr(routes, "_MEDIA_PICK_CACHE", {"at": None, "picks": {}})+ monkeypatch.setattr(hw, "_cuda_order_matches_smi", lambda: True)+ return routes, planned++[email protected]("visible, identical", [((0,), True), ((0,), None), ((0, 1), True)])+def test_catalog_refresh_does_not_rank_on_ordinary_hosts(monkeypatch, visible, identical):+ routes, planned = _catalog(monkeypatch)+ _ordinary(monkeypatch, visible = visible)+ monkeypatch.setattr(hw, "_cuda_cards_identical", lambda gpu_ids = None: identical)+ _counting_target(monkeypatch, dd)+ ranks = _counting_rank(monkeypatch)+ routes._validated_media_picks(1.0)+ assert ranks == []+ assert planned and set(planned) == {None}++[email protected]("identical", [False, None])+def test_catalog_refresh_ranks_once_on_mixed_or_unknown_cards(monkeypatch, identical):+ routes, planned = _catalog(monkeypatch)+ _ordinary(monkeypatch, visible = (0, 1))+ monkeypatch.setattr(hw, "_cuda_cards_identical", lambda gpu_ids = None: identical)+ _counting_target(monkeypatch, dd)+ ranks = _counting_rank(monkeypatch, answer = 1)+ routes._validated_media_picks(1.0)+ assert len(ranks) == 1+ assert set(planned) == {1}+ from core.inference.media_locality import planned_gpu_ordinal++ assert planned_gpu_ordinal() == (False, None)+++# The route's gate answers from memory, or runs on a worker thread.+++ "device, count, mask, cached",+ [+ (DeviceType.CUDA, 1, None, False), # counted single GPU: answered inline+ (DeviceType.MLX, None, None, False), # Apple: answered inline+ (None, None, None, None), # detection not run yet+ (DeviceType.CUDA, None, None, None), # unmasked, never counted+ (DeviceType.CUDA, 2, "GPU-abc,GPU-def", None), # UUID mask may run nvidia-smi+ ],+)+def test_cached_gate_defers_anything_that_could_probe(monkeypatch, device, count, mask, cached):+ monkeypatch.setattr(hw, "IS_ROCM", False)+ monkeypatch.setattr(hw, "DEVICE", device)+ monkeypatch.setattr(hw, "_physical_gpu_count", count)+ monkeypatch.setattr(hw, "get_device", lambda: device)+ monkeypatch.setattr(hw, "get_parent_visible_gpu_ids", lambda: list(range(count or 0)))+ env = {} if mask is None else {"CUDA_VISIBLE_DEVICES": mask}+ with patch.dict(os.environ, env, clear = True):+ assert dd.auto_cuda_ranking_possible_cached() is cached+++def test_route_runs_an_uncached_gate_off_the_event_loop(monkeypatch):+ from routes import inference as routes++ monkeypatch.setattr(dd, "auto_cuda_ranking_possible_cached", lambda: None)+ loop_thread = []++ def _gate():+ import threading+ loop_thread.append(threading.current_thread() is threading.main_thread())+ return False++ monkeypatch.setattr(dd, "auto_cuda_ranking_possible", _gate)+ targets = _counting_target(monkeypatch, dd)+ assert asyncio.run(routes._selected_gpu_ordinal(None)) is None+ assert loop_thread == [False]+ assert targets == []+++def test_identical_rows_with_a_mig_card_are_not_identical(monkeypatch):+ from utils.hardware import nvidia++ row = ("NVIDIA A100-SXM4-80GB", "81920 MiB", "8.0", "1410 MHz")+ monkeypatch.setattr(hw, "IS_ROCM", False)+ monkeypatch.setattr(hw, "get_physical_gpu_count", lambda: 2)+ monkeypatch.setattr(nvidia, "get_card_identities", lambda: {0: row, 1: row})+ monkeypatch.setattr(hw, "_cuda_card_identities", {})+ monkeypatch.setattr(hw, "_nvidia_mig_gpu_indices_or_none", lambda: {1})+ assert hw._cuda_cards_identical() is False+ assert hw._cuda_cards_identical([0]) is True+++def test_catalog_compares_every_card_when_mask_ids_are_not_smi_rows(monkeypatch):+ # FASTEST_FIRST subset mask "0,2": those are CUDA ordinals, not nvidia-smi rows 0 and 2.+ routes, planned = _catalog(monkeypatch)+ _ordinary(monkeypatch, visible = (0, 2))+ monkeypatch.setattr(hw, "_cuda_order_matches_smi", lambda: False)+ asked = []+ # Rows 0 and 2 match, row 1 differs: only the all-cards answer is trustworthy.+ monkeypatch.setattr(+ hw,+ "_cuda_cards_identical",+ lambda gpu_ids = None: asked.append(gpu_ids) or gpu_ids is not None,+ )+ _counting_target(monkeypatch, dd)+ ranks = _counting_rank(monkeypatch, answer = 0)+ routes._validated_media_picks(1.0)+ assert asked == [None]+ assert len(ranks) == 1+++ "listing",+ [+ types.SimpleNamespace(returncode = 1, stdout = ""), # nvidia-smi -L failed+ types.SimpleNamespace(returncode = 0, stdout = ""), # empty output+ "timeout",+ ],+)+def test_failed_mig_discovery_is_unknown_not_identical(monkeypatch, listing):+ import subprocess++ from utils.hardware import nvidia++ def _run(argv, **_k):+ if listing == "timeout":+ raise subprocess.TimeoutExpired(argv, 5)+ return listing++ row = ("NVIDIA A100-SXM4-80GB", "81920 MiB", "8.0", "1410 MHz")+ monkeypatch.setattr(hw, "IS_ROCM", False)+ monkeypatch.setattr(hw, "get_physical_gpu_count", lambda: 2)+ monkeypatch.setattr(nvidia, "get_card_identities", lambda: {0: row, 1: row})+ monkeypatch.setattr(hw, "_cuda_card_identities", {})+ monkeypatch.setattr(nvidia.gpu_query, "run_nvidia_smi", _run)+ assert hw._nvidia_mig_gpu_indices_or_none() is None+ assert hw._cuda_cards_identical() is None+ # The load path keeps card 0 explicit when MIG is unknown.+ _two_cards(monkeypatch, [_dev(0, 80.0, 0.5), _dev(1, 80.0, 0.5)], {0: (8, 0), 1: (8, 0)})+ monkeypatch.setattr(hw, "_nvidia_mig_gpu_indices_or_none", lambda: None)+ assert dd.resolve_auto_cuda_ordinal() == 0+++def test_mig_listing_parses_slices():+ from unittest.mock import patch as _patch++ from utils.hardware import nvidia++ out = "GPU 0: A100 (UUID: GPU-a)\n MIG 3g.40gb Device 0: (UUID: MIG-x)\nGPU 1: A100 (UUID: GPU-b)\n"+ with _patch.object(+ nvidia.gpu_query,+ "run_nvidia_smi",+ lambda argv, **_k: types.SimpleNamespace(returncode = 0, stdout = out),+ ):+ assert hw._nvidia_mig_gpu_indices_or_none() == {0}+++def test_fastest_first_mask_with_an_integrated_card_keeps_torch(monkeypatch):+ # [] = mapping known but an integrated card is mapped: its nvidia-smi row is a carve-out.+ torch_calls, reads = _fastest_first_host(monkeypatch, [])+ with patch.dict(os.environ, _FASTEST, clear = True):+ assert hw._smi_rows_for_cuda_ordinals([0, 1]) is None+ assert hw._smi_rows_for_cuda_ordinals([0, 1]) is None+ hw.auto_select_gpu_ids("m", required_override_gb = 8.0)+ assert torch_calls == [[0, 1]]+ assert reads == [1] # cached, the child is not re-run+++ "child_out, expected",+ [+ ("0000:3b:00.0 0\n0000:1a:00.0 0\n", [1, 0]),+ ("0000:3b:00.0 0\n0000:1a:00.0 1\n", []), # integrated card mapped+ ("0000:3b:00.0\n", None), # no attribute: unreadable+ ],+)+def test_cuda_ordinal_rows_parse_bus_ids_and_integrated(monkeypatch, child_out, expected):+ import subprocess++ from utils.hardware import nvidia++ child = types.SimpleNamespace(returncode = 0, stdout = child_out)+ smi = types.SimpleNamespace(returncode = 0, stdout = "0, 00000000:1A:00.0\n1, 00000000:3B:00.0\n")+ monkeypatch.setattr(subprocess, "run", lambda *a, **k: child)+ monkeypatch.setattr(nvidia.gpu_query, "run_nvidia_smi", lambda *a, **k: smi)+ assert nvidia.get_cuda_ordinal_smi_rows() == expected+++def test_identity_compares_every_listed_row_not_a_stale_count(monkeypatch):+ # A torch-fallback count of 2 must not hide a third, different card nvidia-smi lists.+ from utils.hardware import nvidia++ row = ("NVIDIA A100-SXM4-80GB", "81920 MiB", "8.0", "1410 MHz")+ other = ("NVIDIA RTX A6000", "49140 MiB", "8.6", "2100 MHz")+ monkeypatch.setattr(hw, "IS_ROCM", False)+ monkeypatch.setattr(hw, "get_physical_gpu_count", lambda: 2)+ monkeypatch.setattr(nvidia, "get_card_identities", lambda: {0: row, 1: row, 2: other})+ monkeypatch.setattr(hw, "_cuda_card_identities", {})+ monkeypatch.setattr(hw, "_nvidia_mig_gpu_indices_or_none", lambda: set())+ assert hw._cuda_cards_identical() is False+ assert hw._cuda_cards_identical([0, 1]) is True+++def test_hardware_redetection_retires_the_ordinal_mapping(monkeypatch):+ from utils.hardware import gpu_query++ _torch_calls, reads = _fastest_first_host(monkeypatch, [0, 1])+ assert _auto(_FASTEST) == [1]+ gpu_query.invalidate_static("test: eGPU attached")+ assert _auto(_FASTEST) == [1]+ assert reads == [1, 1]+++def test_cold_rocm_detection_is_settled_before_the_gate_reads_it(monkeypatch):+ _ordinary(monkeypatch, visible = (0, 1))+ monkeypatch.setattr(hw, "IS_ROCM", False)++ def cold_detect():+ hw.IS_ROCM = True # ROCm reports as CUDA once detection has run+ return DeviceType.CUDA++ monkeypatch.setattr(hw, "get_device", cold_detect)+ assert dd.auto_cuda_ranking_possible() is False+++def test_card_zero_stays_explicit_once_a_thread_was_pinned_elsewhere(monkeypatch):+ import torch++ _two_cards(monkeypatch, [_dev(0, 80.0, 0.5), _dev(1, 80.0, 40.0)], {0: (8, 0), 1: (8, 0)})+ monkeypatch.setattr(torch.cuda, "set_device", lambda ordinal: None)+ dd.pin_cuda_ordinal(0)+ assert dd.resolve_auto_cuda_ordinal() is None # pinning card 0 leaves no worker elsewhere+ dd.pin_cuda_ordinal(1)+ # A pooled worker may now sit on card 1: the winner is named so preflight reads card 0.+ assert dd.resolve_auto_cuda_ordinal() == 0+++def test_hardware_redetection_rereads_card_identities(monkeypatch):+ from utils.hardware import gpu_query, nvidia++ a100 = ("NVIDIA A100-SXM4-80GB", "81920 MiB", "8.0", "1410 MHz")+ h100 = ("NVIDIA H100 80GB HBM3", "81559 MiB", "9.0", "1980 MHz")+ inventory = [{0: a100, 1: a100}]+ monkeypatch.setattr(hw, "IS_ROCM", False)+ monkeypatch.setattr(hw, "get_physical_gpu_count", lambda: 2)+ monkeypatch.setattr(nvidia, "get_card_identities", lambda: inventory[0])+ monkeypatch.setattr(hw, "_nvidia_mig_gpu_indices_or_none", lambda: set())+ monkeypatch.setattr(hw, "_cuda_card_identities", {})+ assert hw._cuda_cards_identical() is True+ inventory[0] = {0: a100, 1: h100} # card 1 swapped, same count+ assert hw._cuda_cards_identical() is True # cached until re-detection+ gpu_query.invalidate_static("test: card swapped")+ assert hw._cuda_cards_identical() is Falsediff --git a/studio/backend/tests/test_media_auto_switch.py b/studio/backend/tests/test_media_auto_switch.pyindex b8b19d20b00..87a4c796fb1 100644--- a/studio/backend/tests/test_media_auto_switch.py+++ b/studio/backend/tests/test_media_auto_switch.py@@ -792,6 +792,7 @@ def test_the_load_runs_on_the_card_the_locality_check_planned()ranks = itertools.chain([1, 1], itertools.repeat(0))monkeypatch.setattr(dd, "resolve_auto_cuda_ordinal", lambda: next(ranks))+ monkeypatch.setattr(dd, "auto_cuda_ranking_possible", lambda: True)planned: list = []backend.download_plan = lambda model_path, **kw: planned.append(kw["gpu_ordinal"]) or {"total_bytes": 0diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.pyindex 0bf3b2aed96..ea473d4f104 100644--- a/studio/backend/utils/hardware/hardware.py+++ b/studio/backend/utils/hardware/hardware.py@@ -2054,6 +2054,136 @@ def _mask_names_cuda_ordinals(parent_visible_spec: Dict[str, Any]) -> bool:return not _cuda_order_matches_smi()+_cuda_card_identities: Dict[int, tuple] = {}+_cuda_card_identities_gen: Optional[int] = None+++def _cuda_cards_identical(gpu_ids: Optional[list[int]] = None) -> Optional[bool]:+ """Whether the NVIDIA cards ``gpu_ids`` (default: every physical card) share one identity per nvidia-smi (name, total memory, compute cap, max SM clock) and none is MIG, out of process. None when unknown. Read once and cached: a card's identity does not change, and a grown card count re-reads."""+ global _cuda_card_identities, _cuda_card_identities_gen+ if IS_ROCM:+ return None+ from . import gpu_query++ count = get_physical_gpu_count()+ wanted = list(range(count)) if gpu_ids is None else list(gpu_ids)+ # Hardware re-detection (a swapped card) bumps the static generation.+ if (+ _cuda_card_identities_gen != gpu_query._static_gen+ or not set(wanted) <= set(_cuda_card_identities)+ or len(_cuda_card_identities) < count+ ):+ gen = gpu_query._static_gen+ try:+ from . import nvidia+ identities = nvidia.get_card_identities()+ except Exception:+ return None+ if not identities:+ return None+ _cuda_card_identities, _cuda_card_identities_gen = identities, gen+ if gpu_ids is None:+ # Every row nvidia-smi lists, not just the count: a torch-fallback count can miss cards.+ wanted = sorted(_cuda_card_identities)+ rows = [_cuda_card_identities.get(gpu_id) for gpu_id in wanted]+ if not rows or any(row is None for row in rows):+ return None+ if len(set(rows)) != 1:+ return False+ # A MIG card is seen through a slice whose size the physical row does not show.+ mig = _nvidia_mig_gpu_indices_or_none()+ if mig is None:+ return None+ return not (set(wanted) & mig)+++def _nvidia_mig_gpu_indices_or_none() -> Optional[set[int]]:+ """Physical indices of NVIDIA cards with MIG instances, from ``nvidia-smi -L``, or None when the+ listing failed or came back empty (unknown, unlike a confirmed set with no MIG card)."""+ try:+ from . import nvidia+ result = nvidia.gpu_query.run_nvidia_smi(+ [nvidia._nvidia_smi_executable(), "-L"],+ capture_output = True,+ text = True,+ encoding = "utf-8",+ errors = "replace",+ timeout = 5,+ env = nvidia.child_env_without_native_path_secret(),+ **nvidia._windows_hidden_subprocess_kwargs(),+ )+ except Exception:+ return None+ if result.returncode != 0 or not isinstance(result.stdout, str) or not result.stdout.strip():+ return None+ indices: set[int] = set()+ current: Optional[int] = None+ for line in result.stdout.splitlines():+ if line.startswith("GPU "):+ head = line[4:].split(":", 1)[0].strip()+ current = int(head) if head.isdecimal() else None+ elif current is not None and line.strip().startswith("MIG "):+ indices.add(current)+ return indices+++_cuda_ordinal_rows_cache: Dict[tuple, list[int]] = {}+++def _smi_rows_for_cuda_ordinals(parent_ids: list[int]) -> Optional[Dict[str, Any]]:+ """nvidia-smi utilization keyed by the mask's ids, read from the rows the CUDA ordinals really are.++ For a numeric mask under a non-PCI CUDA_DEVICE_ORDER, where mask entries are CUDA ordinals and not+ nvidia-smi rows. The ordinal -> row join is by PCI bus id, found once per environment out of process+ (no CUDA context here). None when it cannot be established, so the caller keeps the torch path.+ """+ from . import gpu_query++ # The static generation moves on hardware re-detection (eGPU, driver change), retiring old mappings.+ key = (+ os.environ.get("CUDA_VISIBLE_DEVICES"),+ os.environ.get("CUDA_DEVICE_ORDER"),+ gpu_query._static_gen,+ )+ rows = _cuda_ordinal_rows_cache.get(key)+ if rows is None:+ try:+ from . import nvidia+ rows = nvidia.get_cuda_ordinal_smi_rows()+ except Exception:+ rows = None+ if rows is None:+ return None+ _cuda_ordinal_rows_cache.clear()+ _cuda_ordinal_rows_cache[key] = rows+ if not rows: # an integrated card: keep the pool-aware torch telemetry+ return None+ if len(rows) != len(parent_ids) or len(set(rows)) != len(rows):+ return None+ result = _smi_query(+ "get_visible_gpu_utilization", sorted(rows), parent_cuda_visible_devices = None+ )+ # nvidia-smi rows show a MIG card's whole memory, not its slice: with a MIG card among the rows,+ # or MIG discovery failing, keep the slice-aware torch telemetry instead.+ mig = _nvidia_mig_gpu_indices_or_none()+ if mig is None or mig & set(rows):+ return None+ by_row = {d.get("index"): d for d in (result or {}).get("devices", [])}+ if not all(row in by_row for row in rows):+ return None+ return {+ "mig_corrected": True,+ "devices": [+ {+ "index": parent_ids[ordinal],+ "vram_total_gb": by_row[row].get("vram_total_gb"),+ "vram_used_gb": by_row[row].get("vram_used_gb"),+ }+ for ordinal, row in enumerate(rows)+ ],+ }++def _amd_smi_ids_for_hip_ids(hip_ids: Optional[list[int]]) -> Optional[list[int]]:"""Translate visible HIP ordinals to amd-smi physical GPU IDs."""if hip_ids is None or not hip_ids:@@ -6118,9 +6248,20 @@ def auto_select_gpu_ids(metadata["selected_gpu_ids"] = parent_idsreturn parent_ids, metadata- if _mask_names_cuda_ordinals(parent_visible_spec):- # The display keeps the CLI rows; ranking needs the cards the worker will get.+ smi_by_ordinal = (+ _smi_rows_for_cuda_ordinals(get_parent_visible_gpu_ids())+ if _mask_names_cuda_ordinals(parent_visible_spec)+ else None+ )+ if smi_by_ordinal is not None:+ # nvidia-smi telemetry for the cards the ids select, joined by PCI bus id out of process.+ utilization = smi_by_ordinal+ elif _mask_names_cuda_ordinals(parent_visible_spec):+ # The display keeps the CLI rows; ranking needs the cards the worker will get. Torch already+ # reports a MIG slice's own memory, and these rows carry mask ids, not the physical rows the+ # MIG correction keys on, so it must not run on them.utilization = {+ "mig_corrected": True,"devices": [{"index": td["index"],@@ -6128,12 +6269,13 @@ def auto_select_gpu_ids("vram_used_gb": td["used_gb"],}for td in _torch_get_per_device_info(get_parent_visible_gpu_ids())- ]+ ],}else:utilization = get_visible_gpu_utilization()devices = [d for d in utilization.get("devices", []) if d.get("index") not in _uncovered]- devices = _with_mig_instance_memory(devices)+ if not utilization.get("mig_corrected"):+ devices = _with_mig_instance_memory(devices)parent_ids = _covered(get_parent_visible_gpu_ids())if not devices:diff --git a/studio/backend/utils/hardware/nvidia.py b/studio/backend/utils/hardware/nvidia.pyindex b91aa0e05a2..4c6676ab6fd 100644--- a/studio/backend/utils/hardware/nvidia.py+++ b/studio/backend/utils/hardware/nvidia.py@@ -587,6 +587,131 @@ def get_compute_capabilities() -> dict[int, tuple[int, int]]:return capabilities+def get_card_identities() -> dict[int, tuple[str, ...]]:+ """nvidia-smi row index -> (name, total memory, compute cap, max SM clock), no mask and no torch.++ Equal rows pick the same precision and artifacts on either card. {} when unreadable.+ """+ try:+ result = gpu_query.run_nvidia_smi(+ [+ _nvidia_smi_executable(),+ "--query-gpu=index,name,memory.total,compute_cap,clocks.max.sm",+ "--format=csv,noheader",+ ],+ capture_output = True,+ text = True,+ encoding = "utf-8",+ errors = "replace",+ timeout = 5,+ env = child_env_without_native_path_secret(),+ **_windows_hidden_subprocess_kwargs(),+ )+ except (OSError, subprocess.TimeoutExpired):+ return {}+ if result.returncode != 0:+ return {}+ identities: dict[int, tuple[str, ...]] = {}+ for line in result.stdout.strip().splitlines():+ index, *fields = (part.strip() for part in line.split(","))+ if not index.isdigit() or len(fields) != 4:+ return {}+ identities[int(index)] = tuple(fields)+ return identities+++# Driver API only: cuInit enumerates devices under the inherited CUDA_VISIBLE_DEVICES / CUDA_DEVICE_ORDER+# without creating a context, and runs in a child so the Studio process never initialises CUDA.+_CUDA_BUS_ID_SCRIPT = r"""+import ctypes, sys+lib = ctypes.WinDLL("nvcuda.dll") if sys.platform == "win32" else ctypes.CDLL("libcuda.so.1")+if lib.cuInit(0) != 0:+ sys.exit(3)+count = ctypes.c_int(0)+if lib.cuDeviceGetCount(ctypes.byref(count)) != 0:+ sys.exit(4)+for ordinal in range(count.value):+ dev = ctypes.c_int(0)+ buf = ctypes.create_string_buffer(64)+ integrated = ctypes.c_int(0)+ if (+ lib.cuDeviceGet(ctypes.byref(dev), ordinal) != 0+ or lib.cuDeviceGetPCIBusId(buf, 64, dev) != 0+ or lib.cuDeviceGetAttribute(ctypes.byref(integrated), 18, dev) != 0 # CU_DEVICE_ATTRIBUTE_INTEGRATED+ ):+ sys.exit(5)+ print(buf.value.decode(), integrated.value)+"""+++def _normalize_bus_id(bus_id: str) -> Optional[tuple[int, str]]:+ """'00000000:3B:00.0' (nvidia-smi) and '0000:3b:00.0' (driver API) -> (domain, 'BUS:DEV.FN')."""+ domain, sep, rest = bus_id.strip().partition(":")+ if not sep:+ return None+ try:+ return int(domain, 16), rest.upper()+ except ValueError:+ return None+++def get_cuda_ordinal_smi_rows() -> Optional[list[int]]:+ """nvidia-smi row index for each CUDA ordinal of this process's environment, matched by PCI bus id.++ Establishes the ordinal mapping under any CUDA_DEVICE_ORDER / numeric mask without opening a CUDA+ context here: the ordinals come from a short child on the driver API, the rows from nvidia-smi.+ None when either side is unreadable or the two do not join; [] when an integrated card is among them,+ whose nvidia-smi memory is a carve-out rather than its shared pool.+ """+ import sys++ from utils.child_stdio import utf8_child_env++ try:+ child = subprocess.run(+ [sys.executable, "-c", _CUDA_BUS_ID_SCRIPT],+ capture_output = True,+ text = True,+ encoding = "utf-8",+ errors = "replace",+ timeout = 15,+ env = utf8_child_env(child_env_without_native_path_secret()),+ **_windows_hidden_subprocess_kwargs(),+ )+ smi = gpu_query.run_nvidia_smi(+ [_nvidia_smi_executable(), "--query-gpu=index,pci.bus_id", "--format=csv,noheader"],+ capture_output = True,+ text = True,+ encoding = "utf-8",+ errors = "replace",+ timeout = 5,+ env = child_env_without_native_path_secret(),+ **_windows_hidden_subprocess_kwargs(),+ )+ except (OSError, subprocess.TimeoutExpired):+ return None+ if child.returncode != 0 or smi.returncode != 0:+ return None+ row_by_bus: dict[tuple[int, str], int] = {}+ for line in smi.stdout.strip().splitlines():+ index, _, bus_id = (part.strip() for part in line.partition(","))+ key = _normalize_bus_id(bus_id)+ if not index.isdigit() or key is None:+ return None+ row_by_bus[key] = int(index)+ rows = []+ for line in child.stdout.strip().splitlines():+ bus_id, _, integrated = line.strip().partition(" ")+ key = _normalize_bus_id(bus_id)+ if key not in row_by_bus or integrated not in ("0", "1"):+ return None+ if integrated == "1":+ # The row is a carve-out or N/A; only torch reports the shared pool. Known, so cacheable.+ return []+ rows.append(row_by_bus[key])+ return rows or None++def get_physical_gpu_inventory() -> dict[str, Any]:"""Every NVIDIA GPU the driver enumerates, with no visibility mask and no torch. Display-only inventory: ``index`` is nvidia-smi's own row number, a physical id and NOT something a caller may pin, because the whole point of this probe is that PyTorch cannot open these devices. A failed probe comes back as a structured unavailable result, so this never raises out of an endpoint."""rows = _query_gpu_inventory("get_physical_gpu_inventory")