Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: size MIG CUDA instances from their own slice, never a physical card row (#15465)
Daniel Han89a08ba998b5
diff --git a/studio/backend/core/inference/diffusion_device.py b/studio/backend/core/inference/diffusion_device.pyindex 0420d99605b..9937bb8d713 100644--- a/studio/backend/core/inference/diffusion_device.py+++ b/studio/backend/core/inference/diffusion_device.py@@ -419,6 +419,9 @@ def resolve_auto_cuda_ordinal() -> Optional[int]:visible = hw.get_parent_visible_gpu_ids()if len(visible) < 2:return None+ if hw._cuda_instance_ids_active():+ # MIG instance ordinals are not nvidia-smi rows; keep card 0 explicit.+ return 0if not hw._cuda_order_matches_smi():# No ranking (rows are not ordinals), but on cards that are not provably identical keep the# current card explicit, so a pooled thread left on another card cannot preflight against it.@@ -494,11 +497,16 @@ def resolve_auto_cuda_ordinal() -> Optional[int]:def smi_free_total_mib(ordinal: int) -> tuple[Optional[int], Optional[int]]:"""(free, total) MiB of torch ordinal ``ordinal`` from nvidia-smi, so a card can be sized without a CUDA contexton it. (None, None) wherever ``resolve_auto_cuda_ordinal`` would not trust the rows (ROCm, a CUDA order other- than nvidia-smi's, a mask naming no card, an integrated or MIG card, missing telemetry). Never raises."""+ than nvidia-smi's, a mask naming no card, an integrated or MIG card, missing telemetry). An R570+ MIG instance+ ordinal is sized from that instance's own slice row, or (None, None) when it was not read. Never raises."""try:from utils.hardware import hardware as hw- if hw.IS_ROCM or not hw._cuda_order_matches_smi():+ if hw.IS_ROCM:+ return None, None+ if hw._cuda_instance_ids_active():+ return hw._cuda_instance_free_total_mib(int(ordinal))+ if not hw._cuda_order_matches_smi():return None, Nonevisible = hw.get_parent_visible_gpu_ids()if not 0 <= int(ordinal) < len(visible):diff --git a/studio/backend/tests/test_mig_instance_memory_readers.py b/studio/backend/tests/test_mig_instance_memory_readers.pynew file mode 100644index 00000000000..f0ec64fbe10--- /dev/null+++ b/studio/backend/tests/test_mig_instance_memory_readers.py@@ -0,0 +1,194 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""R570+ MIG: ids in the ``cuda_device`` namespace are CUDA instance ordinals, so every free-memory reader+sizes an instance from its own slice row, never from the nvidia-smi row of a physical card that happens to+share the number. CPU fixtures only: nvidia-smi, the NVML probe and torch are faked."""++from __future__ import annotations++import contextlib+import copy+import importlib.util+import sys+import types+from pathlib import Path+from unittest.mock import patch++import pytest++from core.inference import diffusion_device as dd+from core.inference import diffusion_multigpu as multi+from utils.hardware import gpu_query, nvidia+++def _fixture_module():+ # The MIG host fixture lives beside this file; "tests" also names the repo-root package.+ name = "_studio_mig_multi_instance_fixture"+ if name not in sys.modules:+ spec = importlib.util.spec_from_file_location(+ name, Path(__file__).with_name("test_mig_multi_instance.py")+ )+ module = importlib.util.module_from_spec(spec)+ sys.modules[name] = module+ spec.loader.exec_module(module)+ return sys.modules[name]+++fx = _fixture_module()++hw = fx.hw++OTHER = "GPU-dddddddd-0000-0000-0000-000000000000"+THIRD = "GPU-eeeeeeee-0000-0000-0000-000000000000"+SLICE_B = (18000, 19968) # instance B's own free / total MiB++[email protected](autouse = True)+def _probe_script(monkeypatch):+ import utils.prebuilt.update_flow as update_flow+ monkeypatch.setattr(update_flow, "find_installer_script", lambda **k: Path("mock-probe.py"))+++def mig_host(mig_cards = frozenset({0}), b_row = None):+ """Two inherited instances (A, B) plus whole cards 1 and 2, masked out. nvidia-smi rows 1 and 2+ report far more free memory than either slice, so a physical-row join is visible."""+ rows = copy.deepcopy(fx.ROWS) + [+ {**fx.row(OTHER, 72000, 81920), "index": "1"},+ {**fx.row(THIRD, 60000, 81920), "index": "2"},+ ]+ if b_row is not None:+ rows[2] = b_row+ real_run = gpu_query.run_nvidia_smi++ def run_smi(cmd, *args, **kwargs):+ if any("utilization.gpu" in str(arg) for arg in cmd):+ return types.SimpleNamespace(+ returncode = 0,+ stderr = "",+ stdout = (+ "0, 10, 40, 40000, 81920, 100, 400\n"+ "1, 10, 40, 9920, 81920, 100, 400\n"+ "2, 10, 40, 21920, 81920, 100, 400\n"+ ),+ )+ return real_run(cmd, *args, **kwargs)++ listing = (+ fx.LISTING+ + f"GPU 1: NVIDIA A100-SXM4-80GB (UUID: {OTHER})\n"+ + f"GPU 2: NVIDIA A100-SXM4-80GB (UUID: {THIRD})\n"+ )+ real_smi_visible = hw._smi_visible_utilization+ with contextlib.ExitStack() as s:+ s.enter_context(patch.object(fx, "LISTING", listing))+ s.enter_context(fx.host(mask = f"{fx.A},{fx.B_UUID}", rows = rows))+ s.enter_context(+ patch.object(nvidia, "get_mig_mode_gpu_indices_or_none", lambda: set(mig_cards))+ )+ s.enter_context(patch.object(nvidia, "get_mig_gpu_indices_or_none", lambda: set(mig_cards)))+ s.enter_context(+ patch.object(nvidia, "get_compute_capabilities", lambda: {i: (8, 0) for i in range(3)})+ )+ s.enter_context(+ patch.object(+ nvidia,+ "get_card_identities",+ lambda: {i: ("NVIDIA A100-SXM4-80GB", "81920", "8.0", "1410") for i in range(3)},+ )+ )+ s.enter_context(patch.object(hw, "_integrated_cuda_rows", lambda ids: ({}, "index")))+ s.enter_context(patch.object(hw, "_smi_visible_utilization", real_smi_visible))+ s.enter_context(patch.object(gpu_query, "run_nvidia_smi", run_smi))+ s.enter_context(patch.object(hw, "_cuda_card_identities", {}))+ s.enter_context(patch.object(hw, "_cuda_card_identities_gen", None))+ s.enter_context(+ patch.object(dd, "_PLANNED_ORDINAL", dd.ContextVar("t", default = dd._UNPLANNED))+ )+ s.enter_context(patch.object(dd, "_PINNED_AWAY_FROM_ZERO", False))+ assert hw.get_parent_visible_gpu_ids() == [0, 1]+ assert hw._get_parent_visible_gpu_spec()["index_kind"] == "cuda_device"+ yield+++def test_diffusion_smi_reader_sizes_the_instance_by_its_slice():+ with mig_host():+ assert dd.smi_free_total_mib(1) == SLICE_B+ assert dd.smi_free_total_mib(0) == (36000, 40192)+ assert dd.smi_free_total_mib(2) == (None, None)+++def test_automatic_companion_budget_reads_the_slice():+ with mig_host():+ token = multi._SELECTION.set(multi.MultiGpuSelection(0, (1,), automatic = True))+ try:+ assert multi._free_total_mib(1) == SLICE_B+ finally:+ multi._SELECTION.reset(token)+++def test_an_unread_slice_is_unknown_not_another_cards_memory():+ # A failed refresh keeps identities with zero memory: that is unknown, never physical card 1.+ with mig_host(b_row = fx.row(fx.B_UUID, 0, 0, 1)):+ assert dd.smi_free_total_mib(1) == (None, None)+ token = multi._SELECTION.set(multi.MultiGpuSelection(0, (1,), automatic = True))+ try:+ assert multi._free_total_mib(1) == (None, None)+ finally:+ multi._SELECTION.reset(token)+++def test_auto_ordinal_never_ranks_instances_by_physical_rows():+ # The instances sit on card 2, so instance ordinals 0 and 1 collide with whole cards 0 and 1, and+ # whole card 1 has the most free memory: ranking those rows would move the load to instance 1.+ with mig_host(mig_cards = {2}):+ assert dd.resolve_auto_cuda_ordinal() == 0+++def test_instance_ids_are_never_proven_identical_from_physical_rows():+ with mig_host(mig_cards = {2}):+ assert hw._cuda_cards_identical([0, 1]) is None+ assert hw._cuda_cards_identical() is False+++def test_training_utilization_lists_instance_rows():+ with mig_host():+ util = hw.get_gpu_utilization()+ assert util["index_kind"] == "cuda_device"+ totals = [d["vram_total_gb"] for d in util["devices"]]+ assert totals == [round(40192 / 1024, 2), round(19968 / 1024, 2)]+++def test_context_free_reader_uses_the_slice_row():+ with mig_host():+ total = SLICE_B[1] * 1024**2+ free = hw._context_free_cuda_memory_info(1, total)+ assert free == SLICE_B[0] * 1024**2+ with mig_host(b_row = fx.row(fx.B_UUID, 0, 0, 1)):+ assert hw._context_free_cuda_memory_info(1, SLICE_B[1] * 1024**2) is None+++def test_a_single_whole_card_keeps_its_nvidia_smi_reading(monkeypatch):+ spec = {+ "raw": None,+ "numeric_ids": [0],+ "supports_explicit_gpu_ids": True,+ "index_kind": "physical",+ }+ monkeypatch.setattr(hw, "IS_ROCM", False)+ monkeypatch.setattr(hw, "_get_parent_visible_gpu_spec", lambda: dict(spec))+ monkeypatch.setattr(hw, "get_parent_visible_gpu_ids", lambda: [0])+ monkeypatch.setattr(hw, "_cuda_order_matches_smi", lambda: True)+ monkeypatch.setattr(hw, "_integrated_cuda_rows", lambda ids: ({}, "index"))+ monkeypatch.setattr(hw, "_nvidia_mig_gpu_indices_or_none", lambda: set())+ monkeypatch.setattr(+ hw,+ "_smi_visible_utilization",+ lambda s: {"devices": [{"index": 0, "vram_total_gb": 24.0, "vram_used_gb": 4.0}]},+ )+ monkeypatch.setattr(dd, "_PLANNED_ORDINAL", dd.ContextVar("t", default = dd._UNPLANNED))+ assert hw._cuda_instance_ids_active() is False+ assert dd.smi_free_total_mib(0) == (20480, 24576)+ assert dd.smi_free_total_mib(1) == (None, None)+ assert dd.resolve_auto_cuda_ordinal() is Nonediff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.pyindex 1b89baf8f52..69960aa7323 100644--- a/studio/backend/utils/hardware/hardware.py+++ b/studio/backend/utils/hardware/hardware.py@@ -2391,6 +2391,8 @@ def _cuda_cards_identical(gpu_ids: Optional[list[int]] = None) -> Optional[bool]return Nonefrom . import gpu_query+ if gpu_ids is not None and _cuda_instance_ids_active():+ return Nonecount = get_physical_gpu_count()wanted = list(range(count)) if gpu_ids is None else list(gpu_ids)# Hardware re-detection (a swapped card) bumps the static generation.@@ -2440,6 +2442,34 @@ def _nvidia_mig_gpu_indices_or_none() -> Optional[set[int]]:return None+def _cuda_instance_ids_active() -> bool:+ """Whether GPU ids are R570+ MIG CUDA instance ordinals, which nvidia-smi rows (physical cards) never key."""+ try:+ return _get_parent_visible_gpu_spec().get("index_kind") == "cuda_device"+ except Exception:+ return False+++def _cuda_instance_free_total_mib(ordinal: int) -> tuple[Optional[int], Optional[int]]:+ """(free, total) MiB of CUDA device ``ordinal`` from its own MIG instance row (NVML slice memory), or+ (None, None) when that slice was not read. Never another card's figures. Never raises."""+ try:+ from . import mig++ rows = mig.visible_devices(fresh = True)+ if rows is None or not 0 <= int(ordinal) < len(rows):+ return None, None+ row = rows[int(ordinal)]+ total = int(row.get("memory_total_mib") or 0)+ free = int(row.get("memory_free_mib") or 0)+ # A failed refresh keeps identities with zero memory: unknown, not a full card.+ if total <= 0:+ return None, None+ return min(max(free, 0), total), total+ except Exception:+ return None, None++_cuda_ordinal_rows_cache: Dict[tuple, tuple[Optional[list[int]], float]] = {}_CUDA_ORDINAL_ROWS_RETRY_S = 300.0@@ -2623,6 +2653,14 @@ def _context_free_cuda_memory_info(return Nonereturn _free_in_torch_scope(total_bytes, used_bytes / (1024**3))+ if not IS_ROCM and parent_visible_spec.get("index_kind") == "cuda_device":+ free_mib, total_mib = _cuda_instance_free_total_mib(idx)+ if free_mib is None or total_mib is None:+ return None+ if abs(total_mib * 1024**2 - total_bytes) > max(total_bytes // 100, 16 * 1024**2):+ return None+ return _free_in_torch_scope(total_bytes, (total_mib - free_mib) / 1024)+# Prefer the vendor CLI: both nvidia-smi and amd-smi run out of process, so querying an idle backend leaves no context resident here.visible_ids = parent_visible_spec["numeric_ids"]if IS_ROCM:@@ -4574,6 +4612,15 @@ def get_gpu_utilization() -> Dict[str, Any]:if device == DeviceType.CUDA:parent_visible_spec = _get_parent_visible_gpu_spec()+ if not IS_ROCM and parent_visible_spec.get("index_kind") == "cuda_device":+ result = get_visible_gpu_utilization()+ if result.get("index_kind") == "cuda_device":+ return _gpu_utilization_payload(+ device,+ result.get("devices", []),+ parent_visible_gpu_ids = result.get("parent_visible_gpu_ids", []),+ index_kind = "cuda_device",+ )result = _smi_visible_utilization(parent_visible_spec)if result is not None and "devices" in result:devices = result["devices"]