Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: size MIG CUDA instances from their own slice, never a physical card row (#15465)

Daniel Han89a08ba998b5
diff --git a/studio/backend/core/inference/diffusion_device.py b/studio/backend/core/inference/diffusion_device.py
index 0420d99605b..9937bb8d713 100644
--- a/studio/backend/core/inference/diffusion_device.py
+++ b/studio/backend/core/inference/diffusion_device.py
@@ -419,6 +419,9 @@ def resolve_auto_cuda_ordinal() -> Optional[int]:
visible = hw.get_parent_visible_gpu_ids()
if len(visible) < 2:
return None
+ if hw._cuda_instance_ids_active():
+ # MIG instance ordinals are not nvidia-smi rows; keep card 0 explicit.
+ return 0
if not hw._cuda_order_matches_smi():
# No ranking (rows are not ordinals), but on cards that are not provably identical keep the
# current card explicit, so a pooled thread left on another card cannot preflight against it.
@@ -494,11 +497,16 @@ def resolve_auto_cuda_ordinal() -> Optional[int]:
def smi_free_total_mib(ordinal: int) -> tuple[Optional[int], Optional[int]]:
"""(free, total) MiB of torch ordinal ``ordinal`` from nvidia-smi, so a card can be sized without a CUDA context
on it. (None, None) wherever ``resolve_auto_cuda_ordinal`` would not trust the rows (ROCm, a CUDA order other
- than nvidia-smi's, a mask naming no card, an integrated or MIG card, missing telemetry). Never raises."""
+ than nvidia-smi's, a mask naming no card, an integrated or MIG card, missing telemetry). An R570+ MIG instance
+ ordinal is sized from that instance's own slice row, or (None, None) when it was not read. Never raises."""
try:
from utils.hardware import hardware as hw
- if hw.IS_ROCM or not hw._cuda_order_matches_smi():
+ if hw.IS_ROCM:
+ return None, None
+ if hw._cuda_instance_ids_active():
+ return hw._cuda_instance_free_total_mib(int(ordinal))
+ if not hw._cuda_order_matches_smi():
return None, None
visible = hw.get_parent_visible_gpu_ids()
if not 0 <= int(ordinal) < len(visible):
diff --git a/studio/backend/tests/test_mig_instance_memory_readers.py b/studio/backend/tests/test_mig_instance_memory_readers.py
new file mode 100644
index 00000000000..f0ec64fbe10
--- /dev/null
+++ b/studio/backend/tests/test_mig_instance_memory_readers.py
@@ -0,0 +1,194 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""R570+ MIG: ids in the ``cuda_device`` namespace are CUDA instance ordinals, so every free-memory reader
+sizes an instance from its own slice row, never from the nvidia-smi row of a physical card that happens to
+share the number. CPU fixtures only: nvidia-smi, the NVML probe and torch are faked."""
+
+from __future__ import annotations
+
+import contextlib
+import copy
+import importlib.util
+import sys
+import types
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+
+from core.inference import diffusion_device as dd
+from core.inference import diffusion_multigpu as multi
+from utils.hardware import gpu_query, nvidia
+
+
+def _fixture_module():
+ # The MIG host fixture lives beside this file; "tests" also names the repo-root package.
+ name = "_studio_mig_multi_instance_fixture"
+ if name not in sys.modules:
+ spec = importlib.util.spec_from_file_location(
+ name, Path(__file__).with_name("test_mig_multi_instance.py")
+ )
+ module = importlib.util.module_from_spec(spec)
+ sys.modules[name] = module
+ spec.loader.exec_module(module)
+ return sys.modules[name]
+
+
+fx = _fixture_module()
+
+hw = fx.hw
+
+OTHER = "GPU-dddddddd-0000-0000-0000-000000000000"
+THIRD = "GPU-eeeeeeee-0000-0000-0000-000000000000"
+SLICE_B = (18000, 19968) # instance B's own free / total MiB
+
+
[email protected](autouse = True)
+def _probe_script(monkeypatch):
+ import utils.prebuilt.update_flow as update_flow
+ monkeypatch.setattr(update_flow, "find_installer_script", lambda **k: Path("mock-probe.py"))
+
+
+def mig_host(mig_cards = frozenset({0}), b_row = None):
+ """Two inherited instances (A, B) plus whole cards 1 and 2, masked out. nvidia-smi rows 1 and 2
+ report far more free memory than either slice, so a physical-row join is visible."""
+ rows = copy.deepcopy(fx.ROWS) + [
+ {**fx.row(OTHER, 72000, 81920), "index": "1"},
+ {**fx.row(THIRD, 60000, 81920), "index": "2"},
+ ]
+ if b_row is not None:
+ rows[2] = b_row
+ real_run = gpu_query.run_nvidia_smi
+
+ def run_smi(cmd, *args, **kwargs):
+ if any("utilization.gpu" in str(arg) for arg in cmd):
+ return types.SimpleNamespace(
+ returncode = 0,
+ stderr = "",
+ stdout = (
+ "0, 10, 40, 40000, 81920, 100, 400\n"
+ "1, 10, 40, 9920, 81920, 100, 400\n"
+ "2, 10, 40, 21920, 81920, 100, 400\n"
+ ),
+ )
+ return real_run(cmd, *args, **kwargs)
+
+ listing = (
+ fx.LISTING
+ + f"GPU 1: NVIDIA A100-SXM4-80GB (UUID: {OTHER})\n"
+ + f"GPU 2: NVIDIA A100-SXM4-80GB (UUID: {THIRD})\n"
+ )
+ real_smi_visible = hw._smi_visible_utilization
+ with contextlib.ExitStack() as s:
+ s.enter_context(patch.object(fx, "LISTING", listing))
+ s.enter_context(fx.host(mask = f"{fx.A},{fx.B_UUID}", rows = rows))
+ s.enter_context(
+ patch.object(nvidia, "get_mig_mode_gpu_indices_or_none", lambda: set(mig_cards))
+ )
+ s.enter_context(patch.object(nvidia, "get_mig_gpu_indices_or_none", lambda: set(mig_cards)))
+ s.enter_context(
+ patch.object(nvidia, "get_compute_capabilities", lambda: {i: (8, 0) for i in range(3)})
+ )
+ s.enter_context(
+ patch.object(
+ nvidia,
+ "get_card_identities",
+ lambda: {i: ("NVIDIA A100-SXM4-80GB", "81920", "8.0", "1410") for i in range(3)},
+ )
+ )
+ s.enter_context(patch.object(hw, "_integrated_cuda_rows", lambda ids: ({}, "index")))
+ s.enter_context(patch.object(hw, "_smi_visible_utilization", real_smi_visible))
+ s.enter_context(patch.object(gpu_query, "run_nvidia_smi", run_smi))
+ s.enter_context(patch.object(hw, "_cuda_card_identities", {}))
+ s.enter_context(patch.object(hw, "_cuda_card_identities_gen", None))
+ s.enter_context(
+ patch.object(dd, "_PLANNED_ORDINAL", dd.ContextVar("t", default = dd._UNPLANNED))
+ )
+ s.enter_context(patch.object(dd, "_PINNED_AWAY_FROM_ZERO", False))
+ assert hw.get_parent_visible_gpu_ids() == [0, 1]
+ assert hw._get_parent_visible_gpu_spec()["index_kind"] == "cuda_device"
+ yield
+
+
+def test_diffusion_smi_reader_sizes_the_instance_by_its_slice():
+ with mig_host():
+ assert dd.smi_free_total_mib(1) == SLICE_B
+ assert dd.smi_free_total_mib(0) == (36000, 40192)
+ assert dd.smi_free_total_mib(2) == (None, None)
+
+
+def test_automatic_companion_budget_reads_the_slice():
+ with mig_host():
+ token = multi._SELECTION.set(multi.MultiGpuSelection(0, (1,), automatic = True))
+ try:
+ assert multi._free_total_mib(1) == SLICE_B
+ finally:
+ multi._SELECTION.reset(token)
+
+
+def test_an_unread_slice_is_unknown_not_another_cards_memory():
+ # A failed refresh keeps identities with zero memory: that is unknown, never physical card 1.
+ with mig_host(b_row = fx.row(fx.B_UUID, 0, 0, 1)):
+ assert dd.smi_free_total_mib(1) == (None, None)
+ token = multi._SELECTION.set(multi.MultiGpuSelection(0, (1,), automatic = True))
+ try:
+ assert multi._free_total_mib(1) == (None, None)
+ finally:
+ multi._SELECTION.reset(token)
+
+
+def test_auto_ordinal_never_ranks_instances_by_physical_rows():
+ # The instances sit on card 2, so instance ordinals 0 and 1 collide with whole cards 0 and 1, and
+ # whole card 1 has the most free memory: ranking those rows would move the load to instance 1.
+ with mig_host(mig_cards = {2}):
+ assert dd.resolve_auto_cuda_ordinal() == 0
+
+
+def test_instance_ids_are_never_proven_identical_from_physical_rows():
+ with mig_host(mig_cards = {2}):
+ assert hw._cuda_cards_identical([0, 1]) is None
+ assert hw._cuda_cards_identical() is False
+
+
+def test_training_utilization_lists_instance_rows():
+ with mig_host():
+ util = hw.get_gpu_utilization()
+ assert util["index_kind"] == "cuda_device"
+ totals = [d["vram_total_gb"] for d in util["devices"]]
+ assert totals == [round(40192 / 1024, 2), round(19968 / 1024, 2)]
+
+
+def test_context_free_reader_uses_the_slice_row():
+ with mig_host():
+ total = SLICE_B[1] * 1024**2
+ free = hw._context_free_cuda_memory_info(1, total)
+ assert free == SLICE_B[0] * 1024**2
+ with mig_host(b_row = fx.row(fx.B_UUID, 0, 0, 1)):
+ assert hw._context_free_cuda_memory_info(1, SLICE_B[1] * 1024**2) is None
+
+
+def test_a_single_whole_card_keeps_its_nvidia_smi_reading(monkeypatch):
+ spec = {
+ "raw": None,
+ "numeric_ids": [0],
+ "supports_explicit_gpu_ids": True,
+ "index_kind": "physical",
+ }
+ monkeypatch.setattr(hw, "IS_ROCM", False)
+ monkeypatch.setattr(hw, "_get_parent_visible_gpu_spec", lambda: dict(spec))
+ monkeypatch.setattr(hw, "get_parent_visible_gpu_ids", lambda: [0])
+ monkeypatch.setattr(hw, "_cuda_order_matches_smi", lambda: True)
+ monkeypatch.setattr(hw, "_integrated_cuda_rows", lambda ids: ({}, "index"))
+ monkeypatch.setattr(hw, "_nvidia_mig_gpu_indices_or_none", lambda: set())
+ monkeypatch.setattr(
+ hw,
+ "_smi_visible_utilization",
+ lambda s: {"devices": [{"index": 0, "vram_total_gb": 24.0, "vram_used_gb": 4.0}]},
+ )
+ monkeypatch.setattr(dd, "_PLANNED_ORDINAL", dd.ContextVar("t", default = dd._UNPLANNED))
+ assert hw._cuda_instance_ids_active() is False
+ assert dd.smi_free_total_mib(0) == (20480, 24576)
+ assert dd.smi_free_total_mib(1) == (None, None)
+ assert dd.resolve_auto_cuda_ordinal() is None
diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.py
index 1b89baf8f52..69960aa7323 100644
--- a/studio/backend/utils/hardware/hardware.py
+++ b/studio/backend/utils/hardware/hardware.py
@@ -2391,6 +2391,8 @@ def _cuda_cards_identical(gpu_ids: Optional[list[int]] = None) -> Optional[bool]
return None
from . import gpu_query
+ if gpu_ids is not None and _cuda_instance_ids_active():
+ return None
count = get_physical_gpu_count()
wanted = list(range(count)) if gpu_ids is None else list(gpu_ids)
# Hardware re-detection (a swapped card) bumps the static generation.
@@ -2440,6 +2442,34 @@ def _nvidia_mig_gpu_indices_or_none() -> Optional[set[int]]:
return None
+def _cuda_instance_ids_active() -> bool:
+ """Whether GPU ids are R570+ MIG CUDA instance ordinals, which nvidia-smi rows (physical cards) never key."""
+ try:
+ return _get_parent_visible_gpu_spec().get("index_kind") == "cuda_device"
+ except Exception:
+ return False
+
+
+def _cuda_instance_free_total_mib(ordinal: int) -> tuple[Optional[int], Optional[int]]:
+ """(free, total) MiB of CUDA device ``ordinal`` from its own MIG instance row (NVML slice memory), or
+ (None, None) when that slice was not read. Never another card's figures. Never raises."""
+ try:
+ from . import mig
+
+ rows = mig.visible_devices(fresh = True)
+ if rows is None or not 0 <= int(ordinal) < len(rows):
+ return None, None
+ row = rows[int(ordinal)]
+ total = int(row.get("memory_total_mib") or 0)
+ free = int(row.get("memory_free_mib") or 0)
+ # A failed refresh keeps identities with zero memory: unknown, not a full card.
+ if total <= 0:
+ return None, None
+ return min(max(free, 0), total), total
+ except Exception:
+ return None, None
+
+
_cuda_ordinal_rows_cache: Dict[tuple, tuple[Optional[list[int]], float]] = {}
_CUDA_ORDINAL_ROWS_RETRY_S = 300.0
@@ -2623,6 +2653,14 @@ def _context_free_cuda_memory_info(
return None
return _free_in_torch_scope(total_bytes, used_bytes / (1024**3))
+ if not IS_ROCM and parent_visible_spec.get("index_kind") == "cuda_device":
+ free_mib, total_mib = _cuda_instance_free_total_mib(idx)
+ if free_mib is None or total_mib is None:
+ return None
+ if abs(total_mib * 1024**2 - total_bytes) > max(total_bytes // 100, 16 * 1024**2):
+ return None
+ return _free_in_torch_scope(total_bytes, (total_mib - free_mib) / 1024)
+
# Prefer the vendor CLI: both nvidia-smi and amd-smi run out of process, so querying an idle backend leaves no context resident here.
visible_ids = parent_visible_spec["numeric_ids"]
if IS_ROCM:
@@ -4574,6 +4612,15 @@ def get_gpu_utilization() -> Dict[str, Any]:
if device == DeviceType.CUDA:
parent_visible_spec = _get_parent_visible_gpu_spec()
+ if not IS_ROCM and parent_visible_spec.get("index_kind") == "cuda_device":
+ result = get_visible_gpu_utilization()
+ if result.get("index_kind") == "cuda_device":
+ return _gpu_utilization_payload(
+ device,
+ result.get("devices", []),
+ parent_visible_gpu_ids = result.get("parent_visible_gpu_ids", []),
+ index_kind = "cuda_device",
+ )
result = _smi_visible_utilization(parent_visible_spec)
if result is not None and "devices" in result:
devices = result["devices"]