Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: honour MIG masks and size MIG cards by their slice (#13927)

Daniel Han23bbee09b180
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 662c7ef117d..54978e964c5 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -14975,45 +14975,55 @@ class LlamaCppBackend:
def _nvml_rows_visible(rows: list[dict]) -> list[dict]:
"""The NVML rows CUDA_VISIBLE_DEVICES permits, in mask order: an index or a GPU-/MIG-
uuid prefix per entry, as the CUDA runtime reads it. An entry naming no single device
- ends the mask there: "0,2,-1,1" exposes 0 and 2, "-1" alone hides every GPU. A GPU in
- MIG mode is one slice to CUDA (the first, when the mask does not name one), so its
- first slice row stands in for the parent's whole-card memory, and a second slice of
- the same card is dropped: the rows are keyed by physical index, and one slice per
- card is what the driver exposed before R570 and what the child is pinned to."""
+ ends the mask there: "0,2,-1,1" exposes 0 and 2, "-1" alone hides every GPU, and a
+ repeated index empties it, as CUDA does. A GPU in MIG mode is one row backed by one
+ slice, which the child is pinned to: the first slice the mask names, else the one with
+ the least free memory, so the budget never exceeds the device the child gets (R570+
+ enumerates every slice, but one per card is sized and pinned)."""
raw = os.environ.get("CUDA_VISIBLE_DEVICES")
parents = [r for r in rows if not r.get("mig")]
+ def free_mib(r: dict) -> int:
+ try:
+ return int(r.get("memory_free_mib") or 0)
+ except (TypeError, ValueError):
+ return 0
+
def as_cuda_sees(r: dict) -> dict:
if r.get("mig"):
return r # a slice the mask named stays that slice
slices = [
s for s in rows if s.get("mig") and str(s.get("index")) == str(r.get("index"))
]
- return slices[0] if slices else r
+ return min(slices, key = free_mib) if slices else r
tokens = [t.strip() for t in (raw or "").split(",") if t.strip()]
picked: list[dict] = []
if raw is None:
picked = [as_cuda_sees(r) for r in parents]
+ seen: set[int] = set()
for token in tokens:
- if token.isdigit():
- hits = [r for r in parents if str(r.get("index")) == token]
- elif token.startswith("GPU-"):
+ prefix = re.match(r"\+?\d+", token)
+ if token.startswith("GPU-"):
hits = [r for r in parents if str(r.get("uuid", "")).startswith(token)]
elif token.startswith("MIG-"):
# A MIG- entry names a slice row the probe lists under its parent.
hits = [
r for r in rows if r.get("mig") and str(r.get("uuid", "")).startswith(token)
]
+ elif prefix:
+ # CUDA reads an index strtoul-style: "1gpu2" is GPU 1.
+ index = int(prefix.group())
+ if index in seen:
+ return []
+ seen.add(index)
+ hits = [r for r in parents if str(r.get("index")) == str(index)]
else:
hits = []
if len(hits) != 1:
break
- picked.extend(
- r
- for r in map(as_cuda_sees, hits)
- if str(r.get("index")) not in {str(p.get("index")) for p in picked}
- )
+ if str(hits[0].get("index")) not in {str(p.get("index")) for p in picked}:
+ picked.append(as_cuda_sees(hits[0]))
return picked
@staticmethod
@@ -15039,6 +15049,52 @@ class LlamaCppBackend:
return ",".join(by_index[i] for i in ids)
return ",".join(str(i) for i in ids)
+ @staticmethod
+ def _pinned_child_visibility(gpu_indices) -> Optional[str]:
+ """``_child_visibility_for``, or None when the inherited CUDA mask must stand: a MIG
+ or unresolvable UUID mask the ids cannot be mapped back to, where a numeric pin would
+ be re-read against the whole host (a card the parent hid, or the whole MIG card)."""
+ cvd = os.environ.get("CUDA_VISIBLE_DEVICES")
+ if (
+ LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES is False
+ and cvd is not None
+ and cvd.strip()
+ and LlamaCppBackend._visibility_mask_is_unmappable()
+ ):
+ # Torch ordinals: map through the mask itself, never through NVML rows cached by
+ # an earlier probe (their indices are physical, so ordinal 0 could name another slice).
+ return LlamaCppBackend._mig_tokens_for_ordinals(cvd, gpu_indices)
+ mask = LlamaCppBackend._child_visibility_for(gpu_indices)
+ if (
+ cvd is not None
+ and cvd.strip()
+ and mask == ",".join(str(int(i)) for i in gpu_indices)
+ and LlamaCppBackend._visible_devices_mask("CUDA_VISIBLE_DEVICES") is None
+ and LlamaCppBackend._visibility_mask_is_unmappable()
+ ):
+ return LlamaCppBackend._mig_tokens_for_ordinals(cvd, gpu_indices)
+ return mask
+
+ @staticmethod
+ def _mig_tokens_for_ordinals(cvd: str, gpu_indices) -> Optional[str]:
+ """The inherited MIG- entries behind torch ordinals, or None (keep the whole mask). Only
+ when torch enumerated every entry, so ordinal i is entry i in mask order."""
+ tokens = [t.strip() for t in cvd.split(",") if t.strip()]
+ if LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES is not False or not all(
+ t.startswith("MIG-") for t in tokens
+ ):
+ return None
+ try:
+ import torch
+ if torch.cuda.device_count() != len(tokens):
+ return None
+ ids = [int(i) for i in gpu_indices]
+ except Exception:
+ return None
+ if not ids or len(set(ids)) != len(ids) or not all(0 <= i < len(tokens) for i in ids):
+ return None
+ return ",".join(tokens[i] for i in ids)
+
@staticmethod
def _get_gpu_memory_nvml() -> list[tuple[int, int, int]]:
"""Free and total memory per NVIDIA GPU from NVML (studio/nvidia_probe.py in a child
@@ -15121,6 +15177,7 @@ class LlamaCppBackend:
return LlamaCppBackend._get_gpu_free_memory_vulkan(binary)
# ── NVIDIA via nvidia-smi ────────────────────────────────────
unsized_ids: list[int] = []
+ mig_card_rows: list[tuple[int, int, int]] = []
try:
from utils.hardware import gpu_query
@@ -15141,6 +15198,10 @@ class LlamaCppBackend:
)
if result.returncode == 0:
allowed = LlamaCppBackend._visible_devices_mask("CUDA_VISIBLE_DEVICES")
+ if allowed is None and (os.environ.get("CUDA_VISIBLE_DEVICES") or "").strip():
+ # CUDA's numeric-prefix reading ("1gpu2" is 1, "0,1,abc" is 0,1).
+ _ids = LlamaCppBackend._resolve_visible_physical_ids()
+ allowed = set(_ids) if _ids is not None else None
gpus: list[tuple[int, int, int]] = []
for line in result.stdout.strip().splitlines():
parts = [p.strip() for p in line.split(",")]
@@ -15170,6 +15231,23 @@ class LlamaCppBackend:
gpus.append((idx, free_mib, total_mib))
# Match the docstring's sort-by-id guarantee (driver order isn't).
gpus.sort(key = lambda g: g[0])
+ _cvd = os.environ.get("CUDA_VISIBLE_DEVICES")
+ if allowed is None and _cvd is not None and _cvd.strip():
+ # A MIG / unresolvable UUID mask names devices these rows cannot be
+ # matched to: listing every card sizes against GPUs the parent hid.
+ # The NVML branch names slices and uuids.
+ gpus = []
+ elif gpus:
+ from utils.hardware import nvidia
+ mig = nvidia.get_mig_gpu_indices() & {g[0] for g in gpus}
+ if mig:
+ # A MIG card's row is the whole card: swap in its NVML slice reading,
+ # keeping the whole cards. No reading at all: the later probes decide,
+ # then these rows as before.
+ slices = {g[0]: g for g in LlamaCppBackend._get_gpu_memory_nvml()}
+ mig_card_rows = gpus
+ gpus = [slices.get(g[0], g) if g[0] in mig else g for g in gpus]
+ gpus = [g for g in gpus if g[0] not in mig or g[0] in slices]
if gpus:
LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES = True
if for_llama_server:
@@ -15197,6 +15275,9 @@ class LlamaCppBackend:
# ── Torch fallback (covers AMD ROCm and missing nvidia-smi) ──
gpus = LlamaCppBackend._get_gpu_memory_torch(binary, for_llama_server)
+ if not gpus and mig_card_rows:
+ LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES = True
+ return LlamaCppBackend._widen_integrated_cuda_rows(mig_card_rows)
if gpus or not (for_llama_server and unsized_ids):
return gpus
return LlamaCppBackend._tegra_unified_memory_rows(unsized_ids)
@@ -31992,11 +32073,16 @@ class LlamaCppBackend:
_sm_gate = self._cuda_sm_gate_error(binary, _pin_ids)
if _sm_gate:
raise RuntimeError(_sm_gate)
- self._emit_child_gpu_visibility(
- env,
- LlamaCppBackend._child_visibility_for(_pin_ids),
- prefer_rocr = True,
- )
+ _child_mask = LlamaCppBackend._pinned_child_visibility(_pin_ids)
+ if _child_mask is None:
+ logger.info(
+ "Keeping the inherited CUDA_VISIBLE_DEVICES=%s: GPU ids %s "
+ "cannot be mapped to it.",
+ os.environ.get("CUDA_VISIBLE_DEVICES"),
+ _pin_ids,
+ )
+ else:
+ self._emit_child_gpu_visibility(env, _child_mask, prefer_rocr = True)
_child_gpu_physical_ids = tuple(int(i) for i in _pin_ids)
_launch_pinned_ids = list(_pin_ids)
elif not is_vulkan_backend and not gpu_ids:
diff --git a/studio/backend/tests/test_gpu_memory_nvml_fallback.py b/studio/backend/tests/test_gpu_memory_nvml_fallback.py
index 95b56540bc8..4b77bafc251 100644
--- a/studio/backend/tests/test_gpu_memory_nvml_fallback.py
+++ b/studio/backend/tests/test_gpu_memory_nvml_fallback.py
@@ -185,7 +185,8 @@ class TestTheMemoryProbeFallsBackToNvml:
raise AssertionError("no other probe should run")
monkeypatch.setattr(mod.subprocess, "run", smi_ok)
- assert LlamaCppBackend._get_gpu_memory() == [(0, 60000, 81920), (1, 20000, 24576)]
+ # nvidia-smi answering cannot place an unresolved uuid mask: no whole-card rows.
+ assert LlamaCppBackend._get_gpu_memory() == []
# The inherited uuid mask is still what the child gets, a MIG parent still its slice.
assert LlamaCppBackend._child_visibility_for([1, 0]) == "GPU-bbbb2222-1,MIG-cccc3333-0"
# The mask changing underneath is read as it is now, not as it was at the probe.
diff --git a/studio/backend/tests/test_mig_gpu_masks.py b/studio/backend/tests/test_mig_gpu_masks.py
new file mode 100644
index 00000000000..2a1e07a804c
--- /dev/null
+++ b/studio/backend/tests/test_mig_gpu_masks.py
@@ -0,0 +1,382 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+"""MIG hosts: a MIG mask is honoured, MIG slices are not GPUs, and a MIG card is sized by its slice.
+
+nvidia-smi rows are whole cards and ``nvidia-smi -L`` lists each MIG instance as an indented
+row under its card. CUDA sees a MIG card as one slice, so whole-card numbers over-fit it, and
+a numeric pin over a MIG mask hands the child the whole host.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+import types
+
+import pytest
+
+from core.inference import llama_cpp as mod
+from core.inference.llama_cpp import LlamaCppBackend
+import utils.hardware.hardware as hw
+from utils.hardware import nvidia
+
+GPU0 = "GPU-5c89852c-d268-c3f3-1b07-005d5ae1dc3f"
+GPU1 = "GPU-7d3a9e11-4b2c-11ee-8c90-0242ac120002"
+GPU2 = "GPU-0f1e2d3c-aaaa-bbbb-cccc-123456789abc"
+MIG0A = "MIG-c6d4f1ef-42e4-5de3-91c7-45d71c87eb3f"
+MIG0B = "MIG-1b2c3d4e-0000-5abc-8def-0123456789ab"
+MIG0C = "MIG-99aa88bb-7777-5ccc-9ddd-fedcba987654"
+
+A100_MIB = 40960
+
+
+def _smi_list(cards):
+ """``nvidia-smi -L``: cards = [(uuid, [(profile, mig_uuid), ...]), ...]; old_form uses MIG-GPU-<uuid>/gi/ci."""
+ lines = []
+ for i, (uuid, slices) in enumerate(cards):
+ lines.append(f"GPU {i}: NVIDIA A100-SXM4-40GB (UUID: {uuid})")
+ for j, (profile, mig_uuid) in enumerate(slices):
+ lines.append(f" MIG {profile:<12} Device {j}: (UUID: {mig_uuid})")
+ return "\n".join(lines) + "\n"
+
+
+def _nvml_row(
+ index,
+ uuid,
+ free,
+ total,
+ mig = False,
+):
+ row = {
+ "index": str(index),
+ "uuid": uuid,
+ "name": "NVIDIA A100-SXM4-40GB" + (" MIG" if mig else ""),
+ "compute_cap": "8.0",
+ "memory_total_mib": str(total),
+ "memory_free_mib": str(free),
+ }
+ if mig:
+ row["mig"] = "1"
+ return row
+
+
+def host(tmp_path, monkeypatch):
+ """A fake nvidia-smi (``-L``, memory and uuid queries) plus a fake NVML probe script."""
+ for var in ("CUDA_VISIBLE_DEVICES", "HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES"):
+ monkeypatch.delenv(var, raising = False)
+ monkeypatch.setenv("CUDA_DEVICE_ORDER", "PCI_BUS_ID")
+ monkeypatch.setenv("UNSLOTH_GPU_QUERY_CACHE", "0")
+ monkeypatch.setattr(nvidia, "_uuid_mask_cache", {})
+ monkeypatch.setattr(LlamaCppBackend, "_NVML_ROWS", [])
+ monkeypatch.setattr(LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda b = None: False))
+ monkeypatch.setattr(
+ LlamaCppBackend, "_find_llama_server_binary", staticmethod(lambda: "/opt/llama-server")
+ )
+ monkeypatch.setattr(
+ LlamaCppBackend, "_get_gpu_memory_amd_smi", staticmethod(lambda *a, **k: [])
+ )
+ monkeypatch.setattr(LlamaCppBackend, "_get_gpu_memory_torch", staticmethod(lambda *a, **k: []))
+ monkeypatch.setitem(sys.modules, "torch", None)
+ monkeypatch.setattr(mod.sys, "platform", "linux")
+ monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA)
+
+ payload_path = tmp_path / "payload.json"
+ script = tmp_path / "nvidia_probe.py"
+ script.write_text(
+ f"import sys; sys.stdout.write(open({str(payload_path)!r}).read())\n", encoding = "utf-8"
+ )
+ monkeypatch.setenv("UNSLOTH_NVIDIA_PROBE", str(script))
+ monkeypatch.delenv("UNSLOTH_NVIDIA_LIBRARY_PROBE", raising = False)
+
+ state = {"cards": [], "free": [], "nvml": None}
+ real_run = mod.subprocess.run
+
+ def run(cmd, *args, **kwargs):
+ if not cmd or os.path.basename(str(cmd[0])) != "nvidia-smi":
+ return real_run(cmd, *args, **kwargs)
+ ok = lambda out: types.SimpleNamespace(returncode = 0, stdout = out, stderr = "")
+ if "-L" in cmd:
+ return ok(_smi_list(state["cards"]))
+ query = next((a for a in cmd if a.startswith("--query-gpu=")), "")
+ if query == "--query-gpu=index,uuid":
+ return ok("".join(f"{i}, {u}\n" for i, (u, _) in enumerate(state["cards"])))
+ if query == "--query-gpu=index,memory.free,memory.total":
+ # Whole-card numbers, also for a card in MIG mode.
+ return ok("".join(f"{i}, {f}, {A100_MIB}\n" for i, f in enumerate(state["free"])))
+ return types.SimpleNamespace(returncode = 1, stdout = "", stderr = "")
+
+ monkeypatch.setattr(mod.subprocess, "run", run)
+
+ def set_host(
+ cards,
+ free,
+ nvml_rows,
+ driver = "",
+ ):
+ state["cards"], state["free"] = cards, free
+ payload = (
+ {"source": "nvml", "driver_version": driver, "devices": nvml_rows}
+ if nvml_rows is not None
+ else None
+ )
+ payload_path.write_text(json.dumps(payload), encoding = "utf-8")
+
+ return set_host
+
+
+# An A100 split into two 3g.20gb slices (19968 MiB each).
+TWO_SLICES = [(GPU0, [("3g.20gb", MIG0A), ("3g.20gb", MIG0B)])]
+TWO_SLICES_NVML = [
+ _nvml_row(0, GPU0, 38000, A100_MIB),
+ _nvml_row(0, MIG0A, 15000, 19968, mig = True),
+ _nvml_row(0, MIG0B, 19000, 19968, mig = True),
+]
+# Three whole A100s, card 0 split into three 2g.10gb slices (9856 MiB each).
+MIXED = [
+ (GPU0, [("2g.10gb", MIG0A), ("2g.10gb", MIG0B), ("2g.10gb", MIG0C)]),
+ (GPU1, []),
+ (GPU2, []),
+]
+MIXED_NVML = [
+ _nvml_row(0, GPU0, 39000, A100_MIB),
+ _nvml_row(0, MIG0A, 9000, 9856, mig = True),
+ _nvml_row(0, MIG0B, 9500, 9856, mig = True),
+ _nvml_row(0, MIG0C, 9800, 9856, mig = True),
+ _nvml_row(1, GPU1, 30000, A100_MIB),
+ _nvml_row(2, GPU2, 40000, A100_MIB),
+]
+
+
+class TestAMigMaskIsHonouredByLlamaCpp:
+ @pytest.mark.parametrize("mask", [MIG0B, MIG0B[:12]])
+ def test_a_mig_mask_is_sized_by_its_slice_and_kept_for_the_child(self, host, monkeypatch, mask):
+ host(MIXED, [39000, 30000, 40000], MIXED_NVML)
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", mask)
+ # Not three whole 40 GB cards the parent never exposed.
+ assert LlamaCppBackend._get_gpu_memory() == [(0, 9500, 9856)]
+ assert LlamaCppBackend._pinned_child_visibility([0]) == MIG0B
+
+ def test_the_old_mig_gpu_form_is_never_widened_to_a_numeric_pin(self, host, monkeypatch):
+ # Pre-R470 form: NVML on a newer driver names the slice MIG-<uuid>, so nothing matches.
+ host(TWO_SLICES, [38000], TWO_SLICES_NVML)
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", f"MIG-{GPU0}/1/0")
+ assert LlamaCppBackend._get_gpu_memory() == []
+ # Placement from another source (torch ordinal 0) must not become CUDA_VISIBLE_DEVICES=0.
+ assert LlamaCppBackend._pinned_child_visibility([0]) is None
+
+ def test_a_mig_mask_without_nvml_keeps_the_inherited_mask(self, host, monkeypatch):
+ host(TWO_SLICES, [38000], None)
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", MIG0A)
+ assert LlamaCppBackend._get_gpu_memory() == []
+ assert LlamaCppBackend._pinned_child_visibility([0]) is None
+
+ def test_whole_card_masks_still_pin_numerically(self, host, monkeypatch):
+ host(MIXED, [39000, 30000, 40000], MIXED_NVML)
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "1,2")
+ assert LlamaCppBackend._get_gpu_memory() == [(1, 30000, A100_MIB), (2, 40000, A100_MIB)]
+ assert LlamaCppBackend._pinned_child_visibility([1, 2]) == "1,2"
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", f"{GPU2},{GPU1}")
+ assert LlamaCppBackend._pinned_child_visibility([1, 2]) == "1,2"
+
+
+class TestAMigCardIsSizedByItsSlice:
+ def test_a_numeric_mask_on_a_mig_card_reads_the_slice(self, host, monkeypatch):
+ host(TWO_SLICES, [38000], TWO_SLICES_NVML)
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0")
+ # CUDA exposes the first slice, not the 38 GB free on the card.
+ assert LlamaCppBackend._get_gpu_memory() == [(0, 15000, 19968)]
+
+ def test_an_unmasked_mixed_host_reads_slice_and_whole_cards(self, host):
+ host(MIXED, [39000, 30000, 40000], MIXED_NVML)
+ assert LlamaCppBackend._get_gpu_memory() == [
+ (0, 9000, 9856),
+ (1, 30000, A100_MIB),
+ (2, 40000, A100_MIB),
+ ]
+
+ def test_auto_select_ranks_a_mig_card_by_its_slice(self, host, monkeypatch):
+ host(MIXED, [39000, 30000, 40000], MIXED_NVML)
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0,1")
+ monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA)
+ monkeypatch.setattr(hw, "IS_ROCM", False)
+ monkeypatch.setattr(
+ hw,
+ "estimate_required_model_memory_gb",
+ lambda *a, **k: (20.0, {"required_gb": 20.0, "model_size_source": "config"}),
+ )
+ # Whole-card telemetry: card 0 looks freer (39 GB) than card 1 (30 GB).
+ monkeypatch.setattr(
+ hw,
+ "get_visible_gpu_utilization",
+ lambda: {
+ "devices": [
+ {"index": 0, "vram_total_gb": 40.0, "vram_used_gb": 1.0},
+ {"index": 1, "vram_total_gb": 40.0, "vram_used_gb": 10.0},
+ ]
+ },
+ )
+ selected, metadata = hw.auto_select_gpu_ids("unsloth/test")
+ # The 9.6 GB slice cannot hold 20 GB; card 1 can, alone.
+ assert selected == [1]
+ assert metadata["selection_mode"] == "auto"
+
+ def test_auto_select_without_a_slice_reading_does_not_trust_the_card(self, host, monkeypatch):
+ host(MIXED, [39000, 30000, 40000], None)
+ devices = [
+ {"index": 0, "vram_total_gb": 40.0, "vram_used_gb": 1.0},
+ {"index": 1, "vram_total_gb": 40.0, "vram_used_gb": 10.0},
+ ]
+ out = hw._with_mig_instance_memory(devices)
+ assert out[0]["vram_total_gb"] is None and out[0]["vram_used_gb"] is None
+ assert out[1] == devices[1]
+
+ def test_a_host_without_mig_is_unchanged(self, host):
+ host([(GPU0, []), (GPU1, [])], [39000, 30000], None)
+ devices = [
+ {"index": 0, "vram_total_gb": 40.0, "vram_used_gb": 1.0},
+ {"index": 1, "vram_total_gb": 40.0, "vram_used_gb": 10.0},
+ ]
+ assert hw._with_mig_instance_memory(devices) == devices
+ assert LlamaCppBackend._get_gpu_memory() == [(0, 39000, A100_MIB), (1, 30000, A100_MIB)]
+
+
+MIG1A = "MIG-2e3f4a5b-1111-5bcd-9eef-112233445566"
+# Two GPU instances on one card, as an R570+ host enumerates them.
+TWO_GPU_INSTANCES = [dict(r) for r in TWO_SLICES_NVML]
+TWO_GPU_INSTANCES[1]["gpu_instance_id"], TWO_GPU_INSTANCES[2]["gpu_instance_id"] = "1", "2"
+
+
+class TestOneSlicePerMigCard:
+ # R570+ CUDA enumerates every slice; Studio still sizes and pins one per card, so a
+ # tensor split or a single-device load never counts memory the child cannot use.
+ @pytest.mark.parametrize(
+ "mask, expected, uuid",
+ [
+ (None, (0, 15000, 19968), MIG0A),
+ ("0", (0, 15000, 19968), MIG0A),
+ (f"{MIG0A},{MIG0B}", (0, 15000, 19968), MIG0A),
+ (f"{MIG0B},{MIG0A}", (0, 19000, 19968), MIG0B),
+ ],
+ )
+ def test_a_mig_card_is_one_slice(self, host, monkeypatch, mask, expected, uuid):
+ host(TWO_SLICES, [38000], TWO_GPU_INSTANCES, driver = "570.86.15")
+ if mask is not None:
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", mask)
+ assert LlamaCppBackend._get_gpu_memory() == [expected]
+ assert LlamaCppBackend._pinned_child_visibility([0]) == uuid
+
+ @pytest.mark.parametrize("mask", [None, "0"])
+ def test_an_unnamed_slice_is_the_least_free_one(self, host, monkeypatch, mask):
+ # The budget must not exceed the slice the child is pinned to, whichever CUDA lists first.
+ rows = [dict(r) for r in TWO_SLICES_NVML]
+ rows[1]["memory_free_mib"], rows[2]["memory_free_mib"] = "19000", "4000"
+ host(TWO_SLICES, [38000], rows)
+ if mask is not None:
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", mask)
+ assert LlamaCppBackend._get_gpu_memory() == [(0, 4000, 19968)]
+ assert LlamaCppBackend._pinned_child_visibility([0]) == MIG0B
+
+ @pytest.mark.parametrize("mask", ["1,1", "1gpu,1gpu", "0,2,0"])
+ def test_a_repeated_index_hides_every_gpu(self, host, monkeypatch, mask):
+ # CUDA empties the device set on a repeat (torch's _parse_visible_devices agrees).
+ host(MIXED, [39000, 30000, 40000], MIXED_NVML)
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", mask)
+ assert LlamaCppBackend._nvml_rows_visible(MIXED_NVML) == []
+
+ def test_torch_ordinals_ignore_rows_cached_by_an_earlier_probe(self, host, monkeypatch):
+ # Mask order is card 1's slice then card 0's: torch ordinal 0 is MIG1A, while the cached
+ # rows are keyed by physical index and would name card 0's slice.
+ rows = [
+ _nvml_row(0, GPU0, 38000, A100_MIB),
+ _nvml_row(0, MIG0A, 15000, 19968, mig = True),
+ _nvml_row(1, GPU1, 38000, A100_MIB),
+ _nvml_row(1, MIG1A, 17000, 19968, mig = True),
+ ]
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", f"{MIG1A},{MIG0A}")
+ monkeypatch.setattr(LlamaCppBackend, "_NVML_ROWS", rows)
+ monkeypatch.setattr(LlamaCppBackend, "_GPU_IDS_ARE_PCI_INDICES", False)
+ fake = types.SimpleNamespace(cuda = types.SimpleNamespace(device_count = lambda: 2))
+ monkeypatch.setitem(sys.modules, "torch", fake)
+ assert LlamaCppBackend._pinned_child_visibility([0]) == MIG1A
+
+ @pytest.mark.parametrize(
+ "count, picked, expected", [(2, [1], MIG0B), (2, [0, 1], None), (1, [0], None)]
+ )
+ def test_torch_ordinals_narrow_a_multi_mig_mask(
+ self, host, monkeypatch, count, picked, expected
+ ):
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", f"{MIG0A},{MIG0B}")
+ fake = types.SimpleNamespace(cuda = types.SimpleNamespace(device_count = lambda: count))
+ monkeypatch.setitem(sys.modules, "torch", fake)
+ monkeypatch.setattr(LlamaCppBackend, "_GPU_IDS_ARE_PCI_INDICES", False)
+ if expected is None and len(picked) == 2:
+ expected = f"{MIG0A},{MIG0B}"
+ assert LlamaCppBackend._pinned_child_visibility(picked) == expected
+
+
+class TestTelemetryFailuresNeverDropCards:
+ @pytest.mark.parametrize("mask, expected", [("1gpu2", [1]), ("1,2,abc", [1, 2])])
+ def test_a_numeric_prefix_mask_keeps_its_cards(self, host, monkeypatch, mask, expected):
+ host(MIXED, [39000, 30000, 40000], None)
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", mask)
+ free = {1: 30000, 2: 40000}
+ assert LlamaCppBackend._get_gpu_memory() == [(i, free[i], A100_MIB) for i in expected]
+
+ def test_whole_cards_survive_a_missing_slice_reading(self, host):
+ host(MIXED, [39000, 30000, 40000], None)
+ assert LlamaCppBackend._get_gpu_memory() == [(1, 30000, A100_MIB), (2, 40000, A100_MIB)]
+
+ def test_a_mig_only_host_without_any_reading_keeps_the_card_rows(self, host):
+ host(TWO_SLICES, [38000], None)
+ assert LlamaCppBackend._get_gpu_memory() == [(0, 38000, A100_MIB)]
+
+
+class TestMigSlicesAreNotGpus:
+ @pytest.fixture
+ def count_state(self, monkeypatch):
+ monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA)
+ monkeypatch.setattr(hw, "IS_ROCM", False)
+ monkeypatch.setattr(hw, "_physical_gpu_count", None)
+ monkeypatch.setattr(hw, "_physical_gpu_count_from_smi", False)
+ monkeypatch.setattr(hw, "_physical_gpu_count_checked_at", None, raising = False)
+
+ @pytest.mark.parametrize(
+ "cards, expected",
+ [
+ (TWO_SLICES, 1),
+ (MIXED, 3),
+ ([(GPU0, [("3g.20gb", f"MIG-{GPU0}/1/0"), ("3g.20gb", f"MIG-{GPU0}/2/0")])], 1),
+ ],
+ )
+ def test_the_physical_count_skips_mig_rows(self, host, count_state, cards, expected):
+ host(cards, [38000] * len(cards), None)
+ assert hw.get_physical_gpu_count() == expected
+ assert hw.get_parent_visible_gpu_ids() == list(range(expected))
+ with pytest.raises(ValueError):
+ hw.resolve_requested_gpu_ids([expected])
+
+ def test_mig_gpu_indices(self, host):
+ host(MIXED, [0, 0, 0], None)
+ assert nvidia.get_mig_gpu_indices() == {0}
+ host([(GPU0, []), (GPU1, [("1g.5gb", MIG0A)])], [0, 0], None)
+ assert nvidia.get_mig_gpu_indices() == {1}
+
+
+def test_auto_select_and_llama_cpp_budget_the_same_slice(host):
+ # One selected id is one worker device: training and llama.cpp both see one slice per card.
+ host(TWO_SLICES, [38000], TWO_GPU_INSTANCES, driver = "570.86.15")
+ devices = [{"index": 0, "vram_total_gb": 40.0, "vram_used_gb": 2.0}]
+ out = hw._with_mig_instance_memory(devices)
+ assert out[0]["vram_total_gb"] == round(19968 / 1024, 2)
+ assert out[0]["vram_used_gb"] == round((19968 - 15000) / 1024, 2)
+ assert LlamaCppBackend._get_gpu_memory() == [(0, 15000, 19968)]
+
+
+def test_mig_overlay_skips_a_non_cuda_backend(host, monkeypatch):
+ # Intel XPU indices are not NVIDIA indices: an NVIDIA MIG card at 0 must not touch XPU 0.
+ host(TWO_SLICES, [38000], TWO_SLICES_NVML)
+ monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.XPU)
+ devices = [{"index": 0, "vram_total_gb": 24.0, "vram_used_gb": 2.0}]
+ assert hw._with_mig_instance_memory(devices) == devices
diff --git a/studio/backend/tests/test_physical_gpu_count_refresh.py b/studio/backend/tests/test_physical_gpu_count_refresh.py
index b807917d806..958c68c2f47 100644
--- a/studio/backend/tests/test_physical_gpu_count_refresh.py
+++ b/studio/backend/tests/test_physical_gpu_count_refresh.py
@@ -82,6 +82,18 @@ def test_an_smi_that_answers_later_replaces_the_torch_fallback(smi):
assert hw._physical_gpu_count_from_smi is True
+def test_a_lower_smi_count_replaces_a_torch_count(smi, monkeypatch):
+ # R570+ torch counts each MIG slice; the first SMI answer is the physical count.
+ monkeypatch.setattr(hw, "_torch_get_physical_gpu_count", lambda: 7)
+ smi(None, seconds = 0.0)
+ assert hw.get_physical_gpu_count() == 7
+ smi(1)
+ assert hw.get_physical_gpu_count() == 1
+ assert hw._physical_gpu_count_from_smi is True
+ smi(0)
+ assert hw.get_physical_gpu_count() == 1
+
+
def test_non_cuda_counts_are_never_reprobed(smi, monkeypatch):
monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.MLX)
assert hw.get_physical_gpu_count() == 1
@@ -128,7 +140,7 @@ def test_the_refresh_asks_for_physical_rows_only(smi, monkeypatch):
hw.get_physical_gpu_count()
smi(1)
hw.get_physical_gpu_count()
- assert seen == [False, True]
+ assert seen == [True, True]
def test_the_ttl_starts_after_the_probe_returns(smi, monkeypatch):
diff --git a/studio/backend/tests/test_system_poll_no_cuda_context.py b/studio/backend/tests/test_system_poll_no_cuda_context.py
index e31a1a25cf5..2fa91b128a1 100644
--- a/studio/backend/tests/test_system_poll_no_cuda_context.py
+++ b/studio/backend/tests/test_system_poll_no_cuda_context.py
@@ -720,13 +720,13 @@ def test_physical_gpu_count_records_whether_the_smi_answered(monkeypatch):
monkeypatch.setattr(hw, "_physical_gpu_count_from_smi", False)
monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA)
monkeypatch.setattr(hw, "IS_ROCM", False)
- monkeypatch.setattr(nvidia, "get_physical_gpu_count", lambda: 4)
+ monkeypatch.setattr(nvidia, "get_physical_gpu_count", lambda gpu_rows_only = False: 4)
assert hw.get_physical_gpu_count() == 4
assert hw._physical_gpu_count_from_smi is True
monkeypatch.setattr(hw, "_physical_gpu_count", None)
monkeypatch.setattr(hw, "_physical_gpu_count_from_smi", False)
- monkeypatch.setattr(nvidia, "get_physical_gpu_count", lambda: None)
+ monkeypatch.setattr(nvidia, "get_physical_gpu_count", lambda gpu_rows_only = False: None)
monkeypatch.setattr(hw, "_torch_get_physical_gpu_count", lambda: 1)
assert hw.get_physical_gpu_count() == 1
assert hw._physical_gpu_count_from_smi is False
diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.py
index e485dc2c488..836e2782085 100644
--- a/studio/backend/utils/hardware/hardware.py
+++ b/studio/backend/utils/hardware/hardware.py
@@ -6120,6 +6120,7 @@ def auto_select_gpu_ids(
else:
utilization = get_visible_gpu_utilization()
devices = [d for d in utilization.get("devices", []) if d.get("index") not in _uncovered]
+ devices = _with_mig_instance_memory(devices)
parent_ids = _covered(get_parent_visible_gpu_ids())
if not devices:
@@ -6276,6 +6277,40 @@ def prepare_gpu_selection(
return selected_gpu_ids, metadata
+def _with_mig_instance_memory(devices: list[Dict[str, Any]]) -> list[Dict[str, Any]]:
+ """Replace a MIG card's whole-card nvidia-smi memory with the slice CUDA exposes
+ for it (NVML), or drop the reading when no slice reading exists so the ranking
+ skips it. Devices on cards without MIG instances are returned unchanged."""
+ if IS_ROCM or not devices or get_device() != DeviceType.CUDA:
+ return devices
+ try:
+ from . import nvidia
+ mig = nvidia.get_mig_gpu_indices() & {d.get("index") for d in devices}
+ except Exception:
+ return devices
+ if not mig:
+ return devices
+ slices: Dict[int, tuple[int, int]] = {}
+ try:
+ from core.inference.llama_cpp import LlamaCppBackend
+ slices = {idx: (free, total) for idx, free, total in LlamaCppBackend._get_gpu_memory_nvml()}
+ except Exception as e:
+ logger.debug("NVML MIG slice probe failed: %s", e)
+ out = []
+ for device in devices:
+ idx = device.get("index")
+ if idx in mig:
+ free_mib, total_mib = slices.get(idx, (0, 0))
+ known = total_mib > 0
+ device = {
+ **device,
+ "vram_total_gb": round(total_mib / 1024, 2) if known else None,
+ "vram_used_gb": round(max(0, total_mib - free_mib) / 1024, 2) if known else None,
+ }
+ out.append(device)
+ return out
+
+
def _nvidia_physical_gpu_row_count() -> Optional[int]:
# Physical rows only: MIG instances created after startup add indented `nvidia-smi -L` rows, not GPUs.
try:
@@ -6286,15 +6321,17 @@ def _nvidia_physical_gpu_row_count() -> Optional[int]:
def _run_physical_gpu_count_probe() -> None:
- """Keep a re-read NVIDIA count only if it grew: a failed probe or a lower count keeps the old one, so detection never narrows."""
+ """Keep a re-read NVIDIA count only if it grew: a failed probe or a lower count keeps the old one, so detection never narrows. The first SMI answer replaces a torch fallback count even when lower: torch counts CUDA devices, and R570+ enumerates each MIG slice as one."""
global _physical_gpu_count, _physical_gpu_count_from_smi, _physical_gpu_count_checked_at
count = _nvidia_physical_gpu_row_count()
# Stamped after the probe, so the next expiry also outlives gpu_query's own cache entry for it.
_physical_gpu_count_checked_at = time.monotonic()
previous = _physical_gpu_count
- if count is None or previous is None or count <= previous:
+ if count is None or previous is None:
+ return
+ if count <= previous and (_physical_gpu_count_from_smi or count < 1):
return
- logger.info("Physical GPU count grew from %d to %d; using the new GPUs", previous, count)
+ logger.info("Physical GPU count changed from %d to %d (nvidia-smi)", previous, count)
_physical_gpu_count = count
_physical_gpu_count_from_smi = True
@@ -6327,10 +6364,11 @@ def get_physical_gpu_count() -> int:
count = None
try:
if IS_ROCM:
- from . import amd as _smi_mod
+ from . import amd
+ count = amd.get_physical_gpu_count()
else:
- from . import nvidia as _smi_mod
- count = _smi_mod.get_physical_gpu_count()
+ # MIG slices are indented `-L` rows under their card, not GPUs to offer.
+ count = _nvidia_physical_gpu_row_count()
except Exception:
pass
if not IS_ROCM:
diff --git a/studio/backend/utils/hardware/nvidia.py b/studio/backend/utils/hardware/nvidia.py
index e853a915b06..4512f68010f 100644
--- a/studio/backend/utils/hardware/nvidia.py
+++ b/studio/backend/utils/hardware/nvidia.py
@@ -150,6 +150,36 @@ def get_physical_gpu_count(gpu_rows_only: bool = False) -> Optional[int]:
return None
+def get_mig_gpu_indices() -> set[int]:
+ """Physical indices of GPUs with MIG instances (indented ``MIG`` rows under their
+ ``GPU N:`` row in ``nvidia-smi -L``). CUDA sees such a card as one slice, so its
+ ``--query-gpu`` memory is the whole card's, not the device's. Empty on failure."""
+ try:
+ result = gpu_query.run_nvidia_smi(
+ [_nvidia_smi_executable(), "-L"],
+ capture_output = True,
+ text = True,
+ encoding = "utf-8",
+ errors = "replace",
+ timeout = 5,
+ env = child_env_without_native_path_secret(),
+ **_windows_hidden_subprocess_kwargs(),
+ )
+ except Exception:
+ return set()
+ if result.returncode != 0 or not isinstance(result.stdout, str):
+ return set()
+ indices: set[int] = set()
+ current: Optional[int] = None
+ for line in result.stdout.splitlines():
+ if line.startswith("GPU "):
+ head = line[4:].split(":", 1)[0].strip()
+ current = int(head) if head.isdecimal() else None
+ elif current is not None and line.strip().startswith("MIG "):
+ indices.add(current)
+ return indices
+
+
def get_primary_gpu_utilization() -> dict[str, Any]:
try:
result = gpu_query.run_nvidia_smi(