Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: honour MIG masks and size MIG cards by their slice (#13927)
Daniel Han23bbee09b180
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 662c7ef117d..54978e964c5 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -14975,45 +14975,55 @@ class LlamaCppBackend:def _nvml_rows_visible(rows: list[dict]) -> list[dict]:"""The NVML rows CUDA_VISIBLE_DEVICES permits, in mask order: an index or a GPU-/MIG-uuid prefix per entry, as the CUDA runtime reads it. An entry naming no single device- ends the mask there: "0,2,-1,1" exposes 0 and 2, "-1" alone hides every GPU. A GPU in- MIG mode is one slice to CUDA (the first, when the mask does not name one), so its- first slice row stands in for the parent's whole-card memory, and a second slice of- the same card is dropped: the rows are keyed by physical index, and one slice per- card is what the driver exposed before R570 and what the child is pinned to."""+ ends the mask there: "0,2,-1,1" exposes 0 and 2, "-1" alone hides every GPU, and a+ repeated index empties it, as CUDA does. A GPU in MIG mode is one row backed by one+ slice, which the child is pinned to: the first slice the mask names, else the one with+ the least free memory, so the budget never exceeds the device the child gets (R570++ enumerates every slice, but one per card is sized and pinned)."""raw = os.environ.get("CUDA_VISIBLE_DEVICES")parents = [r for r in rows if not r.get("mig")]+ def free_mib(r: dict) -> int:+ try:+ return int(r.get("memory_free_mib") or 0)+ except (TypeError, ValueError):+ return 0+def as_cuda_sees(r: dict) -> dict:if r.get("mig"):return r # a slice the mask named stays that sliceslices = [s for s in rows if s.get("mig") and str(s.get("index")) == str(r.get("index"))]- return slices[0] if slices else r+ return min(slices, key = free_mib) if slices else rtokens = [t.strip() for t in (raw or "").split(",") if t.strip()]picked: list[dict] = []if raw is None:picked = [as_cuda_sees(r) for r in parents]+ seen: set[int] = set()for token in tokens:- if token.isdigit():- hits = [r for r in parents if str(r.get("index")) == token]- elif token.startswith("GPU-"):+ prefix = re.match(r"\+?\d+", token)+ if token.startswith("GPU-"):hits = [r for r in parents if str(r.get("uuid", "")).startswith(token)]elif token.startswith("MIG-"):# A MIG- entry names a slice row the probe lists under its parent.hits = [r for r in rows if r.get("mig") and str(r.get("uuid", "")).startswith(token)]+ elif prefix:+ # CUDA reads an index strtoul-style: "1gpu2" is GPU 1.+ index = int(prefix.group())+ if index in seen:+ return []+ seen.add(index)+ hits = [r for r in parents if str(r.get("index")) == str(index)]else:hits = []if len(hits) != 1:break- picked.extend(- r- for r in map(as_cuda_sees, hits)- if str(r.get("index")) not in {str(p.get("index")) for p in picked}- )+ if str(hits[0].get("index")) not in {str(p.get("index")) for p in picked}:+ picked.append(as_cuda_sees(hits[0]))return picked@staticmethod@@ -15039,6 +15049,52 @@ class LlamaCppBackend:return ",".join(by_index[i] for i in ids)return ",".join(str(i) for i in ids)+ @staticmethod+ def _pinned_child_visibility(gpu_indices) -> Optional[str]:+ """``_child_visibility_for``, or None when the inherited CUDA mask must stand: a MIG+ or unresolvable UUID mask the ids cannot be mapped back to, where a numeric pin would+ be re-read against the whole host (a card the parent hid, or the whole MIG card)."""+ cvd = os.environ.get("CUDA_VISIBLE_DEVICES")+ if (+ LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES is False+ and cvd is not None+ and cvd.strip()+ and LlamaCppBackend._visibility_mask_is_unmappable()+ ):+ # Torch ordinals: map through the mask itself, never through NVML rows cached by+ # an earlier probe (their indices are physical, so ordinal 0 could name another slice).+ return LlamaCppBackend._mig_tokens_for_ordinals(cvd, gpu_indices)+ mask = LlamaCppBackend._child_visibility_for(gpu_indices)+ if (+ cvd is not None+ and cvd.strip()+ and mask == ",".join(str(int(i)) for i in gpu_indices)+ and LlamaCppBackend._visible_devices_mask("CUDA_VISIBLE_DEVICES") is None+ and LlamaCppBackend._visibility_mask_is_unmappable()+ ):+ return LlamaCppBackend._mig_tokens_for_ordinals(cvd, gpu_indices)+ return mask++ @staticmethod+ def _mig_tokens_for_ordinals(cvd: str, gpu_indices) -> Optional[str]:+ """The inherited MIG- entries behind torch ordinals, or None (keep the whole mask). Only+ when torch enumerated every entry, so ordinal i is entry i in mask order."""+ tokens = [t.strip() for t in cvd.split(",") if t.strip()]+ if LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES is not False or not all(+ t.startswith("MIG-") for t in tokens+ ):+ return None+ try:+ import torch+ if torch.cuda.device_count() != len(tokens):+ return None+ ids = [int(i) for i in gpu_indices]+ except Exception:+ return None+ if not ids or len(set(ids)) != len(ids) or not all(0 <= i < len(tokens) for i in ids):+ return None+ return ",".join(tokens[i] for i in ids)+@staticmethoddef _get_gpu_memory_nvml() -> list[tuple[int, int, int]]:"""Free and total memory per NVIDIA GPU from NVML (studio/nvidia_probe.py in a child@@ -15121,6 +15177,7 @@ class LlamaCppBackend:return LlamaCppBackend._get_gpu_free_memory_vulkan(binary)# ── NVIDIA via nvidia-smi ────────────────────────────────────unsized_ids: list[int] = []+ mig_card_rows: list[tuple[int, int, int]] = []try:from utils.hardware import gpu_query@@ -15141,6 +15198,10 @@ class LlamaCppBackend:)if result.returncode == 0:allowed = LlamaCppBackend._visible_devices_mask("CUDA_VISIBLE_DEVICES")+ if allowed is None and (os.environ.get("CUDA_VISIBLE_DEVICES") or "").strip():+ # CUDA's numeric-prefix reading ("1gpu2" is 1, "0,1,abc" is 0,1).+ _ids = LlamaCppBackend._resolve_visible_physical_ids()+ allowed = set(_ids) if _ids is not None else Nonegpus: list[tuple[int, int, int]] = []for line in result.stdout.strip().splitlines():parts = [p.strip() for p in line.split(",")]@@ -15170,6 +15231,23 @@ class LlamaCppBackend:gpus.append((idx, free_mib, total_mib))# Match the docstring's sort-by-id guarantee (driver order isn't).gpus.sort(key = lambda g: g[0])+ _cvd = os.environ.get("CUDA_VISIBLE_DEVICES")+ if allowed is None and _cvd is not None and _cvd.strip():+ # A MIG / unresolvable UUID mask names devices these rows cannot be+ # matched to: listing every card sizes against GPUs the parent hid.+ # The NVML branch names slices and uuids.+ gpus = []+ elif gpus:+ from utils.hardware import nvidia+ mig = nvidia.get_mig_gpu_indices() & {g[0] for g in gpus}+ if mig:+ # A MIG card's row is the whole card: swap in its NVML slice reading,+ # keeping the whole cards. No reading at all: the later probes decide,+ # then these rows as before.+ slices = {g[0]: g for g in LlamaCppBackend._get_gpu_memory_nvml()}+ mig_card_rows = gpus+ gpus = [slices.get(g[0], g) if g[0] in mig else g for g in gpus]+ gpus = [g for g in gpus if g[0] not in mig or g[0] in slices]if gpus:LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES = Trueif for_llama_server:@@ -15197,6 +15275,9 @@ class LlamaCppBackend:# ── Torch fallback (covers AMD ROCm and missing nvidia-smi) ──gpus = LlamaCppBackend._get_gpu_memory_torch(binary, for_llama_server)+ if not gpus and mig_card_rows:+ LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES = True+ return LlamaCppBackend._widen_integrated_cuda_rows(mig_card_rows)if gpus or not (for_llama_server and unsized_ids):return gpusreturn LlamaCppBackend._tegra_unified_memory_rows(unsized_ids)@@ -31992,11 +32073,16 @@ class LlamaCppBackend:_sm_gate = self._cuda_sm_gate_error(binary, _pin_ids)if _sm_gate:raise RuntimeError(_sm_gate)- self._emit_child_gpu_visibility(- env,- LlamaCppBackend._child_visibility_for(_pin_ids),- prefer_rocr = True,- )+ _child_mask = LlamaCppBackend._pinned_child_visibility(_pin_ids)+ if _child_mask is None:+ logger.info(+ "Keeping the inherited CUDA_VISIBLE_DEVICES=%s: GPU ids %s "+ "cannot be mapped to it.",+ os.environ.get("CUDA_VISIBLE_DEVICES"),+ _pin_ids,+ )+ else:+ self._emit_child_gpu_visibility(env, _child_mask, prefer_rocr = True)_child_gpu_physical_ids = tuple(int(i) for i in _pin_ids)_launch_pinned_ids = list(_pin_ids)elif not is_vulkan_backend and not gpu_ids:diff --git a/studio/backend/tests/test_gpu_memory_nvml_fallback.py b/studio/backend/tests/test_gpu_memory_nvml_fallback.pyindex 95b56540bc8..4b77bafc251 100644--- a/studio/backend/tests/test_gpu_memory_nvml_fallback.py+++ b/studio/backend/tests/test_gpu_memory_nvml_fallback.py@@ -185,7 +185,8 @@ class TestTheMemoryProbeFallsBackToNvml:raise AssertionError("no other probe should run")monkeypatch.setattr(mod.subprocess, "run", smi_ok)- assert LlamaCppBackend._get_gpu_memory() == [(0, 60000, 81920), (1, 20000, 24576)]+ # nvidia-smi answering cannot place an unresolved uuid mask: no whole-card rows.+ assert LlamaCppBackend._get_gpu_memory() == []# The inherited uuid mask is still what the child gets, a MIG parent still its slice.assert LlamaCppBackend._child_visibility_for([1, 0]) == "GPU-bbbb2222-1,MIG-cccc3333-0"# The mask changing underneath is read as it is now, not as it was at the probe.diff --git a/studio/backend/tests/test_mig_gpu_masks.py b/studio/backend/tests/test_mig_gpu_masks.pynew file mode 100644index 00000000000..2a1e07a804c--- /dev/null+++ b/studio/backend/tests/test_mig_gpu_masks.py@@ -0,0 +1,382 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0+"""MIG hosts: a MIG mask is honoured, MIG slices are not GPUs, and a MIG card is sized by its slice.++nvidia-smi rows are whole cards and ``nvidia-smi -L`` lists each MIG instance as an indented+row under its card. CUDA sees a MIG card as one slice, so whole-card numbers over-fit it, and+a numeric pin over a MIG mask hands the child the whole host.+"""++from __future__ import annotations++import json+import os+import sys+import types++import pytest++from core.inference import llama_cpp as mod+from core.inference.llama_cpp import LlamaCppBackend+import utils.hardware.hardware as hw+from utils.hardware import nvidia++GPU0 = "GPU-5c89852c-d268-c3f3-1b07-005d5ae1dc3f"+GPU1 = "GPU-7d3a9e11-4b2c-11ee-8c90-0242ac120002"+GPU2 = "GPU-0f1e2d3c-aaaa-bbbb-cccc-123456789abc"+MIG0A = "MIG-c6d4f1ef-42e4-5de3-91c7-45d71c87eb3f"+MIG0B = "MIG-1b2c3d4e-0000-5abc-8def-0123456789ab"+MIG0C = "MIG-99aa88bb-7777-5ccc-9ddd-fedcba987654"++A100_MIB = 40960+++def _smi_list(cards):+ """``nvidia-smi -L``: cards = [(uuid, [(profile, mig_uuid), ...]), ...]; old_form uses MIG-GPU-<uuid>/gi/ci."""+ lines = []+ for i, (uuid, slices) in enumerate(cards):+ lines.append(f"GPU {i}: NVIDIA A100-SXM4-40GB (UUID: {uuid})")+ for j, (profile, mig_uuid) in enumerate(slices):+ lines.append(f" MIG {profile:<12} Device {j}: (UUID: {mig_uuid})")+ return "\n".join(lines) + "\n"+++def _nvml_row(+ index,+ uuid,+ free,+ total,+ mig = False,+):+ row = {+ "index": str(index),+ "uuid": uuid,+ "name": "NVIDIA A100-SXM4-40GB" + (" MIG" if mig else ""),+ "compute_cap": "8.0",+ "memory_total_mib": str(total),+ "memory_free_mib": str(free),+ }+ if mig:+ row["mig"] = "1"+ return row+++def host(tmp_path, monkeypatch):+ """A fake nvidia-smi (``-L``, memory and uuid queries) plus a fake NVML probe script."""+ for var in ("CUDA_VISIBLE_DEVICES", "HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES"):+ monkeypatch.delenv(var, raising = False)+ monkeypatch.setenv("CUDA_DEVICE_ORDER", "PCI_BUS_ID")+ monkeypatch.setenv("UNSLOTH_GPU_QUERY_CACHE", "0")+ monkeypatch.setattr(nvidia, "_uuid_mask_cache", {})+ monkeypatch.setattr(LlamaCppBackend, "_NVML_ROWS", [])+ monkeypatch.setattr(LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda b = None: False))+ monkeypatch.setattr(+ LlamaCppBackend, "_find_llama_server_binary", staticmethod(lambda: "/opt/llama-server")+ )+ monkeypatch.setattr(+ LlamaCppBackend, "_get_gpu_memory_amd_smi", staticmethod(lambda *a, **k: [])+ )+ monkeypatch.setattr(LlamaCppBackend, "_get_gpu_memory_torch", staticmethod(lambda *a, **k: []))+ monkeypatch.setitem(sys.modules, "torch", None)+ monkeypatch.setattr(mod.sys, "platform", "linux")+ monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA)++ payload_path = tmp_path / "payload.json"+ script = tmp_path / "nvidia_probe.py"+ script.write_text(+ f"import sys; sys.stdout.write(open({str(payload_path)!r}).read())\n", encoding = "utf-8"+ )+ monkeypatch.setenv("UNSLOTH_NVIDIA_PROBE", str(script))+ monkeypatch.delenv("UNSLOTH_NVIDIA_LIBRARY_PROBE", raising = False)++ state = {"cards": [], "free": [], "nvml": None}+ real_run = mod.subprocess.run++ def run(cmd, *args, **kwargs):+ if not cmd or os.path.basename(str(cmd[0])) != "nvidia-smi":+ return real_run(cmd, *args, **kwargs)+ ok = lambda out: types.SimpleNamespace(returncode = 0, stdout = out, stderr = "")+ if "-L" in cmd:+ return ok(_smi_list(state["cards"]))+ query = next((a for a in cmd if a.startswith("--query-gpu=")), "")+ if query == "--query-gpu=index,uuid":+ return ok("".join(f"{i}, {u}\n" for i, (u, _) in enumerate(state["cards"])))+ if query == "--query-gpu=index,memory.free,memory.total":+ # Whole-card numbers, also for a card in MIG mode.+ return ok("".join(f"{i}, {f}, {A100_MIB}\n" for i, f in enumerate(state["free"])))+ return types.SimpleNamespace(returncode = 1, stdout = "", stderr = "")++ monkeypatch.setattr(mod.subprocess, "run", run)++ def set_host(+ cards,+ free,+ nvml_rows,+ driver = "",+ ):+ state["cards"], state["free"] = cards, free+ payload = (+ {"source": "nvml", "driver_version": driver, "devices": nvml_rows}+ if nvml_rows is not None+ else None+ )+ payload_path.write_text(json.dumps(payload), encoding = "utf-8")++ return set_host+++# An A100 split into two 3g.20gb slices (19968 MiB each).+TWO_SLICES = [(GPU0, [("3g.20gb", MIG0A), ("3g.20gb", MIG0B)])]+TWO_SLICES_NVML = [+ _nvml_row(0, GPU0, 38000, A100_MIB),+ _nvml_row(0, MIG0A, 15000, 19968, mig = True),+ _nvml_row(0, MIG0B, 19000, 19968, mig = True),+]+# Three whole A100s, card 0 split into three 2g.10gb slices (9856 MiB each).+MIXED = [+ (GPU0, [("2g.10gb", MIG0A), ("2g.10gb", MIG0B), ("2g.10gb", MIG0C)]),+ (GPU1, []),+ (GPU2, []),+]+MIXED_NVML = [+ _nvml_row(0, GPU0, 39000, A100_MIB),+ _nvml_row(0, MIG0A, 9000, 9856, mig = True),+ _nvml_row(0, MIG0B, 9500, 9856, mig = True),+ _nvml_row(0, MIG0C, 9800, 9856, mig = True),+ _nvml_row(1, GPU1, 30000, A100_MIB),+ _nvml_row(2, GPU2, 40000, A100_MIB),+]+++class TestAMigMaskIsHonouredByLlamaCpp:+ @pytest.mark.parametrize("mask", [MIG0B, MIG0B[:12]])+ def test_a_mig_mask_is_sized_by_its_slice_and_kept_for_the_child(self, host, monkeypatch, mask):+ host(MIXED, [39000, 30000, 40000], MIXED_NVML)+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", mask)+ # Not three whole 40 GB cards the parent never exposed.+ assert LlamaCppBackend._get_gpu_memory() == [(0, 9500, 9856)]+ assert LlamaCppBackend._pinned_child_visibility([0]) == MIG0B++ def test_the_old_mig_gpu_form_is_never_widened_to_a_numeric_pin(self, host, monkeypatch):+ # Pre-R470 form: NVML on a newer driver names the slice MIG-<uuid>, so nothing matches.+ host(TWO_SLICES, [38000], TWO_SLICES_NVML)+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", f"MIG-{GPU0}/1/0")+ assert LlamaCppBackend._get_gpu_memory() == []+ # Placement from another source (torch ordinal 0) must not become CUDA_VISIBLE_DEVICES=0.+ assert LlamaCppBackend._pinned_child_visibility([0]) is None++ def test_a_mig_mask_without_nvml_keeps_the_inherited_mask(self, host, monkeypatch):+ host(TWO_SLICES, [38000], None)+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", MIG0A)+ assert LlamaCppBackend._get_gpu_memory() == []+ assert LlamaCppBackend._pinned_child_visibility([0]) is None++ def test_whole_card_masks_still_pin_numerically(self, host, monkeypatch):+ host(MIXED, [39000, 30000, 40000], MIXED_NVML)+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "1,2")+ assert LlamaCppBackend._get_gpu_memory() == [(1, 30000, A100_MIB), (2, 40000, A100_MIB)]+ assert LlamaCppBackend._pinned_child_visibility([1, 2]) == "1,2"+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", f"{GPU2},{GPU1}")+ assert LlamaCppBackend._pinned_child_visibility([1, 2]) == "1,2"+++class TestAMigCardIsSizedByItsSlice:+ def test_a_numeric_mask_on_a_mig_card_reads_the_slice(self, host, monkeypatch):+ host(TWO_SLICES, [38000], TWO_SLICES_NVML)+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0")+ # CUDA exposes the first slice, not the 38 GB free on the card.+ assert LlamaCppBackend._get_gpu_memory() == [(0, 15000, 19968)]++ def test_an_unmasked_mixed_host_reads_slice_and_whole_cards(self, host):+ host(MIXED, [39000, 30000, 40000], MIXED_NVML)+ assert LlamaCppBackend._get_gpu_memory() == [+ (0, 9000, 9856),+ (1, 30000, A100_MIB),+ (2, 40000, A100_MIB),+ ]++ def test_auto_select_ranks_a_mig_card_by_its_slice(self, host, monkeypatch):+ host(MIXED, [39000, 30000, 40000], MIXED_NVML)+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0,1")+ monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA)+ monkeypatch.setattr(hw, "IS_ROCM", False)+ monkeypatch.setattr(+ hw,+ "estimate_required_model_memory_gb",+ lambda *a, **k: (20.0, {"required_gb": 20.0, "model_size_source": "config"}),+ )+ # Whole-card telemetry: card 0 looks freer (39 GB) than card 1 (30 GB).+ monkeypatch.setattr(+ hw,+ "get_visible_gpu_utilization",+ lambda: {+ "devices": [+ {"index": 0, "vram_total_gb": 40.0, "vram_used_gb": 1.0},+ {"index": 1, "vram_total_gb": 40.0, "vram_used_gb": 10.0},+ ]+ },+ )+ selected, metadata = hw.auto_select_gpu_ids("unsloth/test")+ # The 9.6 GB slice cannot hold 20 GB; card 1 can, alone.+ assert selected == [1]+ assert metadata["selection_mode"] == "auto"++ def test_auto_select_without_a_slice_reading_does_not_trust_the_card(self, host, monkeypatch):+ host(MIXED, [39000, 30000, 40000], None)+ devices = [+ {"index": 0, "vram_total_gb": 40.0, "vram_used_gb": 1.0},+ {"index": 1, "vram_total_gb": 40.0, "vram_used_gb": 10.0},+ ]+ out = hw._with_mig_instance_memory(devices)+ assert out[0]["vram_total_gb"] is None and out[0]["vram_used_gb"] is None+ assert out[1] == devices[1]++ def test_a_host_without_mig_is_unchanged(self, host):+ host([(GPU0, []), (GPU1, [])], [39000, 30000], None)+ devices = [+ {"index": 0, "vram_total_gb": 40.0, "vram_used_gb": 1.0},+ {"index": 1, "vram_total_gb": 40.0, "vram_used_gb": 10.0},+ ]+ assert hw._with_mig_instance_memory(devices) == devices+ assert LlamaCppBackend._get_gpu_memory() == [(0, 39000, A100_MIB), (1, 30000, A100_MIB)]+++MIG1A = "MIG-2e3f4a5b-1111-5bcd-9eef-112233445566"+# Two GPU instances on one card, as an R570+ host enumerates them.+TWO_GPU_INSTANCES = [dict(r) for r in TWO_SLICES_NVML]+TWO_GPU_INSTANCES[1]["gpu_instance_id"], TWO_GPU_INSTANCES[2]["gpu_instance_id"] = "1", "2"+++class TestOneSlicePerMigCard:+ # R570+ CUDA enumerates every slice; Studio still sizes and pins one per card, so a+ # tensor split or a single-device load never counts memory the child cannot use.+ @pytest.mark.parametrize(+ "mask, expected, uuid",+ [+ (None, (0, 15000, 19968), MIG0A),+ ("0", (0, 15000, 19968), MIG0A),+ (f"{MIG0A},{MIG0B}", (0, 15000, 19968), MIG0A),+ (f"{MIG0B},{MIG0A}", (0, 19000, 19968), MIG0B),+ ],+ )+ def test_a_mig_card_is_one_slice(self, host, monkeypatch, mask, expected, uuid):+ host(TWO_SLICES, [38000], TWO_GPU_INSTANCES, driver = "570.86.15")+ if mask is not None:+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", mask)+ assert LlamaCppBackend._get_gpu_memory() == [expected]+ assert LlamaCppBackend._pinned_child_visibility([0]) == uuid++ @pytest.mark.parametrize("mask", [None, "0"])+ def test_an_unnamed_slice_is_the_least_free_one(self, host, monkeypatch, mask):+ # The budget must not exceed the slice the child is pinned to, whichever CUDA lists first.+ rows = [dict(r) for r in TWO_SLICES_NVML]+ rows[1]["memory_free_mib"], rows[2]["memory_free_mib"] = "19000", "4000"+ host(TWO_SLICES, [38000], rows)+ if mask is not None:+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", mask)+ assert LlamaCppBackend._get_gpu_memory() == [(0, 4000, 19968)]+ assert LlamaCppBackend._pinned_child_visibility([0]) == MIG0B++ @pytest.mark.parametrize("mask", ["1,1", "1gpu,1gpu", "0,2,0"])+ def test_a_repeated_index_hides_every_gpu(self, host, monkeypatch, mask):+ # CUDA empties the device set on a repeat (torch's _parse_visible_devices agrees).+ host(MIXED, [39000, 30000, 40000], MIXED_NVML)+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", mask)+ assert LlamaCppBackend._nvml_rows_visible(MIXED_NVML) == []++ def test_torch_ordinals_ignore_rows_cached_by_an_earlier_probe(self, host, monkeypatch):+ # Mask order is card 1's slice then card 0's: torch ordinal 0 is MIG1A, while the cached+ # rows are keyed by physical index and would name card 0's slice.+ rows = [+ _nvml_row(0, GPU0, 38000, A100_MIB),+ _nvml_row(0, MIG0A, 15000, 19968, mig = True),+ _nvml_row(1, GPU1, 38000, A100_MIB),+ _nvml_row(1, MIG1A, 17000, 19968, mig = True),+ ]+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", f"{MIG1A},{MIG0A}")+ monkeypatch.setattr(LlamaCppBackend, "_NVML_ROWS", rows)+ monkeypatch.setattr(LlamaCppBackend, "_GPU_IDS_ARE_PCI_INDICES", False)+ fake = types.SimpleNamespace(cuda = types.SimpleNamespace(device_count = lambda: 2))+ monkeypatch.setitem(sys.modules, "torch", fake)+ assert LlamaCppBackend._pinned_child_visibility([0]) == MIG1A++ @pytest.mark.parametrize(+ "count, picked, expected", [(2, [1], MIG0B), (2, [0, 1], None), (1, [0], None)]+ )+ def test_torch_ordinals_narrow_a_multi_mig_mask(+ self, host, monkeypatch, count, picked, expected+ ):+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", f"{MIG0A},{MIG0B}")+ fake = types.SimpleNamespace(cuda = types.SimpleNamespace(device_count = lambda: count))+ monkeypatch.setitem(sys.modules, "torch", fake)+ monkeypatch.setattr(LlamaCppBackend, "_GPU_IDS_ARE_PCI_INDICES", False)+ if expected is None and len(picked) == 2:+ expected = f"{MIG0A},{MIG0B}"+ assert LlamaCppBackend._pinned_child_visibility(picked) == expected+++class TestTelemetryFailuresNeverDropCards:+ @pytest.mark.parametrize("mask, expected", [("1gpu2", [1]), ("1,2,abc", [1, 2])])+ def test_a_numeric_prefix_mask_keeps_its_cards(self, host, monkeypatch, mask, expected):+ host(MIXED, [39000, 30000, 40000], None)+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", mask)+ free = {1: 30000, 2: 40000}+ assert LlamaCppBackend._get_gpu_memory() == [(i, free[i], A100_MIB) for i in expected]++ def test_whole_cards_survive_a_missing_slice_reading(self, host):+ host(MIXED, [39000, 30000, 40000], None)+ assert LlamaCppBackend._get_gpu_memory() == [(1, 30000, A100_MIB), (2, 40000, A100_MIB)]++ def test_a_mig_only_host_without_any_reading_keeps_the_card_rows(self, host):+ host(TWO_SLICES, [38000], None)+ assert LlamaCppBackend._get_gpu_memory() == [(0, 38000, A100_MIB)]+++class TestMigSlicesAreNotGpus:+ @pytest.fixture+ def count_state(self, monkeypatch):+ monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA)+ monkeypatch.setattr(hw, "IS_ROCM", False)+ monkeypatch.setattr(hw, "_physical_gpu_count", None)+ monkeypatch.setattr(hw, "_physical_gpu_count_from_smi", False)+ monkeypatch.setattr(hw, "_physical_gpu_count_checked_at", None, raising = False)++ @pytest.mark.parametrize(+ "cards, expected",+ [+ (TWO_SLICES, 1),+ (MIXED, 3),+ ([(GPU0, [("3g.20gb", f"MIG-{GPU0}/1/0"), ("3g.20gb", f"MIG-{GPU0}/2/0")])], 1),+ ],+ )+ def test_the_physical_count_skips_mig_rows(self, host, count_state, cards, expected):+ host(cards, [38000] * len(cards), None)+ assert hw.get_physical_gpu_count() == expected+ assert hw.get_parent_visible_gpu_ids() == list(range(expected))+ with pytest.raises(ValueError):+ hw.resolve_requested_gpu_ids([expected])++ def test_mig_gpu_indices(self, host):+ host(MIXED, [0, 0, 0], None)+ assert nvidia.get_mig_gpu_indices() == {0}+ host([(GPU0, []), (GPU1, [("1g.5gb", MIG0A)])], [0, 0], None)+ assert nvidia.get_mig_gpu_indices() == {1}+++def test_auto_select_and_llama_cpp_budget_the_same_slice(host):+ # One selected id is one worker device: training and llama.cpp both see one slice per card.+ host(TWO_SLICES, [38000], TWO_GPU_INSTANCES, driver = "570.86.15")+ devices = [{"index": 0, "vram_total_gb": 40.0, "vram_used_gb": 2.0}]+ out = hw._with_mig_instance_memory(devices)+ assert out[0]["vram_total_gb"] == round(19968 / 1024, 2)+ assert out[0]["vram_used_gb"] == round((19968 - 15000) / 1024, 2)+ assert LlamaCppBackend._get_gpu_memory() == [(0, 15000, 19968)]+++def test_mig_overlay_skips_a_non_cuda_backend(host, monkeypatch):+ # Intel XPU indices are not NVIDIA indices: an NVIDIA MIG card at 0 must not touch XPU 0.+ host(TWO_SLICES, [38000], TWO_SLICES_NVML)+ monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.XPU)+ devices = [{"index": 0, "vram_total_gb": 24.0, "vram_used_gb": 2.0}]+ assert hw._with_mig_instance_memory(devices) == devicesdiff --git a/studio/backend/tests/test_physical_gpu_count_refresh.py b/studio/backend/tests/test_physical_gpu_count_refresh.pyindex b807917d806..958c68c2f47 100644--- a/studio/backend/tests/test_physical_gpu_count_refresh.py+++ b/studio/backend/tests/test_physical_gpu_count_refresh.py@@ -82,6 +82,18 @@ def test_an_smi_that_answers_later_replaces_the_torch_fallback(smi):assert hw._physical_gpu_count_from_smi is True+def test_a_lower_smi_count_replaces_a_torch_count(smi, monkeypatch):+ # R570+ torch counts each MIG slice; the first SMI answer is the physical count.+ monkeypatch.setattr(hw, "_torch_get_physical_gpu_count", lambda: 7)+ smi(None, seconds = 0.0)+ assert hw.get_physical_gpu_count() == 7+ smi(1)+ assert hw.get_physical_gpu_count() == 1+ assert hw._physical_gpu_count_from_smi is True+ smi(0)+ assert hw.get_physical_gpu_count() == 1++def test_non_cuda_counts_are_never_reprobed(smi, monkeypatch):monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.MLX)assert hw.get_physical_gpu_count() == 1@@ -128,7 +140,7 @@ def test_the_refresh_asks_for_physical_rows_only(smi, monkeypatch):hw.get_physical_gpu_count()smi(1)hw.get_physical_gpu_count()- assert seen == [False, True]+ assert seen == [True, True]def test_the_ttl_starts_after_the_probe_returns(smi, monkeypatch):diff --git a/studio/backend/tests/test_system_poll_no_cuda_context.py b/studio/backend/tests/test_system_poll_no_cuda_context.pyindex e31a1a25cf5..2fa91b128a1 100644--- a/studio/backend/tests/test_system_poll_no_cuda_context.py+++ b/studio/backend/tests/test_system_poll_no_cuda_context.py@@ -720,13 +720,13 @@ def test_physical_gpu_count_records_whether_the_smi_answered(monkeypatch):monkeypatch.setattr(hw, "_physical_gpu_count_from_smi", False)monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA)monkeypatch.setattr(hw, "IS_ROCM", False)- monkeypatch.setattr(nvidia, "get_physical_gpu_count", lambda: 4)+ monkeypatch.setattr(nvidia, "get_physical_gpu_count", lambda gpu_rows_only = False: 4)assert hw.get_physical_gpu_count() == 4assert hw._physical_gpu_count_from_smi is Truemonkeypatch.setattr(hw, "_physical_gpu_count", None)monkeypatch.setattr(hw, "_physical_gpu_count_from_smi", False)- monkeypatch.setattr(nvidia, "get_physical_gpu_count", lambda: None)+ monkeypatch.setattr(nvidia, "get_physical_gpu_count", lambda gpu_rows_only = False: None)monkeypatch.setattr(hw, "_torch_get_physical_gpu_count", lambda: 1)assert hw.get_physical_gpu_count() == 1assert hw._physical_gpu_count_from_smi is Falsediff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.pyindex e485dc2c488..836e2782085 100644--- a/studio/backend/utils/hardware/hardware.py+++ b/studio/backend/utils/hardware/hardware.py@@ -6120,6 +6120,7 @@ def auto_select_gpu_ids(else:utilization = get_visible_gpu_utilization()devices = [d for d in utilization.get("devices", []) if d.get("index") not in _uncovered]+ devices = _with_mig_instance_memory(devices)parent_ids = _covered(get_parent_visible_gpu_ids())if not devices:@@ -6276,6 +6277,40 @@ def prepare_gpu_selection(return selected_gpu_ids, metadata+def _with_mig_instance_memory(devices: list[Dict[str, Any]]) -> list[Dict[str, Any]]:+ """Replace a MIG card's whole-card nvidia-smi memory with the slice CUDA exposes+ for it (NVML), or drop the reading when no slice reading exists so the ranking+ skips it. Devices on cards without MIG instances are returned unchanged."""+ if IS_ROCM or not devices or get_device() != DeviceType.CUDA:+ return devices+ try:+ from . import nvidia+ mig = nvidia.get_mig_gpu_indices() & {d.get("index") for d in devices}+ except Exception:+ return devices+ if not mig:+ return devices+ slices: Dict[int, tuple[int, int]] = {}+ try:+ from core.inference.llama_cpp import LlamaCppBackend+ slices = {idx: (free, total) for idx, free, total in LlamaCppBackend._get_gpu_memory_nvml()}+ except Exception as e:+ logger.debug("NVML MIG slice probe failed: %s", e)+ out = []+ for device in devices:+ idx = device.get("index")+ if idx in mig:+ free_mib, total_mib = slices.get(idx, (0, 0))+ known = total_mib > 0+ device = {+ **device,+ "vram_total_gb": round(total_mib / 1024, 2) if known else None,+ "vram_used_gb": round(max(0, total_mib - free_mib) / 1024, 2) if known else None,+ }+ out.append(device)+ return out++def _nvidia_physical_gpu_row_count() -> Optional[int]:# Physical rows only: MIG instances created after startup add indented `nvidia-smi -L` rows, not GPUs.try:@@ -6286,15 +6321,17 @@ def _nvidia_physical_gpu_row_count() -> Optional[int]:def _run_physical_gpu_count_probe() -> None:- """Keep a re-read NVIDIA count only if it grew: a failed probe or a lower count keeps the old one, so detection never narrows."""+ """Keep a re-read NVIDIA count only if it grew: a failed probe or a lower count keeps the old one, so detection never narrows. The first SMI answer replaces a torch fallback count even when lower: torch counts CUDA devices, and R570+ enumerates each MIG slice as one."""global _physical_gpu_count, _physical_gpu_count_from_smi, _physical_gpu_count_checked_atcount = _nvidia_physical_gpu_row_count()# Stamped after the probe, so the next expiry also outlives gpu_query's own cache entry for it._physical_gpu_count_checked_at = time.monotonic()previous = _physical_gpu_count- if count is None or previous is None or count <= previous:+ if count is None or previous is None:+ return+ if count <= previous and (_physical_gpu_count_from_smi or count < 1):return- logger.info("Physical GPU count grew from %d to %d; using the new GPUs", previous, count)+ logger.info("Physical GPU count changed from %d to %d (nvidia-smi)", previous, count)_physical_gpu_count = count_physical_gpu_count_from_smi = True@@ -6327,10 +6364,11 @@ def get_physical_gpu_count() -> int:count = Nonetry:if IS_ROCM:- from . import amd as _smi_mod+ from . import amd+ count = amd.get_physical_gpu_count()else:- from . import nvidia as _smi_mod- count = _smi_mod.get_physical_gpu_count()+ # MIG slices are indented `-L` rows under their card, not GPUs to offer.+ count = _nvidia_physical_gpu_row_count()except Exception:passif not IS_ROCM:diff --git a/studio/backend/utils/hardware/nvidia.py b/studio/backend/utils/hardware/nvidia.pyindex e853a915b06..4512f68010f 100644--- a/studio/backend/utils/hardware/nvidia.py+++ b/studio/backend/utils/hardware/nvidia.py@@ -150,6 +150,36 @@ def get_physical_gpu_count(gpu_rows_only: bool = False) -> Optional[int]:return None+def get_mig_gpu_indices() -> set[int]:+ """Physical indices of GPUs with MIG instances (indented ``MIG`` rows under their+ ``GPU N:`` row in ``nvidia-smi -L``). CUDA sees such a card as one slice, so its+ ``--query-gpu`` memory is the whole card's, not the device's. Empty on failure."""+ try:+ result = gpu_query.run_nvidia_smi(+ [_nvidia_smi_executable(), "-L"],+ capture_output = True,+ text = True,+ encoding = "utf-8",+ errors = "replace",+ timeout = 5,+ env = child_env_without_native_path_secret(),+ **_windows_hidden_subprocess_kwargs(),+ )+ except Exception:+ return set()+ if result.returncode != 0 or not isinstance(result.stdout, str):+ return set()+ indices: set[int] = set()+ current: Optional[int] = None+ for line in result.stdout.splitlines():+ if line.startswith("GPU "):+ head = line[4:].split(":", 1)[0].strip()+ current = int(head) if head.isdecimal() else None+ elif current is not None and line.strip().startswith("MIG "):+ indices.add(current)+ return indices++def get_primary_gpu_utilization() -> dict[str, Any]:try:result = gpu_query.run_nvidia_smi(