Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: offer, size and pin every MIG GPU instance CUDA enumerates on R570+ drivers (#14981)

Daniel Han674f7f437220
diff --git a/studio/backend/core/inference/engine_adapters.py b/studio/backend/core/inference/engine_adapters.py
index e7c00cf8c36..2971dfa96ce 100644
--- a/studio/backend/core/inference/engine_adapters.py
+++ b/studio/backend/core/inference/engine_adapters.py
@@ -438,7 +438,23 @@ def gpu_memory_fraction(
def _nvidia_memory_rows(gpu_ids: list[int]) -> list[tuple[float, float]]:
+ # R570+ MIG ids are CUDA devices: budget each slice; nvidia-smi --id takes physical ids.
+ try:
+ from utils.hardware import mig
+ slices = mig.rows_for(gpu_ids)
+ except Exception:
+ slices = None
+ if slices is not None:
+ if not all(float(r.get("memory_total_mib") or 0) > 0 for r in slices):
+ raise RuntimeError(
+ "The selected MIG GPU instance has no memory reading right now because the GPU "
+ "probe did not answer. Retry in a few seconds."
+ )
+ return [
+ (float(r["memory_total_mib"]), float(r.get("memory_free_mib") or 0)) for r in slices
+ ]
from utils.hardware.nvidia import _nvidia_smi_executable
+
result = subprocess.run(
[
_nvidia_smi_executable(),
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 811aad00f5c..fb891895001 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -12452,6 +12452,9 @@ class LlamaCppBackend:
spec = _get_parent_visible_gpu_spec()
except Exception:
return None
+ if spec.get("index_kind") == "cuda_device":
+ # R570+ MIG ids are CUDA devices; the MIG paths pin through the snapshot.
+ return None
return spec["numeric_ids"] if spec.get("raw") == cvd.strip() else None
@staticmethod
@@ -12598,6 +12601,17 @@ class LlamaCppBackend:
sort. The visible set itself never changes. No mask, an empty mask, or a
UUID/MIG mask (which resolves to None) is left alone -- the multi-GPU
controls are hidden for the latter."""
+ mig_rows = LlamaCppBackend._mig_snapshot_rows()
+ if mig_rows is not None:
+ # R570+ MIG: pin UUIDs; numeric ids would resolve against the whole host.
+ from utils.hardware import mig
+
+ # Snapshot order = picker order; never sort (a numeric mask 2,0 is not ascending).
+ order = [int(r["index"]) for r in mig_rows]
+ if not order:
+ return None
+ env["CUDA_VISIBLE_DEVICES"] = mig.visibility_for(mig_rows, order)
+ return tuple(order)
env["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
inherited = LlamaCppBackend._resolve_visible_physical_ids()
if not inherited:
@@ -13906,6 +13920,17 @@ class LlamaCppBackend:
return None
return any(not kind.startswith("IDENTITY") for kind in kinds)
+ @staticmethod
+ def _mig_snapshot_rows():
+ """R570+ MIG rows _get_gpu_memory sized under this mask and order, else None (index = CUDA id)."""
+ snapshot = getattr(LlamaCppBackend, "_MIG_DEVICE_SNAPSHOT", None)
+ if snapshot is not None and snapshot[:2] == (
+ os.environ.get("CUDA_VISIBLE_DEVICES"),
+ os.environ.get("CUDA_DEVICE_ORDER"),
+ ):
+ return snapshot[2]
+ return None
+
@classmethod
def _p2p_veto_reason(
cls,
@@ -13953,6 +13978,17 @@ class LlamaCppBackend:
return "disabled by UNSLOTH_DISABLE_DC_P2P=1"
if os.environ.get("UNSLOTH_FORCE_DC_P2P") == "1":
return None
+ mig_rows = cls._mig_snapshot_rows()
+ if mig_rows is not None:
+ # MIG instances have no NVLink matrix row; whole cards keep it via physical_index.
+ by_id = {int(r["index"]): r for r in mig_rows}
+ selected = (
+ sorted(by_id) if gpu_indices is None else sorted(set(int(i) for i in gpu_indices))
+ )
+ if any(i not in by_id or by_id[i].get("mig") for i in selected):
+ return "MIG device UUIDs cannot be verified by the physical-card NVLink matrix"
+ gpu_indices = [int(by_id[i]["physical_index"]) for i in selected]
+ ids_are_pci_indices = True
def _pcie(reason: str) -> str:
"""Append the IOMMU diagnosis to a veto that implies a PCIe peer path.
@@ -14356,6 +14392,16 @@ class LlamaCppBackend:
def _cuda_compute_caps() -> dict:
"""Physical id -> SM (90 for sm_90) via nvidia-smi, honoring
CUDA_VISIBLE_DEVICES; {} when unknown. Never raises."""
+ mig_rows = LlamaCppBackend._mig_snapshot_rows()
+ if mig_rows is not None:
+ caps = {}
+ for row in mig_rows:
+ try:
+ major, minor = row.get("compute_cap", "").split(".")
+ caps[int(row["index"])] = int(major) * 10 + int(minor)
+ except (TypeError, ValueError):
+ pass
+ return caps
try:
from utils.hardware import gpu_query
@@ -15687,6 +15733,10 @@ class LlamaCppBackend:
or unresolvable UUID mask the ids cannot be mapped back to, where a numeric pin would
be re-read against the whole host (a card the parent hid, or the whole MIG card)."""
cvd = os.environ.get("CUDA_VISIBLE_DEVICES")
+ mig_rows = LlamaCppBackend._mig_snapshot_rows()
+ if mig_rows is not None:
+ from utils.hardware import mig
+ return mig.visibility_for(mig_rows, gpu_indices)
if (
LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES is False
and cvd is not None
@@ -15879,6 +15929,27 @@ class LlamaCppBackend:
binary = binary or LlamaCppBackend._find_llama_server_binary()
if LlamaCppBackend._is_vulkan_backend(binary):
return LlamaCppBackend._get_gpu_free_memory_vulkan(binary)
+ from utils.hardware import mig
+
+ mig_rows = mig.visible_devices(fresh = True, listing = True)
+ LlamaCppBackend._MIG_DEVICE_SNAPSHOT = None
+ if mig_rows is not None:
+ # CUDA device ids; unknown memory stays zero rather than dropping a device.
+ LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES = False
+ LlamaCppBackend._MIG_DEVICE_SNAPSHOT = (
+ os.environ.get("CUDA_VISIBLE_DEVICES"),
+ os.environ.get("CUDA_DEVICE_ORDER"),
+ mig_rows,
+ )
+ gpus = sorted(
+ (
+ int(r["index"]),
+ int(r.get("memory_free_mib") or 0),
+ int(r.get("memory_total_mib") or 0),
+ )
+ for r in mig_rows
+ )
+ return LlamaCppBackend._cuda_sm_gate_keep(binary, gpus) if for_llama_server else gpus
# ── NVIDIA via nvidia-smi ────────────────────────────────────
unsized_ids: list[int] = []
mig_card_rows: list[tuple[int, int, int]] = []
@@ -21122,6 +21193,14 @@ class LlamaCppBackend:
shim_cmd, _visual_bin, _extra_pythonpath = assets
return _shim_supports_ngl(shim_cmd)
+ @staticmethod
+ def _mig_pinned_gpu_token(gpu: str) -> str:
+ """On an R570+ MIG host, numeric ``gpu`` ids become the instance UUIDs; anything else as is."""
+ if gpu and all(token.strip().isdigit() for token in gpu.split(",")):
+ from utils.hardware import mig
+ return mig.mask_for([int(token) for token in gpu.split(",")]) or gpu
+ return gpu
+
@staticmethod
def _diffusion_gpu_arg(
gpu_ids: Optional[List[int]],
@@ -21363,6 +21442,7 @@ class LlamaCppBackend:
holds_no_gpu = True
gpu = self._diffusion_gpu_arg(gpu_ids, cpu_only = cpu_only, force_cpu = True)
+ gpu = self._mig_pinned_gpu_token(gpu)
cmd = list(shim_cmd) + [
"--gguf",
model_path,
diff --git a/studio/backend/core/inference/managed_engine.py b/studio/backend/core/inference/managed_engine.py
index 4f079157347..d457b1b0816 100644
--- a/studio/backend/core/inference/managed_engine.py
+++ b/studio/backend/core/inference/managed_engine.py
@@ -56,6 +56,33 @@ def _offline(env) -> bool:
)
+def _mig_rows(gpu_ids: list[int]):
+ """R570+ MIG rows for these ids, or None (untranslatable: keep the physical-id behaviour)."""
+ try:
+ from utils.hardware import mig
+
+ from .engine_install import gpu_platform
+
+ if gpu_platform() == "rocm":
+ return None
+ rows = mig.rows_for(gpu_ids)
+ if rows is None or mig.mask_for(gpu_ids) is None:
+ return None
+ return rows
+ except Exception:
+ return None
+
+
+def _child_cuda_mask(gpu_ids: list[int]) -> str:
+ """Instance UUIDs on an R570+ MIG host (numeric ids would name the whole host), else the numeric mask."""
+ try:
+ from utils.hardware import mig
+ pinned = mig.mask_for(gpu_ids)
+ except Exception:
+ pinned = None
+ return pinned or ",".join(str(i) for i in gpu_ids)
+
+
def validate_load(engine: str, request) -> list[int]:
"""Returns the physical GPU ids to launch on, all inside the GPUs Studio may use."""
profile(engine)
@@ -88,8 +115,10 @@ def validate_load(engine: str, request) -> list[int]:
f"{ENGINE_NAMES[engine]} cannot select AMD GPUs while Unsloth runs with {mask}. "
"Use only HIP_VISIBLE_DEVICES or only ROCR_VISIBLE_DEVICES instead."
)
- for gpu_id in gpu_ids:
- reason = support_reason(engine, gpu_id)
+ mig_rows = _mig_rows(gpu_ids)
+ for position, gpu_id in enumerate(gpu_ids):
+ checked = int(mig_rows[position]["physical_index"]) if mig_rows else gpu_id
+ reason = support_reason(engine, checked)
if reason:
raise ValueError(f"GPU {gpu_id}: {reason}")
info = installed(engine)
@@ -682,7 +711,7 @@ class ManagedEngine:
child_env.get("PATH", os.defpath),
]
)
- child_env["CUDA_VISIBLE_DEVICES"] = ",".join(str(i) for i in (gpu_ids or [0]))
+ child_env["CUDA_VISIBLE_DEVICES"] = _child_cuda_mask(gpu_ids or [0])
rocm = info.get("platform") == "rocm"
if rocm:
child_env.update(_rocm_visibility(child_env, gpu_ids or [0]))
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 9506fa54a83..accd9771d81 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -16700,7 +16700,10 @@ async def _override_gpu_ids_still_resolve(
)
device, is_vulkan, resolved = await asyncio.to_thread(_device_and_resolution)
- if stored_index_kind != ("vulkan" if is_vulkan else "physical"):
+ from utils.hardware.hardware import current_gpu_index_kind
+
+ current_kind = await asyncio.to_thread(current_gpu_index_kind, is_vulkan)
+ if stored_index_kind != current_kind:
return False
if device == DeviceType.XPU and not is_vulkan:
# Rejected outright on XPU.
diff --git a/studio/backend/routes/settings.py b/studio/backend/routes/settings.py
index da3df81bcf4..f8d732c3f02 100644
--- a/studio/backend/routes/settings.py
+++ b/studio/backend/routes/settings.py
@@ -1105,7 +1105,7 @@ class ModelOverridePayload(BaseModel):
gpu_ids: Optional[list[int]] = Field(default = None, max_length = MAX_GPU_IDS)
# Which index space gpu_ids is in. Absent means physical, the only thing a client
# written before this field could have meant.
- gpu_index_kind: Optional[Literal["physical", "vulkan"]] = None
+ gpu_index_kind: Optional[Literal["physical", "vulkan", "cuda_device"]] = None
# An all-default save carries no fields, like a forget; None keeps the legacy contract.
remove: Optional[bool] = None
# Fill in, don't replace: the backfill reads the map once then writes each model.
diff --git a/studio/backend/tests/test_mig_multi_instance.py b/studio/backend/tests/test_mig_multi_instance.py
new file mode 100644
index 00000000000..57d0a9e36a8
--- /dev/null
+++ b/studio/backend/tests/test_mig_multi_instance.py
@@ -0,0 +1,836 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""R570+ MIG: CUDA enumerates one device per GPU instance, so Studio offers, sizes and pins each
+one instead of collapsing a card's instances to a single slice. Older drivers keep the old rule.
+CPU fixtures only: every NVIDIA boundary (nvidia-smi, the NVML / CUDA driver probe, torch) is faked."""
+
+from __future__ import annotations
+
+import contextlib
+import copy
+import ctypes
+import importlib.util
+import json
+import os
+import subprocess
+import sys
+import types
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+
+import utils.hardware.hardware as hw
+from core.inference import llama_cpp as lc
+from utils.hardware import gpu_query, mig, nvidia
+
+B = lc.LlamaCppBackend
+
+GPU = "GPU-00000000-0000-0000-0000-000000000000"
+A, B_UUID, C = [f"MIG-{c * 8}-0000-0000-0000-000000000000" for c in "abc"]
+LISTING = f"""GPU 0: NVIDIA A100-SXM4-80GB (UUID: {GPU})
+ MIG 3g.40gb Device 0: (UUID: {A})
+ MIG 2g.20gb Device 1: (UUID: {B_UUID})
+ MIG 2g.20gb Device 2: (UUID: {C})
+"""
+
+
+def row(
+ uuid,
+ free,
+ total,
+ gi = None,
+):
+ r = dict(
+ index = "0",
+ uuid = uuid,
+ name = "NVIDIA A100-SXM4-80GB",
+ compute_cap = "8.0",
+ memory_free_mib = str(free),
+ memory_total_mib = str(total),
+ )
+ if gi is not None:
+ r.update(mig = "1", gpu_instance_id = str(gi))
+ return r
+
+
+ROWS = [
+ row(GPU, 78000, 81920),
+ row(A, 36000, 40192, 0),
+ row(B_UUID, 18000, 19968, 1),
+ row(C, 17000, 19968, 2),
+]
+
+
+def payload(
+ rows = None,
+ uuids = None,
+ driver = "570.86.15",
+):
+ return dict(
+ source = "nvml",
+ driver_version = driver,
+ devices = copy.deepcopy(ROWS if rows is None else rows),
+ cuda_devices = [
+ dict(uuid = u[4:], index = str(i))
+ for i, u in enumerate([A, B_UUID, C] if uuids is None else uuids)
+ ],
+ )
+
+
+def host(
+ mask = None,
+ driver = "570.86.15",
+ rows = None,
+):
+ """One A100 80GB with three GPU instances; CUDA enumerates the mask's instances."""
+ probe = dict(
+ source = "nvml",
+ driver_version = driver,
+ cuda_driver_version = [12, 8],
+ devices = copy.deepcopy(ROWS if rows is None else rows),
+ )
+ visible = (
+ [A, B_UUID, C]
+ if mask is None
+ else [
+ [A, B_UUID, C][int(t)] if t.isdigit() else t
+ for t in (x.strip() for x in mask.split(","))
+ if t and t != "-1"
+ ]
+ )
+ probe["cuda_devices"] = [{"index": str(i), "uuid": u[4:]} for i, u in enumerate(visible)]
+
+ def result(text):
+ return types.SimpleNamespace(returncode = 0, stdout = text, stderr = "")
+
+ def run(cmd, *args, **kw):
+ if os.path.basename(str(cmd[0])) == "nvidia-smi":
+ if "-L" in cmd:
+ return result(LISTING)
+ query = next((x for x in cmd if x.startswith("--query-gpu=")), "")
+ if query == "--query-gpu=index,uuid":
+ return result(f"0, {GPU}\n")
+ if "memory.free" in query:
+ return result(
+ "0, 78000, 81920" + (f", {GPU}" if query.endswith(",uuid") else "") + "\n"
+ )
+ return types.SimpleNamespace(returncode = 1, stdout = "", stderr = "unsupported")
+ if "--json" in cmd:
+ return result(json.dumps(probe))
+ raise AssertionError(f"Unmocked subprocess: {cmd}")
+
+ torch_count = 3 if mask is None else len(mask.split(",")) if mask else 0
+ torch = types.SimpleNamespace(
+ cuda = types.SimpleNamespace(device_count = lambda: torch_count),
+ version = types.SimpleNamespace(hip = None, cuda = "12.8"),
+ __version__ = "2.9.0",
+ )
+ with contextlib.ExitStack() as s:
+ s.enter_context(
+ patch.dict(
+ os.environ,
+ {
+ "CUDA_DEVICE_ORDER": "PCI_BUS_ID",
+ "UNSLOTH_GPU_QUERY_CACHE": "0",
+ # conftest turns the NVML probe off for every test; this host's answers come from it.
+ "UNSLOTH_NVIDIA_LIBRARY_PROBE": "1",
+ },
+ )
+ )
+ for key in (
+ "CUDA_VISIBLE_DEVICES",
+ "HIP_VISIBLE_DEVICES",
+ "ROCR_VISIBLE_DEVICES",
+ mig._ENV,
+ ):
+ os.environ.pop(key, None)
+ if mask is not None:
+ os.environ["CUDA_VISIBLE_DEVICES"] = mask
+ s.enter_context(patch.dict(sys.modules, {"torch": torch}))
+ s.enter_context(patch.object(subprocess, "run", run))
+ s.enter_context(patch.object(mig, "possible", lambda: True))
+ for name, value in [
+ ("DEVICE", hw.DeviceType.CUDA),
+ ("IS_ROCM", False),
+ ("_physical_gpu_count", None),
+ ("_physical_gpu_count_from_smi", False),
+ ("_physical_gpu_count_checked_at", None),
+ ("_visible_gpu_count", None),
+ ]:
+ s.enter_context(patch.object(hw, name, value, create = True))
+ s.enter_context(patch.object(hw, "get_device", lambda: hw.DeviceType.CUDA))
+ s.enter_context(patch.object(hw, "_torch_get_physical_gpu_count", lambda: torch_count))
+ s.enter_context(patch.object(hw, "_mask_names_cuda_ordinals", lambda *a: False))
+ s.enter_context(patch.object(hw, "rocm_gpu_ids_without_torch_kernels", lambda: set()))
+ s.enter_context(
+ patch.object(
+ hw,
+ "_smi_visible_utilization",
+ lambda spec: {
+ "devices": [{"index": 0, "vram_total_gb": 80.0, "vram_used_gb": 2.0}]
+ },
+ )
+ )
+ for name, value in [
+ ("_NVML_ROWS", []),
+ ("_SMI_UUID_ROWS", []),
+ ("_SMI_MIG_INDICES", None),
+ ("_GPU_IDS_ARE_PCI_INDICES", None),
+ ("_MIG_DEVICE_SNAPSHOT", None),
+ ("_is_vulkan_backend", staticmethod(lambda *a: False)),
+ ("_find_llama_server_binary", staticmethod(lambda: "/mock/llama-server")),
+ ("_nvidia_probe_script", staticmethod(lambda: Path("mock-probe.py"))),
+ ("_get_gpu_memory_amd_smi", staticmethod(lambda *a, **k: [])),
+ ("_get_gpu_memory_torch", staticmethod(lambda *a, **k: [])),
+ ("_widen_integrated_cuda_rows", staticmethod(lambda rows: rows)),
+ ]:
+ s.enter_context(patch.object(B, name, value, create = True))
+ gpu_query.invalidate_static("MIG fixture host")
+ s.enter_context(patch.object(nvidia, "_mig_entry", None))
+ s.enter_context(patch.object(mig, "_cache", {}))
+ yield probe
+
+
[email protected](autouse = True)
+def _probe_script(monkeypatch):
+ import utils.prebuilt.update_flow as update_flow
+ monkeypatch.setattr(update_flow, "find_installer_script", lambda **k: Path("mock-probe.py"))
+
+
+def test_training_and_gguf_offer_and_pin_every_gpu_instance():
+ for mask, count in ((None, 3), (f"{A},{B_UUID}", 2)):
+ with host(mask):
+ assert hw.get_parent_visible_gpu_ids() == list(range(count))
+ selection, _ = hw.auto_select_gpu_ids("fixture", required_override_gb = 65)
+ assert selection == list(range(count))
+ expected = ",".join([A, B_UUID, C][:count])
+ before = os.environ.get("CUDA_VISIBLE_DEVICES")
+ hw.apply_gpu_ids(selection, backend = "cuda")
+ assert os.environ["CUDA_VISIBLE_DEVICES"] == expected
+ if before is None:
+ os.environ.pop("CUDA_VISIBLE_DEVICES")
+ else:
+ os.environ["CUDA_VISIBLE_DEVICES"] = before
+ gpus = B._get_gpu_memory()
+ assert len(gpus) == count
+ assert B._pinned_child_visibility([g[0] for g in gpus]) == expected
+
+
+def test_each_instance_is_sized_by_its_own_memory():
+ with host():
+ assert B._get_gpu_memory() == [(0, 36000, 40192), (1, 18000, 19968), (2, 17000, 19968)]
+ assert B._pinned_child_visibility([2, 0]) == f"{C},{A}"
+
+
+def test_utilization_lists_every_instance():
+ with host():
+ util = hw.get_visible_gpu_utilization()
+ assert util["index_kind"] == "cuda_device"
+ assert util["parent_visible_gpu_ids"] == [0, 1, 2]
+ assert [d["uuid"] for d in util["devices"]] == [A, B_UUID, C]
+ assert util["devices"][0]["vram_total_gb"] == round(40192 / 1024, 2)
+
+
+def test_reordered_explicit_mig_mask_keeps_order_and_membership():
+ with host(f"{B_UUID},{A}"):
+ assert B._get_gpu_memory() == [(0, 18000, 19968), (1, 36000, 40192)]
+ assert B._pinned_child_visibility([0, 1]) == f"{B_UUID},{A}"
+ hw.apply_gpu_ids([0, 1], backend = "cuda")
+ assert os.environ["CUDA_VISIBLE_DEVICES"] == f"{B_UUID},{A}"
+
+
+def test_a_pick_outside_the_inherited_mask_is_refused():
+ with host(f"{A},{B_UUID}"):
+ assert hw.get_parent_visible_gpu_ids() == [0, 1]
+ with pytest.raises(ValueError):
+ hw.apply_gpu_ids([2], backend = "cuda")
+ assert os.environ["CUDA_VISIBLE_DEVICES"] == f"{A},{B_UUID}"
+
+
+def test_a_fresh_worker_pins_the_instances_the_parent_sized_without_probing():
+ with host():
+ assert hw.get_parent_visible_gpu_ids() == [0, 1, 2]
+ inherited = os.environ[mig._ENV]
+ # A spawned worker: cold caches, the parent's environment, and no probe may run.
+ with (
+ patch.object(mig, "_cache", {}),
+ patch.object(nvidia, "_mig_entry", None),
+ patch.object(subprocess, "run", side_effect = AssertionError("worker probed")),
+ ):
+ assert os.environ[mig._ENV] == inherited
+ hw.apply_gpu_ids([1, 2], backend = "cuda")
+ assert os.environ["CUDA_VISIBLE_DEVICES"] == f"{B_UUID},{C}"
+
+
+def test_an_inherited_map_from_another_mask_is_ignored():
+ with host():
+ hw.get_parent_visible_gpu_ids()
+ with patch.object(mig, "_cache", {}), patch.object(nvidia, "_mig_entry", None):
+ os.environ["CUDA_DEVICE_ORDER"] = "FASTEST_FIRST"
+ assert mig._inherited_uuids() is None
+ assert mig.mask_for([0]) is None
+
+
+ sys.platform == "darwin",
+ reason = "main never runs the NVML probe on macOS, so the pre-R570 slice reading this checks does not exist there",
+)
+def test_old_drivers_keep_the_one_slice_rule():
+ with host(driver = "565.57.01"):
+ assert hw.get_parent_visible_gpu_ids() == [0]
+ assert B._get_gpu_memory() == [(0, 17000, 19968)]
+ assert B._pinned_child_visibility([0]) == C
+ assert mig._ENV not in os.environ
+
+
+def test_an_instance_without_a_memory_reading_keeps_its_identity():
+ rows = copy.deepcopy(ROWS)
+ rows[2]["memory_free_mib"] = rows[2]["memory_total_mib"] = "0"
+ with host(rows = rows):
+ assert len(hw.get_parent_visible_gpu_ids()) == 3
+ assert B._get_gpu_memory() == [(0, 36000, 40192), (1, 0, 0), (2, 17000, 19968)]
+ assert B._pinned_child_visibility([0, 1, 2]) == f"{A},{B_UUID},{C}"
+
+
+def _age_cache(seconds):
+ for key, entry in list(mig._cache.items()):
+ mig._cache[key] = (entry[0] - seconds, *entry[1:])
+
+
+def test_a_failed_refresh_keeps_known_identities_without_stale_memory():
+ with host() as probe:
+ assert len(hw.get_parent_visible_gpu_ids()) == 3
+ probe["source"] = "failed"
+ _age_cache(mig._FRESH_SHARE_S + 1)
+ assert B._get_gpu_memory() == [(0, 0, 0), (1, 0, 0), (2, 0, 0)]
+ assert B._pinned_child_visibility([0, 1, 2]) == f"{A},{B_UUID},{C}"
+
+
+def test_full_numeric_mask_keeps_its_ids_for_training_and_gguf():
+ with host("2,0"):
+ assert hw.get_parent_visible_gpu_ids() == [2, 0]
+ assert B._get_gpu_memory() == [(0, 36000, 40192), (2, 17000, 19968)]
+ assert B._pinned_child_visibility([2, 0]) == f"{C},{A}"
+ hw.apply_gpu_ids([2, 0], backend = "cuda")
+ assert os.environ["CUDA_VISIBLE_DEVICES"] == f"{C},{A}"
+
+
+def test_several_compute_instances_of_one_gpu_instance_count_once():
+ rows = copy.deepcopy(ROWS)
+ rows[2]["gpu_instance_id"] = rows[1]["gpu_instance_id"]
+ # The CUDA driver, not NVML's number of CI handles, picks the representatives.
+ result = mig.rows_from_payload(payload(rows = rows, uuids = [A, C]))
+ assert [r["uuid"] for r in result] == [A, C]
+
+
+def test_mixed_mig_and_whole_gpu_host():
+ other = "GPU-dddddddd-0000-0000-0000-000000000000"
+ rows = copy.deepcopy(ROWS) + [{**row(other, 72000, 81920), "index": "1"}]
+ result = mig.rows_from_payload(payload(rows = rows, uuids = [A, B_UUID, C, other]))
+ assert [r["uuid"] for r in result] == [A, B_UUID, C, other]
+ assert [r["index"] for r in result] == ["0", "1", "2", "3"]
+
+
+def test_numeric_mask_ids_stay_mask_ids():
+ with patch.dict(os.environ, {"CUDA_VISIBLE_DEVICES": "2,0"}):
+ result = mig.rows_from_payload(payload(uuids = [C, A]))
+ assert [r["index"] for r in result] == ["2", "0"]
+ assert mig.visibility_for(result, [2, 0]) == f"{C},{A}"
+
+
+def test_no_driver_enumeration_means_the_old_rule():
+ p = payload()
+ p["cuda_devices"] = None
+ assert mig.rows_from_payload(p) is None
+ p["cuda_devices"] = [{"uuid": ""}]
+ assert mig.rows_from_payload(p) is None
+ assert mig.rows_from_payload(payload(driver = "565.57.01")) is None
+ assert mig.rows_from_payload(payload(rows = [ROWS[0]], uuids = [GPU])) is None # no MIG
+
+
+def test_other_hosts_never_start_the_probe():
+ with (
+ patch.object(mig, "possible", lambda: False),
+ patch.object(mig.subprocess, "run", side_effect = AssertionError),
+ ):
+ assert mig.visible_devices() is None
+ # An NVIDIA Linux host whose cached listing shows no MIG instance, or could not answer.
+ for listed in (set(), None):
+ with (
+ patch.object(mig, "possible", lambda: True),
+ patch.dict(os.environ, {}, clear = True),
+ patch.object(nvidia, "get_mig_gpu_indices_or_none", lambda: listed),
+ patch.object(mig.subprocess, "run", side_effect = AssertionError),
+ ):
+ assert mig.visible_devices() is None
+
+
+def test_a_worker_never_probes_for_its_pin():
+ with (
+ patch.dict(os.environ, {}, clear = True),
+ patch.object(nvidia, "get_mig_gpu_indices_or_none", side_effect = AssertionError("listed")),
+ patch.object(mig.subprocess, "run", side_effect = AssertionError("probed")),
+ ):
+ assert mig.mask_for([0]) is None
+
+
+def test_gguf_memory_skips_the_mig_listing_off_nvidia_linux(monkeypatch):
+ monkeypatch.setattr(mig, "possible", lambda: False)
+ monkeypatch.setattr(
+ nvidia,
+ "get_mig_gpu_indices_or_none",
+ lambda: (_ for _ in ()).throw(AssertionError("listed")),
+ )
+ monkeypatch.setattr(B, "_is_vulkan_backend", staticmethod(lambda *a: False))
+ monkeypatch.setattr(B, "_find_llama_server_binary", staticmethod(lambda: "/mock/llama-server"))
+ monkeypatch.setattr(
+ gpu_query, "run_nvidia_smi", lambda *a, **k: (_ for _ in ()).throw(FileNotFoundError())
+ )
+ monkeypatch.setattr(B, "_get_gpu_memory_amd_smi", staticmethod(lambda *a, **k: []))
+ monkeypatch.setattr(B, "_get_gpu_memory_torch", staticmethod(lambda *a, **k: []))
+ monkeypatch.setattr(B, "_get_gpu_memory_nvml", staticmethod(lambda *a, **k: []))
+ monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)
+ assert B._get_gpu_memory() == []
+
+
+def test_compute_caps_follow_the_instances():
+ with host():
+ with patch.object(
+ B, "_installed_llama_cuda_sms", return_value = frozenset([80]), create = True
+ ):
+ assert len(B._get_gpu_memory(for_llama_server = True)) == 3
+ assert B._cuda_compute_caps() == {0: 80, 1: 80, 2: 80}
+
+
+def test_mig_instances_cannot_use_the_physical_nvlink_matrix():
+ with (
+ host(),
+ patch.dict(os.environ, {"UNSLOTH_DISABLE_DC_P2P": "0", "UNSLOTH_FORCE_DC_P2P": "0"}),
+ ):
+ B._get_gpu_memory()
+ assert "MIG" in B._p2p_veto_reason_inner([0, 1, 2], True, True)
+
+
+def _probe_module():
+ path = Path(__file__).resolve().parents[2] / "nvidia_probe.py"
+ spec = importlib.util.spec_from_file_location("mig_test_nvidia_probe", path)
+ module = importlib.util.module_from_spec(spec)
+ sys.modules[spec.name] = module
+ spec.loader.exec_module(module)
+ return module
+
+
+def _set_int(
+ ptr,
+ value,
+ kind = ctypes.c_int,
+):
+ ctypes.cast(ptr, ctypes.POINTER(kind))[0] = value
+ return 0
+
+
+class _Cuda:
+ raw = bytes.fromhex(A[4:].replace("-", ""))
+
+ def cuInit(self, _):
+ return 0
+
+ def cuDriverGetVersion(self, p):
+ return _set_int(p, 12080)
+
+ def cuDeviceGetCount(self, p):
+ return _set_int(p, 1)
+
+ def cuDeviceGet(self, p, i):
+ return _set_int(p, i)
+
+ def cuDeviceGetName(self, p, n, d):
+ p.value = b"A100 MIG"
+ return 0
+
+ def cuDeviceGetAttribute(self, p, a, d):
+ return _set_int(p, 8 if a == 75 else 0)
+
+ def cuDeviceGetUuid_v2(self, p, d):
+ ctypes.memmove(p, self.raw, 16)
+ return 0
+
+
+def test_strict_cuda_enumeration_reads_instance_uuids():
+ module = _probe_module()
+ with patch.object(module, "_load", return_value = _Cuda()):
+ result = module._probe_cuda_driver(strict = True)
+ assert result["devices"][0]["uuid"] == A[4:]
+ assert result["devices"][0]["memory_total_mib"] == "0"
+
+
+def test_the_cuda_driver_fallback_is_unchanged():
+ module = _probe_module()
+ with patch.object(module, "_load", return_value = _Cuda()):
+ assert module._probe_cuda_driver()["devices"][0]["uuid"] == ""
+
+ class NoCount(_Cuda):
+ def cuDeviceGetCount(self, p):
+ return 1
+
+ with patch.object(module, "_load", return_value = NoCount()):
+ assert module._probe_cuda_driver()["devices"] == []
+ assert module._probe_cuda_driver(strict = True) is None
+
+
+def test_nvml_probe_enumerates_cuda_uuids_only_for_r570_mig():
+ module = _probe_module()
+
+ class NVML:
+ version = b"570.86.15"
+
+ def nvmlInit_v2(self):
+ return 0
+
+ def nvmlShutdown(self):
+ return 0
+
+ def nvmlSystemGetDriverVersion(self, p, n):
+ p.value = self.version
+ return 0
+
+ def nvmlSystemGetCudaDriverVersion(self, p):
+ return _set_int(p, 12080, ctypes.c_uint)
+
+ def nvmlDeviceGetCount_v2(self, p):
+ return _set_int(p, 1, ctypes.c_uint)
+
+ def nvmlDeviceGetHandleByIndex_v2(self, i, p):
+ return 0
+
+ def nvmlDeviceGetName(self, h, p, n):
+ p.value = b"A100"
+ return 0
+
+ def nvmlDeviceGetUUID(self, h, p, n):
+ p.value = GPU.encode()
+ return 0
+
+ def nvmlDeviceGetCudaComputeCapability(self, h, a, b):
+ _set_int(a, 8)
+ return _set_int(b, 0)
+
+ def nvmlDeviceGetMemoryInfo(self, h, p):
+ return 0
+
+ nvml = NVML()
+ memory = module._NvmlMemory(total = 40192 * 1024 * 1024, free = 36000 * 1024 * 1024, used = 0)
+ cuda = dict(devices = [dict(uuid = A[4:], index = "0")])
+ with (
+ patch.object(module, "_load", return_value = nvml),
+ patch.object(module, "_mig_children", return_value = [(None, A, memory)]),
+ patch.object(module, "_probe_cuda_driver", return_value = cuda) as reader,
+ patch.object(module.sys, "platform", "linux"),
+ ):
+ result = module._probe_nvml()
+ assert result["cuda_devices"] == cuda["devices"]
+ reader.assert_called_once_with(strict = True)
+ reader.reset_mock()
+ nvml.version = b"565.57.01"
+ assert module._probe_nvml()["cuda_devices"] is None
+ reader.assert_not_called()
+ with (
+ patch.object(module, "_load", return_value = nvml),
+ patch.object(module, "_mig_children", return_value = []),
+ patch.object(module, "_probe_cuda_driver", return_value = cuda) as reader,
+ ):
+ nvml.version = b"570.86.15"
+ assert module._probe_nvml()["cuda_devices"] is None # no MIG: no extra enumeration
+ reader.assert_not_called()
+
+
+def test_only_placement_relists_after_a_listing_found_no_instance():
+ # The last listing (any age, here from before a hardware re-detect) found no MIG instance.
+ stale = (gpu_query._static_gen - 1, 0, 0.0, frozenset())
+ with (
+ patch.object(mig, "possible", lambda: True),
+ patch.dict(os.environ, {"UNSLOTH_NVIDIA_LIBRARY_PROBE": "1"}, clear = True),
+ patch.object(nvidia, "_mig_entry", stale),
+ patch.object(mig.subprocess, "run", side_effect = AssertionError("probed")),
+ ):
+ with patch.object(
+ nvidia, "get_mig_gpu_indices_or_none", side_effect = AssertionError("listed")
+ ):
+ assert mig.visible_devices() is None
+ assert mig.visible_devices(fresh = True) is None
+ listed = []
+ with patch.object(nvidia, "get_mig_gpu_indices_or_none", lambda: listed.append(1) or set()):
+ assert mig.visible_devices(fresh = True, listing = True) is None
+ assert listed == [1]
+
+
+def _counting_probe(gate = None):
+ """Count NVML probe children (``--json``); optionally hold each one until ``gate`` is set."""
+ inner = subprocess.run
+ calls = []
+
+ def run(cmd, *args, **kw):
+ if "--json" in cmd:
+ calls.append(cmd)
+ if gate is not None:
+ gate.wait(10)
+ return inner(cmd, *args, **kw)
+
+ with patch.object(subprocess, "run", run):
+ yield calls
+
+
+def test_the_backend_visible_inventory_lists_every_gpu_instance():
+ with host():
+ info = hw._probe_backend_visible_gpu_info(hw.DeviceType.CUDA)
+ assert info["index_kind"] == "cuda_device"
+ assert [d["index"] for d in info["devices"]] == [0, 1, 2]
+ assert [d["uuid"] for d in info["devices"]] == [A, B_UUID, C]
+ assert [d["memory_total_gb"] for d in info["devices"]] == [39.25, 19.5, 19.5]
+
+
+def test_a_manual_split_under_an_inherited_uuid_mask_pins_those_uuids():
+ with host(f"{B_UUID},{A}"):
+ assert len(B._get_gpu_memory()) == 2
+ env = dict(os.environ)
+ assert B._pin_visible_gpu_order_for_split(env) == (0, 1)
+ assert env["CUDA_VISIBLE_DEVICES"] == f"{B_UUID},{A}"
+ # CUDA device ids never pass for physical card ids.
+ assert B._resolve_visible_physical_ids() is None
+
+
+def test_p2p_vetoes_mig_instances_but_whole_cards_keep_the_physical_matrix(monkeypatch):
+ rows = [
+ dict(index = "0", physical_index = "1", uuid = GPU, name = "NVIDIA A100-SXM4-80GB"),
+ dict(index = "1", physical_index = "0", uuid = A, name = "NVIDIA A100-SXM4-80GB", mig = "1"),
+ ]
+ monkeypatch.delenv("UNSLOTH_DISABLE_DC_P2P", raising = False)
+ monkeypatch.delenv("UNSLOTH_FORCE_DC_P2P", raising = False)
+ monkeypatch.setattr(
+ B,
+ "_MIG_DEVICE_SNAPSHOT",
+ (os.environ.get("CUDA_VISIBLE_DEVICES"), os.environ.get("CUDA_DEVICE_ORDER"), rows),
+ raising = False,
+ )
+ seen = []
+
+ def match(pattern, gpu_indices):
+ seen.append(list(gpu_indices))
+ return False
+
+ monkeypatch.setattr(B, "_all_selected_gpus_match", staticmethod(match))
+ mig_reason = "MIG device UUIDs cannot be verified by the physical-card NVLink matrix"
+ assert B._p2p_veto_reason_inner([1]) == mig_reason
+ assert B._p2p_veto_reason_inner(None) == mig_reason
+ assert B._p2p_veto_reason_inner([0]) != mig_reason
+ assert seen == [[1]] # the whole card's nvidia-smi index, not its CUDA device id
+
+
+def test_a_failed_refresh_backs_off_for_fresh_callers():
+ with host() as probe:
+ assert len(B._get_gpu_memory()) == 3
+ probe["source"] = "failed"
+ _age_cache(mig._FRESH_SHARE_S + 1)
+ with _counting_probe() as calls:
+ for _ in range(5):
+ rows = mig.visible_devices(fresh = True)
+ assert [r["memory_total_mib"] for r in rows] == ["0", "0", "0"]
+ assert len(calls) == 1
+
+
+def test_concurrent_fresh_callers_share_one_probe_child():
+ import threading
+ with host():
+ assert len(B._get_gpu_memory()) == 3
+ _age_cache(mig._FRESH_SHARE_S + 1)
+ gate = threading.Event()
+ with _counting_probe(gate) as calls:
+ results = []
+ threads = [
+ threading.Thread(target = lambda: results.append(mig.visible_devices(fresh = True)))
+ for _ in range(8)
+ ]
+ for t in threads:
+ t.start()
+ while not calls:
+ threading.Event().wait(0.01)
+ threading.Event().wait(0.2)
+ gate.set()
+ for t in threads:
+ t.join(10)
+ assert len(calls) == 1
+ assert len(results) == 8 and all(r is not None and len(r) == 3 for r in results)
+
+
+def test_polling_inside_the_share_window_starts_no_child():
+ with host():
+ assert len(B._get_gpu_memory()) == 3
+ with _counting_probe() as calls:
+ for _ in range(10):
+ mig.visible_devices(fresh = True)
+ assert calls == []
+
+
+def test_a_legacy_answer_is_not_re_probed_by_fresh_callers():
+ with host(driver = "565.57.01"):
+ with _counting_probe() as calls:
+ for _ in range(3):
+ assert mig.visible_devices(fresh = True, listing = True) is None
+ _age_cache(mig._FAILED_BACKOFF_S + 1)
+ assert len(calls) == 1
+
+
+def test_the_physical_card_correction_never_touches_instance_rows(monkeypatch):
+ def correction(devices):
+ raise AssertionError("physical-card MIG correction ran on CUDA device rows")
+
+ with host():
+ monkeypatch.setattr(hw, "_with_mig_instance_memory", correction)
+ selection, _ = hw.auto_select_gpu_ids("fixture", required_override_gb = 30)
+ assert selection == [0]
+
+
+def test_diffusion_gemma_pins_the_instance_uuid_not_a_numeric_id():
+ with host(f"{B_UUID},{A}"):
+ assert len(B._get_gpu_memory()) == 2
+ assert B._mig_pinned_gpu_token("0") == B_UUID
+ assert B._mig_pinned_gpu_token("1,0") == f"{A},{B_UUID}"
+ assert B._mig_pinned_gpu_token("") == ""
+ assert B._mig_pinned_gpu_token(B_UUID) == B_UUID
+
+
+def test_diffusion_gemma_launch_uses_the_pinned_token():
+ import inspect
+ assert "gpu = self._mig_pinned_gpu_token(gpu)" in inspect.getsource(lc)
+
+
+def test_optional_engines_pin_the_instance_uuid_and_budget_its_slice(monkeypatch):
+ from core.inference import engine_adapters, engine_install, managed_engine
+
+ monkeypatch.setattr(engine_install, "gpu_platform", lambda: "cuda")
+ checked = []
+
+ def support_reason(engine, gpu_id, **kwargs):
+ checked.append(gpu_id)
+ return "stop here"
+
+ monkeypatch.setattr(managed_engine, "support_reason", support_reason)
+ request = types.SimpleNamespace(gpu_ids = [1], gguf_variant = None, model_path = "org/model")
+ with host(f"{B_UUID},{A}"):
+ with pytest.raises(ValueError, match = "GPU 1: stop here"):
+ managed_engine.validate_load("vllm", request)
+ # CUDA device 1 is instance A on physical card 0: the driver check reads that card.
+ assert checked == [0]
+ assert managed_engine._child_cuda_mask([1]) == A
+ assert managed_engine._child_cuda_mask([1, 0]) == f"{A},{B_UUID}"
+ # The budget is the selected slice's memory, not nvidia-smi --id on a physical card.
+ assert engine_adapters._nvidia_memory_rows([1]) == [(40192.0, 36000.0)]
+
+
+def test_a_reordered_numeric_mask_keeps_its_order_for_a_manual_split():
+ with host("2,0"):
+ assert [g[0] for g in B._get_gpu_memory()] == [0, 2]
+ env = dict(os.environ)
+ assert B._pin_visible_gpu_order_for_split(env) == (2, 0)
+ assert env["CUDA_VISIBLE_DEVICES"] == f"{C},{A}"
+
+
+def test_an_r570_mig_payload_without_its_cuda_enumeration_is_a_failed_refresh():
+ with host() as probe:
+ assert len(B._get_gpu_memory()) == 3
+ probe["cuda_devices"] = None
+ _age_cache(mig._FRESH_SHARE_S + 1)
+ rows = mig.visible_devices(fresh = True)
+ assert [r["uuid"] for r in rows] == [A, B_UUID, C]
+ assert all(r["memory_total_mib"] == "0" for r in rows)
+ assert next(iter(mig._cache.values()))[2] == "failed"
+
+
+def test_saved_picks_carry_the_cuda_device_namespace():
+ from utils.openai_auto_switch_settings import stored_gpu_index_kind
+
+ assert stored_gpu_index_kind({"gpu_index_kind": "cuda_device"}) == "cuda_device"
+ with host():
+ assert hw.current_gpu_index_kind() == "cuda_device"
+ assert hw.current_gpu_index_kind(is_vulkan = True) == "vulkan"
+ with host(driver = "565.57.01"):
+ assert hw.current_gpu_index_kind() == "physical"
+
+
+def test_a_physical_pin_saved_before_is_not_reused_on_a_mig_host(monkeypatch):
+ import asyncio
+
+ from routes import inference
+
+ monkeypatch.setattr(B, "_is_vulkan_backend", staticmethod(lambda *a: False))
+ with host():
+ assert asyncio.run(inference._override_gpu_ids_still_resolve([1], "physical")) is False
+ assert asyncio.run(inference._override_gpu_ids_still_resolve([1], "cuda_device")) is True
+
+
+def test_a_retired_probe_does_not_replace_the_worker_map():
+ with host() as probe:
+ assert len(B._get_gpu_memory()) == 3
+ current = os.environ[mig._ENV]
+ _age_cache(mig._FRESH_SHARE_S + 1)
+ probe["cuda_devices"] = probe["cuda_devices"][:1]
+ inner = subprocess.run
+
+ def run(cmd, *args, **kw):
+ if "--json" in cmd:
+ # Hardware re-detection lands while this probe runs.
+ gpu_query.invalidate_static("re-detect during probe")
+ return inner(cmd, *args, **kw)
+
+ with patch.object(subprocess, "run", run):
+ assert len(mig.visible_devices(fresh = True)) == 1
+ assert os.environ[mig._ENV] == current
+
+
+def test_untranslatable_optional_engine_ids_keep_the_physical_behaviour(monkeypatch):
+ from core.inference import engine_install, managed_engine
+
+ monkeypatch.setattr(engine_install, "gpu_platform", lambda: "cuda")
+ checked = []
+
+ def support_reason(engine, gpu_id, **kwargs):
+ checked.append(gpu_id)
+ return "stop here"
+
+ monkeypatch.setattr(managed_engine, "support_reason", support_reason)
+ request = types.SimpleNamespace(gpu_ids = [1], gguf_variant = None, model_path = "org/model")
+ with host():
+ monkeypatch.setattr(mig, "mask_for", lambda ids: None)
+ # No refusal: the check and the mask are exactly the pre-MIG ones for this launch.
+ with pytest.raises(ValueError, match = "GPU 1: stop here"):
+ managed_engine.validate_load("vllm", request)
+ assert checked == [1]
+ assert managed_engine._child_cuda_mask([1]) == "1"
+
+
+def test_a_mig_slice_without_a_memory_reading_fails_the_reservation_clearly():
+ from core.inference import engine_adapters
+
+ # Imported here, not first inside host(): its sys.modules patch would drop the module on exit.
+ from utils import vram_budget_settings # noqa: F401
+ with host() as probe:
+ assert len(B._get_gpu_memory()) == 3
+ probe["source"] = "failed"
+ _age_cache(mig._FRESH_SHARE_S + 1)
+ inner = subprocess.run
+ queried = []
+
+ def run(cmd, *args, **kw):
+ if "--query-gpu=memory.total,memory.free" in cmd:
+ queried.append(cmd)
+ return inner(cmd, *args, **kw)
+
+ with patch.object(subprocess, "run", run):
+ with pytest.raises(RuntimeError, match = "MIG GPU instance has no memory reading.*Retry"):
+ engine_adapters.gpu_memory_fraction([0])
+ # CUDA device ids are not physical cards: nvidia-smi --id never stands in for a slice.
+ assert queried == []
diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.py
index 80964175ff8..0be5f13e771 100644
--- a/studio/backend/utils/hardware/hardware.py
+++ b/studio/backend/utils/hardware/hardware.py
@@ -5306,6 +5306,40 @@ def get_visible_gpu_utilization(*, no_context: bool = False) -> Dict[str, Any]:
"""``no_context``: never ask torch (it leaves a resident driver context): CUDA devices are empty without nvidia-smi,
ROCm ones come from _rocm_context_free_visible_devices."""
device = get_device()
+ if device == DeviceType.CUDA and not IS_ROCM:
+ from . import mig
+ rows = mig.visible_devices(fresh = True)
+ if rows is not None:
+ devices = []
+ for ordinal, row in enumerate(rows):
+ free = int(row.get("memory_free_mib") or 0)
+ total = int(row.get("memory_total_mib") or 0)
+ used = max(0, total - free) if total else None
+ devices.append(
+ {
+ "index": int(row["index"]),
+ "index_kind": "cuda_device",
+ "visible_ordinal": ordinal,
+ "name": row.get("name"),
+ "uuid": row["uuid"],
+ "gpu_utilization_pct": None,
+ "temperature_c": None,
+ "vram_used_gb": round(used / 1024, 2) if used is not None else None,
+ "vram_total_gb": round(total / 1024, 2) if total else None,
+ "vram_utilization_pct": round(used / total * 100, 1) if total else None,
+ "power_draw_w": None,
+ "power_limit_w": None,
+ "power_utilization_pct": None,
+ }
+ )
+ return {
+ "available": bool(rows),
+ "backend": _backend_label(device),
+ "parent_visible_gpu_ids": [d["index"] for d in devices],
+ "devices": devices,
+ "index_kind": "cuda_device",
+ "mig_corrected": True,
+ }
if no_context and IS_ROCM and device == DeviceType.CUDA:
devices = _rocm_context_free_visible_devices()
return {
@@ -5519,6 +5553,16 @@ _visible_gpu_count: Optional[int] = None
def _get_parent_visible_gpu_spec() -> Dict[str, Any]:
+ if get_device() == DeviceType.CUDA and not IS_ROCM:
+ from . import mig
+ rows = mig.visible_devices()
+ if rows is not None:
+ return {
+ "raw": os.environ.get("CUDA_VISIBLE_DEVICES"),
+ "numeric_ids": [int(r["index"]) for r in rows],
+ "supports_explicit_gpu_ids": True,
+ "index_kind": "cuda_device",
+ }
# On Intel XPU, visibility is controlled by ZE_AFFINITY_MASK (Level Zero), not CUDA_VISIBLE_DEVICES.
if get_device() == DeviceType.XPU:
xpu_mask_raw = _ze_affinity_mask()
@@ -6733,13 +6777,16 @@ def auto_select_gpu_ids(
smi_by_ordinal = (
_smi_rows_for_cuda_ordinals(get_parent_visible_gpu_ids())
- if _mask_names_cuda_ordinals(parent_visible_spec)
+ if parent_visible_spec.get("index_kind") != "cuda_device"
+ and _mask_names_cuda_ordinals(parent_visible_spec)
else None
)
if smi_by_ordinal is not None:
# nvidia-smi telemetry for the cards the ids select, joined by PCI bus id out of process.
utilization = smi_by_ordinal
- elif _mask_names_cuda_ordinals(parent_visible_spec):
+ elif parent_visible_spec.get("index_kind") != "cuda_device" and _mask_names_cuda_ordinals(
+ parent_visible_spec
+ ):
# The display keeps the CLI rows; ranking needs the cards the worker will get. Torch already
# reports a MIG slice's own memory, and these rows carry mask ids, not the physical rows the
# MIG correction keys on, so it must not run on them.
@@ -6757,7 +6804,8 @@ def auto_select_gpu_ids(
else:
utilization = get_visible_gpu_utilization()
devices = [d for d in utilization.get("devices", []) if d.get("index") not in _uncovered]
- if not utilization.get("mig_corrected"):
+ # Instance rows carry slice memory; the physical-card correction would overwrite it.
+ if not utilization.get("mig_corrected") and utilization.get("index_kind") != "cuda_device":
devices = _with_mig_instance_memory(devices)
parent_ids = _covered(get_parent_visible_gpu_ids())
@@ -7327,6 +7375,47 @@ def get_backend_visible_gpu_info() -> Dict[str, Any]:
return stale
+def current_gpu_index_kind(is_vulkan: bool = False) -> str:
+ """GPU id namespace on this host now: ``vulkan``, ``cuda_device`` (R570+ MIG) or ``physical``."""
+ if is_vulkan:
+ return "vulkan"
+ try:
+ if _get_parent_visible_gpu_spec().get("index_kind") == "cuda_device":
+ return "cuda_device"
+ except Exception:
+ pass
+ return "physical"
+
+
+def _mig_backend_visible_gpu_info(raw_mask) -> Optional[Dict[str, Any]]:
+ """Backend-visible inventory on an R570+ MIG host, one row per CUDA device, else None."""
+ from . import mig
+
+ rows = mig.visible_devices()
+ if rows is None:
+ return None
+ devices = []
+ for ordinal, row in enumerate(rows):
+ total_mib = int(row.get("memory_total_mib") or 0)
+ devices.append(
+ {
+ "index": int(row["index"]),
+ "index_kind": "cuda_device",
+ "visible_ordinal": ordinal,
+ "name": row.get("name"),
+ "uuid": row["uuid"],
+ "memory_total_gb": round(total_mib / 1024, 2) if total_mib else None,
+ }
+ )
+ return {
+ "available": bool(devices),
+ "backend_cuda_visible_devices": raw_mask,
+ "parent_visible_gpu_ids": [d["index"] for d in devices],
+ "devices": devices,
+ "index_kind": "cuda_device",
+ }
+
+
def _probe_backend_visible_gpu_info(device) -> Dict[str, Any]:
if device in (DeviceType.CUDA, DeviceType.XPU):
parent_visible_ids = get_parent_visible_gpu_ids()
@@ -7341,6 +7430,11 @@ def _probe_backend_visible_gpu_info(device) -> Dict[str, Any]:
from . import nvidia
parent_visible_spec = _get_parent_visible_gpu_spec()
+ if parent_visible_spec.get("index_kind") == "cuda_device":
+ mig_result = _mig_backend_visible_gpu_info(parent_visible_spec["raw"])
+ if mig_result is not None:
+ mig_result["backend"] = _backend_label(device)
+ return mig_result
result = nvidia.get_backend_visible_gpu_info(
parent_visible_spec["numeric_ids"],
parent_visible_spec["raw"],
@@ -7616,6 +7710,18 @@ def apply_gpu_ids(gpu_ids, backend: Optional[str] = None) -> None:
global _visible_gpu_count
+ if (
+ backend == DeviceType.CUDA.value or (backend is None and DEVICE == DeviceType.CUDA)
+ ) and not IS_ROCM:
+ from . import mig
+
+ ids = gpu_ids if isinstance(gpu_ids, (list, tuple)) else str(gpu_ids).split(",")
+ mask = mig.mask_for(ids)
+ if mask is not None:
+ os.environ["CUDA_VISIBLE_DEVICES"] = mask
+ _visible_gpu_count = None
+ return
+
if isinstance(gpu_ids, (list, tuple)):
value = ",".join(str(g) for g in gpu_ids)
else:
diff --git a/studio/backend/utils/hardware/mig.py b/studio/backend/utils/hardware/mig.py
new file mode 100644
index 00000000000..38037a2df34
--- /dev/null
+++ b/studio/backend/utils/hardware/mig.py
@@ -0,0 +1,249 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+"""R570+ MIG: CUDA device ordinals (one per GPU instance) joined to NVML slice memory by UUID."""
+
+import json
+import os
+import re
+import subprocess
+import sys
+import threading
+import time
+
+from . import gpu_query, nvidia
+from utils.child_stdio import utf8_child_env
+from utils.native_path_leases import child_env_without_native_path_secret
+from utils.subprocess_compat import windows_hidden_subprocess_kwargs
+
+# key -> (monotonic time, rows, state); state "rows", "legacy" (old rule) or "failed" (rows keep identities, zero memory).
+_cache = {}
+_flights = {}
+_flights_lock = threading.Lock()
+_FAILED_BACKOFF_S = 5.0
+_ROWS_TTL_S = 60.0
+_FRESH_SHARE_S = 2.0
+# Parent's ordinal -> MIG uuid map; workers pin from it and never probe.
+_ENV = "UNSLOTH_MIG_CUDA_DEVICES"
+
+
+def possible() -> bool:
+ """MIG needs bare-metal Linux with the NVIDIA kernel driver (not WSL); elsewhere nothing starts."""
+ return sys.platform == "linux" and os.path.isfile("/proc/driver/nvidia/version")
+
+
+def rows_from_payload(payload):
+ """None = legacy/unknown; [] = CUDA confirmed an empty set. Uses CUDA's UUIDs, not MIG row
+ counts: several CIs share a GI, and containers may expose only some."""
+ if not isinstance(payload, dict) or payload.get("source") != "nvml":
+ return None
+ match = re.match(r"(\d+)\.", str(payload.get("driver_version") or ""))
+ if not match or int(match[1]) < 570:
+ return None
+ cuda_devices = payload.get("cuda_devices")
+ if not isinstance(cuda_devices, list):
+ return None
+ inventory = payload.get("devices") or []
+ if not any(r.get("mig") for r in inventory):
+ return None
+ by_uuid = {str(r.get("uuid", ""))[-36:].lower(): r for r in inventory if r.get("uuid")}
+ raw = os.environ.get("CUDA_VISIBLE_DEVICES")
+ tokens = [t.strip() for t in (raw or "").split(",")]
+ numeric = raw is not None and all(re.fullmatch(r"\d+", t) for t in tokens)
+ ids = [int(t) for t in tokens] if numeric and len(tokens) == len(cuda_devices) else None
+ rows = []
+ seen = set()
+ for ordinal, device in enumerate(cuda_devices):
+ key = str(device.get("uuid", ""))[-36:].lower()
+ row = by_uuid.get(key)
+ if row is None or key in seen:
+ return None # never guess an ordinal-to-UUID mapping
+ seen.add(key)
+ rows.append(
+ {
+ **row,
+ "physical_index": row["index"],
+ "index": str(ids[ordinal] if ids is not None else ordinal),
+ }
+ )
+ return rows
+
+
+def _mig_instances(listing):
+ """Cards with MIG instances from the cached ``nvidia-smi -L``. Only ``listing`` callers (GGUF
+ placement) may always relist; others list once per hardware generation, then reuse the answer."""
+ if listing:
+ return nvidia.get_mig_gpu_indices_or_none()
+ cached = nvidia._mig_cached()
+ if cached is not nvidia._MISS:
+ return cached
+ entry = nvidia._mig_entry
+ if entry is not None and entry[3] is not None and not entry[3]:
+ return None
+ return nvidia.get_mig_gpu_indices_or_none()
+
+
+def visible_devices(*, fresh = False, listing = False):
+ """R570+ MIG rows from a bounded out-of-process probe, or None (old rule). ``fresh`` for memory."""
+ if not possible() or os.environ.get("UNSLOTH_NVIDIA_LIBRARY_PROBE", "1") == "0":
+ return None
+ raw = os.environ.get("CUDA_VISIBLE_DEVICES")
+ if not (raw and "mig-" in raw.lower()) and not _mig_instances(listing):
+ os.environ.pop(_ENV, None)
+ return None
+ key = _current_key()
+ hit = _cached_answer(key, fresh)
+ if hit is not _MISS:
+ return hit
+ # Single flight per key.
+ with _flights_lock:
+ flight = _flights.get(key)
+ leader = flight is None
+ if leader:
+ flight = _flights[key] = threading.Event()
+ if not leader:
+ flight.wait(15)
+ entry = _cache.get(key)
+ return entry[1] if entry is not None else None
+ try:
+ return _refresh(key, raw)
+ finally:
+ with _flights_lock:
+ _flights.pop(key, None)
+ flight.set()
+
+
+_MISS = object()
+
+
+def _current_key():
+ return (
+ os.environ.get("CUDA_VISIBLE_DEVICES"),
+ os.environ.get("CUDA_DEVICE_ORDER"),
+ gpu_query._static_gen,
+ os.environ.get("UNSLOTH_NVIDIA_PROBE"),
+ )
+
+
+def _cached_answer(key, fresh):
+ entry = _cache.get(key)
+ if entry is None:
+ return _MISS
+ age = time.monotonic() - entry[0]
+ state = entry[2]
+ if state == "failed":
+ return entry[1] if age < _FAILED_BACKOFF_S else _MISS
+ if state == "legacy":
+ return None if age < _ROWS_TTL_S else _MISS
+ if age < _FRESH_SHARE_S or (not fresh and age < _ROWS_TTL_S):
+ return entry[1]
+ return _MISS
+
+
+def _probe():
+ """(rows, answered). ``answered`` is False when the probe child gave no NVML answer."""
+ try:
+ from utils.prebuilt.update_flow import find_installer_script
+
+ script = find_installer_script(
+ env_var = "UNSLOTH_NVIDIA_PROBE", script_name = "nvidia_probe.py"
+ )
+ if script is None:
+ return None, False
+ result = subprocess.run(
+ [sys.executable, "-I", str(script), "--json"],
+ capture_output = True,
+ text = True,
+ encoding = "utf-8",
+ errors = "replace",
+ timeout = 10,
+ env = utf8_child_env(child_env_without_native_path_secret()),
+ **windows_hidden_subprocess_kwargs(),
+ )
+ if result.returncode != 0:
+ return None, False
+ payload = json.loads(result.stdout)
+ # Only NVML answers; an R570+ MIG payload without ``cuda_devices`` has not answered either.
+ answered = isinstance(payload, dict) and payload.get("source") == "nvml"
+ if answered and not isinstance(payload.get("cuda_devices"), list):
+ match = re.match(r"(\d+)\.", str(payload.get("driver_version") or ""))
+ if (
+ match
+ and int(match[1]) >= 570
+ and any(r.get("mig") for r in payload.get("devices") or [])
+ ):
+ answered = False
+ return rows_from_payload(payload), answered
+ except Exception:
+ return None, False
+
+
+def _refresh(key, raw):
+ previous = _cache.get(key)
+ rows, answered = _probe()
+ if rows is not None:
+ state = "rows"
+ elif answered:
+ state = "legacy"
+ else:
+ state = "failed"
+ if previous is not None and previous[1] is not None:
+ # Never collapse a known set to physical cards; keep identities, drop stale memory.
+ rows = [{**r, "memory_free_mib": "0", "memory_total_mib": "0"} for r in previous[1]]
+ _cache[key] = (time.monotonic(), rows, state)
+ if key != _current_key():
+ # Retired while it ran: the worker map belongs to the current key only.
+ return rows
+ if rows is not None:
+ os.environ[_ENV] = json.dumps(
+ {
+ "key": [raw, os.environ.get("CUDA_DEVICE_ORDER")],
+ "uuids": {str(r["index"]): r["uuid"] for r in rows},
+ }
+ )
+ else:
+ os.environ.pop(_ENV, None)
+ return rows
+
+
+def _inherited_uuids():
+ """The parent's map if made under this process's mask and order, else None."""
+ try:
+ data = json.loads(os.environ.get(_ENV) or "null")
+ if data["key"] != [
+ os.environ.get("CUDA_VISIBLE_DEVICES"),
+ os.environ.get("CUDA_DEVICE_ORDER"),
+ ]:
+ return None
+ return {int(k): str(v) for k, v in data["uuids"].items()}
+ except Exception:
+ return None
+
+
+def mask_for(gpu_ids):
+ """CUDA_VISIBLE_DEVICES pinning these ids on an R570+ MIG host, else None. Reads only the
+ published map, never probes."""
+ by_id = _inherited_uuids()
+ return None if by_id is None else _pin(by_id, gpu_ids)
+
+
+def rows_for(gpu_ids):
+ """MIG rows for these ids in order, None off R570+ MIG; ValueError outside the visible set."""
+ rows = visible_devices(fresh = True)
+ if rows is None:
+ return None
+ by_id = {int(r["index"]): r for r in rows}
+ ids = [int(i) for i in gpu_ids]
+ if any(i not in by_id for i in ids):
+ raise ValueError(f"GPU IDs {ids} are outside the visible CUDA device set {list(by_id)}")
+ return [by_id[i] for i in ids]
+
+
+def visibility_for(rows, gpu_ids):
+ return _pin({int(r["index"]): r["uuid"] for r in rows}, gpu_ids)
+
+
+def _pin(by_id, gpu_ids):
+ ids = [int(i) for i in gpu_ids]
+ if len(ids) != len(set(ids)) or any(i not in by_id for i in ids):
+ raise ValueError(f"GPU IDs {ids} are outside the visible CUDA device set {list(by_id)}")
+ return ",".join(by_id[i] for i in ids)
diff --git a/studio/backend/utils/openai_auto_switch_settings.py b/studio/backend/utils/openai_auto_switch_settings.py
index e37778525ce..aed604ddaf2 100644
--- a/studio/backend/utils/openai_auto_switch_settings.py
+++ b/studio/backend/utils/openai_auto_switch_settings.py
@@ -395,7 +395,8 @@ MAX_CHAT_TEMPLATE_OVERRIDE_BYTES = MAX_CHAT_TEMPLATE_BYTES
MAX_GPU_ID = 1024
# Which index space a stored gpu_ids belongs to: the same integers are ggml Vulkan ordinals under a Vulkan build and physical device ids elsewhere, so the namespace travels with the ids or a pin addresses another card. Mirrors GpuIndexKind in hooks/gpu-selection.ts, legacy rule included: an absent kind is "physical".
-VALID_GPU_INDEX_KINDS = frozenset({"physical", "vulkan"})
+# cuda_device: R570+ MIG, one CUDA device per GPU instance.
+VALID_GPU_INDEX_KINDS = frozenset({"physical", "vulkan", "cuda_device"})
LEGACY_GPU_INDEX_KIND = "physical"
diff --git a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts
index 04b873dcbe2..e157450d75d 100644
--- a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts
+++ b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts
@@ -2381,7 +2381,11 @@ export function loadedGpuMemoryFields(resp: {
const gpuIds = managed
? (requestedGpuIdsFromResponse(resp) ?? resp.gpu_ids ?? [0])
: null;
- const indexKind = managed ? ("physical" as const) : null;
+ const indexKind = managed
+ ? cachedPinnableGpuIndexKind() === "cuda_device"
+ ? ("cuda_device" as const)
+ : ("physical" as const)
+ : null;
return {
selectedGpuIds: gpuIds,
selectedGpuIndexKind: indexKind,
diff --git a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts b/studio/frontend/src/features/model-picker/model-config/per-model-config.ts
index 609412e285b..ad4d4818d13 100644
--- a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts
+++ b/studio/frontend/src/features/model-picker/model-config/per-model-config.ts
@@ -576,6 +576,7 @@ function normalizeGpuFields(partial: RawConfig): {
if (
partial.selectedGpuIndexKind === "physical" ||
partial.selectedGpuIndexKind === "vulkan" ||
+ partial.selectedGpuIndexKind === "cuda_device" ||
partial.selectedGpuIndexKind === null
) {
out.selectedGpuIndexKind = partial.selectedGpuIndexKind;
diff --git a/studio/frontend/src/features/model-picker/sharing/fields.ts b/studio/frontend/src/features/model-picker/sharing/fields.ts
index 09b1e7b8a87..4372896caa9 100644
--- a/studio/frontend/src/features/model-picker/sharing/fields.ts
+++ b/studio/frontend/src/features/model-picker/sharing/fields.ts
@@ -147,7 +147,10 @@ export const SHARED_CONFIG_FIELDS: Record<SharedConfigKey, Field> = {
},
selectedGpuIndexKind: {
label: "GPU device index type",
- valid: nullable((value) => value === "physical" || value === "vulkan"),
+ valid: nullable(
+ (value) =>
+ value === "physical" || value === "vulkan" || value === "cuda_device",
+ ),
},
};
diff --git a/studio/frontend/src/hooks/gpu-selection.ts b/studio/frontend/src/hooks/gpu-selection.ts
index f0fb758ecd9..e9e5531acb7 100644
--- a/studio/frontend/src/hooks/gpu-selection.ts
+++ b/studio/frontend/src/hooks/gpu-selection.ts
@@ -1,7 +1,8 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-export type GpuIndexKind = "physical" | "vulkan";
+/** "cuda_device": R570+ MIG, ids name GPU instances, not cards. */
+export type GpuIndexKind = "physical" | "vulkan" | "cuda_device";
export interface SystemGpuDevice {
index: number;
diff --git a/studio/frontend/src/hooks/use-gpu-info.ts b/studio/frontend/src/hooks/use-gpu-info.ts
index d788bf30a1a..fe3caf4220e 100644
--- a/studio/frontend/src/hooks/use-gpu-info.ts
+++ b/studio/frontend/src/hooks/use-gpu-info.ts
@@ -299,7 +299,9 @@ function toGpuDevices(
.map((d) => ({
index: d.index as number,
indexKind:
- d.index_kind === "physical" || d.index_kind === "vulkan"
+ d.index_kind === "physical" ||
+ d.index_kind === "vulkan" ||
+ d.index_kind === "cuda_device"
? d.index_kind
: null,
name: d.name ?? `GPU ${d.index}`,
@@ -316,8 +318,11 @@ function toGpuDevices(
pinnable:
pinnableBackend &&
(d.index_kind === "vulkan" ||
- (data?.device_backend !== "xpu" && d.index_kind === "physical")),
- diffusionPinnable: diffusionBackend && d.index_kind === "physical",
+ (data?.device_backend !== "xpu" &&
+ (d.index_kind === "physical" || d.index_kind === "cuda_device"))),
+ diffusionPinnable:
+ diffusionBackend &&
+ (d.index_kind === "physical" || d.index_kind === "cuda_device"),
}));
}
diff --git a/studio/frontend/tests/mig-gpu-index-kind-persistence.test.ts b/studio/frontend/tests/mig-gpu-index-kind-persistence.test.ts
new file mode 100644
index 00000000000..380542e0da7
--- /dev/null
+++ b/studio/frontend/tests/mig-gpu-index-kind-persistence.test.ts
@@ -0,0 +1,37 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import assert from "node:assert/strict";
+import test from "node:test";
+import {
+ installLocalStorageFake,
+ registerStoreStubResolver,
+} from "./helpers/kit.ts";
+
+registerStoreStubResolver();
+installLocalStorageFake();
+const { normalizePerModelConfig } = await import(
+ "../src/features/model-picker/model-config/per-model-config.ts"
+);
+const { toApiOverride } = await import(
+ "../src/features/model-picker/api/model-overrides.ts"
+);
+
+test("a MIG instance pick keeps its CUDA-device namespace through normalization", () => {
+ const config = normalizePerModelConfig({
+ selectedGpuIds: [2, 0],
+ selectedGpuIndexKind: "cuda_device",
+ });
+ assert.deepEqual(config.selectedGpuIds, [2, 0]);
+ assert.equal(config.selectedGpuIndexKind, "cuda_device");
+ // And the saved override carries it, so the server never reads these ids as physical cards.
+ assert.equal(toApiOverride(config).gpu_index_kind, "cuda_device");
+});
+
+test("an unknown namespace is still dropped", () => {
+ const config = normalizePerModelConfig({
+ selectedGpuIds: [0],
+ selectedGpuIndexKind: "relative",
+ });
+ assert.equal(config.selectedGpuIndexKind, undefined);
+});
diff --git a/studio/frontend/tests/mig-gpu-instances.test.ts b/studio/frontend/tests/mig-gpu-instances.test.ts
new file mode 100644
index 00000000000..c896fd4d541
--- /dev/null
+++ b/studio/frontend/tests/mig-gpu-instances.test.ts
@@ -0,0 +1,96 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import assert from "node:assert/strict";
+import test from "node:test";
+import type * as GpuHooks from "../src/hooks/use-gpu-info.ts";
+import {
+ pinnableGpuContext,
+ reconcileGpuSelection,
+} from "../src/hooks/gpu-selection.ts";
+import { normalizeDenseQuantSchemes } from "../src/lib/dense-quant-schemes.ts";
+import { normalizeReportedOffloadFitTiers } from "../src/lib/offload-fit-tiers.ts";
+import { loadWithStubs } from "./helpers/module-stubs.ts";
+
+let snapshot: unknown = null;
+const hooks = loadWithStubs<typeof GpuHooks>(
+ new URL("../src/hooks/use-gpu-info.ts", import.meta.url),
+ {
+ react: {
+ useState: (initial: unknown) => [
+ typeof initial === "function" ? initial() : initial,
+ () => {},
+ ],
+ useEffect: () => {},
+ useMemo: (read: () => unknown) => read(),
+ },
+ "./use-system": { getCachedSystemInfo: () => snapshot },
+ "@/lib/dense-quant-schemes": { normalizeDenseQuantSchemes },
+ "@/lib/offload-fit-tiers": { normalizeReportedOffloadFitTiers },
+ },
+ { relativePassthrough: true },
+);
+
+// One A100 split into three GPU instances on an R570+ driver: /api/system lists each CUDA device.
+function migHost() {
+ const devices = [
+ { index: 0, index_kind: "cuda_device", name: "NVIDIA A100-SXM4-80GB", memory_total_gb: 39.25 },
+ { index: 1, index_kind: "cuda_device", name: "NVIDIA A100-SXM4-80GB", memory_total_gb: 19.5 },
+ { index: 2, index_kind: "cuda_device", name: "NVIDIA A100-SXM4-80GB", memory_total_gb: 19.5 },
+ ];
+ const gpu = { available: true, backend: "cuda", devices };
+ snapshot = {
+ status: "ready",
+ device_backend: "cuda",
+ gpu,
+ inference_gpu: gpu,
+ memory: { total_gb: 128, available_gb: 100 },
+ };
+}
+
+test("every MIG instance is pickable and tensor-splittable in the CUDA-device namespace", () => {
+ migHost();
+ const devices = hooks.useGpuDevices();
+ assert.deepEqual(
+ devices.map((d) => [d.index, d.indexKind, d.pinnable]),
+ [
+ [0, "cuda_device", true],
+ [1, "cuda_device", true],
+ [2, "cuda_device", true],
+ ],
+ );
+ assert.deepEqual(pinnableGpuContext(devices), {
+ devices,
+ ids: [0, 1, 2],
+ indexKind: "cuda_device",
+ });
+});
+
+test("DiffusionGemma and the optional engines can pin a MIG instance", () => {
+ migHost();
+ assert.deepEqual(
+ hooks
+ .useGpuDevices(true)
+ .filter(hooks.isEngineGpuDevice)
+ .map((d) => d.index),
+ [0, 1, 2],
+ );
+});
+
+test("a pick saved in another namespace is never read as a CUDA-device id", () => {
+ for (const saved of ["physical", "vulkan", undefined] as const) {
+ assert.deepEqual(reconcileGpuSelection([1], saved, "cuda_device", [0, 1, 2]), {
+ ids: null,
+ indexKind: null,
+ });
+ }
+ assert.deepEqual(reconcileGpuSelection([1], "cuda_device", "cuda_device", [0, 1, 2]), {
+ ids: [1],
+ indexKind: "cuda_device",
+ });
+ // And the reverse: a CUDA-device pick does not survive a move to physical ids.
+ assert.deepEqual(reconcileGpuSelection([1], "cuda_device", "physical", [0, 1]), {
+ ids: null,
+ indexKind: null,
+ });
+});
diff --git a/studio/nvidia_probe.py b/studio/nvidia_probe.py
index 3c4663deb94..02e81d0aa36 100644
--- a/studio/nvidia_probe.py
+++ b/studio/nvidia_probe.py
@@ -24,6 +24,7 @@ import os
import re
import subprocess
import sys
+import uuid as uuid_lib
from dataclasses import dataclass, field
@@ -178,7 +179,16 @@ def _probe_nvml() -> dict | None:
"mig": "1",
}
)
+ # R570+ enumerates one CI per GI: read CUDA's UUID order here (cuInit, no context).
+ cuda_devices = None
+ if sys.platform == "linux" and any(r.get("mig") for r in devices):
+ match = re.match(r"(\d+)\.", version)
+ if match and int(match[1]) >= 570:
+ enumerated = _probe_cuda_driver(strict = True)
+ if enumerated is not None and all(r.get("uuid") for r in enumerated["devices"]):
+ cuda_devices = enumerated["devices"]
return {
+ "cuda_devices": cuda_devices,
"source": "nvml",
"cuda_driver_version": list(cuda) if cuda else None,
"driver_version": version,
@@ -191,7 +201,8 @@ def _probe_nvml() -> dict | None:
pass
-def _probe_cuda_driver() -> dict | None:
+def _probe_cuda_driver(strict: bool = False) -> dict | None:
+ """``strict``: None when any device cannot be read, so a MIG ordinal map is never partial."""
cuda = _load("cuda")
if cuda is None or cuda.cuInit(0) != 0:
return None
@@ -201,11 +212,15 @@ def _probe_cuda_driver() -> dict | None:
version = _split_cuda_version(packed.value)
count = ctypes.c_int(0)
if cuda.cuDeviceGetCount(ctypes.byref(count)) != 0:
+ if strict:
+ return None
count.value = 0
devices = []
for index in range(count.value):
device = ctypes.c_int(0)
if cuda.cuDeviceGet(ctypes.byref(device), index) != 0:
+ if strict:
+ return None
continue
name = ctypes.create_string_buffer(96)
cuda.cuDeviceGetName(name, 96, device)
@@ -213,10 +228,17 @@ def _probe_cuda_driver() -> dict | None:
# CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR = 75, _MINOR = 76.
ok = cuda.cuDeviceGetAttribute(ctypes.byref(major), 75, device) == 0
ok = ok and cuda.cuDeviceGetAttribute(ctypes.byref(minor), 76, device) == 0
+ device_uuid = ""
+ # The instance's MIG UUID on a MIG device (CUDA 11.4+).
+ get_uuid = getattr(cuda, "cuDeviceGetUuid_v2", None) if strict else None
+ if get_uuid is not None:
+ raw_uuid = (ctypes.c_ubyte * 16)()
+ if get_uuid(ctypes.byref(raw_uuid), device) == 0:
+ device_uuid = str(uuid_lib.UUID(bytes = bytes(raw_uuid)))
devices.append(
{
"index": str(index),
- "uuid": "",
+ "uuid": device_uuid,
"name": name.value.decode("utf-8", "replace"),
"compute_cap": f"{major.value}.{minor.value}" if ok else "",
# Memory needs a context, which this probe never creates.
diff --git a/tests/studio/test_model_picker_contracts.py b/tests/studio/test_model_picker_contracts.py
index aa33da91560..24e0c97ce44 100644
--- a/tests/studio/test_model_picker_contracts.py
+++ b/tests/studio/test_model_picker_contracts.py
@@ -2478,14 +2478,18 @@ def test_vulkan_inference_devices_are_the_pickable_set():
assert (
"pinnable: pinnableBackend && "
'(d.index_kind === "vulkan" || '
- '(data?.device_backend !== "xpu" && d.index_kind === "physical")),' in src
+ '(data?.device_backend !== "xpu" && '
+ '(d.index_kind === "physical" || d.index_kind === "cuda_device"))),' in src
)
# ROCm reuses torch.cuda.* and physical IDs, so excluding it hides the multi-GPU picker.
assert (
"const diffusionBackend = "
'data?.device_backend === "cuda" || data?.device_backend === "rocm";' in src
)
- assert 'diffusionPinnable: diffusionBackend && d.index_kind === "physical",' in src
+ assert (
+ "diffusionPinnable: diffusionBackend && "
+ '(d.index_kind === "physical" || d.index_kind === "cuda_device"),' in src
+ )
def test_chat_autoload_records_every_validation_failure():