Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: read whole-GPU uuid masks from nvidia-smi rows and cache the MIG set (#14017)

Daniel Haneb0617b62bb5
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 54978e964c5..e2048ce18a7 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -11906,6 +11906,10 @@ class LlamaCppBackend:
emit, so callers can treat non-None as "this changed something".
"""
inherited = LlamaCppBackend._resolve_visible_physical_ids()
+ if not inherited:
+ # A uuid mask the ordinal resolvers leave open (FASTEST_FIRST) still orders
+ # the cards the memory probe matched by uuid.
+ inherited = LlamaCppBackend._smi_uuid_mask_order()
if not inherited:
return None
wanted = [int(i) for i in pinned_ids]
@@ -13208,6 +13212,9 @@ class LlamaCppBackend:
_GPU_IDS_ARE_PCI_INDICES = None
# index -> inherited uuid, when CUDA_VISIBLE_DEVICES named the NVML rows by uuid.
_NVML_ROWS: list = [] # the last NVML inventory; _child_visibility_for reads it
+ _SMI_UUID_ROWS: list = [] # (index, uuid) from the last nvidia-smi memory probe
+ # MIG cards that same probe saw (None = discovery unknown), so pinning uses its snapshot.
+ _SMI_MIG_INDICES: Optional[frozenset] = None
# Boot-time property: read once, not per launch (the #10613 host has 175
# groups). Only the default root is cached; an explicit root (tests) re-reads.
@@ -14997,19 +15004,25 @@ class LlamaCppBackend:
]
return min(slices, key = free_mib) if slices else r
- tokens = [t.strip() for t in (raw or "").split(",") if t.strip()]
+ tokens = [t.strip() for t in (raw or "").split(",")] if raw else []
picked: list[dict] = []
if raw is None:
picked = [as_cuda_sees(r) for r in parents]
seen: set[int] = set()
for token in tokens:
+ if not token:
+ break # an empty entry ends the mask, as CUDA reads it
prefix = re.match(r"\+?\d+", token)
- if token.startswith("GPU-"):
- hits = [r for r in parents if str(r.get("uuid", "")).startswith(token)]
- elif token.startswith("MIG-"):
+ # Case-insensitive, as the nvidia-smi uuid matcher reads the same mask.
+ low = token.lower()
+ if low.startswith("gpu-"):
+ hits = [r for r in parents if str(r.get("uuid", "")).lower().startswith(low)]
+ elif low.startswith("mig-"):
# A MIG- entry names a slice row the probe lists under its parent.
hits = [
- r for r in rows if r.get("mig") and str(r.get("uuid", "")).startswith(token)
+ r
+ for r in rows
+ if r.get("mig") and str(r.get("uuid", "")).lower().startswith(low)
]
elif prefix:
# CUDA reads an index strtoul-style: "1gpu2" is GPU 1.
@@ -15072,9 +15085,59 @@ class LlamaCppBackend:
and LlamaCppBackend._visible_devices_mask("CUDA_VISIBLE_DEVICES") is None
and LlamaCppBackend._visibility_mask_is_unmappable()
):
+ # Physical rows the memory probe matched by uuid: hand the child those uuids.
+ uuids = LlamaCppBackend._smi_uuids_for(cvd, gpu_indices)
+ if uuids is not None:
+ return uuids
return LlamaCppBackend._mig_tokens_for_ordinals(cvd, gpu_indices)
return mask
+ @staticmethod
+ def _smi_uuid_mask_order() -> Optional[list[int]]:
+ """Physical ids of a whole-GPU uuid mask in mask order, from the last nvidia-smi
+ rows; None unless the probe's ids are physical and every token matched a row."""
+ cvd = os.environ.get("CUDA_VISIBLE_DEVICES")
+ if LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES is not True or not (cvd or "").strip():
+ return None
+ from utils.hardware import nvidia
+
+ visible = nvidia._uuid_visible_ordinal_map(cvd, LlamaCppBackend._SMI_UUID_ROWS)
+ return sorted(visible, key = visible.__getitem__) if visible else None
+
+ @staticmethod
+ def _smi_uuids_for(cvd: str, gpu_indices) -> Optional[str]:
+ """The uuids of these physical ids when the current whole-GPU uuid mask names each
+ of them in the last nvidia-smi rows, else None. Never for a MIG card: CUDA hands a
+ GPU- entry on one a slice, which only the NVML rows name."""
+ if LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES is not True:
+ return None
+ from utils.hardware import nvidia
+
+ rows = LlamaCppBackend._SMI_UUID_ROWS
+ visible = nvidia._uuid_visible_ordinal_map(cvd, rows)
+ try:
+ ids = [int(i) for i in gpu_indices]
+ except (TypeError, ValueError):
+ return None
+ if not visible or not ids or len(set(ids)) != len(ids):
+ return None
+ if any(i not in visible for i in ids):
+ return None
+ # The probe that chose these rows classified them; a later recheck failing must
+ # not widen the child to the whole inherited mask.
+ mig = LlamaCppBackend._SMI_MIG_INDICES
+ if mig is None:
+ # Unknown then too: NVML rows from that probe still name MIG cards.
+ mig = {
+ int(r["index"])
+ for r in LlamaCppBackend._NVML_ROWS
+ if r.get("mig") and str(r.get("index", "")).isdigit()
+ }
+ if set(ids) & mig:
+ return None
+ by_index = dict(rows)
+ return ",".join(by_index[i] for i in ids)
+
@staticmethod
def _mig_tokens_for_ordinals(cvd: str, gpu_indices) -> Optional[str]:
"""The inherited MIG- entries behind torch ordinals, or None (keep the whole mask). Only
@@ -15142,6 +15205,28 @@ class LlamaCppBackend:
gpus.sort(key = lambda g: g[0])
return gpus
+ @staticmethod
+ def _whole_gpu_mask_rows(
+ cvd: str, uuid_rows: list[tuple[int, str]], physical_ids: set[int]
+ ) -> Optional[set[int]]:
+ """Physical rows behind a whole-GPU uuid mask the ordinal resolvers left open.
+ A uuid names its card whatever CUDA_DEVICE_ORDER says, so only membership is
+ needed. None for a MIG mask or a token no row verifies, so the caller keeps
+ its conservative path."""
+ from utils.hardware import nvidia
+
+ tokens = [t.strip().lower() for t in cvd.split(",") if t.strip()]
+ if not tokens or any(not t.startswith("gpu-") for t in tokens):
+ return None
+ if uuid_rows:
+ visible = nvidia._uuid_visible_ordinal_map(cvd, uuid_rows)
+ return set(visible) if visible is not None else None
+ # No uuid column: verify with the separate uuid query, never assume. Only a
+ # one-card mask: these rows carry no uuids to order or pin a multi-card subset,
+ # while the inherited mask already pins a single card.
+ ids = nvidia.resolve_uuid_mask(cvd)
+ return set(ids) if ids and len(ids) == 1 and set(ids) <= physical_ids else None
+
@staticmethod
def _get_gpu_memory(
binary: Optional[str] = None, *, for_llama_server: bool = False
@@ -15185,7 +15270,9 @@ class LlamaCppBackend:
result = gpu_query.run_nvidia_smi(
[
"nvidia-smi",
- "--query-gpu=index,memory.free,memory.total",
+ # uuid in the same call: a whole-GPU uuid mask resolves without
+ # CUDA_DEVICE_ORDER or a second query.
+ "--query-gpu=index,memory.free,memory.total,uuid",
"--format=csv,noheader,nounits",
],
capture_output = True,
@@ -15197,16 +15284,19 @@ class LlamaCppBackend:
**_windows_hidden_subprocess_kwargs(),
)
if result.returncode == 0:
+ from utils.hardware import nvidia
+
allowed = LlamaCppBackend._visible_devices_mask("CUDA_VISIBLE_DEVICES")
- if allowed is None and (os.environ.get("CUDA_VISIBLE_DEVICES") or "").strip():
- # CUDA's numeric-prefix reading ("1gpu2" is 1, "0,1,abc" is 0,1).
- _ids = LlamaCppBackend._resolve_visible_physical_ids()
- allowed = set(_ids) if _ids is not None else None
- gpus: list[tuple[int, int, int]] = []
+ _cvd = (os.environ.get("CUDA_VISIBLE_DEVICES") or "").strip()
+ rows: list[tuple[int, int, int]] = []
+ unsized_rows: list[int] = []
+ uuid_rows: list[tuple[int, str]] = []
for line in result.stdout.strip().splitlines():
parts = [p.strip() for p in line.split(",")]
if len(parts) < 2:
continue
+ if len(parts) >= 4 and parts[0].isdigit() and parts[3]:
+ uuid_rows.append((int(parts[0]), parts[3]))
# Index and free required; skip a bad line rather than abandon
# the probe to the torch fallback.
try:
@@ -15214,8 +15304,8 @@ class LlamaCppBackend:
free_mib = int(parts[1])
except ValueError:
# Jetson answers "[N/A]": no dedicated VRAM (#4862).
- if parts[0].isdigit() and (allowed is None or int(parts[0]) in allowed):
- unsized_ids.append(int(parts[0]))
+ if parts[0].isdigit():
+ unsized_rows.append(int(parts[0]))
continue
# Total parsed separately: a two-column line or a non-integer
# total ("N/A" on MIG/vGPU) keeps the GPU at total 0 (fit uses
@@ -15226,20 +15316,45 @@ class LlamaCppBackend:
total_mib = int(parts[2])
except ValueError:
total_mib = 0
- if allowed is not None and idx not in allowed:
- continue
- gpus.append((idx, free_mib, total_mib))
+ rows.append((idx, free_mib, total_mib))
+ LlamaCppBackend._SMI_UUID_ROWS = uuid_rows
+ LlamaCppBackend._SMI_MIG_INDICES = None
+ if allowed is None and _cvd:
+ allowed = LlamaCppBackend._whole_gpu_mask_rows(
+ _cvd, uuid_rows, {r[0] for r in rows} | set(unsized_rows)
+ )
+ if allowed is None and _cvd:
+ # CUDA's numeric-prefix reading ("1gpu2" is 1, "0,1,abc" is 0,1).
+ _ids = LlamaCppBackend._resolve_visible_physical_ids()
+ allowed = set(_ids) if _ids is not None else None
+ unsized_ids = [i for i in unsized_rows if allowed is None or i in allowed]
+ gpus = [r for r in rows if allowed is None or r[0] in allowed]
# Match the docstring's sort-by-id guarantee (driver order isn't).
gpus.sort(key = lambda g: g[0])
- _cvd = os.environ.get("CUDA_VISIBLE_DEVICES")
- if allowed is None and _cvd is not None and _cvd.strip():
+ if allowed is None and _cvd:
# A MIG / unresolvable UUID mask names devices these rows cannot be
# matched to: listing every card sizes against GPUs the parent hid.
# The NVML branch names slices and uuids.
gpus = []
elif gpus:
- from utils.hardware import nvidia
- mig = nvidia.get_mig_gpu_indices() & {g[0] for g in gpus}
+ mig_known = nvidia.get_mig_gpu_indices_or_none()
+ if mig_known is not None:
+ LlamaCppBackend._SMI_MIG_INDICES = frozenset(mig_known)
+ mig = (mig_known or set()) & {g[0] for g in gpus}
+ if mig_known is None:
+ # MIG discovery failed: a row may be a whole MIG card. NVML names
+ # slices and whole cards alike, so its reading wins; without one
+ # the CLI rows stand, as before MIG support (no CUDA context).
+ nvml = {g[0]: g for g in LlamaCppBackend._get_gpu_memory_nvml()}
+ if nvml:
+ gpus = [nvml.get(g[0], g) for g in gpus]
+ else:
+ # No current NVML inventory: an older one must not name devices.
+ LlamaCppBackend._NVML_ROWS = []
+ elif not mig:
+ # The CLI now answers for these cards: an earlier NVML inventory
+ # (slices since removed) must not name the child's devices.
+ LlamaCppBackend._NVML_ROWS = []
if mig:
# A MIG card's row is the whole card: swap in its NVML slice reading,
# keeping the whole cards. No reading at all: the later probes decide,
diff --git a/studio/backend/tests/test_mig_gpu_masks.py b/studio/backend/tests/test_mig_gpu_masks.py
index 2a1e07a804c..970b8021520 100644
--- a/studio/backend/tests/test_mig_gpu_masks.py
+++ b/studio/backend/tests/test_mig_gpu_masks.py
@@ -11,6 +11,7 @@ from __future__ import annotations
import json
import os
+import subprocess
import sys
import types
@@ -70,6 +71,8 @@ def host(tmp_path, monkeypatch):
monkeypatch.setenv("UNSLOTH_GPU_QUERY_CACHE", "0")
monkeypatch.setattr(nvidia, "_uuid_mask_cache", {})
monkeypatch.setattr(LlamaCppBackend, "_NVML_ROWS", [])
+ monkeypatch.setattr(LlamaCppBackend, "_SMI_UUID_ROWS", [], raising = False)
+ monkeypatch.setattr(LlamaCppBackend, "_SMI_MIG_INDICES", None, raising = False)
monkeypatch.setattr(LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda b = None: False))
monkeypatch.setattr(
LlamaCppBackend, "_find_llama_server_binary", staticmethod(lambda: "/opt/llama-server")
@@ -102,9 +105,15 @@ def host(tmp_path, monkeypatch):
query = next((a for a in cmd if a.startswith("--query-gpu=")), "")
if query == "--query-gpu=index,uuid":
return ok("".join(f"{i}, {u}\n" for i, (u, _) in enumerate(state["cards"])))
- if query == "--query-gpu=index,memory.free,memory.total":
+ if query.startswith("--query-gpu=index,memory.free,memory.total"):
# Whole-card numbers, also for a card in MIG mode.
- return ok("".join(f"{i}, {f}, {A100_MIB}\n" for i, f in enumerate(state["free"])))
+ uuids = [u for u, _ in state["cards"]] if query.endswith(",uuid") else None
+ return ok(
+ "".join(
+ f"{i}, {f}, {A100_MIB}" + (f", {uuids[i]}" if uuids else "") + "\n"
+ for i, f in enumerate(state["free"])
+ )
+ )
return types.SimpleNamespace(returncode = 1, stdout = "", stderr = "")
monkeypatch.setattr(mod.subprocess, "run", run)
@@ -361,6 +370,9 @@ class TestMigSlicesAreNotGpus:
host(MIXED, [0, 0, 0], None)
assert nvidia.get_mig_gpu_indices() == {0}
host([(GPU0, []), (GPU1, [("1g.5gb", MIG0A)])], [0, 0], None)
+ # Cached until the inventory generation moves (hotplug, driver reset).
+ assert nvidia.get_mig_gpu_indices() == {0}
+ nvidia.gpu_query.invalidate_static("test")
assert nvidia.get_mig_gpu_indices() == {1}
@@ -380,3 +392,433 @@ def test_mig_overlay_skips_a_non_cuda_backend(host, monkeypatch):
monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.XPU)
devices = [{"index": 0, "vram_total_gb": 24.0, "vram_used_gb": 2.0}]
assert hw._with_mig_instance_memory(devices) == devices
+
+
+T4_MIB = 15360
+
+
+def calls(monkeypatch):
+ seen = {
+ "torch": 0,
+ "list": 0,
+ "uuid_query_fails": False,
+ "list_fails": False,
+ "list_empty": False,
+ }
+ run = mod.subprocess.run
+
+ def counting_run(cmd, *args, **kwargs):
+ if cmd and os.path.basename(str(cmd[0])) == "nvidia-smi":
+ if "-L" in cmd:
+ seen["list"] += 1
+ if seen["list_fails"]:
+ return types.SimpleNamespace(returncode = 9, stdout = "", stderr = "")
+ if seen["list_empty"]:
+ return types.SimpleNamespace(returncode = 0, stdout = "", stderr = "")
+ if seen["uuid_query_fails"] and "--query-gpu=index,uuid" in cmd:
+ return types.SimpleNamespace(returncode = 1, stdout = "", stderr = "")
+ return run(cmd, *args, **kwargs)
+
+ monkeypatch.setattr(mod.subprocess, "run", counting_run)
+
+ def torch_probe(*a, **k):
+ seen["torch"] += 1
+ return [(0, 1, 1)]
+
+ monkeypatch.setattr(LlamaCppBackend, "_get_gpu_memory_torch", staticmethod(torch_probe))
+ return seen
+
+
+class TestOrdinaryHostsKeepTheirReadings:
+ """Whole-GPU uuid masks and plain polls on hosts without MIG behave as before MIG support."""
+
+ @pytest.mark.parametrize("token", [GPU0, GPU0[:12], GPU0.lower()])
+ def test_a_uuid_mask_under_fastest_first_reads_its_card(self, host, monkeypatch, calls, token):
+ host([(GPU0, [])], [14000], None)
+ monkeypatch.setenv("CUDA_DEVICE_ORDER", "FASTEST_FIRST")
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", token)
+ assert LlamaCppBackend._get_gpu_memory() == [(0, 14000, A100_MIB)]
+ assert calls["torch"] == 0
+
+ def test_a_uuid_mask_reads_its_card_when_the_uuid_query_fails(self, host, monkeypatch, calls):
+ host([(GPU0, [])], [14000], None)
+ calls["uuid_query_fails"] = True
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", GPU0)
+ assert LlamaCppBackend._get_gpu_memory() == [(0, 14000, A100_MIB)]
+ assert calls["torch"] == 0
+
+ def test_a_multi_gpu_uuid_mask_under_fastest_first_keeps_only_its_cards(
+ self, host, monkeypatch, calls
+ ):
+ host([(GPU0, []), (GPU1, []), (GPU2, [])], [39000, 30000, 40000], None)
+ monkeypatch.setenv("CUDA_DEVICE_ORDER", "FASTEST_FIRST")
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", f"{GPU2},{GPU1}")
+ assert LlamaCppBackend._get_gpu_memory() == [(1, 30000, A100_MIB), (2, 40000, A100_MIB)]
+ assert calls["torch"] == 0
+
+ def test_an_unknown_uuid_still_reads_nothing_from_the_cli(self, host, monkeypatch, calls):
+ host([(GPU0, []), (GPU1, [])], [39000, 30000], None)
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", GPU2)
+ assert LlamaCppBackend._get_gpu_memory() == [(0, 1, 1)]
+
+ def test_without_a_uuid_column_the_mask_is_verified_or_unresolved(
+ self, host, monkeypatch, calls
+ ):
+ host([(GPU0, [])], [14000], None)
+ monkeypatch.setenv("CUDA_DEVICE_ORDER", "FASTEST_FIRST")
+ assert LlamaCppBackend._whole_gpu_mask_rows(GPU0, [], {0}) == {0}
+ # A uuid no card has hides every device, even on a one-card host.
+ assert LlamaCppBackend._whole_gpu_mask_rows(GPU1, [], {0}) is None
+ assert LlamaCppBackend._whole_gpu_mask_rows(MIG0A, [], {0}) is None
+ nvidia._uuid_mask_cache.clear()
+ calls["uuid_query_fails"] = True
+ assert LlamaCppBackend._whole_gpu_mask_rows(GPU0, [], {0}) is None
+
+ def test_a_subset_of_a_uuid_mask_pins_the_child_by_uuid(self, host, monkeypatch, calls):
+ host([(GPU0, []), (GPU1, []), (GPU2, [])], [39000, 30000, 40000], None)
+ monkeypatch.setenv("CUDA_DEVICE_ORDER", "FASTEST_FIRST")
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", f"{GPU2},{GPU1}")
+ assert LlamaCppBackend._get_gpu_memory() == [(1, 30000, A100_MIB), (2, 40000, A100_MIB)]
+ assert LlamaCppBackend._pinned_child_visibility([2]) == GPU2
+ assert LlamaCppBackend._pinned_child_visibility([2, 1]) == f"{GPU2},{GPU1}"
+ # A card the mask does not name is never pinned.
+ assert LlamaCppBackend._pinned_child_visibility([0]) is None
+ # The mask is re-read: a card the new mask drops is no longer mapped.
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", GPU1)
+ assert LlamaCppBackend._pinned_child_visibility([2]) is None
+
+ def test_a_uuid_mask_on_a_mig_card_keeps_the_inherited_mask(self, host, monkeypatch, calls):
+ host(MIXED, [39000, 30000, 40000], None)
+ monkeypatch.setenv("CUDA_DEVICE_ORDER", "FASTEST_FIRST")
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", f"{GPU0},{GPU1}")
+ LlamaCppBackend._get_gpu_memory()
+ assert LlamaCppBackend._pinned_child_visibility([0, 1]) is None
+ assert LlamaCppBackend._pinned_child_visibility([1]) == GPU1
+
+ def test_a_listing_raced_by_an_invalidation_is_not_cached(self, host, monkeypatch, calls):
+ host([(GPU0, [])], [14000], None)
+ real = nvidia._run_mig_listing
+
+ def racing():
+ out = real()
+ nvidia.gpu_query.invalidate_static("hotplug during the query")
+ return out
+
+ monkeypatch.setattr(nvidia, "_run_mig_listing", racing)
+ assert nvidia.get_mig_gpu_indices_or_none() is None
+ monkeypatch.setattr(nvidia, "_run_mig_listing", real)
+ nvidia.get_mig_gpu_indices()
+ assert calls["list"] == 2
+ nvidia.get_mig_gpu_indices()
+ assert calls["list"] == 2
+
+ def test_memory_polls_reuse_one_mig_listing(self, host, calls):
+ host([(GPU0, [])], [14000], None)
+ devices = [{"index": 0, "vram_total_gb": 15.0, "vram_used_gb": 1.0}]
+ for _ in range(3):
+ assert LlamaCppBackend._get_gpu_memory() == [(0, 14000, A100_MIB)]
+ assert hw._with_mig_instance_memory(devices) == devices
+ assert calls["list"] == 1
+
+ def test_the_physical_count_probe_publishes_the_mig_set(self, host, monkeypatch, calls):
+ monkeypatch.setattr(hw, "IS_ROCM", False)
+ monkeypatch.setattr(hw, "_physical_gpu_count", None)
+ monkeypatch.setattr(hw, "_physical_gpu_count_from_smi", False)
+ monkeypatch.setattr(hw, "_physical_gpu_count_checked_at", None, raising = False)
+ host([(GPU0, [])], [14000], None)
+ assert hw.get_physical_gpu_count() == 1
+ assert calls["list"] == 1
+ for _ in range(3):
+ assert LlamaCppBackend._get_gpu_memory() == [(0, 14000, A100_MIB)]
+ assert calls["list"] == 1
+
+ def test_a_raised_physical_count_probe_leaves_mig_unknown(self, host, monkeypatch, calls):
+ host([(GPU0, [])], [14000], None)
+ assert nvidia.get_mig_gpu_indices_or_none() == frozenset()
+ real = nvidia.gpu_query.run_nvidia_smi
+
+ def boom(cmd, *a, **k):
+ if cmd[-1] == "-L":
+ raise subprocess.TimeoutExpired(cmd, 5)
+ return real(cmd, *a, **k)
+
+ monkeypatch.setattr(nvidia.gpu_query, "run_nvidia_smi", boom)
+ assert nvidia.get_physical_gpu_count() is None
+ # The newest probe failed: the older "no MIG" answer is no longer served.
+ assert nvidia._mig_entry[3] is None
+ monkeypatch.setattr(nvidia.gpu_query, "run_nvidia_smi", real)
+ assert nvidia.get_mig_gpu_indices_or_none() is None
+
+
+class TestMigListingFreshness:
+ @pytest.fixture
+ def clock(self, monkeypatch):
+ now = [1000.0]
+ monkeypatch.setattr(nvidia.time, "monotonic", lambda: now[0])
+ return now
+
+ def test_a_cached_listing_keeps_its_age(self, host, monkeypatch, clock, calls):
+ monkeypatch.setenv("UNSLOTH_GPU_QUERY_CACHE", "1")
+ host(TWO_SLICES, [38000], None)
+ assert nvidia.get_physical_gpu_count(gpu_rows_only = True) == 1
+ monkeypatch.setattr(nvidia, "_mig_entry", None)
+ clock[0] += 59
+ # Served from gpu_query's own cache: stamped with that answer's time, not now.
+ assert nvidia.get_mig_gpu_indices() == {0}
+ assert calls["list"] == 1
+ assert nvidia._mig_entry[2] == 1000.0
+ clock[0] += 2
+ assert nvidia.get_mig_gpu_indices() == {0}
+ assert calls["list"] == 2
+
+ def test_a_failed_listing_is_unknown_and_backs_off(self, host, monkeypatch, clock, calls):
+ host(TWO_SLICES, [38000], None)
+ assert nvidia.get_mig_gpu_indices() == {0}
+ clock[0] += 61
+ calls["list_fails"] = True
+ for _ in range(4):
+ assert nvidia.get_mig_gpu_indices_or_none() is None
+ assert calls["list"] == 2
+ clock[0] += nvidia._MIG_FAILURE_RETRY_S
+ calls["list_fails"] = False
+ host([(GPU0, [])], [38000], None)
+ assert nvidia.get_mig_gpu_indices() == set()
+ assert calls["list"] == 3
+
+ def test_a_cold_failure_is_retried_after_the_backoff(self, host, monkeypatch, clock, calls):
+ host(TWO_SLICES, [38000], None)
+ calls["list_fails"] = True
+ assert nvidia.get_mig_gpu_indices() == set()
+ assert nvidia.get_mig_gpu_indices() == set()
+ assert calls["list"] == 1
+ calls["list_fails"] = False
+ clock[0] += nvidia._MIG_FAILURE_RETRY_S
+ assert nvidia.get_mig_gpu_indices() == {0}
+ assert calls["list"] == 2
+
+
+def test_a_reordered_uuid_mask_orders_the_pinned_child(host, monkeypatch):
+ host([(GPU0, []), (GPU1, []), (GPU2, [])], [39000, 30000, 40000], None)
+ monkeypatch.setenv("CUDA_DEVICE_ORDER", "FASTEST_FIRST")
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", f"{GPU2},{GPU1}")
+ LlamaCppBackend._get_gpu_memory()
+ # Auto fit hands ascending ids; the launch reorders them (and their shares) to the mask.
+ assert LlamaCppBackend._inherited_child_gpu_order([1, 2]) == [2, 1]
+ assert LlamaCppBackend._pinned_child_visibility([2, 1]) == f"{GPU2},{GPU1}"
+ assert LlamaCppBackend._inherited_child_gpu_order([2]) is None
+
+
+class TestMigListingHonoursTheStaticTtl:
+ @pytest.mark.parametrize("ttl, expected", [("10", [1, 1, 2]), ("0", [1, 2, 3])])
+ def test_the_configured_ttl_bounds_the_mig_set(self, host, monkeypatch, calls, ttl, expected):
+ now = [1000.0]
+ monkeypatch.setattr(nvidia.time, "monotonic", lambda: now[0])
+ monkeypatch.setenv("UNSLOTH_GPU_QUERY_STATIC_TTL", ttl)
+ host(TWO_SLICES, [38000], None)
+ seen = []
+ for step in (0, 5, 6):
+ now[0] += step
+ assert nvidia.get_mig_gpu_indices() == {0}
+ seen.append(calls["list"])
+ assert seen == expected
+
+
+ "mask, expected, child",
+ [
+ (GPU0.lower(), (0, 9000, 9856), MIG0A),
+ (MIG0B.lower(), (0, 9500, 9856), MIG0B),
+ ],
+)
+def test_a_lowercase_uuid_reads_the_slice(host, monkeypatch, mask, expected, child):
+ host(MIXED, [39000, 30000, 40000], MIXED_NVML)
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", mask)
+ assert LlamaCppBackend._get_gpu_memory() == [expected]
+ assert LlamaCppBackend._pinned_child_visibility([0]) == child
+
+
+def test_an_empty_listing_is_a_failed_probe(host, monkeypatch, calls):
+ now = [1000.0]
+ monkeypatch.setattr(nvidia.time, "monotonic", lambda: now[0])
+ host(TWO_SLICES, [38000], None)
+ assert nvidia.get_mig_gpu_indices() == {0}
+ now[0] += 61
+ calls["list_empty"] = True
+ assert nvidia.get_mig_gpu_indices_or_none() is None
+ assert nvidia._mig_entry[3] is None
+ calls["list_empty"] = False
+ now[0] += nvidia._MIG_FAILURE_RETRY_S
+ assert nvidia.get_mig_gpu_indices() == {0}
+ assert calls["list"] == 3
+
+
+def test_a_superseded_listing_is_not_cached(host, monkeypatch, calls):
+ monkeypatch.setenv("UNSLOTH_GPU_QUERY_CACHE", "1")
+ host(TWO_SLICES, [38000], None)
+ assert nvidia.get_physical_gpu_count(gpu_rows_only = True) == 1
+ assert nvidia._listing_taken_at(1e9, _smi_list(TWO_SLICES)) is not None
+ assert nvidia._listing_taken_at(1e9, _smi_list([(GPU0, [])])) is None
+ # The query hands back an older listing than the one the cache now holds.
+ real = nvidia.gpu_query.run_nvidia_smi
+
+ def stale(argv, **kw):
+ real(argv, **kw)
+ return types.SimpleNamespace(returncode = 0, stdout = _smi_list([(GPU0, [])]), stderr = "")
+
+ monkeypatch.setattr(nvidia, "_mig_entry", None)
+ monkeypatch.setattr(nvidia.gpu_query, "run_nvidia_smi", stale)
+ # Neither published nor acted on: discovery is unknown, not "no MIG".
+ assert nvidia.get_mig_gpu_indices_or_none() is None
+ assert nvidia._mig_entry is None
+
+
+class TestAnEmptyEntryEndsAUuidMask:
+ @pytest.mark.parametrize(
+ "mask, expected",
+ [
+ (f",{GPU1}", [(0, 1, 1)]),
+ (f"{GPU2},,{GPU1}", [(2, 40000, A100_MIB)]),
+ (f"{GPU2},{GPU1},", [(1, 30000, A100_MIB), (2, 40000, A100_MIB)]),
+ ],
+ )
+ def test_the_cli_rows(self, host, monkeypatch, calls, mask, expected):
+ host([(GPU0, []), (GPU1, []), (GPU2, [])], [39000, 30000, 40000], None)
+ monkeypatch.setenv("CUDA_DEVICE_ORDER", "FASTEST_FIRST")
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", mask)
+ assert LlamaCppBackend._get_gpu_memory() == expected
+ if mask.startswith(","):
+ # Nothing visible: no uuid order or uuid pin is derived from the rows.
+ monkeypatch.setattr(LlamaCppBackend, "_GPU_IDS_ARE_PCI_INDICES", True)
+ assert LlamaCppBackend._smi_uuid_mask_order() is None
+ assert LlamaCppBackend._smi_uuids_for(mask, [1]) is None
+
+ @pytest.mark.parametrize(
+ "mask, expected", [(f",{GPU1}", []), (f"{GPU1},,{GPU2}", ["1"]), ("1,,2", ["1"])]
+ )
+ def test_the_nvml_rows(self, host, monkeypatch, mask, expected):
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", mask)
+ assert [r["index"] for r in LlamaCppBackend._nvml_rows_visible(MIXED_NVML)] == expected
+
+
+def test_without_a_uuid_column_only_a_one_card_mask_resolves(host, monkeypatch):
+ # No uuids in the rows: a multi-card subset could be neither ordered nor pinned.
+ host([(GPU0, []), (GPU1, [])], [39000, 30000], None)
+ mask = f"{GPU0},{GPU1}"
+ assert LlamaCppBackend._whole_gpu_mask_rows(mask, [], {0, 1}) is None
+ assert LlamaCppBackend._whole_gpu_mask_rows(mask, [], {0}) is None
+ assert LlamaCppBackend._whole_gpu_mask_rows(GPU1, [], {0, 1}) == {1}
+
+
+class TestTheLatestStartedProbeWins:
+ LISTING = types.SimpleNamespace(returncode = 0, stdout = _smi_list(TWO_SLICES), stderr = "")
+ FAILED = types.SimpleNamespace(returncode = 9, stdout = "", stderr = "")
+
+ def test_an_older_failure_keeps_a_newer_answer(self, host):
+ old, new = nvidia._mig_probe_start(), nvidia._mig_probe_start()
+ assert nvidia._mig_finish(new, self.LISTING) == {0}
+ assert nvidia._mig_finish(old, self.FAILED) == {0}
+ assert nvidia.get_mig_gpu_indices_or_none() == {0}
+
+ def test_an_older_answer_keeps_a_newer_failure(self, host):
+ old, new = nvidia._mig_probe_start(), nvidia._mig_probe_start()
+ assert nvidia._mig_finish(new, self.FAILED) is None
+ assert nvidia._mig_finish(old, self.LISTING) is None
+ assert nvidia.get_mig_gpu_indices_or_none() is None
+
+ def test_a_probe_from_an_old_generation_is_unknown(self, host):
+ probe = nvidia._mig_probe_start()
+ nvidia.gpu_query.invalidate_static("test")
+ assert nvidia._mig_finish(probe, self.LISTING) is None
+ assert nvidia._mig_cached() is nvidia._MISS
+
+
+def test_unknown_mig_discovery_is_not_no_mig(host, monkeypatch, calls):
+ now = [1000.0]
+ monkeypatch.setattr(nvidia.time, "monotonic", lambda: now[0])
+ host(TWO_SLICES, [38000], None)
+ calls["list_fails"] = True
+ assert nvidia.get_mig_gpu_indices_or_none() is None
+ assert nvidia.get_mig_gpu_indices() == set()
+ calls["list_fails"] = False
+ now[0] += nvidia._MIG_FAILURE_RETRY_S
+ assert nvidia.get_mig_gpu_indices_or_none() == {0}
+ now[0] += 61
+ calls["list_empty"] = True
+ assert nvidia.get_mig_gpu_indices_or_none() is None
+ assert nvidia.get_mig_gpu_indices() == set()
+ calls["list_empty"] = False
+ nvidia.gpu_query.invalidate_static("test")
+ host([(GPU0, [])], [38000], None)
+ assert nvidia.get_mig_gpu_indices_or_none() == set()
+
+
+class TestUnknownMigDiscovery:
+ def test_placement_reads_nvml_when_the_listing_fails(self, host, calls):
+ host(TWO_SLICES, [38000], TWO_SLICES_NVML)
+ calls["list_fails"] = True
+ assert LlamaCppBackend._get_gpu_memory() == [(0, 15000, 19968)]
+ devices = [{"index": 0, "vram_total_gb": 40.0, "vram_used_gb": 2.0}]
+ out = hw._with_mig_instance_memory(devices)
+ assert out[0]["vram_total_gb"] == round(19968 / 1024, 2)
+
+ def test_without_nvml_the_cli_rows_stand_and_cuda_is_not_touched(self, host, calls):
+ host([(GPU0, [])], [14000], None)
+ calls["list_fails"] = True
+ assert LlamaCppBackend._get_gpu_memory() == [(0, 14000, A100_MIB)]
+ devices = [{"index": 0, "vram_total_gb": 40.0, "vram_used_gb": 2.0}]
+ assert hw._with_mig_instance_memory(devices) == devices
+ assert calls["torch"] == 0
+
+
+def test_cli_rows_retire_an_old_nvml_inventory(host, monkeypatch):
+ # MIG was on (NVML rows cached), then disabled: the CLI now reports whole cards.
+ host([(GPU0, []), (GPU1, [])], [39000, 30000], None)
+ monkeypatch.setattr(LlamaCppBackend, "_NVML_ROWS", [dict(r) for r in MIXED_NVML])
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", GPU0)
+ assert LlamaCppBackend._get_gpu_memory() == [(0, 39000, A100_MIB)]
+ assert LlamaCppBackend._NVML_ROWS == []
+ assert LlamaCppBackend._pinned_child_visibility([0]) == "0"
+
+
+def test_unknown_discovery_without_nvml_retires_old_nvml_rows(host, monkeypatch, calls):
+ host([(GPU0, []), (GPU1, [])], [39000, 30000], None)
+ monkeypatch.setattr(LlamaCppBackend, "_NVML_ROWS", [dict(r) for r in MIXED_NVML])
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", GPU0)
+ calls["list_fails"] = True
+ assert LlamaCppBackend._get_gpu_memory() == [(0, 39000, A100_MIB)]
+ assert LlamaCppBackend._NVML_ROWS == []
+ assert LlamaCppBackend._pinned_child_visibility([0]) == "0"
+
+
+class TestAUuidSubsetPinSurvivesAFailedMigRecheck:
+ def _narrowed(self, host, monkeypatch):
+ host([(GPU0, []), (GPU1, []), (GPU2, [])], [39000, 30000, 40000], None)
+ monkeypatch.setenv("CUDA_DEVICE_ORDER", "FASTEST_FIRST")
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", f"{GPU2},{GPU1}")
+
+ def test_the_memory_probes_snapshot_is_used(self, host, monkeypatch, calls):
+ self._narrowed(host, monkeypatch)
+ assert LlamaCppBackend._get_gpu_memory() == [(1, 30000, A100_MIB), (2, 40000, A100_MIB)]
+ calls["list_fails"] = True
+ nvidia.gpu_query.invalidate_static("recheck after the probe")
+ assert LlamaCppBackend._pinned_child_visibility([2]) == GPU2
+ assert calls["list"] == 1
+
+ def test_unknown_at_probe_time_still_pins_the_fitted_subset(self, host, monkeypatch, calls):
+ self._narrowed(host, monkeypatch)
+ calls["list_fails"] = True
+ assert LlamaCppBackend._get_gpu_memory() == [(1, 30000, A100_MIB), (2, 40000, A100_MIB)]
+ assert LlamaCppBackend._pinned_child_visibility([2]) == GPU2
+
+ def test_unknown_at_probe_time_never_pins_a_card_nvml_calls_mig(self, host, monkeypatch, calls):
+ host(MIXED, [39000, 30000, 40000], None)
+ monkeypatch.setenv("CUDA_DEVICE_ORDER", "FASTEST_FIRST")
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", f"{GPU0},{GPU1}")
+ calls["list_fails"] = True
+ LlamaCppBackend._get_gpu_memory()
+ monkeypatch.setattr(LlamaCppBackend, "_NVML_ROWS", [dict(r) for r in MIXED_NVML])
+ # The MIG card is pinned to its slice through the NVML rows, never its whole uuid.
+ assert LlamaCppBackend._smi_uuids_for(f"{GPU0},{GPU1}", [0]) is None
+ assert LlamaCppBackend._pinned_child_visibility([0]) == MIG0A
+ assert LlamaCppBackend._smi_uuids_for(f"{GPU0},{GPU1}", [1]) == GPU1
diff --git a/studio/backend/utils/hardware/gpu_query.py b/studio/backend/utils/hardware/gpu_query.py
index 4120e646679..71374b79bde 100644
--- a/studio/backend/utils/hardware/gpu_query.py
+++ b/studio/backend/utils/hardware/gpu_query.py
@@ -369,6 +369,24 @@ def _fallback(key: tuple, kind: str) -> Optional[subprocess.CompletedProcess]:
return None
+def answered_at(
+ argv: Sequence[str],
+ stdout: Any,
+ *,
+ text_mode: bool = True,
+) -> Optional[float]:
+ """Monotonic start of the cached answer for ``argv`` in this static generation when its
+ stdout is exactly ``stdout``; None when no such entry is cached (absent or superseded)."""
+ key = (subprocess.run, _resolved(argv[0]), tuple(argv), text_mode)
+ with _lock:
+ entry = _cache.get(key)
+ if entry is None or entry.result is None or entry.static_gen != _static_gen:
+ return None
+ if getattr(entry.result, "stdout", None) != stdout:
+ return None
+ return entry.at
+
+
def run_nvidia_smi(
argv: Sequence[str],
*,
diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.py
index f4866df0334..0bf3b2aed96 100644
--- a/studio/backend/utils/hardware/hardware.py
+++ b/studio/backend/utils/hardware/hardware.py
@@ -6298,10 +6298,12 @@ def _with_mig_instance_memory(devices: list[Dict[str, Any]]) -> list[Dict[str, A
return devices
try:
from . import nvidia
- mig = nvidia.get_mig_gpu_indices() & {d.get("index") for d in devices}
+ mig_known = nvidia.get_mig_gpu_indices_or_none()
except Exception:
return devices
- if not mig:
+ indices = {d.get("index") for d in devices}
+ mig = None if mig_known is None else mig_known & indices
+ if mig is not None and not mig:
return devices
slices: Dict[int, tuple[int, int]] = {}
try:
@@ -6309,6 +6311,12 @@ def _with_mig_instance_memory(devices: list[Dict[str, Any]]) -> list[Dict[str, A
slices = {idx: (free, total) for idx, free, total in LlamaCppBackend._get_gpu_memory_nvml()}
except Exception as e:
logger.debug("NVML MIG slice probe failed: %s", e)
+ if mig is None:
+ # MIG discovery failed: NVML reads slices and whole cards alike, so its readings
+ # replace the CLI's; without any the devices stand as before MIG support.
+ if not slices:
+ return devices
+ mig = set(slices) & indices
out = []
for device in devices:
idx = device.get("index")
diff --git a/studio/backend/utils/hardware/nvidia.py b/studio/backend/utils/hardware/nvidia.py
index 4512f68010f..b91aa0e05a2 100644
--- a/studio/backend/utils/hardware/nvidia.py
+++ b/studio/backend/utils/hardware/nvidia.py
@@ -59,11 +59,10 @@ def _uuid_visible_ordinal_map(
parent_cuda_visible_devices: Optional[str], gpu_rows: list[tuple[int, str]]
) -> Optional[dict[int, int]]:
"""Resolve an ordered full-GPU UUID mask against nvidia-smi rows."""
- tokens = [
- token.strip().lower()
- for token in (parent_cuda_visible_devices or "").split(",")
- if token.strip()
- ]
+ tokens = [token.strip().lower() for token in (parent_cuda_visible_devices or "").split(",")]
+ # CUDA stops at the first empty entry: a leading one hides every GPU, so leave it unresolved.
+ if "" in tokens:
+ tokens = tokens[: tokens.index("")]
if not tokens or any(not token.startswith("gpu-") for token in tokens):
return None
@@ -125,6 +124,8 @@ def _query_uuid_mask(parent_cuda_visible_devices: str) -> Optional[list[int]]:
def get_physical_gpu_count(gpu_rows_only: bool = False) -> Optional[int]:
"""Return physical GPU count via nvidia-smi, or None on failure. ``gpu_rows_only`` skips the indented MIG child rows."""
+ probe = _mig_probe_start()
+ published = False
try:
result = gpu_query.run_nvidia_smi(
[_nvidia_smi_executable(), "-L"],
@@ -136,6 +137,9 @@ def get_physical_gpu_count(gpu_rows_only: bool = False) -> Optional[int]:
env = child_env_without_native_path_secret(),
**_windows_hidden_subprocess_kwargs(),
)
+ # Same listing get_mig_gpu_indices parses: publish it so memory polls reuse it.
+ published = True
+ _mig_finish(probe, result)
if result.returncode == 0 and result.stdout.strip():
lines = result.stdout.strip().splitlines()
if gpu_rows_only:
@@ -146,16 +150,124 @@ def get_physical_gpu_count(gpu_rows_only: bool = False) -> Optional[int]:
result.returncode,
)
except Exception as e:
+ if not published:
+ # This probe is the latest started: a raise must leave MIG unknown, not the old answer.
+ _mig_finish(probe, None)
logger.warning("nvidia-smi -L failed: %s; caller should fall back to torch", e)
return None
+# MIG discovery cache: one entry (gen, seq, at, indices), indices None meaning unknown
+# (the probe failed, answered empty, or its listing was already superseded). Every probe
+# takes a sequence number when it starts and may only replace an entry of its own static
+# generation from an earlier-started probe, so the latest-started probe always wins. An
+# answer lives for gpu_query's static TTL from when its listing was taken; an unknown
+# for a short backoff, so a broken nvidia-smi adds no work per poll.
+_MIG_FAILURE_RETRY_S = 5.0
+_mig_entry: Optional[tuple[int, int, float, Optional[frozenset[int]]]] = None
+_mig_seq = 0
+_mig_lock = threading.Lock()
+_MISS = object()
+
+
+def _mig_probe_start() -> tuple[int, int, float]:
+ global _mig_seq
+ with _mig_lock:
+ _mig_seq += 1
+ return gpu_query._static_gen, _mig_seq, time.monotonic()
+
+
+def _mig_cached() -> Any:
+ """The fresh entry's indices (None = unknown), or _MISS when a probe is due."""
+ entry = _mig_entry
+ if entry is None or entry[0] != gpu_query._static_gen:
+ return _MISS
+ # Same lifetime as gpu_query's static inventory: UNSLOTH_GPU_QUERY_STATIC_TTL (0 too).
+ ttl = gpu_query.ttl_for(gpu_query.STATIC)
+ life = ttl if entry[3] is not None else min(ttl, _MIG_FAILURE_RETRY_S)
+ return entry[3] if time.monotonic() - entry[2] < life else _MISS
+
+
+def _mig_finish(probe: tuple[int, int, float], result: Any) -> Optional[frozenset[int]]:
+ """Publish one probe's outcome (``result`` None when the query raised) and return the
+ indices callers should use, None for unknown."""
+ global _mig_entry
+ gen, seq, started = probe
+ stdout = getattr(result, "stdout", None)
+ indices: Optional[frozenset[int]] = None
+ at = time.monotonic()
+ if getattr(result, "returncode", 1) == 0 and isinstance(stdout, str) and stdout.strip():
+ taken = _listing_taken_at(started, stdout)
+ if taken is None:
+ # A newer listing replaced this one: never publish or act on it.
+ cached = _mig_cached()
+ return None if cached is _MISS else cached
+ indices, at = _parse_mig_gpu_indices(stdout), taken
+ with _mig_lock:
+ if gen != gpu_query._static_gen:
+ return None
+ cur = _mig_entry
+ if cur is not None and cur[0] == gen and cur[1] > seq:
+ return cur[3]
+ _mig_entry = (gen, seq, at, indices)
+ return indices
+
+
+def _listing_taken_at(started: float, stdout: Any) -> Optional[float]:
+ """When the ``-L`` answer ``stdout`` was produced: the start of the gpu_query entry
+ holding exactly that answer, or when this query began with the cache off. None when
+ the cache no longer holds it (a newer listing replaced it)."""
+ if not gpu_query.enabled():
+ return started
+ try:
+ at = gpu_query.answered_at([_nvidia_smi_executable(), "-L"], stdout)
+ except Exception:
+ return None
+ return None if at is None else min(at, started)
+
+
+def _parse_mig_gpu_indices(stdout: Any) -> frozenset[int]:
+ if not isinstance(stdout, str):
+ return frozenset()
+ indices: set[int] = set()
+ current: Optional[int] = None
+ for line in stdout.splitlines():
+ if line.startswith("GPU "):
+ head = line[4:].split(":", 1)[0].strip()
+ current = int(head) if head.isdecimal() else None
+ elif current is not None and line.strip().startswith("MIG "):
+ indices.add(current)
+ return frozenset(indices)
+
+
def get_mig_gpu_indices() -> set[int]:
"""Physical indices of GPUs with MIG instances (indented ``MIG`` rows under their
``GPU N:`` row in ``nvidia-smi -L``). CUDA sees such a card as one slice, so its
- ``--query-gpu`` memory is the whole card's, not the device's. Empty on failure."""
+ ``--query-gpu`` memory is the whole card's, not the device's. Empty when discovery
+ is unknown: placement uses get_mig_gpu_indices_or_none to tell the two apart."""
+ return set(_mig_gpu_lookup() or ())
+
+
+def get_mig_gpu_indices_or_none() -> Optional[set[int]]:
+ """``get_mig_gpu_indices``, but None when MIG discovery is unknown: the latest-started
+ probe of this generation failed, answered empty or was superseded, or none has run.
+ An empty set means a listing answered and no card has MIG instances."""
+ indices = _mig_gpu_lookup()
+ return None if indices is None else set(indices)
+
+
+def _mig_gpu_lookup() -> Optional[frozenset[int]]:
+ cached = _mig_cached()
+ if cached is not _MISS:
+ return cached
+ probe = _mig_probe_start()
+ return _mig_finish(probe, _run_mig_listing())
+
+
+def _run_mig_listing() -> Any:
+ """``nvidia-smi -L`` through gpu_query, or None when it raised."""
try:
- result = gpu_query.run_nvidia_smi(
+ return gpu_query.run_nvidia_smi(
[_nvidia_smi_executable(), "-L"],
capture_output = True,
text = True,
@@ -166,18 +278,7 @@ def get_mig_gpu_indices() -> set[int]:
**_windows_hidden_subprocess_kwargs(),
)
except Exception:
- return set()
- if result.returncode != 0 or not isinstance(result.stdout, str):
- return set()
- indices: set[int] = set()
- current: Optional[int] = None
- for line in result.stdout.splitlines():
- if line.startswith("GPU "):
- head = line[4:].split(":", 1)[0].strip()
- current = int(head) if head.isdecimal() else None
- elif current is not None and line.strip().startswith("MIG "):
- indices.add(current)
- return indices
+ return None
def get_primary_gpu_utilization() -> dict[str, Any]: