Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: read whole-GPU uuid masks from nvidia-smi rows and cache the MIG set (#14017)
Daniel Haneb0617b62bb5
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 54978e964c5..e2048ce18a7 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -11906,6 +11906,10 @@ class LlamaCppBackend:emit, so callers can treat non-None as "this changed something"."""inherited = LlamaCppBackend._resolve_visible_physical_ids()+ if not inherited:+ # A uuid mask the ordinal resolvers leave open (FASTEST_FIRST) still orders+ # the cards the memory probe matched by uuid.+ inherited = LlamaCppBackend._smi_uuid_mask_order()if not inherited:return Nonewanted = [int(i) for i in pinned_ids]@@ -13208,6 +13212,9 @@ class LlamaCppBackend:_GPU_IDS_ARE_PCI_INDICES = None# index -> inherited uuid, when CUDA_VISIBLE_DEVICES named the NVML rows by uuid._NVML_ROWS: list = [] # the last NVML inventory; _child_visibility_for reads it+ _SMI_UUID_ROWS: list = [] # (index, uuid) from the last nvidia-smi memory probe+ # MIG cards that same probe saw (None = discovery unknown), so pinning uses its snapshot.+ _SMI_MIG_INDICES: Optional[frozenset] = None# Boot-time property: read once, not per launch (the #10613 host has 175# groups). Only the default root is cached; an explicit root (tests) re-reads.@@ -14997,19 +15004,25 @@ class LlamaCppBackend:]return min(slices, key = free_mib) if slices else r- tokens = [t.strip() for t in (raw or "").split(",") if t.strip()]+ tokens = [t.strip() for t in (raw or "").split(",")] if raw else []picked: list[dict] = []if raw is None:picked = [as_cuda_sees(r) for r in parents]seen: set[int] = set()for token in tokens:+ if not token:+ break # an empty entry ends the mask, as CUDA reads itprefix = re.match(r"\+?\d+", token)- if token.startswith("GPU-"):- hits = [r for r in parents if str(r.get("uuid", "")).startswith(token)]- elif token.startswith("MIG-"):+ # Case-insensitive, as the nvidia-smi uuid matcher reads the same mask.+ low = token.lower()+ if low.startswith("gpu-"):+ hits = [r for r in parents if str(r.get("uuid", "")).lower().startswith(low)]+ elif low.startswith("mig-"):# A MIG- entry names a slice row the probe lists under its parent.hits = [- r for r in rows if r.get("mig") and str(r.get("uuid", "")).startswith(token)+ r+ for r in rows+ if r.get("mig") and str(r.get("uuid", "")).lower().startswith(low)]elif prefix:# CUDA reads an index strtoul-style: "1gpu2" is GPU 1.@@ -15072,9 +15085,59 @@ class LlamaCppBackend:and LlamaCppBackend._visible_devices_mask("CUDA_VISIBLE_DEVICES") is Noneand LlamaCppBackend._visibility_mask_is_unmappable()):+ # Physical rows the memory probe matched by uuid: hand the child those uuids.+ uuids = LlamaCppBackend._smi_uuids_for(cvd, gpu_indices)+ if uuids is not None:+ return uuidsreturn LlamaCppBackend._mig_tokens_for_ordinals(cvd, gpu_indices)return mask+ @staticmethod+ def _smi_uuid_mask_order() -> Optional[list[int]]:+ """Physical ids of a whole-GPU uuid mask in mask order, from the last nvidia-smi+ rows; None unless the probe's ids are physical and every token matched a row."""+ cvd = os.environ.get("CUDA_VISIBLE_DEVICES")+ if LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES is not True or not (cvd or "").strip():+ return None+ from utils.hardware import nvidia++ visible = nvidia._uuid_visible_ordinal_map(cvd, LlamaCppBackend._SMI_UUID_ROWS)+ return sorted(visible, key = visible.__getitem__) if visible else None++ @staticmethod+ def _smi_uuids_for(cvd: str, gpu_indices) -> Optional[str]:+ """The uuids of these physical ids when the current whole-GPU uuid mask names each+ of them in the last nvidia-smi rows, else None. Never for a MIG card: CUDA hands a+ GPU- entry on one a slice, which only the NVML rows name."""+ if LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES is not True:+ return None+ from utils.hardware import nvidia++ rows = LlamaCppBackend._SMI_UUID_ROWS+ visible = nvidia._uuid_visible_ordinal_map(cvd, rows)+ try:+ ids = [int(i) for i in gpu_indices]+ except (TypeError, ValueError):+ return None+ if not visible or not ids or len(set(ids)) != len(ids):+ return None+ if any(i not in visible for i in ids):+ return None+ # The probe that chose these rows classified them; a later recheck failing must+ # not widen the child to the whole inherited mask.+ mig = LlamaCppBackend._SMI_MIG_INDICES+ if mig is None:+ # Unknown then too: NVML rows from that probe still name MIG cards.+ mig = {+ int(r["index"])+ for r in LlamaCppBackend._NVML_ROWS+ if r.get("mig") and str(r.get("index", "")).isdigit()+ }+ if set(ids) & mig:+ return None+ by_index = dict(rows)+ return ",".join(by_index[i] for i in ids)+@staticmethoddef _mig_tokens_for_ordinals(cvd: str, gpu_indices) -> Optional[str]:"""The inherited MIG- entries behind torch ordinals, or None (keep the whole mask). Only@@ -15142,6 +15205,28 @@ class LlamaCppBackend:gpus.sort(key = lambda g: g[0])return gpus+ @staticmethod+ def _whole_gpu_mask_rows(+ cvd: str, uuid_rows: list[tuple[int, str]], physical_ids: set[int]+ ) -> Optional[set[int]]:+ """Physical rows behind a whole-GPU uuid mask the ordinal resolvers left open.+ A uuid names its card whatever CUDA_DEVICE_ORDER says, so only membership is+ needed. None for a MIG mask or a token no row verifies, so the caller keeps+ its conservative path."""+ from utils.hardware import nvidia++ tokens = [t.strip().lower() for t in cvd.split(",") if t.strip()]+ if not tokens or any(not t.startswith("gpu-") for t in tokens):+ return None+ if uuid_rows:+ visible = nvidia._uuid_visible_ordinal_map(cvd, uuid_rows)+ return set(visible) if visible is not None else None+ # No uuid column: verify with the separate uuid query, never assume. Only a+ # one-card mask: these rows carry no uuids to order or pin a multi-card subset,+ # while the inherited mask already pins a single card.+ ids = nvidia.resolve_uuid_mask(cvd)+ return set(ids) if ids and len(ids) == 1 and set(ids) <= physical_ids else None+@staticmethoddef _get_gpu_memory(binary: Optional[str] = None, *, for_llama_server: bool = False@@ -15185,7 +15270,9 @@ class LlamaCppBackend:result = gpu_query.run_nvidia_smi(["nvidia-smi",- "--query-gpu=index,memory.free,memory.total",+ # uuid in the same call: a whole-GPU uuid mask resolves without+ # CUDA_DEVICE_ORDER or a second query.+ "--query-gpu=index,memory.free,memory.total,uuid","--format=csv,noheader,nounits",],capture_output = True,@@ -15197,16 +15284,19 @@ class LlamaCppBackend:**_windows_hidden_subprocess_kwargs(),)if result.returncode == 0:+ from utils.hardware import nvidia+allowed = LlamaCppBackend._visible_devices_mask("CUDA_VISIBLE_DEVICES")- if allowed is None and (os.environ.get("CUDA_VISIBLE_DEVICES") or "").strip():- # CUDA's numeric-prefix reading ("1gpu2" is 1, "0,1,abc" is 0,1).- _ids = LlamaCppBackend._resolve_visible_physical_ids()- allowed = set(_ids) if _ids is not None else None- gpus: list[tuple[int, int, int]] = []+ _cvd = (os.environ.get("CUDA_VISIBLE_DEVICES") or "").strip()+ rows: list[tuple[int, int, int]] = []+ unsized_rows: list[int] = []+ uuid_rows: list[tuple[int, str]] = []for line in result.stdout.strip().splitlines():parts = [p.strip() for p in line.split(",")]if len(parts) < 2:continue+ if len(parts) >= 4 and parts[0].isdigit() and parts[3]:+ uuid_rows.append((int(parts[0]), parts[3]))# Index and free required; skip a bad line rather than abandon# the probe to the torch fallback.try:@@ -15214,8 +15304,8 @@ class LlamaCppBackend:free_mib = int(parts[1])except ValueError:# Jetson answers "[N/A]": no dedicated VRAM (#4862).- if parts[0].isdigit() and (allowed is None or int(parts[0]) in allowed):- unsized_ids.append(int(parts[0]))+ if parts[0].isdigit():+ unsized_rows.append(int(parts[0]))continue# Total parsed separately: a two-column line or a non-integer# total ("N/A" on MIG/vGPU) keeps the GPU at total 0 (fit uses@@ -15226,20 +15316,45 @@ class LlamaCppBackend:total_mib = int(parts[2])except ValueError:total_mib = 0- if allowed is not None and idx not in allowed:- continue- gpus.append((idx, free_mib, total_mib))+ rows.append((idx, free_mib, total_mib))+ LlamaCppBackend._SMI_UUID_ROWS = uuid_rows+ LlamaCppBackend._SMI_MIG_INDICES = None+ if allowed is None and _cvd:+ allowed = LlamaCppBackend._whole_gpu_mask_rows(+ _cvd, uuid_rows, {r[0] for r in rows} | set(unsized_rows)+ )+ if allowed is None and _cvd:+ # CUDA's numeric-prefix reading ("1gpu2" is 1, "0,1,abc" is 0,1).+ _ids = LlamaCppBackend._resolve_visible_physical_ids()+ allowed = set(_ids) if _ids is not None else None+ unsized_ids = [i for i in unsized_rows if allowed is None or i in allowed]+ gpus = [r for r in rows if allowed is None or r[0] in allowed]# Match the docstring's sort-by-id guarantee (driver order isn't).gpus.sort(key = lambda g: g[0])- _cvd = os.environ.get("CUDA_VISIBLE_DEVICES")- if allowed is None and _cvd is not None and _cvd.strip():+ if allowed is None and _cvd:# A MIG / unresolvable UUID mask names devices these rows cannot be# matched to: listing every card sizes against GPUs the parent hid.# The NVML branch names slices and uuids.gpus = []elif gpus:- from utils.hardware import nvidia- mig = nvidia.get_mig_gpu_indices() & {g[0] for g in gpus}+ mig_known = nvidia.get_mig_gpu_indices_or_none()+ if mig_known is not None:+ LlamaCppBackend._SMI_MIG_INDICES = frozenset(mig_known)+ mig = (mig_known or set()) & {g[0] for g in gpus}+ if mig_known is None:+ # MIG discovery failed: a row may be a whole MIG card. NVML names+ # slices and whole cards alike, so its reading wins; without one+ # the CLI rows stand, as before MIG support (no CUDA context).+ nvml = {g[0]: g for g in LlamaCppBackend._get_gpu_memory_nvml()}+ if nvml:+ gpus = [nvml.get(g[0], g) for g in gpus]+ else:+ # No current NVML inventory: an older one must not name devices.+ LlamaCppBackend._NVML_ROWS = []+ elif not mig:+ # The CLI now answers for these cards: an earlier NVML inventory+ # (slices since removed) must not name the child's devices.+ LlamaCppBackend._NVML_ROWS = []if mig:# A MIG card's row is the whole card: swap in its NVML slice reading,# keeping the whole cards. No reading at all: the later probes decide,diff --git a/studio/backend/tests/test_mig_gpu_masks.py b/studio/backend/tests/test_mig_gpu_masks.pyindex 2a1e07a804c..970b8021520 100644--- a/studio/backend/tests/test_mig_gpu_masks.py+++ b/studio/backend/tests/test_mig_gpu_masks.py@@ -11,6 +11,7 @@ from __future__ import annotationsimport jsonimport os+import subprocessimport sysimport types@@ -70,6 +71,8 @@ def host(tmp_path, monkeypatch):monkeypatch.setenv("UNSLOTH_GPU_QUERY_CACHE", "0")monkeypatch.setattr(nvidia, "_uuid_mask_cache", {})monkeypatch.setattr(LlamaCppBackend, "_NVML_ROWS", [])+ monkeypatch.setattr(LlamaCppBackend, "_SMI_UUID_ROWS", [], raising = False)+ monkeypatch.setattr(LlamaCppBackend, "_SMI_MIG_INDICES", None, raising = False)monkeypatch.setattr(LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda b = None: False))monkeypatch.setattr(LlamaCppBackend, "_find_llama_server_binary", staticmethod(lambda: "/opt/llama-server")@@ -102,9 +105,15 @@ def host(tmp_path, monkeypatch):query = next((a for a in cmd if a.startswith("--query-gpu=")), "")if query == "--query-gpu=index,uuid":return ok("".join(f"{i}, {u}\n" for i, (u, _) in enumerate(state["cards"])))- if query == "--query-gpu=index,memory.free,memory.total":+ if query.startswith("--query-gpu=index,memory.free,memory.total"):# Whole-card numbers, also for a card in MIG mode.- return ok("".join(f"{i}, {f}, {A100_MIB}\n" for i, f in enumerate(state["free"])))+ uuids = [u for u, _ in state["cards"]] if query.endswith(",uuid") else None+ return ok(+ "".join(+ f"{i}, {f}, {A100_MIB}" + (f", {uuids[i]}" if uuids else "") + "\n"+ for i, f in enumerate(state["free"])+ )+ )return types.SimpleNamespace(returncode = 1, stdout = "", stderr = "")monkeypatch.setattr(mod.subprocess, "run", run)@@ -361,6 +370,9 @@ class TestMigSlicesAreNotGpus:host(MIXED, [0, 0, 0], None)assert nvidia.get_mig_gpu_indices() == {0}host([(GPU0, []), (GPU1, [("1g.5gb", MIG0A)])], [0, 0], None)+ # Cached until the inventory generation moves (hotplug, driver reset).+ assert nvidia.get_mig_gpu_indices() == {0}+ nvidia.gpu_query.invalidate_static("test")assert nvidia.get_mig_gpu_indices() == {1}@@ -380,3 +392,433 @@ def test_mig_overlay_skips_a_non_cuda_backend(host, monkeypatch):monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.XPU)devices = [{"index": 0, "vram_total_gb": 24.0, "vram_used_gb": 2.0}]assert hw._with_mig_instance_memory(devices) == devices+++T4_MIB = 15360+++def calls(monkeypatch):+ seen = {+ "torch": 0,+ "list": 0,+ "uuid_query_fails": False,+ "list_fails": False,+ "list_empty": False,+ }+ run = mod.subprocess.run++ def counting_run(cmd, *args, **kwargs):+ if cmd and os.path.basename(str(cmd[0])) == "nvidia-smi":+ if "-L" in cmd:+ seen["list"] += 1+ if seen["list_fails"]:+ return types.SimpleNamespace(returncode = 9, stdout = "", stderr = "")+ if seen["list_empty"]:+ return types.SimpleNamespace(returncode = 0, stdout = "", stderr = "")+ if seen["uuid_query_fails"] and "--query-gpu=index,uuid" in cmd:+ return types.SimpleNamespace(returncode = 1, stdout = "", stderr = "")+ return run(cmd, *args, **kwargs)++ monkeypatch.setattr(mod.subprocess, "run", counting_run)++ def torch_probe(*a, **k):+ seen["torch"] += 1+ return [(0, 1, 1)]++ monkeypatch.setattr(LlamaCppBackend, "_get_gpu_memory_torch", staticmethod(torch_probe))+ return seen+++class TestOrdinaryHostsKeepTheirReadings:+ """Whole-GPU uuid masks and plain polls on hosts without MIG behave as before MIG support."""++ @pytest.mark.parametrize("token", [GPU0, GPU0[:12], GPU0.lower()])+ def test_a_uuid_mask_under_fastest_first_reads_its_card(self, host, monkeypatch, calls, token):+ host([(GPU0, [])], [14000], None)+ monkeypatch.setenv("CUDA_DEVICE_ORDER", "FASTEST_FIRST")+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", token)+ assert LlamaCppBackend._get_gpu_memory() == [(0, 14000, A100_MIB)]+ assert calls["torch"] == 0++ def test_a_uuid_mask_reads_its_card_when_the_uuid_query_fails(self, host, monkeypatch, calls):+ host([(GPU0, [])], [14000], None)+ calls["uuid_query_fails"] = True+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", GPU0)+ assert LlamaCppBackend._get_gpu_memory() == [(0, 14000, A100_MIB)]+ assert calls["torch"] == 0++ def test_a_multi_gpu_uuid_mask_under_fastest_first_keeps_only_its_cards(+ self, host, monkeypatch, calls+ ):+ host([(GPU0, []), (GPU1, []), (GPU2, [])], [39000, 30000, 40000], None)+ monkeypatch.setenv("CUDA_DEVICE_ORDER", "FASTEST_FIRST")+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", f"{GPU2},{GPU1}")+ assert LlamaCppBackend._get_gpu_memory() == [(1, 30000, A100_MIB), (2, 40000, A100_MIB)]+ assert calls["torch"] == 0++ def test_an_unknown_uuid_still_reads_nothing_from_the_cli(self, host, monkeypatch, calls):+ host([(GPU0, []), (GPU1, [])], [39000, 30000], None)+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", GPU2)+ assert LlamaCppBackend._get_gpu_memory() == [(0, 1, 1)]++ def test_without_a_uuid_column_the_mask_is_verified_or_unresolved(+ self, host, monkeypatch, calls+ ):+ host([(GPU0, [])], [14000], None)+ monkeypatch.setenv("CUDA_DEVICE_ORDER", "FASTEST_FIRST")+ assert LlamaCppBackend._whole_gpu_mask_rows(GPU0, [], {0}) == {0}+ # A uuid no card has hides every device, even on a one-card host.+ assert LlamaCppBackend._whole_gpu_mask_rows(GPU1, [], {0}) is None+ assert LlamaCppBackend._whole_gpu_mask_rows(MIG0A, [], {0}) is None+ nvidia._uuid_mask_cache.clear()+ calls["uuid_query_fails"] = True+ assert LlamaCppBackend._whole_gpu_mask_rows(GPU0, [], {0}) is None++ def test_a_subset_of_a_uuid_mask_pins_the_child_by_uuid(self, host, monkeypatch, calls):+ host([(GPU0, []), (GPU1, []), (GPU2, [])], [39000, 30000, 40000], None)+ monkeypatch.setenv("CUDA_DEVICE_ORDER", "FASTEST_FIRST")+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", f"{GPU2},{GPU1}")+ assert LlamaCppBackend._get_gpu_memory() == [(1, 30000, A100_MIB), (2, 40000, A100_MIB)]+ assert LlamaCppBackend._pinned_child_visibility([2]) == GPU2+ assert LlamaCppBackend._pinned_child_visibility([2, 1]) == f"{GPU2},{GPU1}"+ # A card the mask does not name is never pinned.+ assert LlamaCppBackend._pinned_child_visibility([0]) is None+ # The mask is re-read: a card the new mask drops is no longer mapped.+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", GPU1)+ assert LlamaCppBackend._pinned_child_visibility([2]) is None++ def test_a_uuid_mask_on_a_mig_card_keeps_the_inherited_mask(self, host, monkeypatch, calls):+ host(MIXED, [39000, 30000, 40000], None)+ monkeypatch.setenv("CUDA_DEVICE_ORDER", "FASTEST_FIRST")+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", f"{GPU0},{GPU1}")+ LlamaCppBackend._get_gpu_memory()+ assert LlamaCppBackend._pinned_child_visibility([0, 1]) is None+ assert LlamaCppBackend._pinned_child_visibility([1]) == GPU1++ def test_a_listing_raced_by_an_invalidation_is_not_cached(self, host, monkeypatch, calls):+ host([(GPU0, [])], [14000], None)+ real = nvidia._run_mig_listing++ def racing():+ out = real()+ nvidia.gpu_query.invalidate_static("hotplug during the query")+ return out++ monkeypatch.setattr(nvidia, "_run_mig_listing", racing)+ assert nvidia.get_mig_gpu_indices_or_none() is None+ monkeypatch.setattr(nvidia, "_run_mig_listing", real)+ nvidia.get_mig_gpu_indices()+ assert calls["list"] == 2+ nvidia.get_mig_gpu_indices()+ assert calls["list"] == 2++ def test_memory_polls_reuse_one_mig_listing(self, host, calls):+ host([(GPU0, [])], [14000], None)+ devices = [{"index": 0, "vram_total_gb": 15.0, "vram_used_gb": 1.0}]+ for _ in range(3):+ assert LlamaCppBackend._get_gpu_memory() == [(0, 14000, A100_MIB)]+ assert hw._with_mig_instance_memory(devices) == devices+ assert calls["list"] == 1++ def test_the_physical_count_probe_publishes_the_mig_set(self, host, monkeypatch, calls):+ monkeypatch.setattr(hw, "IS_ROCM", False)+ monkeypatch.setattr(hw, "_physical_gpu_count", None)+ monkeypatch.setattr(hw, "_physical_gpu_count_from_smi", False)+ monkeypatch.setattr(hw, "_physical_gpu_count_checked_at", None, raising = False)+ host([(GPU0, [])], [14000], None)+ assert hw.get_physical_gpu_count() == 1+ assert calls["list"] == 1+ for _ in range(3):+ assert LlamaCppBackend._get_gpu_memory() == [(0, 14000, A100_MIB)]+ assert calls["list"] == 1++ def test_a_raised_physical_count_probe_leaves_mig_unknown(self, host, monkeypatch, calls):+ host([(GPU0, [])], [14000], None)+ assert nvidia.get_mig_gpu_indices_or_none() == frozenset()+ real = nvidia.gpu_query.run_nvidia_smi++ def boom(cmd, *a, **k):+ if cmd[-1] == "-L":+ raise subprocess.TimeoutExpired(cmd, 5)+ return real(cmd, *a, **k)++ monkeypatch.setattr(nvidia.gpu_query, "run_nvidia_smi", boom)+ assert nvidia.get_physical_gpu_count() is None+ # The newest probe failed: the older "no MIG" answer is no longer served.+ assert nvidia._mig_entry[3] is None+ monkeypatch.setattr(nvidia.gpu_query, "run_nvidia_smi", real)+ assert nvidia.get_mig_gpu_indices_or_none() is None+++class TestMigListingFreshness:+ @pytest.fixture+ def clock(self, monkeypatch):+ now = [1000.0]+ monkeypatch.setattr(nvidia.time, "monotonic", lambda: now[0])+ return now++ def test_a_cached_listing_keeps_its_age(self, host, monkeypatch, clock, calls):+ monkeypatch.setenv("UNSLOTH_GPU_QUERY_CACHE", "1")+ host(TWO_SLICES, [38000], None)+ assert nvidia.get_physical_gpu_count(gpu_rows_only = True) == 1+ monkeypatch.setattr(nvidia, "_mig_entry", None)+ clock[0] += 59+ # Served from gpu_query's own cache: stamped with that answer's time, not now.+ assert nvidia.get_mig_gpu_indices() == {0}+ assert calls["list"] == 1+ assert nvidia._mig_entry[2] == 1000.0+ clock[0] += 2+ assert nvidia.get_mig_gpu_indices() == {0}+ assert calls["list"] == 2++ def test_a_failed_listing_is_unknown_and_backs_off(self, host, monkeypatch, clock, calls):+ host(TWO_SLICES, [38000], None)+ assert nvidia.get_mig_gpu_indices() == {0}+ clock[0] += 61+ calls["list_fails"] = True+ for _ in range(4):+ assert nvidia.get_mig_gpu_indices_or_none() is None+ assert calls["list"] == 2+ clock[0] += nvidia._MIG_FAILURE_RETRY_S+ calls["list_fails"] = False+ host([(GPU0, [])], [38000], None)+ assert nvidia.get_mig_gpu_indices() == set()+ assert calls["list"] == 3++ def test_a_cold_failure_is_retried_after_the_backoff(self, host, monkeypatch, clock, calls):+ host(TWO_SLICES, [38000], None)+ calls["list_fails"] = True+ assert nvidia.get_mig_gpu_indices() == set()+ assert nvidia.get_mig_gpu_indices() == set()+ assert calls["list"] == 1+ calls["list_fails"] = False+ clock[0] += nvidia._MIG_FAILURE_RETRY_S+ assert nvidia.get_mig_gpu_indices() == {0}+ assert calls["list"] == 2+++def test_a_reordered_uuid_mask_orders_the_pinned_child(host, monkeypatch):+ host([(GPU0, []), (GPU1, []), (GPU2, [])], [39000, 30000, 40000], None)+ monkeypatch.setenv("CUDA_DEVICE_ORDER", "FASTEST_FIRST")+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", f"{GPU2},{GPU1}")+ LlamaCppBackend._get_gpu_memory()+ # Auto fit hands ascending ids; the launch reorders them (and their shares) to the mask.+ assert LlamaCppBackend._inherited_child_gpu_order([1, 2]) == [2, 1]+ assert LlamaCppBackend._pinned_child_visibility([2, 1]) == f"{GPU2},{GPU1}"+ assert LlamaCppBackend._inherited_child_gpu_order([2]) is None+++class TestMigListingHonoursTheStaticTtl:+ @pytest.mark.parametrize("ttl, expected", [("10", [1, 1, 2]), ("0", [1, 2, 3])])+ def test_the_configured_ttl_bounds_the_mig_set(self, host, monkeypatch, calls, ttl, expected):+ now = [1000.0]+ monkeypatch.setattr(nvidia.time, "monotonic", lambda: now[0])+ monkeypatch.setenv("UNSLOTH_GPU_QUERY_STATIC_TTL", ttl)+ host(TWO_SLICES, [38000], None)+ seen = []+ for step in (0, 5, 6):+ now[0] += step+ assert nvidia.get_mig_gpu_indices() == {0}+ seen.append(calls["list"])+ assert seen == expected+++ "mask, expected, child",+ [+ (GPU0.lower(), (0, 9000, 9856), MIG0A),+ (MIG0B.lower(), (0, 9500, 9856), MIG0B),+ ],+)+def test_a_lowercase_uuid_reads_the_slice(host, monkeypatch, mask, expected, child):+ host(MIXED, [39000, 30000, 40000], MIXED_NVML)+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", mask)+ assert LlamaCppBackend._get_gpu_memory() == [expected]+ assert LlamaCppBackend._pinned_child_visibility([0]) == child+++def test_an_empty_listing_is_a_failed_probe(host, monkeypatch, calls):+ now = [1000.0]+ monkeypatch.setattr(nvidia.time, "monotonic", lambda: now[0])+ host(TWO_SLICES, [38000], None)+ assert nvidia.get_mig_gpu_indices() == {0}+ now[0] += 61+ calls["list_empty"] = True+ assert nvidia.get_mig_gpu_indices_or_none() is None+ assert nvidia._mig_entry[3] is None+ calls["list_empty"] = False+ now[0] += nvidia._MIG_FAILURE_RETRY_S+ assert nvidia.get_mig_gpu_indices() == {0}+ assert calls["list"] == 3+++def test_a_superseded_listing_is_not_cached(host, monkeypatch, calls):+ monkeypatch.setenv("UNSLOTH_GPU_QUERY_CACHE", "1")+ host(TWO_SLICES, [38000], None)+ assert nvidia.get_physical_gpu_count(gpu_rows_only = True) == 1+ assert nvidia._listing_taken_at(1e9, _smi_list(TWO_SLICES)) is not None+ assert nvidia._listing_taken_at(1e9, _smi_list([(GPU0, [])])) is None+ # The query hands back an older listing than the one the cache now holds.+ real = nvidia.gpu_query.run_nvidia_smi++ def stale(argv, **kw):+ real(argv, **kw)+ return types.SimpleNamespace(returncode = 0, stdout = _smi_list([(GPU0, [])]), stderr = "")++ monkeypatch.setattr(nvidia, "_mig_entry", None)+ monkeypatch.setattr(nvidia.gpu_query, "run_nvidia_smi", stale)+ # Neither published nor acted on: discovery is unknown, not "no MIG".+ assert nvidia.get_mig_gpu_indices_or_none() is None+ assert nvidia._mig_entry is None+++class TestAnEmptyEntryEndsAUuidMask:+ @pytest.mark.parametrize(+ "mask, expected",+ [+ (f",{GPU1}", [(0, 1, 1)]),+ (f"{GPU2},,{GPU1}", [(2, 40000, A100_MIB)]),+ (f"{GPU2},{GPU1},", [(1, 30000, A100_MIB), (2, 40000, A100_MIB)]),+ ],+ )+ def test_the_cli_rows(self, host, monkeypatch, calls, mask, expected):+ host([(GPU0, []), (GPU1, []), (GPU2, [])], [39000, 30000, 40000], None)+ monkeypatch.setenv("CUDA_DEVICE_ORDER", "FASTEST_FIRST")+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", mask)+ assert LlamaCppBackend._get_gpu_memory() == expected+ if mask.startswith(","):+ # Nothing visible: no uuid order or uuid pin is derived from the rows.+ monkeypatch.setattr(LlamaCppBackend, "_GPU_IDS_ARE_PCI_INDICES", True)+ assert LlamaCppBackend._smi_uuid_mask_order() is None+ assert LlamaCppBackend._smi_uuids_for(mask, [1]) is None++ @pytest.mark.parametrize(+ "mask, expected", [(f",{GPU1}", []), (f"{GPU1},,{GPU2}", ["1"]), ("1,,2", ["1"])]+ )+ def test_the_nvml_rows(self, host, monkeypatch, mask, expected):+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", mask)+ assert [r["index"] for r in LlamaCppBackend._nvml_rows_visible(MIXED_NVML)] == expected+++def test_without_a_uuid_column_only_a_one_card_mask_resolves(host, monkeypatch):+ # No uuids in the rows: a multi-card subset could be neither ordered nor pinned.+ host([(GPU0, []), (GPU1, [])], [39000, 30000], None)+ mask = f"{GPU0},{GPU1}"+ assert LlamaCppBackend._whole_gpu_mask_rows(mask, [], {0, 1}) is None+ assert LlamaCppBackend._whole_gpu_mask_rows(mask, [], {0}) is None+ assert LlamaCppBackend._whole_gpu_mask_rows(GPU1, [], {0, 1}) == {1}+++class TestTheLatestStartedProbeWins:+ LISTING = types.SimpleNamespace(returncode = 0, stdout = _smi_list(TWO_SLICES), stderr = "")+ FAILED = types.SimpleNamespace(returncode = 9, stdout = "", stderr = "")++ def test_an_older_failure_keeps_a_newer_answer(self, host):+ old, new = nvidia._mig_probe_start(), nvidia._mig_probe_start()+ assert nvidia._mig_finish(new, self.LISTING) == {0}+ assert nvidia._mig_finish(old, self.FAILED) == {0}+ assert nvidia.get_mig_gpu_indices_or_none() == {0}++ def test_an_older_answer_keeps_a_newer_failure(self, host):+ old, new = nvidia._mig_probe_start(), nvidia._mig_probe_start()+ assert nvidia._mig_finish(new, self.FAILED) is None+ assert nvidia._mig_finish(old, self.LISTING) is None+ assert nvidia.get_mig_gpu_indices_or_none() is None++ def test_a_probe_from_an_old_generation_is_unknown(self, host):+ probe = nvidia._mig_probe_start()+ nvidia.gpu_query.invalidate_static("test")+ assert nvidia._mig_finish(probe, self.LISTING) is None+ assert nvidia._mig_cached() is nvidia._MISS+++def test_unknown_mig_discovery_is_not_no_mig(host, monkeypatch, calls):+ now = [1000.0]+ monkeypatch.setattr(nvidia.time, "monotonic", lambda: now[0])+ host(TWO_SLICES, [38000], None)+ calls["list_fails"] = True+ assert nvidia.get_mig_gpu_indices_or_none() is None+ assert nvidia.get_mig_gpu_indices() == set()+ calls["list_fails"] = False+ now[0] += nvidia._MIG_FAILURE_RETRY_S+ assert nvidia.get_mig_gpu_indices_or_none() == {0}+ now[0] += 61+ calls["list_empty"] = True+ assert nvidia.get_mig_gpu_indices_or_none() is None+ assert nvidia.get_mig_gpu_indices() == set()+ calls["list_empty"] = False+ nvidia.gpu_query.invalidate_static("test")+ host([(GPU0, [])], [38000], None)+ assert nvidia.get_mig_gpu_indices_or_none() == set()+++class TestUnknownMigDiscovery:+ def test_placement_reads_nvml_when_the_listing_fails(self, host, calls):+ host(TWO_SLICES, [38000], TWO_SLICES_NVML)+ calls["list_fails"] = True+ assert LlamaCppBackend._get_gpu_memory() == [(0, 15000, 19968)]+ devices = [{"index": 0, "vram_total_gb": 40.0, "vram_used_gb": 2.0}]+ out = hw._with_mig_instance_memory(devices)+ assert out[0]["vram_total_gb"] == round(19968 / 1024, 2)++ def test_without_nvml_the_cli_rows_stand_and_cuda_is_not_touched(self, host, calls):+ host([(GPU0, [])], [14000], None)+ calls["list_fails"] = True+ assert LlamaCppBackend._get_gpu_memory() == [(0, 14000, A100_MIB)]+ devices = [{"index": 0, "vram_total_gb": 40.0, "vram_used_gb": 2.0}]+ assert hw._with_mig_instance_memory(devices) == devices+ assert calls["torch"] == 0+++def test_cli_rows_retire_an_old_nvml_inventory(host, monkeypatch):+ # MIG was on (NVML rows cached), then disabled: the CLI now reports whole cards.+ host([(GPU0, []), (GPU1, [])], [39000, 30000], None)+ monkeypatch.setattr(LlamaCppBackend, "_NVML_ROWS", [dict(r) for r in MIXED_NVML])+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", GPU0)+ assert LlamaCppBackend._get_gpu_memory() == [(0, 39000, A100_MIB)]+ assert LlamaCppBackend._NVML_ROWS == []+ assert LlamaCppBackend._pinned_child_visibility([0]) == "0"+++def test_unknown_discovery_without_nvml_retires_old_nvml_rows(host, monkeypatch, calls):+ host([(GPU0, []), (GPU1, [])], [39000, 30000], None)+ monkeypatch.setattr(LlamaCppBackend, "_NVML_ROWS", [dict(r) for r in MIXED_NVML])+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", GPU0)+ calls["list_fails"] = True+ assert LlamaCppBackend._get_gpu_memory() == [(0, 39000, A100_MIB)]+ assert LlamaCppBackend._NVML_ROWS == []+ assert LlamaCppBackend._pinned_child_visibility([0]) == "0"+++class TestAUuidSubsetPinSurvivesAFailedMigRecheck:+ def _narrowed(self, host, monkeypatch):+ host([(GPU0, []), (GPU1, []), (GPU2, [])], [39000, 30000, 40000], None)+ monkeypatch.setenv("CUDA_DEVICE_ORDER", "FASTEST_FIRST")+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", f"{GPU2},{GPU1}")++ def test_the_memory_probes_snapshot_is_used(self, host, monkeypatch, calls):+ self._narrowed(host, monkeypatch)+ assert LlamaCppBackend._get_gpu_memory() == [(1, 30000, A100_MIB), (2, 40000, A100_MIB)]+ calls["list_fails"] = True+ nvidia.gpu_query.invalidate_static("recheck after the probe")+ assert LlamaCppBackend._pinned_child_visibility([2]) == GPU2+ assert calls["list"] == 1++ def test_unknown_at_probe_time_still_pins_the_fitted_subset(self, host, monkeypatch, calls):+ self._narrowed(host, monkeypatch)+ calls["list_fails"] = True+ assert LlamaCppBackend._get_gpu_memory() == [(1, 30000, A100_MIB), (2, 40000, A100_MIB)]+ assert LlamaCppBackend._pinned_child_visibility([2]) == GPU2++ def test_unknown_at_probe_time_never_pins_a_card_nvml_calls_mig(self, host, monkeypatch, calls):+ host(MIXED, [39000, 30000, 40000], None)+ monkeypatch.setenv("CUDA_DEVICE_ORDER", "FASTEST_FIRST")+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", f"{GPU0},{GPU1}")+ calls["list_fails"] = True+ LlamaCppBackend._get_gpu_memory()+ monkeypatch.setattr(LlamaCppBackend, "_NVML_ROWS", [dict(r) for r in MIXED_NVML])+ # The MIG card is pinned to its slice through the NVML rows, never its whole uuid.+ assert LlamaCppBackend._smi_uuids_for(f"{GPU0},{GPU1}", [0]) is None+ assert LlamaCppBackend._pinned_child_visibility([0]) == MIG0A+ assert LlamaCppBackend._smi_uuids_for(f"{GPU0},{GPU1}", [1]) == GPU1diff --git a/studio/backend/utils/hardware/gpu_query.py b/studio/backend/utils/hardware/gpu_query.pyindex 4120e646679..71374b79bde 100644--- a/studio/backend/utils/hardware/gpu_query.py+++ b/studio/backend/utils/hardware/gpu_query.py@@ -369,6 +369,24 @@ def _fallback(key: tuple, kind: str) -> Optional[subprocess.CompletedProcess]:return None+def answered_at(+ argv: Sequence[str],+ stdout: Any,+ *,+ text_mode: bool = True,+) -> Optional[float]:+ """Monotonic start of the cached answer for ``argv`` in this static generation when its+ stdout is exactly ``stdout``; None when no such entry is cached (absent or superseded)."""+ key = (subprocess.run, _resolved(argv[0]), tuple(argv), text_mode)+ with _lock:+ entry = _cache.get(key)+ if entry is None or entry.result is None or entry.static_gen != _static_gen:+ return None+ if getattr(entry.result, "stdout", None) != stdout:+ return None+ return entry.at++def run_nvidia_smi(argv: Sequence[str],*,diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.pyindex f4866df0334..0bf3b2aed96 100644--- a/studio/backend/utils/hardware/hardware.py+++ b/studio/backend/utils/hardware/hardware.py@@ -6298,10 +6298,12 @@ def _with_mig_instance_memory(devices: list[Dict[str, Any]]) -> list[Dict[str, Areturn devicestry:from . import nvidia- mig = nvidia.get_mig_gpu_indices() & {d.get("index") for d in devices}+ mig_known = nvidia.get_mig_gpu_indices_or_none()except Exception:return devices- if not mig:+ indices = {d.get("index") for d in devices}+ mig = None if mig_known is None else mig_known & indices+ if mig is not None and not mig:return devicesslices: Dict[int, tuple[int, int]] = {}try:@@ -6309,6 +6311,12 @@ def _with_mig_instance_memory(devices: list[Dict[str, Any]]) -> list[Dict[str, Aslices = {idx: (free, total) for idx, free, total in LlamaCppBackend._get_gpu_memory_nvml()}except Exception as e:logger.debug("NVML MIG slice probe failed: %s", e)+ if mig is None:+ # MIG discovery failed: NVML reads slices and whole cards alike, so its readings+ # replace the CLI's; without any the devices stand as before MIG support.+ if not slices:+ return devices+ mig = set(slices) & indicesout = []for device in devices:idx = device.get("index")diff --git a/studio/backend/utils/hardware/nvidia.py b/studio/backend/utils/hardware/nvidia.pyindex 4512f68010f..b91aa0e05a2 100644--- a/studio/backend/utils/hardware/nvidia.py+++ b/studio/backend/utils/hardware/nvidia.py@@ -59,11 +59,10 @@ def _uuid_visible_ordinal_map(parent_cuda_visible_devices: Optional[str], gpu_rows: list[tuple[int, str]]) -> Optional[dict[int, int]]:"""Resolve an ordered full-GPU UUID mask against nvidia-smi rows."""- tokens = [- token.strip().lower()- for token in (parent_cuda_visible_devices or "").split(",")- if token.strip()- ]+ tokens = [token.strip().lower() for token in (parent_cuda_visible_devices or "").split(",")]+ # CUDA stops at the first empty entry: a leading one hides every GPU, so leave it unresolved.+ if "" in tokens:+ tokens = tokens[: tokens.index("")]if not tokens or any(not token.startswith("gpu-") for token in tokens):return None@@ -125,6 +124,8 @@ def _query_uuid_mask(parent_cuda_visible_devices: str) -> Optional[list[int]]:def get_physical_gpu_count(gpu_rows_only: bool = False) -> Optional[int]:"""Return physical GPU count via nvidia-smi, or None on failure. ``gpu_rows_only`` skips the indented MIG child rows."""+ probe = _mig_probe_start()+ published = Falsetry:result = gpu_query.run_nvidia_smi([_nvidia_smi_executable(), "-L"],@@ -136,6 +137,9 @@ def get_physical_gpu_count(gpu_rows_only: bool = False) -> Optional[int]:env = child_env_without_native_path_secret(),**_windows_hidden_subprocess_kwargs(),)+ # Same listing get_mig_gpu_indices parses: publish it so memory polls reuse it.+ published = True+ _mig_finish(probe, result)if result.returncode == 0 and result.stdout.strip():lines = result.stdout.strip().splitlines()if gpu_rows_only:@@ -146,16 +150,124 @@ def get_physical_gpu_count(gpu_rows_only: bool = False) -> Optional[int]:result.returncode,)except Exception as e:+ if not published:+ # This probe is the latest started: a raise must leave MIG unknown, not the old answer.+ _mig_finish(probe, None)logger.warning("nvidia-smi -L failed: %s; caller should fall back to torch", e)return None+# MIG discovery cache: one entry (gen, seq, at, indices), indices None meaning unknown+# (the probe failed, answered empty, or its listing was already superseded). Every probe+# takes a sequence number when it starts and may only replace an entry of its own static+# generation from an earlier-started probe, so the latest-started probe always wins. An+# answer lives for gpu_query's static TTL from when its listing was taken; an unknown+# for a short backoff, so a broken nvidia-smi adds no work per poll.+_MIG_FAILURE_RETRY_S = 5.0+_mig_entry: Optional[tuple[int, int, float, Optional[frozenset[int]]]] = None+_mig_seq = 0+_mig_lock = threading.Lock()+_MISS = object()+++def _mig_probe_start() -> tuple[int, int, float]:+ global _mig_seq+ with _mig_lock:+ _mig_seq += 1+ return gpu_query._static_gen, _mig_seq, time.monotonic()+++def _mig_cached() -> Any:+ """The fresh entry's indices (None = unknown), or _MISS when a probe is due."""+ entry = _mig_entry+ if entry is None or entry[0] != gpu_query._static_gen:+ return _MISS+ # Same lifetime as gpu_query's static inventory: UNSLOTH_GPU_QUERY_STATIC_TTL (0 too).+ ttl = gpu_query.ttl_for(gpu_query.STATIC)+ life = ttl if entry[3] is not None else min(ttl, _MIG_FAILURE_RETRY_S)+ return entry[3] if time.monotonic() - entry[2] < life else _MISS+++def _mig_finish(probe: tuple[int, int, float], result: Any) -> Optional[frozenset[int]]:+ """Publish one probe's outcome (``result`` None when the query raised) and return the+ indices callers should use, None for unknown."""+ global _mig_entry+ gen, seq, started = probe+ stdout = getattr(result, "stdout", None)+ indices: Optional[frozenset[int]] = None+ at = time.monotonic()+ if getattr(result, "returncode", 1) == 0 and isinstance(stdout, str) and stdout.strip():+ taken = _listing_taken_at(started, stdout)+ if taken is None:+ # A newer listing replaced this one: never publish or act on it.+ cached = _mig_cached()+ return None if cached is _MISS else cached+ indices, at = _parse_mig_gpu_indices(stdout), taken+ with _mig_lock:+ if gen != gpu_query._static_gen:+ return None+ cur = _mig_entry+ if cur is not None and cur[0] == gen and cur[1] > seq:+ return cur[3]+ _mig_entry = (gen, seq, at, indices)+ return indices+++def _listing_taken_at(started: float, stdout: Any) -> Optional[float]:+ """When the ``-L`` answer ``stdout`` was produced: the start of the gpu_query entry+ holding exactly that answer, or when this query began with the cache off. None when+ the cache no longer holds it (a newer listing replaced it)."""+ if not gpu_query.enabled():+ return started+ try:+ at = gpu_query.answered_at([_nvidia_smi_executable(), "-L"], stdout)+ except Exception:+ return None+ return None if at is None else min(at, started)+++def _parse_mig_gpu_indices(stdout: Any) -> frozenset[int]:+ if not isinstance(stdout, str):+ return frozenset()+ indices: set[int] = set()+ current: Optional[int] = None+ for line in stdout.splitlines():+ if line.startswith("GPU "):+ head = line[4:].split(":", 1)[0].strip()+ current = int(head) if head.isdecimal() else None+ elif current is not None and line.strip().startswith("MIG "):+ indices.add(current)+ return frozenset(indices)++def get_mig_gpu_indices() -> set[int]:"""Physical indices of GPUs with MIG instances (indented ``MIG`` rows under their``GPU N:`` row in ``nvidia-smi -L``). CUDA sees such a card as one slice, so its- ``--query-gpu`` memory is the whole card's, not the device's. Empty on failure."""+ ``--query-gpu`` memory is the whole card's, not the device's. Empty when discovery+ is unknown: placement uses get_mig_gpu_indices_or_none to tell the two apart."""+ return set(_mig_gpu_lookup() or ())+++def get_mig_gpu_indices_or_none() -> Optional[set[int]]:+ """``get_mig_gpu_indices``, but None when MIG discovery is unknown: the latest-started+ probe of this generation failed, answered empty or was superseded, or none has run.+ An empty set means a listing answered and no card has MIG instances."""+ indices = _mig_gpu_lookup()+ return None if indices is None else set(indices)+++def _mig_gpu_lookup() -> Optional[frozenset[int]]:+ cached = _mig_cached()+ if cached is not _MISS:+ return cached+ probe = _mig_probe_start()+ return _mig_finish(probe, _run_mig_listing())+++def _run_mig_listing() -> Any:+ """``nvidia-smi -L`` through gpu_query, or None when it raised."""try:- result = gpu_query.run_nvidia_smi(+ return gpu_query.run_nvidia_smi([_nvidia_smi_executable(), "-L"],capture_output = True,text = True,@@ -166,18 +278,7 @@ def get_mig_gpu_indices() -> set[int]:**_windows_hidden_subprocess_kwargs(),)except Exception:- return set()- if result.returncode != 0 or not isinstance(result.stdout, str):- return set()- indices: set[int] = set()- current: Optional[int] = None- for line in result.stdout.splitlines():- if line.startswith("GPU "):- head = line[4:].split(":", 1)[0].strip()- current = int(head) if head.isdecimal() else None- elif current is not None and line.strip().startswith("MIG "):- indices.add(current)- return indices+ return Nonedef get_primary_gpu_utilization() -> dict[str, Any]: