Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio runtime keeps the GPU when nvidia-smi cannot answer, and prefers a GPU-capable llama-server beside a CPU build (#11005)
Daniel Hanc829b33685ea
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex badc49b49d2..066ffa54ee7 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -8194,8 +8194,18 @@ class LlamaCppBackend:# while sibling build layouts are checked. include_denied returns an# unavailable path instead: diffusion asset lookup only needs its dir.non_executable = None+ from utils.llama_cpp_path_settings import prefer_gpu_capable++ statuses = {p: _file_status(p) for p in paths}+ first = next((p for p in paths if statuses[p] == "file"), None)+ # A first hit proven CPU-only yields to a GPU-capable sibling build (#5941), unless+ # a denied candidate precedes it: that denial stops discovery, it is not skipped over.+ if first is not None and not any(+ statuses[p] == "denied" for p in paths[: paths.index(first)]+ ):+ paths = prefer_gpu_capable(paths, lambda p: statuses[p] == "file")for p in paths:- st = _file_status(p)+ st = statuses[p]if st == "file":return str(p), Noneif st == "denied":@@ -8275,7 +8285,9 @@ class LlamaCppBackend:# 5-6. Legacy: in-tree build (older setup.sh / setup.ps1). A fallback,# so a denied candidate here just continues (no no-fallback halt).project_root = Path(__file__).resolve().parents[4]- for p in _layout_candidates(project_root / "llama.cpp"):+ from utils.llama_cpp_path_settings import prefer_gpu_capable++ for p in prefer_gpu_capable(_layout_candidates(project_root / "llama.cpp"), _is_file):if _is_file(p):return str(p)@@ -10552,6 +10564,8 @@ class LlamaCppBackend:# only under PCI_BUS_ID, so a caller pinning the child's device order has to# know which it holds (#10613)._GPU_IDS_ARE_PCI_INDICES = None+ # index -> inherited uuid, when CUDA_VISIBLE_DEVICES named the NVML rows by uuid.+ _NVML_ROWS: list = [] # the last NVML inventory; _child_visibility_for reads it# Boot-time property: read once, not per launch (the #10613 host has 175# groups). Only the default root is cached; an explicit root (tests) re-reads.@@ -11717,6 +11731,127 @@ class LlamaCppBackend:except Exception as e: # noqa: BLE001 -- diagnostics must not break a loadreturn f"the reason could not be determined ({type(e).__name__})"+ @staticmethod+ def _nvidia_probe_script() -> Optional[Path]:+ """studio/nvidia_probe.py, the installers' NVML / CUDA driver reader (UNSLOTH_NVIDIA_PROBE wins)."""+ from utils.prebuilt.update_flow import find_installer_script+ return find_installer_script(env_var = "UNSLOTH_NVIDIA_PROBE", script_name = "nvidia_probe.py")++ @staticmethod+ def _nvml_rows_visible(rows: list[dict]) -> list[dict]:+ """The NVML rows CUDA_VISIBLE_DEVICES permits, in mask order: an index or a GPU-/MIG-+ uuid prefix per entry, as the CUDA runtime reads it. An entry naming no single device+ ends the mask there: "0,2,-1,1" exposes 0 and 2, "-1" alone hides every GPU. A GPU in+ MIG mode is one slice to CUDA (the first, when the mask does not name one), so its+ first slice row stands in for the parent's whole-card memory, and a second slice of+ the same card is dropped: the rows are keyed by physical index, and one slice per+ card is what the driver exposed before R570 and what the child is pinned to."""+ raw = os.environ.get("CUDA_VISIBLE_DEVICES")+ parents = [r for r in rows if not r.get("mig")]++ def as_cuda_sees(r: dict) -> dict:+ if r.get("mig"):+ return r # a slice the mask named stays that slice+ slices = [+ s for s in rows if s.get("mig") and str(s.get("index")) == str(r.get("index"))+ ]+ return slices[0] if slices else r++ tokens = [t.strip() for t in (raw or "").split(",") if t.strip()]+ picked: list[dict] = []+ if raw is None:+ picked = [as_cuda_sees(r) for r in parents]+ for token in tokens:+ if token.isdigit():+ hits = [r for r in parents if str(r.get("index")) == token]+ elif token.startswith("GPU-"):+ hits = [r for r in parents if str(r.get("uuid", "")).startswith(token)]+ elif token.startswith("MIG-"):+ # A MIG- entry names a slice row the probe lists under its parent.+ hits = [+ r for r in rows if r.get("mig") and str(r.get("uuid", "")).startswith(token)+ ]+ else:+ hits = []+ if len(hits) != 1:+ break+ picked.extend(+ r+ for r in map(as_cuda_sees, hits)+ if str(r.get("index")) not in {str(p.get("index")) for p in picked}+ )+ return picked++ @staticmethod+ def _uuid_by_index(rows: list[dict]) -> dict[int, str]:+ """The uuid each visible index stands for: every row when the mask named rows by+ uuid, else only a slice standing in for its MIG parent (a slice has no index of its+ own). Derived from the rows and the mask on each call, so it belongs to no probe."""+ tokens = [t.strip() for t in (os.environ.get("CUDA_VISIBLE_DEVICES") or "").split(",")]+ by_uuid = any(t and not t.isdigit() for t in tokens)+ return {+ int(r["index"]): str(r["uuid"])+ for r in LlamaCppBackend._nvml_rows_visible(rows)+ if str(r.get("index", "")).isdigit() and (by_uuid or r.get("mig"))+ }++ @staticmethod+ def _child_visibility_for(gpu_indices) -> str:+ """The mask for these indices: the inherited uuids when the mask named the rows by+ uuid (a MIG slice must stay a MIG- entry), else the indices themselves."""+ ids = [int(i) for i in gpu_indices]+ by_index = LlamaCppBackend._uuid_by_index(LlamaCppBackend._NVML_ROWS)+ if by_index and all(i in by_index for i in ids):+ return ",".join(by_index[i] for i in ids)+ return ",".join(str(i) for i in ids)++ @staticmethod+ def _get_gpu_memory_nvml() -> list[tuple[int, int, int]]:+ """Free and total memory per NVIDIA GPU from NVML (studio/nvidia_probe.py in a child+ with a deadline), for a host whose nvidia-smi is absent, stale or hung: read as "no+ GPU", it pinned the embedding server to the CPU. Physical indices, masked like the+ nvidia-smi rows; a visible row without a memory reading voids the answer, so the+ torch probe decides."""+ if sys.platform == "darwin" or os.environ.get("UNSLOTH_NVIDIA_LIBRARY_PROBE", "1") == "0":+ return []+ script = LlamaCppBackend._nvidia_probe_script()+ if script is None:+ return []+ try:+ result = subprocess.run(+ [sys.executable, "-I", str(script), "--json"],+ capture_output = True,+ text = True,+ encoding = "utf-8",+ errors = "replace",+ timeout = 10,+ env = utf8_child_env(child_env_without_native_path_secret()),+ **_windows_hidden_subprocess_kwargs(),+ )+ payload = json.loads(result.stdout or "null")+ except Exception as e:+ logger.debug(f"NVML probe failed: {e}")+ return []+ if not isinstance(payload, dict) or payload.get("source") != "nvml":+ return []+ LlamaCppBackend._NVML_ROWS = list(payload.get("devices") or [])+ gpus: list[tuple[int, int, int]] = []+ for row in LlamaCppBackend._nvml_rows_visible(LlamaCppBackend._NVML_ROWS):+ try:+ idx = int(row["index"])+ free_mib = int(row.get("memory_free_mib") or 0)+ total_mib = int(row.get("memory_total_mib") or 0)+ except (KeyError, TypeError, ValueError):+ return []+ if total_mib <= 0:+ # The probe writes a failed reading as total 0. One visible GPU without a+ # reading voids the answer: a partial list would place and split across+ # fewer GPUs than the child enumerates. A full card (free 0) is a reading.+ return []+ gpus.append((idx, free_mib, total_mib))+ gpus.sort(key = lambda g: g[0])+ return gpus+@staticmethoddef _get_gpu_memory(binary: Optional[str] = None, *, for_llama_server: bool = False@@ -11800,6 +11935,12 @@ class LlamaCppBackend:except Exception as e:logger.debug(f"nvidia-smi probe failed: {e}")+ # ── NVIDIA via NVML, when nvidia-smi is absent, stale or hung ────+ nvml_gpus = LlamaCppBackend._get_gpu_memory_nvml()+ if nvml_gpus:+ LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES = True+ return nvml_gpus+# ── AMD ROCm via amd-smi ─────────────────────────────────────rocm_gpus = LlamaCppBackend._get_gpu_memory_amd_smi(binary, for_llama_server = for_llama_server@@ -13944,10 +14085,11 @@ class LlamaCppBackend:# /usr/local/cuda.import glob as _glob+ # glob.escape: a prefix with [brackets] is otherwise read as a pattern.+ _site = os.path.join(_glob.escape(sys.prefix), "lib", "python*", "site-packages")for _nv_pattern in [- os.path.join(sys.prefix, "lib", "python*", "site-packages", "nvidia", _sub, "lib")- for _sub in ("cu*", "cudnn", "nvjitlink")- ]:+ os.path.join(_site, "nvidia", _sub, "lib") for _sub in ("cu*", "cudnn", "nvjitlink")+ ] + [os.path.join(_site, "torch", "lib")]:for _nv_dir in _glob.glob(_nv_pattern):if os.path.isdir(_nv_dir):lib_dirs.append(_nv_dir)@@ -26304,7 +26446,7 @@ class LlamaCppBackend:# enumerates every agent first, which segfaults on a deselected# unsupported GPU (e.g. gfx1036 iGPU under a gfx103X prebuilt).self._emit_child_gpu_visibility(- env, ",".join(str(i) for i in gpu_indices), prefer_rocr = True+ env, LlamaCppBackend._child_visibility_for(gpu_indices), prefer_rocr = True)_child_gpu_physical_ids = tuple(int(i) for i in gpu_indices)_launch_pinned_ids = list(gpu_indices)diff --git a/studio/backend/core/rag/embed_llama_server.py b/studio/backend/core/rag/embed_llama_server.pyindex 5d6f82404ef..4e3862e17b5 100644--- a/studio/backend/core/rag/embed_llama_server.py+++ b/studio/backend/core/rag/embed_llama_server.py@@ -751,9 +751,16 @@ class LlamaServerBackend:# real server and the dylibs sit next to the target (#8566).env = _with_dyld_path(env, _binary_lib_dir(binary))elif use_gpu:- # Left as Path(binary).parent: resolving the entrypoint here would move the first LD_LIBRARY_PATH- # entry for every existing Linux GPU install whose llama-server is a symlink.- self._add_linux_cuda_libs(env, str(Path(binary).parent))+ if sys.platform == "win32":+ # Chat's DLL search path: without it a venv-hosted cudart is never found and the CUDA build runs on the CPU.+ from core.inference.llama_cpp import _llama_lib_dir+ path_dirs = LlamaCppBackend._build_windows_path_dirs(+ str(_llama_lib_dir(binary)), sys.prefix, os.environ.get("CUDA_PATH", "")+ )+ env["PATH"] = ";".join(path_dirs) + ";" + env.get("PATH", "")+ else:+ # Path(binary).parent, unresolved: resolving would move the first LD_LIBRARY_PATH entry of every symlinked install.+ self._add_linux_cuda_libs(env, str(Path(binary).parent))_pinned = self._arch_gated_gpu_ids(binary)if _pinned:from core.inference.llama_cpp import LlamaCppBackend@@ -790,9 +797,12 @@ class LlamaServerBackend:returnarch = platform.machine()lib_dirs = [binary_dir]+ # glob.escape: a prefix with [brackets] is otherwise read as a pattern.+ site = os.path.join(glob.escape(sys.prefix), "lib", "python*", "site-packages")for pattern in (- os.path.join(sys.prefix, "lib", "python*", "site-packages", "nvidia", "cu*", "lib"),- os.path.join(sys.prefix, "lib", "python*", "site-packages", "nvidia", "cudnn", "lib"),+ os.path.join(site, "nvidia", "cu*", "lib"),+ os.path.join(site, "nvidia", "cudnn", "lib"),+ os.path.join(site, "torch", "lib"),):lib_dirs.extend(d for d in glob.glob(pattern) if os.path.isdir(d))for cuda_lib in (diff --git a/studio/backend/tests/test_gpu_memory_nvml_fallback.py b/studio/backend/tests/test_gpu_memory_nvml_fallback.pynew file mode 100644index 00000000000..95b56540bc8--- /dev/null+++ b/studio/backend/tests/test_gpu_memory_nvml_fallback.py@@ -0,0 +1,567 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0+"""The runtime GPU memory probe reads NVML when nvidia-smi cannot answer.++An absent, stale or hung nvidia-smi left _get_gpu_memory empty, and the embedding server+read that as "no GPU": -ngl 0 and a blanked CUDA_VISIBLE_DEVICES around a working CUDA+build. The installers stopped trusting that misread in #10985; this is the runtime half.+"""++from __future__ import annotations++import json+import os+import sys+import types+from pathlib import Path++import pytest++from core.inference import llama_cpp as mod+from core.inference.llama_cpp import LlamaCppBackend+from core.rag import embed_llama_server as embed_mod++# Captured before the autouse fixture stubs it, for the one test that exercises the real finder.+_REAL_FINDER = LlamaCppBackend.__dict__["_find_llama_server_binary"]+++def _payload(rows, source = "nvml"):+ return {+ "source": source,+ "cuda_driver_version": [13, 0],+ "driver_version": "580.65.06",+ "devices": rows,+ }+++def _row(+ index,+ free,+ total = 24576,+ uuid = None,+):+ return {+ "index": str(index),+ "uuid": uuid or f"GPU-{index:04d}",+ "name": "NVIDIA test",+ "compute_cap": "8.9",+ "memory_total_mib": str(total),+ "memory_free_mib": str(free),+ }+++def probe_script(tmp_path, monkeypatch):+ """A stand-in for studio/nvidia_probe.py that prints whatever payload the test wrote."""+ payload_path = tmp_path / "payload.json"+ script = tmp_path / "nvidia_probe.py"+ script.write_text(+ f"import sys; sys.stdout.write(open({str(payload_path)!r}).read())\n", encoding = "utf-8"+ )+ monkeypatch.setenv("UNSLOTH_NVIDIA_PROBE", str(script))+ monkeypatch.delenv("UNSLOTH_NVIDIA_LIBRARY_PROBE", raising = False)+ monkeypatch.setattr(mod.sys, "platform", "linux")++ def write(payload):+ payload_path.write_text(json.dumps(payload), encoding = "utf-8")++ return write++[email protected](autouse = True)+def _no_other_probes(monkeypatch):+ monkeypatch.setattr(LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda b = None: False))+ monkeypatch.setattr(+ LlamaCppBackend, "_find_llama_server_binary", staticmethod(lambda: "/opt/llama-server")+ )+ monkeypatch.setattr(+ LlamaCppBackend, "_get_gpu_memory_amd_smi", staticmethod(lambda *a, **k: [])+ )+ monkeypatch.setitem(sys.modules, "torch", None)+ for var in ("CUDA_VISIBLE_DEVICES", "HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES"):+ monkeypatch.delenv(var, raising = False)+++def _failing_smi(monkeypatch):+ real_run = mod.subprocess.run++ def run(cmd, *args, **kwargs):+ if cmd and os.path.basename(str(cmd[0])) == "nvidia-smi":+ return types.SimpleNamespace(returncode = 1, stdout = "", stderr = "")+ return real_run(cmd, *args, **kwargs)++ monkeypatch.setattr(mod.subprocess, "run", run)+++class TestTheMemoryProbeFallsBackToNvml:+ def test_a_failing_nvidia_smi_no_longer_reads_as_no_gpu(self, monkeypatch, probe_script):+ _failing_smi(monkeypatch)+ probe_script(_payload([_row(1, 20000), _row(0, 8000, 12288)]))+ assert LlamaCppBackend._get_gpu_memory() == [(0, 8000, 12288), (1, 20000, 24576)]+ assert LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES is True++ def test_the_mask_applies_as_it_does_to_nvidia_smi_rows(self, monkeypatch, probe_script):+ _failing_smi(monkeypatch)+ probe_script(_payload([_row(0, 8000), _row(1, 20000)]))+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "1")+ assert LlamaCppBackend._get_gpu_memory() == [(1, 20000, 24576)]+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "")+ assert LlamaCppBackend._get_gpu_memory() == []++ def test_a_uuid_mask_selects_by_the_rows_own_uuid(self, monkeypatch, probe_script):+ _failing_smi(monkeypatch)+ probe_script(+ _payload([_row(0, 8000, uuid = "GPU-aaaa1111-0"), _row(1, 20000, uuid = "GPU-bbbb2222-1")])+ )+ # A full uuid, a prefix, mask order, and a mixed index + uuid mask, as the CUDA runtime reads them.+ for mask, expected in (+ ("GPU-bbbb2222-1", [(1, 20000, 24576)]),+ ("GPU-aaaa", [(0, 8000, 24576)]),+ ("GPU-bbbb,GPU-aaaa", [(0, 8000, 24576), (1, 20000, 24576)]),+ ("1,GPU-aaaa", [(0, 8000, 24576), (1, 20000, 24576)]),+ # An entry naming no single device ends the mask there, as CUDA documents+ # for "0,2,-1,1": the devices before it stay visible, nothing after it does.+ ("nope", []),+ ("GPU-", []),+ ("1,-1,0", [(1, 20000, 24576)]),+ ("0,GPU-,1", [(0, 8000, 24576)]),+ ("0,7,1", [(0, 8000, 24576)]),+ ):+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", mask)+ assert LlamaCppBackend._get_gpu_memory() == expected, mask++ def test_a_mig_assignment_names_the_slice_row_the_probe_lists(self, monkeypatch, probe_script):+ _failing_smi(monkeypatch)+ slice_row = dict(_row(0, 9000, 20480, uuid = "MIG-cccc3333-0"), mig = "1")+ probe_script(_payload([_row(0, 60000, 81920, uuid = "GPU-aaaa1111-0"), slice_row]))+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "MIG-cccc")+ assert LlamaCppBackend._get_gpu_memory() == [(0, 9000, 20480)]+ # An index, a GPU- entry and no mask at all name the parent, and CUDA exposes its+ # first slice then, not the whole card: the slice's memory is what the model gets.+ for mask in ("0", "GPU-aaaa"):+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", mask)+ assert LlamaCppBackend._get_gpu_memory() == [(0, 9000, 20480)], mask+ assert LlamaCppBackend._child_visibility_for([0]) == "MIG-cccc3333-0"+ monkeypatch.delenv("CUDA_VISIBLE_DEVICES")+ assert LlamaCppBackend._get_gpu_memory() == [(0, 9000, 20480)]+ assert LlamaCppBackend._child_visibility_for([0]) == "MIG-cccc3333-0"+ # A slice the probe does not list hides every GPU rather than exposing the parent.+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "MIG-dddd")+ assert LlamaCppBackend._get_gpu_memory() == []+ # A second slice named by the mask is that slice, not the parent's first.+ second = dict(_row(0, 4000, 10240, uuid = "MIG-dddd4444-0"), mig = "1")+ probe_script(_payload([_row(0, 60000, 81920, uuid = "GPU-aaaa1111-0"), slice_row, second]))+ assert LlamaCppBackend._get_gpu_memory() == [(0, 4000, 10240)]+ assert LlamaCppBackend._child_visibility_for([0]) == "MIG-dddd4444-0"+ # Two slices of one card: one row per physical index, the first named, and the+ # child is pinned to that one so the budget and the launch agree.+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "MIG-dddd,MIG-cccc")+ assert LlamaCppBackend._get_gpu_memory() == [(0, 4000, 10240)]+ assert LlamaCppBackend._child_visibility_for([0]) == "MIG-dddd4444-0"++ def test_the_child_pin_belongs_to_no_probe(self, monkeypatch, probe_script):+ """The uuid each index stands for is derived from the last NVML inventory and the+ mask in force when the launch asks, so another query answering in between (nvidia-smi+ recovering, a concurrent preflight) neither blanks it nor leaves a stale one."""+ _failing_smi(monkeypatch)+ slice_row = dict(_row(0, 9000, 20480, uuid = "MIG-cccc3333-0"), mig = "1")+ probe_script(+ _payload(+ [+ _row(0, 60000, 81920, uuid = "GPU-aaaa1111-0"),+ _row(1, 20000, uuid = "GPU-bbbb2222-1"),+ slice_row,+ ]+ )+ )+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "GPU-bbbb,GPU-aaaa")+ assert LlamaCppBackend._get_gpu_memory() == [(0, 9000, 20480), (1, 20000, 24576)]++ def smi_ok(cmd, *args, **kwargs):+ if cmd and os.path.basename(str(cmd[0])) == "nvidia-smi":+ return types.SimpleNamespace(+ returncode = 0, stdout = "0, 60000, 81920\n1, 20000, 24576\n", stderr = ""+ )+ raise AssertionError("no other probe should run")++ monkeypatch.setattr(mod.subprocess, "run", smi_ok)+ assert LlamaCppBackend._get_gpu_memory() == [(0, 60000, 81920), (1, 20000, 24576)]+ # The inherited uuid mask is still what the child gets, a MIG parent still its slice.+ assert LlamaCppBackend._child_visibility_for([1, 0]) == "GPU-bbbb2222-1,MIG-cccc3333-0"+ # The mask changing underneath is read as it is now, not as it was at the probe.+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0,1")+ assert LlamaCppBackend._child_visibility_for([1]) == "1"+ assert LlamaCppBackend._child_visibility_for([0]) == "MIG-cccc3333-0"+ # No NVML inventory at all: indices as before.+ monkeypatch.setattr(LlamaCppBackend, "_NVML_ROWS", [])+ assert LlamaCppBackend._child_visibility_for([0, 1]) == "0,1"++ def test_a_uuid_mask_is_handed_to_the_child_as_uuids(self, monkeypatch, probe_script):+ _failing_smi(monkeypatch)+ slice_row = dict(_row(0, 9000, 20480, uuid = "MIG-cccc3333-0"), mig = "1")+ probe_script(+ _payload(+ [+ _row(0, 60000, 81920, uuid = "GPU-aaaa1111-0"),+ _row(1, 20000, uuid = "GPU-bbbb2222-1"),+ slice_row,+ ]+ )+ )+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "MIG-cccc")+ assert LlamaCppBackend._get_gpu_memory() == [(0, 9000, 20480)]+ # The launch must not turn the slice into its parent's index.+ assert LlamaCppBackend._child_visibility_for([0]) == "MIG-cccc3333-0"+ # The GPU- entry names the MIG parent, which CUDA exposes as its first slice.+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "GPU-bbbb,GPU-aaaa")+ assert LlamaCppBackend._get_gpu_memory() == [(0, 9000, 20480), (1, 20000, 24576)]+ assert LlamaCppBackend._child_visibility_for([1, 0]) == "GPU-bbbb2222-1,MIG-cccc3333-0"+ # A numeric or absent mask re-emits indices as before; only a slice standing in+ # for its MIG parent is named by uuid, and a selection mixing both stays numeric.+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0,1")+ assert LlamaCppBackend._get_gpu_memory() == [(0, 9000, 20480), (1, 20000, 24576)]+ assert LlamaCppBackend._child_visibility_for([1]) == "1"+ assert LlamaCppBackend._child_visibility_for([0]) == "MIG-cccc3333-0"+ monkeypatch.delenv("CUDA_VISIBLE_DEVICES")+ LlamaCppBackend._get_gpu_memory()+ assert LlamaCppBackend._child_visibility_for([0, 1]) == "0,1"++ def test_rows_without_a_memory_reading_are_not_evidence(self, monkeypatch, probe_script):+ _failing_smi(monkeypatch)+ probe_script(_payload([_row(0, 0, 0)], source = "cuda"))+ assert LlamaCppBackend._get_gpu_memory() == []+ probe_script(_payload([_row(0, 0, 0)]))+ assert LlamaCppBackend._get_gpu_memory() == []+ # One visible GPU without a reading voids the answer rather than shrinking the host;+ # a full card (free 0, a total) is a reading and stays listed.+ probe_script(_payload([_row(0, 8000), _row(1, 0, 0)]))+ assert LlamaCppBackend._get_gpu_memory() == []+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0")+ assert LlamaCppBackend._get_gpu_memory() == [(0, 8000, 24576)]+ monkeypatch.delenv("CUDA_VISIBLE_DEVICES")+ probe_script(_payload([_row(0, 8000), _row(1, 0)]))+ assert LlamaCppBackend._get_gpu_memory() == [(0, 8000, 24576), (1, 0, 24576)]++ def test_a_working_nvidia_smi_is_not_second_guessed(self, monkeypatch, probe_script):+ probe_script(_payload([_row(5, 1)]))+ real_run = mod.subprocess.run++ def run(cmd, *args, **kwargs):+ if cmd and os.path.basename(str(cmd[0])) == "nvidia-smi":+ return types.SimpleNamespace(returncode = 0, stdout = "0, 4096, 8192\n", stderr = "")+ return real_run(cmd, *args, **kwargs)++ monkeypatch.setattr(mod.subprocess, "run", run)+ assert LlamaCppBackend._get_gpu_memory() == [(0, 4096, 8192)]++ def test_the_switch_and_a_missing_script_turn_it_off(self, monkeypatch, probe_script):+ _failing_smi(monkeypatch)+ probe_script(_payload([_row(0, 8000)]))+ monkeypatch.setenv("UNSLOTH_NVIDIA_LIBRARY_PROBE", "0")+ assert LlamaCppBackend._get_gpu_memory() == []+ monkeypatch.delenv("UNSLOTH_NVIDIA_LIBRARY_PROBE")+ monkeypatch.setenv("UNSLOTH_NVIDIA_PROBE", "/nonexistent/nvidia_probe.py")+ monkeypatch.setattr(LlamaCppBackend, "_nvidia_probe_script", staticmethod(lambda: None))+ assert LlamaCppBackend._get_gpu_memory() == []++ def test_the_real_script_is_found_next_to_the_installers(self, monkeypatch):+ monkeypatch.delenv("UNSLOTH_NVIDIA_PROBE", raising = False)+ script = LlamaCppBackend._nvidia_probe_script()+ assert script is not None and script.name == "nvidia_probe.py"+ assert (script.parent / "install_llama_prebuilt.py").is_file()+++class TestTheEmbeddingServerKeepsTheGpu:+ def test_embeddings_offload_when_only_nvml_answers(self, monkeypatch, probe_script):+ _failing_smi(monkeypatch)+ probe_script(_payload([_row(0, 8000)]))+ import utils.hardware as hardware++ monkeypatch.setattr(hardware, "is_apple_silicon", lambda: False) # Metal answers on a Mac+ monkeypatch.setattr(embed_mod.config, "embed_device_preference", lambda: "auto")+ monkeypatch.setattr(+ LlamaCppBackend, "_arch_gate_survivors", staticmethod(lambda binary: []), raising = False+ )+ server = embed_mod.LlamaServerBackend.__new__(embed_mod.LlamaServerBackend)+ server._force_cpu = False+ assert server._use_gpu() is True+ cmd = server._build_cmd("/opt/llama-server", "m.gguf", 9999, use_gpu = True)+ assert cmd[-2:] == ["-ngl", "-1"]+ # And the misread this closes: nothing answering pins the server to the CPU.+ probe_script(_payload([]))+ assert server._use_gpu() is False++ def test_windows_embeddings_get_the_same_dll_search_path_as_chat(self, monkeypatch, tmp_path):+ monkeypatch.setattr(embed_mod.sys, "platform", "win32")+ monkeypatch.setattr(mod.sys, "platform", "win32")+ monkeypatch.setattr(+ embed_mod, "child_env_without_native_path_secret", lambda: {"PATH": "inherited"}+ )+ monkeypatch.setattr(LlamaCppBackend, "_sanitize_p2p_env", staticmethod(lambda env: None))+ monkeypatch.setattr(+ LlamaCppBackend, "_arch_gate_survivors", staticmethod(lambda b: []), raising = False+ )+ monkeypatch.setattr(+ LlamaCppBackend,+ "_build_windows_path_dirs",+ staticmethod(+ lambda binary_dir, prefix, cuda_path: [binary_dir, "C:\\venv\\nvidia\\cu13\\bin"]+ ),+ )+ monkeypatch.setattr(mod, "_llama_lib_dir", lambda binary: Path("C:/llama/build/bin"))+ server = embed_mod.LlamaServerBackend.__new__(embed_mod.LlamaServerBackend)+ env = server._build_env("C:/llama/llama-server.exe", use_gpu = True)+ assert env["PATH"].split(";")[:2] == [+ str(Path("C:/llama/build/bin")),+ "C:\\venv\\nvidia\\cu13\\bin",+ ]+ assert env["PATH"].endswith(";inherited")+++class TestAGpuCapableBuildIsPreferred:+ """#5941: a CPU build/ beside a CUDA build-cuda/ ran on the CPU forever."""++ @staticmethod+ def _tree(root, platform, layouts):+ from utils.llama_cpp_path_settings import llama_server_binary_name++ name = llama_server_binary_name(platform)+ made = {}+ for build, libs in layouts.items():+ bindir = root / build / "bin" / ("Release" if platform == "win32" else "")+ bindir.mkdir(parents = True, exist_ok = True)+ exe = bindir / name+ exe.write_text("#!/bin/sh\n", encoding = "utf-8")+ exe.chmod(0o755)+ for lib in libs:+ (bindir / lib).write_bytes(b"")+ made[build] = exe+ return made++ @pytest.mark.parametrize("platform", ["linux", "win32"])+ def test_the_cuda_sibling_build_wins_over_a_cpu_only_first_hit(+ self, tmp_path, platform, monkeypatch+ ):+ from utils import llama_cpp_path_settings as ps++ monkeypatch.setattr(ps, "host_gpu_vendors", lambda: None) # not this host's vendors+ so = ".dll" if platform == "win32" else ".so"+ pre = "" if platform == "win32" else "lib"+ made = self._tree(+ tmp_path,+ platform,+ {+ "build": [f"{pre}ggml-cpu{so}", f"{pre}ggml-base{so}"],+ "build-cuda": [f"{pre}ggml-cuda{so}", f"{pre}ggml-cpu{so}", f"{pre}ggml-base{so}"],+ },+ )+ assert ps.resolve_llama_server_binary(tmp_path, platform = platform) == made["build-cuda"]+ assert ps.binary_gpu_verdict(made["build"]) == "cpu"+ assert ps.binary_gpu_verdict(made["build-cuda"]) == "gpu"++ def test_a_stale_cuda_build_does_not_shadow_the_build_this_host_can_run(+ self, tmp_path, monkeypatch+ ):+ from utils import llama_cpp_path_settings as ps++ made = self._tree(+ tmp_path,+ "linux",+ {+ "build": ["libggml-cpu.so", "libggml-base.so"],+ "build-cuda": ["libggml-cuda.so", "libggml-cpu.so"],+ "build-hip": ["libggml-hip.so", "libggml-cpu.so"],+ "build-vulkan": ["libggml-vulkan.so", "libggml-cpu.so"],+ },+ )+ for vendors, winner in (+ ({"amd"}, "build-hip"),+ ({"nvidia"}, "build-cuda"),+ (None, "build-cuda"),+ ):+ monkeypatch.setattr(ps, "host_gpu_vendors", lambda v = vendors: v)+ assert (+ ps.resolve_llama_server_binary(tmp_path, platform = "linux") == made[winner]+ ), vendors+ # A vendor no build targets still gets the vendor-agnostic Vulkan build.+ monkeypatch.setattr(ps, "host_gpu_vendors", lambda: {"intel"})+ assert ps.resolve_llama_server_binary(tmp_path, platform = "linux") == made["build-vulkan"]++ def test_host_vendors_read_the_drm_sysfs(self, tmp_path, monkeypatch):+ from utils import llama_cpp_path_settings as ps++ monkeypatch.setattr(ps.sys, "platform", "linux")+ monkeypatch.setattr(ps.os.path, "isdir", lambda p: False)+ nodes = {"/dev/nvidiactl", "/dev/kfd"}+ monkeypatch.setattr(ps.os.path, "exists", lambda p: p in nodes)+ monkeypatch.setattr(ps, "_DRM_ROOT", str(tmp_path))+ for var in ("CUDA_VISIBLE_DEVICES", "HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES"):+ monkeypatch.delenv(var, raising = False)+ assert ps.host_gpu_vendors() is None+ for card, vendor in (("card0", "0x1002"), ("card1", "0x10de"), ("card1-DP-1", "0x10de")):+ (tmp_path / card / "device").mkdir(parents = True)+ (tmp_path / card / "device" / "vendor").write_text(vendor + "\n", encoding = "utf-8")+ assert ps.host_gpu_vendors() == {"amd", "nvidia"}+ # A container exposing only the AMD device node, or a mask hiding NVIDIA: not runnable.+ nodes.discard("/dev/nvidiactl")+ assert ps.host_gpu_vendors() == {"amd"}+ nodes.add("/dev/nvidiactl")+ # HIP reads CUDA_VISIBLE_DEVICES when neither HIP_ nor ROCR_ is set, so an empty one+ # hides both vendors: detected but nothing reachable is an empty set, not unknown,+ # and only Vulkan fits then. A set HIP_ mask outranks it.+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "")+ assert ps.host_gpu_vendors() == set()+ assert ps._fits_host({"cuda"}, set()) is False and ps._fits_host({"vulkan"}, set()) is True+ monkeypatch.setenv("HIP_VISIBLE_DEVICES", "0")+ assert ps.host_gpu_vendors() == {"amd"}+ # The two ROCm masks stack: an empty ROCR_ under a set HIP_ still hides every agent.+ monkeypatch.setenv("ROCR_VISIBLE_DEVICES", "")+ assert ps.host_gpu_vendors() == set()+ monkeypatch.setenv("ROCR_VISIBLE_DEVICES", "0")+ assert ps.host_gpu_vendors() == {"amd"}+ monkeypatch.setenv("HIP_VISIBLE_DEVICES", "-1")+ assert ps.host_gpu_vendors() == set()+ assert ps._fits_host({"cuda"}, set()) is False and ps._fits_host({"vulkan"}, set()) is True++ def test_host_vendors_read_the_wsl_runtimes(self, tmp_path, monkeypatch):+ """WSL2 has no DRM card and no /dev/kfd or /dev/nvidiactl: the GPU is behind /dev/dxg+ and the vendor is the runtime that drives it."""+ from utils import llama_cpp_path_settings as ps++ monkeypatch.setattr(ps.sys, "platform", "linux")+ monkeypatch.setattr(ps.os.path, "isdir", lambda p: False)+ monkeypatch.setattr(ps, "_DRM_ROOT", str(tmp_path / "drm"))+ rocm = tmp_path / "rocm"+ wsl_lib = tmp_path / "wsl"+ monkeypatch.setattr(ps, "_WSL_ROCM_LIB_DIRS", (str(rocm),))+ monkeypatch.setattr(ps, "_WSL_CUDA_LIB_DIR", str(wsl_lib))+ nodes = {"/dev/dxg"}+ monkeypatch.setattr(ps.os.path, "exists", lambda p: p in nodes or os.path.lexists(p))+ for var in ("CUDA_VISIBLE_DEVICES", "HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES"):+ monkeypatch.delenv(var, raising = False)+ assert ps.host_gpu_vendors() is None+ rocm.mkdir()+ (rocm / "librocdxg.so.1").write_bytes(b"")+ assert ps.host_gpu_vendors() == {"amd"}+ wsl_lib.mkdir()+ (wsl_lib / "libcuda.so.1.1").write_bytes(b"")+ assert ps.host_gpu_vendors() == {"amd", "nvidia"}+ # The masks still apply, and without /dev/dxg the runtimes prove nothing.+ monkeypatch.setenv("HIP_VISIBLE_DEVICES", "")+ assert ps.host_gpu_vendors() == {"nvidia"}+ nodes.discard("/dev/dxg")+ assert ps.host_gpu_vendors() is None++ def test_a_gpu_first_hit_for_another_vendor_yields_too(self, tmp_path, monkeypatch):+ from utils import llama_cpp_path_settings as ps++ # No build/ at all: a stale build-cuda/ is the first hit on an AMD box.+ made = self._tree(+ tmp_path, "linux", {"build-cuda": ["libggml-cuda.so"], "build-hip": ["libggml-hip.so"]}+ )+ monkeypatch.setattr(ps, "host_gpu_vendors", lambda: {"amd"})+ assert ps.resolve_llama_server_binary(tmp_path, platform = "linux") == made["build-hip"]+ # On the NVIDIA box, or an unknown host, the first hit stands.+ for vendors in ({"nvidia"}, None):+ monkeypatch.setattr(ps, "host_gpu_vendors", lambda v = vendors: v)+ assert ps.resolve_llama_server_binary(tmp_path, platform = "linux") == made["build-cuda"]++ def test_a_first_hit_of_unknown_layout_keeps_its_place(self, tmp_path):+ from utils import llama_cpp_path_settings as ps+ made = self._tree(tmp_path, "linux", {"build": [], "build-cuda": ["libggml-cuda.so"]})+ # A static build or the installer's wrapper: not proven CPU-only, so search order holds.+ assert ps.resolve_llama_server_binary(tmp_path, platform = "linux") == made["build"]++ @pytest.mark.skipif(+ sys.platform == "win32" or os.geteuid() == 0, reason = "needs a real permission denial"+ )+ def test_a_denied_candidate_ahead_of_the_gpu_build_still_stops_discovery(+ self, tmp_path, monkeypatch+ ):+ made = self._tree(+ tmp_path, "linux", {"build": ["libggml-cpu.so"], "build-cuda": ["libggml-cuda.so"]}+ )+ monkeypatch.setattr(LlamaCppBackend, "_find_llama_server_binary", _REAL_FINDER)+ monkeypatch.setattr(mod.sys, "platform", "linux")+ monkeypatch.delenv("LLAMA_SERVER_PATH", raising = False)+ monkeypatch.delenv("UNSLOTH_STUDIO_MANAGED_LLAMA_CPP_PATH", raising = False)+ monkeypatch.setenv("UNSLOTH_LLAMA_CPP_PATH", str(tmp_path))+ # An in-flight replace or an ACL: the pinned layout's build/ cannot be read at all.+ (tmp_path / "build" / "bin").chmod(0)+ try:+ assert LlamaCppBackend._find_llama_server_binary() is None+ finally:+ (tmp_path / "build" / "bin").chmod(0o755)+ assert LlamaCppBackend._find_llama_server_binary() == str(made["build-cuda"])++ def test_the_runtime_finder_agrees(self, tmp_path, monkeypatch):+ made = self._tree(+ tmp_path,+ "linux",+ {"build": ["libggml-cpu.so", "libggml-base.so"], "build-cuda": ["libggml-cuda.so"]},+ )+ monkeypatch.setattr(LlamaCppBackend, "_find_llama_server_binary", _REAL_FINDER)+ monkeypatch.setattr(mod.sys, "platform", "linux")+ monkeypatch.delenv("LLAMA_SERVER_PATH", raising = False)+ monkeypatch.delenv("UNSLOTH_STUDIO_MANAGED_LLAMA_CPP_PATH", raising = False)+ monkeypatch.setenv("UNSLOTH_LLAMA_CPP_PATH", str(tmp_path))+ assert LlamaCppBackend._find_llama_server_binary() == str(made["build-cuda"])++ def test_the_legacy_in_tree_build_prefers_the_gpu_build_too(self, tmp_path, monkeypatch):+ """No managed or custom runtime configured: the project_root/llama.cpp fallback."""+ from utils import llama_cpp_path_settings as ps++ made = self._tree(+ tmp_path / "llama.cpp",+ "linux",+ {"build": ["libggml-cpu.so", "libggml-base.so"], "build-cuda": ["libggml-cuda.so"]},+ )+ monkeypatch.setattr(LlamaCppBackend, "_find_llama_server_binary", _REAL_FINDER)+ monkeypatch.setattr(mod.sys, "platform", "linux")+ monkeypatch.setattr(mod, "__file__", str(tmp_path / "s" / "b" / "c" / "i" / "llama_cpp.py"))+ for var in (+ "LLAMA_SERVER_PATH",+ "UNSLOTH_LLAMA_CPP_PATH",+ "UNSLOTH_STUDIO_MANAGED_LLAMA_CPP_PATH",+ ):+ monkeypatch.delenv(var, raising = False)+ monkeypatch.setattr(ps, "get_stored_custom_llama_cpp_path", lambda: None)+ monkeypatch.setattr(+ LlamaCppBackend,+ "_resolved_studio_root_and_is_legacy",+ staticmethod(lambda: (tmp_path / "no-such-studio", False)),+ )+ monkeypatch.setattr(mod.shutil, "which", lambda name: None)+ assert LlamaCppBackend._find_llama_server_binary() == str(made["build-cuda"])+++ sys.platform == "win32", reason = "the Linux library path is built from posix paths"+)+class TestTheLinuxLibrarySearchPath:+ @pytest.mark.parametrize("prefix_name", ["Studio", "Studio[CUDA]"])+ def test_a_bracketed_prefix_and_torch_lib_are_found(self, monkeypatch, tmp_path, prefix_name):+ prefix = tmp_path / prefix_name+ site = prefix / "lib" / "python3.12" / "site-packages"+ cu = site / "nvidia" / "cu13" / "lib"+ torch_lib = site / "torch" / "lib"+ cu.mkdir(parents = True)+ torch_lib.mkdir(parents = True)+ monkeypatch.setattr(embed_mod.sys, "platform", "linux")+ monkeypatch.setattr(embed_mod.sys, "prefix", str(prefix))+ env: dict[str, str] = {}+ embed_mod.LlamaServerBackend._add_linux_cuda_libs(env, "/opt/llama")+ parts = env["LD_LIBRARY_PATH"].split(":")+ assert str(cu) in parts and str(torch_lib) in parts+ # The chat server's builder reads the same prefix the same way.+ monkeypatch.setattr(mod.sys, "platform", "linux")+ monkeypatch.setattr(mod.sys, "prefix", str(prefix))+ monkeypatch.setattr(mod, "_llama_lib_dir", lambda binary: Path("/opt/llama"))+ monkeypatch.setattr(mod, "_wsl_system_rocm_lib_dirs", lambda: [])+ monkeypatch.setattr(mod, "_native_linux_system_rocm_lib_dirs", lambda binary_dir: [])+ monkeypatch.setattr(+ LlamaCppBackend, "_prefers_bundle_only_rocm", staticmethod(lambda b: True)+ )+ chat_env = LlamaCppBackend._llama_server_env_for_binary("/opt/llama/llama-server")+ chat_parts = chat_env["LD_LIBRARY_PATH"].split(":")+ assert str(cu) in chat_parts and str(torch_lib) in chat_partsdiff --git a/studio/backend/utils/llama_cpp_path_settings.py b/studio/backend/utils/llama_cpp_path_settings.pyindex 361d65d2cb6..f875587e298 100644--- a/studio/backend/utils/llama_cpp_path_settings.py+++ b/studio/backend/utils/llama_cpp_path_settings.py@@ -6,11 +6,12 @@from __future__ import annotationsimport os+import reimport sysimport threadingfrom contextlib import contextmanagerfrom pathlib import Path-from typing import Iterator, Optional+from typing import Any, Callable, Iterable, Iterator, OptionalCUSTOM_LLAMA_CPP_PATH_SETTING_KEY = "custom_llama_cpp_path"MAX_CUSTOM_LLAMA_CPP_PATH_LENGTH = 32767@@ -59,12 +60,13 @@ def llama_server_candidates("""Supported llama.cpp build layouts, in the runtime's search order."""root = Path(directory)binary_name = llama_server_binary_name(platform)- candidates = [- root / binary_name,- root / "build" / "bin" / binary_name,- ]- if (platform or sys.platform) == "win32":- candidates.append(root / "build" / "bin" / "Release" / binary_name)+ windows = (platform or sys.platform) == "win32"+ candidates = [root / binary_name]+ # build/ first, then the per-backend build dirs a source checkout keeps side by side (#5941).+ for build in ("build", "build-cuda", "build-hip", "build-rocm", "build-vulkan"):+ candidates.append(root / build / "bin" / binary_name)+ if windows:+ candidates.append(root / build / "bin" / "Release" / binary_name)return tuple(candidates)@@ -77,18 +79,157 @@ def _usable_binary(path: Path, *, platform: Optional[str] = None) -> bool:return (platform or sys.platform) == "win32" or os.access(path, os.X_OK)+_GPU_BACKEND_LIB_RE = re.compile(+ r"^(?:lib)?ggml-(cann|cuda|hip|metal|musa|opencl|sycl|virtgpu|vulkan)"+ r"(?:\.dll|\.so(?:\.\d+)*|(?:\.\d+)*\.dylib)$"+)+_CPU_BACKEND_LIB_RE = re.compile(r"^(?:lib)?ggml-(?:cpu|base)(?:[-.]|$)")+# Backends bound to one vendor; vulkan, opencl and virtgpu run on any GPU.+_BACKEND_VENDOR = {+ "cuda": "nvidia",+ "hip": "amd",+ "sycl": "intel",+ "metal": "apple",+ "musa": "mthreads",+ "cann": "huawei",+}+_DRM_VENDOR = {"0x10de": "nvidia", "0x1002": "amd", "0x8086": "intel"}+_WINDOWS_VENDOR_DLLS = {+ "nvidia": ("nvcuda.dll",),+ "amd": ("amdhip64*.dll", "amdocl64.dll"),+ "intel": ("ze_intel_gpu64.dll", "igdrcl64.dll"),+}+_DRM_ROOT = "/sys/class/drm"+# WSL2 reaches the GPU through /dev/dxg (no DRM card, no /dev/kfd or /dev/nvidiactl): the+# vendor is the runtime that drives it, librocdxg for AMD and the WSL libcuda for NVIDIA.+_WSL_ROCM_LIB_DIRS = ("/opt/rocm/lib", "/opt/rocm/lib64")+_WSL_CUDA_LIB_DIR = "/usr/lib/wsl/lib"+_HOST: Any = object() # prefer_gpu_capable's default: read the vendors from this host+++def binary_gpu_backends(binary: Path | str) -> Optional[set[str]]:+ """GPU backend libraries beside the binary; empty for a split build with only CPU+ libraries, None when the layout says nothing (a static build or a wrapper)."""+ try:+ names = [p.name for p in Path(binary).resolve().parent.iterdir() if p.is_file()]+ except OSError:+ return None+ gpu = {m.group(1) for m in map(_GPU_BACKEND_LIB_RE.match, names) if m}+ if gpu or any(_CPU_BACKEND_LIB_RE.match(name) for name in names):+ return gpu+ return None+++def binary_gpu_verdict(binary: Path | str) -> str:+ """``gpu``, ``cpu`` (a split-library build with no GPU backend beside it) or ``unknown``."""+ backends = binary_gpu_backends(binary)+ return "unknown" if backends is None else ("gpu" if backends else "cpu")+++def _mask_hides_all(*names: str) -> bool:+ """Whether the first set mask among names (their precedence order) hides every device."""+ mask = next((os.environ[n] for n in names if n in os.environ), "x")+ return mask.strip() in ("", "-1")+++def _amd_mask_hides_all() -> bool:+ """HIP_ and ROCR_VISIBLE_DEVICES stack (ROCR filters the agents HIP then indexes), so an+ empty one at either layer hides everything; CUDA_VISIBLE_DEVICES stands in only when+ neither is set (_active_gpu_visibility_mask)."""+ if any(n in os.environ for n in ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES")):+ return _mask_hides_all("HIP_VISIBLE_DEVICES") or _mask_hides_all("ROCR_VISIBLE_DEVICES")+ return _mask_hides_all("CUDA_VISIBLE_DEVICES")+++def host_gpu_vendors() -> Optional[set[str]]:+ """GPU vendors this process can reach: DRM sysfs vendors whose device node is present and+ whose visibility mask is not empty on Linux, the vendors' driver DLLs on Windows. None when+ nothing was detected at all (an unknown host filters nothing); an empty set when GPUs were+ detected but none is reachable, so only vendor-agnostic backends fit."""+ vendors: set[str] = set()+ if sys.platform == "darwin":+ return {"apple"}+ if sys.platform == "win32":+ system32 = Path(os.environ.get("SystemRoot", r"C:\Windows")) / "System32"+ for vendor, patterns in _WINDOWS_VENDOR_DLLS.items():+ if any(next(system32.glob(pattern), None) for pattern in patterns):+ vendors.add(vendor)+ else:+ for vendor_file in Path(_DRM_ROOT).glob("card*/device/vendor"):+ try:+ vendor = _DRM_VENDOR.get(vendor_file.read_text(encoding = "utf-8").strip().lower())+ except OSError:+ continue+ if vendor:+ vendors.add(vendor)+ if os.path.isdir("/proc/driver/nvidia"):+ vendors.add("nvidia")+ wsl = os.path.exists("/dev/dxg")+ wsl_amd = wsl and any(+ os.path.exists(os.path.join(d, n))+ for d in _WSL_ROCM_LIB_DIRS+ for n in ("librocdxg.so", "librocdxg.so.1")+ )+ wsl_nvidia = wsl and any(Path(_WSL_CUDA_LIB_DIR).glob("libcuda.so*"))+ vendors |= {v for v, on in (("amd", wsl_amd), ("nvidia", wsl_nvidia)) if on}+ detected = set(vendors)+ # A container that exposes only one vendor's device nodes, or a mask hiding a vendor,+ # must not make its backend look runnable on a hybrid box.+ if "nvidia" in vendors and (+ not (os.path.exists("/dev/nvidiactl") or wsl_nvidia)+ or _mask_hides_all("CUDA_VISIBLE_DEVICES")+ ):+ vendors.discard("nvidia")+ if "amd" in vendors and (+ not (os.path.exists("/dev/kfd") or wsl_amd) or _amd_mask_hides_all()+ ):+ vendors.discard("amd")+ return vendors if detected else None+ return vendors or None+++def _fits_host(backends: set[str], vendors: Optional[set[str]]) -> bool:+ if vendors is None:+ return True+ return any(_BACKEND_VENDOR.get(b) is None or _BACKEND_VENDOR[b] in vendors for b in backends)+++def prefer_gpu_capable(+ candidates: Iterable[Path],+ usable: Callable[[Path], bool],+ vendors: Any = _HOST,+) -> list[Path]:+ """Search order, except that a first hit proven CPU-only, or proven built for a GPU vendor+ this host lacks, yields to a later usable build shipping a GPU backend this host's vendor+ runs (#5941: a CPU build/ beside build-cuda/; a stale build-cuda/ must not shadow+ build-hip/ on an AMD box, whether or not a build/ precedes it). Unknown layouts stay put."""+ ordered = list(candidates)+ first = next((c for c in ordered if usable(c)), None)+ if first is None:+ return ordered+ verdict = binary_gpu_verdict(first)+ if verdict == "unknown":+ return ordered+ if vendors is _HOST:+ vendors = host_gpu_vendors()+ if verdict == "gpu" and _fits_host(binary_gpu_backends(first), vendors):+ return ordered+ for candidate in ordered:+ if candidate == first or not usable(candidate):+ continue+ backends = binary_gpu_backends(candidate)+ if backends and _fits_host(backends, vendors):+ return [candidate] + [c for c in ordered if c != candidate]+ return ordered++def resolve_llama_server_binary(directory: Path | str, *, platform: Optional[str] = None) -> Optional[Path]:- """Return the first executable llama-server in a supported layout."""- return next(- (- candidate- for candidate in llama_server_candidates(directory, platform = platform)- if _usable_binary(candidate, platform = platform)- ),- None,- )+ """Return the first executable llama-server in a supported layout, GPU-capable preferred."""+ usable = lambda candidate: _usable_binary(candidate, platform = platform) # noqa: E731+ ordered = prefer_gpu_capable(llama_server_candidates(directory, platform = platform), usable)+ return next((candidate for candidate in ordered if usable(candidate)), None)def get_stored_custom_llama_cpp_path() -> Optional[Path]:diff --git a/studio/nvidia_probe.py b/studio/nvidia_probe.pyindex 9496016a652..3c4663deb94 100644--- a/studio/nvidia_probe.py+++ b/studio/nvidia_probe.py@@ -70,6 +70,43 @@ def _split_cuda_version(packed: int) -> tuple[int, int] | None:return packed // 1000, (packed % 1000) // 10+class _NvmlMemory(ctypes.Structure):+ # nvmlMemory_t (v1): bytes, in this order.+ _fields_ = [+ ("total", ctypes.c_ulonglong),+ ("free", ctypes.c_ulonglong),+ ("used", ctypes.c_ulonglong),+ ]+++def _mig_children(nvml, handle) -> list:+ """(handle, uuid, memory) per MIG instance of a device; empty without MIG or on an NVML+ too old to ask. Unused instance slots answer NOT_FOUND and are skipped."""+ get_max = getattr(nvml, "nvmlDeviceGetMaxMigDeviceCount", None)+ get_mig = getattr(nvml, "nvmlDeviceGetMigDeviceHandleByIndex", None)+ if get_max is None or get_mig is None:+ return []+ children = []+ try:+ count = ctypes.c_uint(0)+ if get_max(handle, ctypes.byref(count)) != 0:+ return []+ for index in range(count.value):+ mig = ctypes.c_void_p()+ if get_mig(handle, index, ctypes.byref(mig)) != 0:+ continue+ uuid = ctypes.create_string_buffer(96)+ if nvml.nvmlDeviceGetUUID(mig, uuid, 96) != 0:+ continue+ memory = _NvmlMemory()+ if nvml.nvmlDeviceGetMemoryInfo(mig, ctypes.byref(memory)) != 0:+ memory.total = memory.free = 0+ children.append((mig, uuid.value.decode("ascii", "replace"), memory))+ except Exception:+ return children+ return children++def _probe_nvml() -> dict | None:nvml = _load("nvml")if nvml is None:@@ -100,7 +137,7 @@ def _probe_nvml() -> dict | None:for index in range(count.value):handle = ctypes.c_void_p()if get_handle(index, ctypes.byref(handle)) != 0:- continue+ return None # a GPU this reader cannot see is a GPU the selectors would missname = ctypes.create_string_buffer(96)uuid = ctypes.create_string_buffer(96)major, minor = ctypes.c_int(0), ctypes.c_int(0)@@ -114,14 +151,33 @@ def _probe_nvml() -> dict | None:== 0):cap = f"{major.value}.{minor.value}"+ memory = _NvmlMemory()+ if nvml.nvmlDeviceGetMemoryInfo(handle, ctypes.byref(memory)) != 0:+ memory.total = memory.free = 0devices.append({"index": str(index),"uuid": uuid.value.decode("ascii", "replace"),"name": name.value.decode("utf-8", "replace"),"compute_cap": cap,+ "memory_total_mib": str(memory.total // (1024 * 1024)),+ "memory_free_mib": str(memory.free // (1024 * 1024)),})+ # MIG slices, so a CUDA_VISIBLE_DEVICES=MIG-... assignment can be named: same+ # parent index and capability, their own uuid and memory, marked "mig".+ for mig, mig_uuid, mig_memory in _mig_children(nvml, handle):+ devices.append(+ {+ "index": str(index),+ "uuid": mig_uuid,+ "name": name.value.decode("utf-8", "replace") + " MIG",+ "compute_cap": cap,+ "memory_total_mib": str(mig_memory.total // (1024 * 1024)),+ "memory_free_mib": str(mig_memory.free // (1024 * 1024)),+ "mig": "1",+ }+ )return {"source": "nvml","cuda_driver_version": list(cuda) if cuda else None,@@ -163,6 +219,9 @@ def _probe_cuda_driver() -> dict | None:"uuid": "","name": name.value.decode("utf-8", "replace"),"compute_cap": f"{major.value}.{minor.value}" if ok else "",+ # Memory needs a context, which this probe never creates.+ "memory_total_mib": "0",+ "memory_free_mib": "0",})return {@@ -190,9 +249,11 @@ def _from_payload(payload: object) -> NvidiaLibraryInventory | None:return Noneversion = payload.get("cuda_driver_version")cuda = tuple(int(part) for part in version[:2]) if isinstance(version, list) else None+ keys = ("index", "uuid", "name", "compute_cap", "memory_total_mib", "memory_free_mib")devices = [- {key: str(row.get(key, "")) for key in ("index", "uuid", "name", "compute_cap")}+ {key: str(row.get(key, "")) for key in keys}for row in payload.get("devices") or []+ if not row.get("mig")if isinstance(row, dict)]return NvidiaLibraryInventory(diff --git a/tests/studio/install/test_nvidia_library_probe.py b/tests/studio/install/test_nvidia_library_probe.pyindex dc5b937a3d1..af023875213 100644--- a/tests/studio/install/test_nvidia_library_probe.py+++ b/tests/studio/install/test_nvidia_library_probe.py@@ -62,16 +62,49 @@ def _inventory(class TestProbeModule:+ def test_mig_rows_are_not_devices_to_the_installers(self):+ payload = {+ "source": "nvml",+ "cuda_driver_version": [13, 0],+ "devices": [+ {"index": "0", "uuid": "GPU-a", "name": "H100", "compute_cap": "9.0"},+ {+ "index": "0",+ "uuid": "MIG-b",+ "name": "H100 MIG",+ "compute_cap": "9.0",+ "mig": "1",+ },+ ],+ }+ inv = PROBE._from_payload(payload)+ assert [d["uuid"] for d in inv.devices] == ["GPU-a"]+def test_the_payload_round_trips(self):payload = {"source": "nvml","cuda_driver_version": [13, 1],"driver_version": "590.48.01",- "devices": [{"index": "0", "uuid": "GPU-x", "name": "B200", "compute_cap": "10.0"}],+ "devices": [+ {+ "index": "0",+ "uuid": "GPU-x",+ "name": "B200",+ "compute_cap": "10.0",+ "memory_total_mib": "183359",+ "memory_free_mib": "182630",+ }+ ],}inv = PROBE._from_payload(payload)assert inv.cuda_driver_version == (13, 1)assert inv.devices == payload["devices"]+ # An older payload without the memory fields still reads; they come back empty.+ old = {+ **payload,+ "devices": [{"index": "0", "uuid": "GPU-x", "name": "B200", "compute_cap": "10.0"}],+ }+ assert PROBE._from_payload(old).devices[0]["memory_free_mib"] == ""assert PROBE._from_payload(None) is Noneassert PROBE._from_payload({"source": "other"}) is None@@ -199,6 +232,11 @@ class TestProbeModule:1 for line in listing.stdout.splitlines() if line.startswith("GPU "))assert inv.cuda_driver_version is not None and inv.cuda_driver_version[0] >= 11+ # The memory reading the runtime probe needs; a busy card may legitimately have none free.+ assert all(int(d["memory_total_mib"]) > 0 for d in inv.devices)+ assert all(+ 0 <= int(d["memory_free_mib"]) <= int(d["memory_total_mib"]) for d in inv.devices+ )# ── detect_host ──