Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio runtime keeps the GPU when nvidia-smi cannot answer, and prefers a GPU-capable llama-server beside a CPU build (#11005)

Daniel Hanc829b33685ea
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index badc49b49d2..066ffa54ee7 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -8194,8 +8194,18 @@ class LlamaCppBackend:
# while sibling build layouts are checked. include_denied returns an
# unavailable path instead: diffusion asset lookup only needs its dir.
non_executable = None
+ from utils.llama_cpp_path_settings import prefer_gpu_capable
+
+ statuses = {p: _file_status(p) for p in paths}
+ first = next((p for p in paths if statuses[p] == "file"), None)
+ # A first hit proven CPU-only yields to a GPU-capable sibling build (#5941), unless
+ # a denied candidate precedes it: that denial stops discovery, it is not skipped over.
+ if first is not None and not any(
+ statuses[p] == "denied" for p in paths[: paths.index(first)]
+ ):
+ paths = prefer_gpu_capable(paths, lambda p: statuses[p] == "file")
for p in paths:
- st = _file_status(p)
+ st = statuses[p]
if st == "file":
return str(p), None
if st == "denied":
@@ -8275,7 +8285,9 @@ class LlamaCppBackend:
# 5-6. Legacy: in-tree build (older setup.sh / setup.ps1). A fallback,
# so a denied candidate here just continues (no no-fallback halt).
project_root = Path(__file__).resolve().parents[4]
- for p in _layout_candidates(project_root / "llama.cpp"):
+ from utils.llama_cpp_path_settings import prefer_gpu_capable
+
+ for p in prefer_gpu_capable(_layout_candidates(project_root / "llama.cpp"), _is_file):
if _is_file(p):
return str(p)
@@ -10552,6 +10564,8 @@ class LlamaCppBackend:
# only under PCI_BUS_ID, so a caller pinning the child's device order has to
# know which it holds (#10613).
_GPU_IDS_ARE_PCI_INDICES = None
+ # index -> inherited uuid, when CUDA_VISIBLE_DEVICES named the NVML rows by uuid.
+ _NVML_ROWS: list = [] # the last NVML inventory; _child_visibility_for reads it
# Boot-time property: read once, not per launch (the #10613 host has 175
# groups). Only the default root is cached; an explicit root (tests) re-reads.
@@ -11717,6 +11731,127 @@ class LlamaCppBackend:
except Exception as e: # noqa: BLE001 -- diagnostics must not break a load
return f"the reason could not be determined ({type(e).__name__})"
+ @staticmethod
+ def _nvidia_probe_script() -> Optional[Path]:
+ """studio/nvidia_probe.py, the installers' NVML / CUDA driver reader (UNSLOTH_NVIDIA_PROBE wins)."""
+ from utils.prebuilt.update_flow import find_installer_script
+ return find_installer_script(env_var = "UNSLOTH_NVIDIA_PROBE", script_name = "nvidia_probe.py")
+
+ @staticmethod
+ def _nvml_rows_visible(rows: list[dict]) -> list[dict]:
+ """The NVML rows CUDA_VISIBLE_DEVICES permits, in mask order: an index or a GPU-/MIG-
+ uuid prefix per entry, as the CUDA runtime reads it. An entry naming no single device
+ ends the mask there: "0,2,-1,1" exposes 0 and 2, "-1" alone hides every GPU. A GPU in
+ MIG mode is one slice to CUDA (the first, when the mask does not name one), so its
+ first slice row stands in for the parent's whole-card memory, and a second slice of
+ the same card is dropped: the rows are keyed by physical index, and one slice per
+ card is what the driver exposed before R570 and what the child is pinned to."""
+ raw = os.environ.get("CUDA_VISIBLE_DEVICES")
+ parents = [r for r in rows if not r.get("mig")]
+
+ def as_cuda_sees(r: dict) -> dict:
+ if r.get("mig"):
+ return r # a slice the mask named stays that slice
+ slices = [
+ s for s in rows if s.get("mig") and str(s.get("index")) == str(r.get("index"))
+ ]
+ return slices[0] if slices else r
+
+ tokens = [t.strip() for t in (raw or "").split(",") if t.strip()]
+ picked: list[dict] = []
+ if raw is None:
+ picked = [as_cuda_sees(r) for r in parents]
+ for token in tokens:
+ if token.isdigit():
+ hits = [r for r in parents if str(r.get("index")) == token]
+ elif token.startswith("GPU-"):
+ hits = [r for r in parents if str(r.get("uuid", "")).startswith(token)]
+ elif token.startswith("MIG-"):
+ # A MIG- entry names a slice row the probe lists under its parent.
+ hits = [
+ r for r in rows if r.get("mig") and str(r.get("uuid", "")).startswith(token)
+ ]
+ else:
+ hits = []
+ if len(hits) != 1:
+ break
+ picked.extend(
+ r
+ for r in map(as_cuda_sees, hits)
+ if str(r.get("index")) not in {str(p.get("index")) for p in picked}
+ )
+ return picked
+
+ @staticmethod
+ def _uuid_by_index(rows: list[dict]) -> dict[int, str]:
+ """The uuid each visible index stands for: every row when the mask named rows by
+ uuid, else only a slice standing in for its MIG parent (a slice has no index of its
+ own). Derived from the rows and the mask on each call, so it belongs to no probe."""
+ tokens = [t.strip() for t in (os.environ.get("CUDA_VISIBLE_DEVICES") or "").split(",")]
+ by_uuid = any(t and not t.isdigit() for t in tokens)
+ return {
+ int(r["index"]): str(r["uuid"])
+ for r in LlamaCppBackend._nvml_rows_visible(rows)
+ if str(r.get("index", "")).isdigit() and (by_uuid or r.get("mig"))
+ }
+
+ @staticmethod
+ def _child_visibility_for(gpu_indices) -> str:
+ """The mask for these indices: the inherited uuids when the mask named the rows by
+ uuid (a MIG slice must stay a MIG- entry), else the indices themselves."""
+ ids = [int(i) for i in gpu_indices]
+ by_index = LlamaCppBackend._uuid_by_index(LlamaCppBackend._NVML_ROWS)
+ if by_index and all(i in by_index for i in ids):
+ return ",".join(by_index[i] for i in ids)
+ return ",".join(str(i) for i in ids)
+
+ @staticmethod
+ def _get_gpu_memory_nvml() -> list[tuple[int, int, int]]:
+ """Free and total memory per NVIDIA GPU from NVML (studio/nvidia_probe.py in a child
+ with a deadline), for a host whose nvidia-smi is absent, stale or hung: read as "no
+ GPU", it pinned the embedding server to the CPU. Physical indices, masked like the
+ nvidia-smi rows; a visible row without a memory reading voids the answer, so the
+ torch probe decides."""
+ if sys.platform == "darwin" or os.environ.get("UNSLOTH_NVIDIA_LIBRARY_PROBE", "1") == "0":
+ return []
+ script = LlamaCppBackend._nvidia_probe_script()
+ if script is None:
+ return []
+ try:
+ result = subprocess.run(
+ [sys.executable, "-I", str(script), "--json"],
+ capture_output = True,
+ text = True,
+ encoding = "utf-8",
+ errors = "replace",
+ timeout = 10,
+ env = utf8_child_env(child_env_without_native_path_secret()),
+ **_windows_hidden_subprocess_kwargs(),
+ )
+ payload = json.loads(result.stdout or "null")
+ except Exception as e:
+ logger.debug(f"NVML probe failed: {e}")
+ return []
+ if not isinstance(payload, dict) or payload.get("source") != "nvml":
+ return []
+ LlamaCppBackend._NVML_ROWS = list(payload.get("devices") or [])
+ gpus: list[tuple[int, int, int]] = []
+ for row in LlamaCppBackend._nvml_rows_visible(LlamaCppBackend._NVML_ROWS):
+ try:
+ idx = int(row["index"])
+ free_mib = int(row.get("memory_free_mib") or 0)
+ total_mib = int(row.get("memory_total_mib") or 0)
+ except (KeyError, TypeError, ValueError):
+ return []
+ if total_mib <= 0:
+ # The probe writes a failed reading as total 0. One visible GPU without a
+ # reading voids the answer: a partial list would place and split across
+ # fewer GPUs than the child enumerates. A full card (free 0) is a reading.
+ return []
+ gpus.append((idx, free_mib, total_mib))
+ gpus.sort(key = lambda g: g[0])
+ return gpus
+
@staticmethod
def _get_gpu_memory(
binary: Optional[str] = None, *, for_llama_server: bool = False
@@ -11800,6 +11935,12 @@ class LlamaCppBackend:
except Exception as e:
logger.debug(f"nvidia-smi probe failed: {e}")
+ # ── NVIDIA via NVML, when nvidia-smi is absent, stale or hung ────
+ nvml_gpus = LlamaCppBackend._get_gpu_memory_nvml()
+ if nvml_gpus:
+ LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES = True
+ return nvml_gpus
+
# ── AMD ROCm via amd-smi ─────────────────────────────────────
rocm_gpus = LlamaCppBackend._get_gpu_memory_amd_smi(
binary, for_llama_server = for_llama_server
@@ -13944,10 +14085,11 @@ class LlamaCppBackend:
# /usr/local/cuda.
import glob as _glob
+ # glob.escape: a prefix with [brackets] is otherwise read as a pattern.
+ _site = os.path.join(_glob.escape(sys.prefix), "lib", "python*", "site-packages")
for _nv_pattern in [
- os.path.join(sys.prefix, "lib", "python*", "site-packages", "nvidia", _sub, "lib")
- for _sub in ("cu*", "cudnn", "nvjitlink")
- ]:
+ os.path.join(_site, "nvidia", _sub, "lib") for _sub in ("cu*", "cudnn", "nvjitlink")
+ ] + [os.path.join(_site, "torch", "lib")]:
for _nv_dir in _glob.glob(_nv_pattern):
if os.path.isdir(_nv_dir):
lib_dirs.append(_nv_dir)
@@ -26304,7 +26446,7 @@ class LlamaCppBackend:
# enumerates every agent first, which segfaults on a deselected
# unsupported GPU (e.g. gfx1036 iGPU under a gfx103X prebuilt).
self._emit_child_gpu_visibility(
- env, ",".join(str(i) for i in gpu_indices), prefer_rocr = True
+ env, LlamaCppBackend._child_visibility_for(gpu_indices), prefer_rocr = True
)
_child_gpu_physical_ids = tuple(int(i) for i in gpu_indices)
_launch_pinned_ids = list(gpu_indices)
diff --git a/studio/backend/core/rag/embed_llama_server.py b/studio/backend/core/rag/embed_llama_server.py
index 5d6f82404ef..4e3862e17b5 100644
--- a/studio/backend/core/rag/embed_llama_server.py
+++ b/studio/backend/core/rag/embed_llama_server.py
@@ -751,9 +751,16 @@ class LlamaServerBackend:
# real server and the dylibs sit next to the target (#8566).
env = _with_dyld_path(env, _binary_lib_dir(binary))
elif use_gpu:
- # Left as Path(binary).parent: resolving the entrypoint here would move the first LD_LIBRARY_PATH
- # entry for every existing Linux GPU install whose llama-server is a symlink.
- self._add_linux_cuda_libs(env, str(Path(binary).parent))
+ if sys.platform == "win32":
+ # Chat's DLL search path: without it a venv-hosted cudart is never found and the CUDA build runs on the CPU.
+ from core.inference.llama_cpp import _llama_lib_dir
+ path_dirs = LlamaCppBackend._build_windows_path_dirs(
+ str(_llama_lib_dir(binary)), sys.prefix, os.environ.get("CUDA_PATH", "")
+ )
+ env["PATH"] = ";".join(path_dirs) + ";" + env.get("PATH", "")
+ else:
+ # Path(binary).parent, unresolved: resolving would move the first LD_LIBRARY_PATH entry of every symlinked install.
+ self._add_linux_cuda_libs(env, str(Path(binary).parent))
_pinned = self._arch_gated_gpu_ids(binary)
if _pinned:
from core.inference.llama_cpp import LlamaCppBackend
@@ -790,9 +797,12 @@ class LlamaServerBackend:
return
arch = platform.machine()
lib_dirs = [binary_dir]
+ # glob.escape: a prefix with [brackets] is otherwise read as a pattern.
+ site = os.path.join(glob.escape(sys.prefix), "lib", "python*", "site-packages")
for pattern in (
- os.path.join(sys.prefix, "lib", "python*", "site-packages", "nvidia", "cu*", "lib"),
- os.path.join(sys.prefix, "lib", "python*", "site-packages", "nvidia", "cudnn", "lib"),
+ os.path.join(site, "nvidia", "cu*", "lib"),
+ os.path.join(site, "nvidia", "cudnn", "lib"),
+ os.path.join(site, "torch", "lib"),
):
lib_dirs.extend(d for d in glob.glob(pattern) if os.path.isdir(d))
for cuda_lib in (
diff --git a/studio/backend/tests/test_gpu_memory_nvml_fallback.py b/studio/backend/tests/test_gpu_memory_nvml_fallback.py
new file mode 100644
index 00000000000..95b56540bc8
--- /dev/null
+++ b/studio/backend/tests/test_gpu_memory_nvml_fallback.py
@@ -0,0 +1,567 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+"""The runtime GPU memory probe reads NVML when nvidia-smi cannot answer.
+
+An absent, stale or hung nvidia-smi left _get_gpu_memory empty, and the embedding server
+read that as "no GPU": -ngl 0 and a blanked CUDA_VISIBLE_DEVICES around a working CUDA
+build. The installers stopped trusting that misread in #10985; this is the runtime half.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+import types
+from pathlib import Path
+
+import pytest
+
+from core.inference import llama_cpp as mod
+from core.inference.llama_cpp import LlamaCppBackend
+from core.rag import embed_llama_server as embed_mod
+
+# Captured before the autouse fixture stubs it, for the one test that exercises the real finder.
+_REAL_FINDER = LlamaCppBackend.__dict__["_find_llama_server_binary"]
+
+
+def _payload(rows, source = "nvml"):
+ return {
+ "source": source,
+ "cuda_driver_version": [13, 0],
+ "driver_version": "580.65.06",
+ "devices": rows,
+ }
+
+
+def _row(
+ index,
+ free,
+ total = 24576,
+ uuid = None,
+):
+ return {
+ "index": str(index),
+ "uuid": uuid or f"GPU-{index:04d}",
+ "name": "NVIDIA test",
+ "compute_cap": "8.9",
+ "memory_total_mib": str(total),
+ "memory_free_mib": str(free),
+ }
+
+
+def probe_script(tmp_path, monkeypatch):
+ """A stand-in for studio/nvidia_probe.py that prints whatever payload the test wrote."""
+ payload_path = tmp_path / "payload.json"
+ script = tmp_path / "nvidia_probe.py"
+ script.write_text(
+ f"import sys; sys.stdout.write(open({str(payload_path)!r}).read())\n", encoding = "utf-8"
+ )
+ monkeypatch.setenv("UNSLOTH_NVIDIA_PROBE", str(script))
+ monkeypatch.delenv("UNSLOTH_NVIDIA_LIBRARY_PROBE", raising = False)
+ monkeypatch.setattr(mod.sys, "platform", "linux")
+
+ def write(payload):
+ payload_path.write_text(json.dumps(payload), encoding = "utf-8")
+
+ return write
+
+
[email protected](autouse = True)
+def _no_other_probes(monkeypatch):
+ monkeypatch.setattr(LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda b = None: False))
+ monkeypatch.setattr(
+ LlamaCppBackend, "_find_llama_server_binary", staticmethod(lambda: "/opt/llama-server")
+ )
+ monkeypatch.setattr(
+ LlamaCppBackend, "_get_gpu_memory_amd_smi", staticmethod(lambda *a, **k: [])
+ )
+ monkeypatch.setitem(sys.modules, "torch", None)
+ for var in ("CUDA_VISIBLE_DEVICES", "HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES"):
+ monkeypatch.delenv(var, raising = False)
+
+
+def _failing_smi(monkeypatch):
+ real_run = mod.subprocess.run
+
+ def run(cmd, *args, **kwargs):
+ if cmd and os.path.basename(str(cmd[0])) == "nvidia-smi":
+ return types.SimpleNamespace(returncode = 1, stdout = "", stderr = "")
+ return real_run(cmd, *args, **kwargs)
+
+ monkeypatch.setattr(mod.subprocess, "run", run)
+
+
+class TestTheMemoryProbeFallsBackToNvml:
+ def test_a_failing_nvidia_smi_no_longer_reads_as_no_gpu(self, monkeypatch, probe_script):
+ _failing_smi(monkeypatch)
+ probe_script(_payload([_row(1, 20000), _row(0, 8000, 12288)]))
+ assert LlamaCppBackend._get_gpu_memory() == [(0, 8000, 12288), (1, 20000, 24576)]
+ assert LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES is True
+
+ def test_the_mask_applies_as_it_does_to_nvidia_smi_rows(self, monkeypatch, probe_script):
+ _failing_smi(monkeypatch)
+ probe_script(_payload([_row(0, 8000), _row(1, 20000)]))
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "1")
+ assert LlamaCppBackend._get_gpu_memory() == [(1, 20000, 24576)]
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "")
+ assert LlamaCppBackend._get_gpu_memory() == []
+
+ def test_a_uuid_mask_selects_by_the_rows_own_uuid(self, monkeypatch, probe_script):
+ _failing_smi(monkeypatch)
+ probe_script(
+ _payload([_row(0, 8000, uuid = "GPU-aaaa1111-0"), _row(1, 20000, uuid = "GPU-bbbb2222-1")])
+ )
+ # A full uuid, a prefix, mask order, and a mixed index + uuid mask, as the CUDA runtime reads them.
+ for mask, expected in (
+ ("GPU-bbbb2222-1", [(1, 20000, 24576)]),
+ ("GPU-aaaa", [(0, 8000, 24576)]),
+ ("GPU-bbbb,GPU-aaaa", [(0, 8000, 24576), (1, 20000, 24576)]),
+ ("1,GPU-aaaa", [(0, 8000, 24576), (1, 20000, 24576)]),
+ # An entry naming no single device ends the mask there, as CUDA documents
+ # for "0,2,-1,1": the devices before it stay visible, nothing after it does.
+ ("nope", []),
+ ("GPU-", []),
+ ("1,-1,0", [(1, 20000, 24576)]),
+ ("0,GPU-,1", [(0, 8000, 24576)]),
+ ("0,7,1", [(0, 8000, 24576)]),
+ ):
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", mask)
+ assert LlamaCppBackend._get_gpu_memory() == expected, mask
+
+ def test_a_mig_assignment_names_the_slice_row_the_probe_lists(self, monkeypatch, probe_script):
+ _failing_smi(monkeypatch)
+ slice_row = dict(_row(0, 9000, 20480, uuid = "MIG-cccc3333-0"), mig = "1")
+ probe_script(_payload([_row(0, 60000, 81920, uuid = "GPU-aaaa1111-0"), slice_row]))
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "MIG-cccc")
+ assert LlamaCppBackend._get_gpu_memory() == [(0, 9000, 20480)]
+ # An index, a GPU- entry and no mask at all name the parent, and CUDA exposes its
+ # first slice then, not the whole card: the slice's memory is what the model gets.
+ for mask in ("0", "GPU-aaaa"):
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", mask)
+ assert LlamaCppBackend._get_gpu_memory() == [(0, 9000, 20480)], mask
+ assert LlamaCppBackend._child_visibility_for([0]) == "MIG-cccc3333-0"
+ monkeypatch.delenv("CUDA_VISIBLE_DEVICES")
+ assert LlamaCppBackend._get_gpu_memory() == [(0, 9000, 20480)]
+ assert LlamaCppBackend._child_visibility_for([0]) == "MIG-cccc3333-0"
+ # A slice the probe does not list hides every GPU rather than exposing the parent.
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "MIG-dddd")
+ assert LlamaCppBackend._get_gpu_memory() == []
+ # A second slice named by the mask is that slice, not the parent's first.
+ second = dict(_row(0, 4000, 10240, uuid = "MIG-dddd4444-0"), mig = "1")
+ probe_script(_payload([_row(0, 60000, 81920, uuid = "GPU-aaaa1111-0"), slice_row, second]))
+ assert LlamaCppBackend._get_gpu_memory() == [(0, 4000, 10240)]
+ assert LlamaCppBackend._child_visibility_for([0]) == "MIG-dddd4444-0"
+ # Two slices of one card: one row per physical index, the first named, and the
+ # child is pinned to that one so the budget and the launch agree.
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "MIG-dddd,MIG-cccc")
+ assert LlamaCppBackend._get_gpu_memory() == [(0, 4000, 10240)]
+ assert LlamaCppBackend._child_visibility_for([0]) == "MIG-dddd4444-0"
+
+ def test_the_child_pin_belongs_to_no_probe(self, monkeypatch, probe_script):
+ """The uuid each index stands for is derived from the last NVML inventory and the
+ mask in force when the launch asks, so another query answering in between (nvidia-smi
+ recovering, a concurrent preflight) neither blanks it nor leaves a stale one."""
+ _failing_smi(monkeypatch)
+ slice_row = dict(_row(0, 9000, 20480, uuid = "MIG-cccc3333-0"), mig = "1")
+ probe_script(
+ _payload(
+ [
+ _row(0, 60000, 81920, uuid = "GPU-aaaa1111-0"),
+ _row(1, 20000, uuid = "GPU-bbbb2222-1"),
+ slice_row,
+ ]
+ )
+ )
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "GPU-bbbb,GPU-aaaa")
+ assert LlamaCppBackend._get_gpu_memory() == [(0, 9000, 20480), (1, 20000, 24576)]
+
+ def smi_ok(cmd, *args, **kwargs):
+ if cmd and os.path.basename(str(cmd[0])) == "nvidia-smi":
+ return types.SimpleNamespace(
+ returncode = 0, stdout = "0, 60000, 81920\n1, 20000, 24576\n", stderr = ""
+ )
+ raise AssertionError("no other probe should run")
+
+ monkeypatch.setattr(mod.subprocess, "run", smi_ok)
+ assert LlamaCppBackend._get_gpu_memory() == [(0, 60000, 81920), (1, 20000, 24576)]
+ # The inherited uuid mask is still what the child gets, a MIG parent still its slice.
+ assert LlamaCppBackend._child_visibility_for([1, 0]) == "GPU-bbbb2222-1,MIG-cccc3333-0"
+ # The mask changing underneath is read as it is now, not as it was at the probe.
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0,1")
+ assert LlamaCppBackend._child_visibility_for([1]) == "1"
+ assert LlamaCppBackend._child_visibility_for([0]) == "MIG-cccc3333-0"
+ # No NVML inventory at all: indices as before.
+ monkeypatch.setattr(LlamaCppBackend, "_NVML_ROWS", [])
+ assert LlamaCppBackend._child_visibility_for([0, 1]) == "0,1"
+
+ def test_a_uuid_mask_is_handed_to_the_child_as_uuids(self, monkeypatch, probe_script):
+ _failing_smi(monkeypatch)
+ slice_row = dict(_row(0, 9000, 20480, uuid = "MIG-cccc3333-0"), mig = "1")
+ probe_script(
+ _payload(
+ [
+ _row(0, 60000, 81920, uuid = "GPU-aaaa1111-0"),
+ _row(1, 20000, uuid = "GPU-bbbb2222-1"),
+ slice_row,
+ ]
+ )
+ )
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "MIG-cccc")
+ assert LlamaCppBackend._get_gpu_memory() == [(0, 9000, 20480)]
+ # The launch must not turn the slice into its parent's index.
+ assert LlamaCppBackend._child_visibility_for([0]) == "MIG-cccc3333-0"
+ # The GPU- entry names the MIG parent, which CUDA exposes as its first slice.
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "GPU-bbbb,GPU-aaaa")
+ assert LlamaCppBackend._get_gpu_memory() == [(0, 9000, 20480), (1, 20000, 24576)]
+ assert LlamaCppBackend._child_visibility_for([1, 0]) == "GPU-bbbb2222-1,MIG-cccc3333-0"
+ # A numeric or absent mask re-emits indices as before; only a slice standing in
+ # for its MIG parent is named by uuid, and a selection mixing both stays numeric.
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0,1")
+ assert LlamaCppBackend._get_gpu_memory() == [(0, 9000, 20480), (1, 20000, 24576)]
+ assert LlamaCppBackend._child_visibility_for([1]) == "1"
+ assert LlamaCppBackend._child_visibility_for([0]) == "MIG-cccc3333-0"
+ monkeypatch.delenv("CUDA_VISIBLE_DEVICES")
+ LlamaCppBackend._get_gpu_memory()
+ assert LlamaCppBackend._child_visibility_for([0, 1]) == "0,1"
+
+ def test_rows_without_a_memory_reading_are_not_evidence(self, monkeypatch, probe_script):
+ _failing_smi(monkeypatch)
+ probe_script(_payload([_row(0, 0, 0)], source = "cuda"))
+ assert LlamaCppBackend._get_gpu_memory() == []
+ probe_script(_payload([_row(0, 0, 0)]))
+ assert LlamaCppBackend._get_gpu_memory() == []
+ # One visible GPU without a reading voids the answer rather than shrinking the host;
+ # a full card (free 0, a total) is a reading and stays listed.
+ probe_script(_payload([_row(0, 8000), _row(1, 0, 0)]))
+ assert LlamaCppBackend._get_gpu_memory() == []
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0")
+ assert LlamaCppBackend._get_gpu_memory() == [(0, 8000, 24576)]
+ monkeypatch.delenv("CUDA_VISIBLE_DEVICES")
+ probe_script(_payload([_row(0, 8000), _row(1, 0)]))
+ assert LlamaCppBackend._get_gpu_memory() == [(0, 8000, 24576), (1, 0, 24576)]
+
+ def test_a_working_nvidia_smi_is_not_second_guessed(self, monkeypatch, probe_script):
+ probe_script(_payload([_row(5, 1)]))
+ real_run = mod.subprocess.run
+
+ def run(cmd, *args, **kwargs):
+ if cmd and os.path.basename(str(cmd[0])) == "nvidia-smi":
+ return types.SimpleNamespace(returncode = 0, stdout = "0, 4096, 8192\n", stderr = "")
+ return real_run(cmd, *args, **kwargs)
+
+ monkeypatch.setattr(mod.subprocess, "run", run)
+ assert LlamaCppBackend._get_gpu_memory() == [(0, 4096, 8192)]
+
+ def test_the_switch_and_a_missing_script_turn_it_off(self, monkeypatch, probe_script):
+ _failing_smi(monkeypatch)
+ probe_script(_payload([_row(0, 8000)]))
+ monkeypatch.setenv("UNSLOTH_NVIDIA_LIBRARY_PROBE", "0")
+ assert LlamaCppBackend._get_gpu_memory() == []
+ monkeypatch.delenv("UNSLOTH_NVIDIA_LIBRARY_PROBE")
+ monkeypatch.setenv("UNSLOTH_NVIDIA_PROBE", "/nonexistent/nvidia_probe.py")
+ monkeypatch.setattr(LlamaCppBackend, "_nvidia_probe_script", staticmethod(lambda: None))
+ assert LlamaCppBackend._get_gpu_memory() == []
+
+ def test_the_real_script_is_found_next_to_the_installers(self, monkeypatch):
+ monkeypatch.delenv("UNSLOTH_NVIDIA_PROBE", raising = False)
+ script = LlamaCppBackend._nvidia_probe_script()
+ assert script is not None and script.name == "nvidia_probe.py"
+ assert (script.parent / "install_llama_prebuilt.py").is_file()
+
+
+class TestTheEmbeddingServerKeepsTheGpu:
+ def test_embeddings_offload_when_only_nvml_answers(self, monkeypatch, probe_script):
+ _failing_smi(monkeypatch)
+ probe_script(_payload([_row(0, 8000)]))
+ import utils.hardware as hardware
+
+ monkeypatch.setattr(hardware, "is_apple_silicon", lambda: False) # Metal answers on a Mac
+ monkeypatch.setattr(embed_mod.config, "embed_device_preference", lambda: "auto")
+ monkeypatch.setattr(
+ LlamaCppBackend, "_arch_gate_survivors", staticmethod(lambda binary: []), raising = False
+ )
+ server = embed_mod.LlamaServerBackend.__new__(embed_mod.LlamaServerBackend)
+ server._force_cpu = False
+ assert server._use_gpu() is True
+ cmd = server._build_cmd("/opt/llama-server", "m.gguf", 9999, use_gpu = True)
+ assert cmd[-2:] == ["-ngl", "-1"]
+ # And the misread this closes: nothing answering pins the server to the CPU.
+ probe_script(_payload([]))
+ assert server._use_gpu() is False
+
+ def test_windows_embeddings_get_the_same_dll_search_path_as_chat(self, monkeypatch, tmp_path):
+ monkeypatch.setattr(embed_mod.sys, "platform", "win32")
+ monkeypatch.setattr(mod.sys, "platform", "win32")
+ monkeypatch.setattr(
+ embed_mod, "child_env_without_native_path_secret", lambda: {"PATH": "inherited"}
+ )
+ monkeypatch.setattr(LlamaCppBackend, "_sanitize_p2p_env", staticmethod(lambda env: None))
+ monkeypatch.setattr(
+ LlamaCppBackend, "_arch_gate_survivors", staticmethod(lambda b: []), raising = False
+ )
+ monkeypatch.setattr(
+ LlamaCppBackend,
+ "_build_windows_path_dirs",
+ staticmethod(
+ lambda binary_dir, prefix, cuda_path: [binary_dir, "C:\\venv\\nvidia\\cu13\\bin"]
+ ),
+ )
+ monkeypatch.setattr(mod, "_llama_lib_dir", lambda binary: Path("C:/llama/build/bin"))
+ server = embed_mod.LlamaServerBackend.__new__(embed_mod.LlamaServerBackend)
+ env = server._build_env("C:/llama/llama-server.exe", use_gpu = True)
+ assert env["PATH"].split(";")[:2] == [
+ str(Path("C:/llama/build/bin")),
+ "C:\\venv\\nvidia\\cu13\\bin",
+ ]
+ assert env["PATH"].endswith(";inherited")
+
+
+class TestAGpuCapableBuildIsPreferred:
+ """#5941: a CPU build/ beside a CUDA build-cuda/ ran on the CPU forever."""
+
+ @staticmethod
+ def _tree(root, platform, layouts):
+ from utils.llama_cpp_path_settings import llama_server_binary_name
+
+ name = llama_server_binary_name(platform)
+ made = {}
+ for build, libs in layouts.items():
+ bindir = root / build / "bin" / ("Release" if platform == "win32" else "")
+ bindir.mkdir(parents = True, exist_ok = True)
+ exe = bindir / name
+ exe.write_text("#!/bin/sh\n", encoding = "utf-8")
+ exe.chmod(0o755)
+ for lib in libs:
+ (bindir / lib).write_bytes(b"")
+ made[build] = exe
+ return made
+
+ @pytest.mark.parametrize("platform", ["linux", "win32"])
+ def test_the_cuda_sibling_build_wins_over_a_cpu_only_first_hit(
+ self, tmp_path, platform, monkeypatch
+ ):
+ from utils import llama_cpp_path_settings as ps
+
+ monkeypatch.setattr(ps, "host_gpu_vendors", lambda: None) # not this host's vendors
+ so = ".dll" if platform == "win32" else ".so"
+ pre = "" if platform == "win32" else "lib"
+ made = self._tree(
+ tmp_path,
+ platform,
+ {
+ "build": [f"{pre}ggml-cpu{so}", f"{pre}ggml-base{so}"],
+ "build-cuda": [f"{pre}ggml-cuda{so}", f"{pre}ggml-cpu{so}", f"{pre}ggml-base{so}"],
+ },
+ )
+ assert ps.resolve_llama_server_binary(tmp_path, platform = platform) == made["build-cuda"]
+ assert ps.binary_gpu_verdict(made["build"]) == "cpu"
+ assert ps.binary_gpu_verdict(made["build-cuda"]) == "gpu"
+
+ def test_a_stale_cuda_build_does_not_shadow_the_build_this_host_can_run(
+ self, tmp_path, monkeypatch
+ ):
+ from utils import llama_cpp_path_settings as ps
+
+ made = self._tree(
+ tmp_path,
+ "linux",
+ {
+ "build": ["libggml-cpu.so", "libggml-base.so"],
+ "build-cuda": ["libggml-cuda.so", "libggml-cpu.so"],
+ "build-hip": ["libggml-hip.so", "libggml-cpu.so"],
+ "build-vulkan": ["libggml-vulkan.so", "libggml-cpu.so"],
+ },
+ )
+ for vendors, winner in (
+ ({"amd"}, "build-hip"),
+ ({"nvidia"}, "build-cuda"),
+ (None, "build-cuda"),
+ ):
+ monkeypatch.setattr(ps, "host_gpu_vendors", lambda v = vendors: v)
+ assert (
+ ps.resolve_llama_server_binary(tmp_path, platform = "linux") == made[winner]
+ ), vendors
+ # A vendor no build targets still gets the vendor-agnostic Vulkan build.
+ monkeypatch.setattr(ps, "host_gpu_vendors", lambda: {"intel"})
+ assert ps.resolve_llama_server_binary(tmp_path, platform = "linux") == made["build-vulkan"]
+
+ def test_host_vendors_read_the_drm_sysfs(self, tmp_path, monkeypatch):
+ from utils import llama_cpp_path_settings as ps
+
+ monkeypatch.setattr(ps.sys, "platform", "linux")
+ monkeypatch.setattr(ps.os.path, "isdir", lambda p: False)
+ nodes = {"/dev/nvidiactl", "/dev/kfd"}
+ monkeypatch.setattr(ps.os.path, "exists", lambda p: p in nodes)
+ monkeypatch.setattr(ps, "_DRM_ROOT", str(tmp_path))
+ for var in ("CUDA_VISIBLE_DEVICES", "HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES"):
+ monkeypatch.delenv(var, raising = False)
+ assert ps.host_gpu_vendors() is None
+ for card, vendor in (("card0", "0x1002"), ("card1", "0x10de"), ("card1-DP-1", "0x10de")):
+ (tmp_path / card / "device").mkdir(parents = True)
+ (tmp_path / card / "device" / "vendor").write_text(vendor + "\n", encoding = "utf-8")
+ assert ps.host_gpu_vendors() == {"amd", "nvidia"}
+ # A container exposing only the AMD device node, or a mask hiding NVIDIA: not runnable.
+ nodes.discard("/dev/nvidiactl")
+ assert ps.host_gpu_vendors() == {"amd"}
+ nodes.add("/dev/nvidiactl")
+ # HIP reads CUDA_VISIBLE_DEVICES when neither HIP_ nor ROCR_ is set, so an empty one
+ # hides both vendors: detected but nothing reachable is an empty set, not unknown,
+ # and only Vulkan fits then. A set HIP_ mask outranks it.
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "")
+ assert ps.host_gpu_vendors() == set()
+ assert ps._fits_host({"cuda"}, set()) is False and ps._fits_host({"vulkan"}, set()) is True
+ monkeypatch.setenv("HIP_VISIBLE_DEVICES", "0")
+ assert ps.host_gpu_vendors() == {"amd"}
+ # The two ROCm masks stack: an empty ROCR_ under a set HIP_ still hides every agent.
+ monkeypatch.setenv("ROCR_VISIBLE_DEVICES", "")
+ assert ps.host_gpu_vendors() == set()
+ monkeypatch.setenv("ROCR_VISIBLE_DEVICES", "0")
+ assert ps.host_gpu_vendors() == {"amd"}
+ monkeypatch.setenv("HIP_VISIBLE_DEVICES", "-1")
+ assert ps.host_gpu_vendors() == set()
+ assert ps._fits_host({"cuda"}, set()) is False and ps._fits_host({"vulkan"}, set()) is True
+
+ def test_host_vendors_read_the_wsl_runtimes(self, tmp_path, monkeypatch):
+ """WSL2 has no DRM card and no /dev/kfd or /dev/nvidiactl: the GPU is behind /dev/dxg
+ and the vendor is the runtime that drives it."""
+ from utils import llama_cpp_path_settings as ps
+
+ monkeypatch.setattr(ps.sys, "platform", "linux")
+ monkeypatch.setattr(ps.os.path, "isdir", lambda p: False)
+ monkeypatch.setattr(ps, "_DRM_ROOT", str(tmp_path / "drm"))
+ rocm = tmp_path / "rocm"
+ wsl_lib = tmp_path / "wsl"
+ monkeypatch.setattr(ps, "_WSL_ROCM_LIB_DIRS", (str(rocm),))
+ monkeypatch.setattr(ps, "_WSL_CUDA_LIB_DIR", str(wsl_lib))
+ nodes = {"/dev/dxg"}
+ monkeypatch.setattr(ps.os.path, "exists", lambda p: p in nodes or os.path.lexists(p))
+ for var in ("CUDA_VISIBLE_DEVICES", "HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES"):
+ monkeypatch.delenv(var, raising = False)
+ assert ps.host_gpu_vendors() is None
+ rocm.mkdir()
+ (rocm / "librocdxg.so.1").write_bytes(b"")
+ assert ps.host_gpu_vendors() == {"amd"}
+ wsl_lib.mkdir()
+ (wsl_lib / "libcuda.so.1.1").write_bytes(b"")
+ assert ps.host_gpu_vendors() == {"amd", "nvidia"}
+ # The masks still apply, and without /dev/dxg the runtimes prove nothing.
+ monkeypatch.setenv("HIP_VISIBLE_DEVICES", "")
+ assert ps.host_gpu_vendors() == {"nvidia"}
+ nodes.discard("/dev/dxg")
+ assert ps.host_gpu_vendors() is None
+
+ def test_a_gpu_first_hit_for_another_vendor_yields_too(self, tmp_path, monkeypatch):
+ from utils import llama_cpp_path_settings as ps
+
+ # No build/ at all: a stale build-cuda/ is the first hit on an AMD box.
+ made = self._tree(
+ tmp_path, "linux", {"build-cuda": ["libggml-cuda.so"], "build-hip": ["libggml-hip.so"]}
+ )
+ monkeypatch.setattr(ps, "host_gpu_vendors", lambda: {"amd"})
+ assert ps.resolve_llama_server_binary(tmp_path, platform = "linux") == made["build-hip"]
+ # On the NVIDIA box, or an unknown host, the first hit stands.
+ for vendors in ({"nvidia"}, None):
+ monkeypatch.setattr(ps, "host_gpu_vendors", lambda v = vendors: v)
+ assert ps.resolve_llama_server_binary(tmp_path, platform = "linux") == made["build-cuda"]
+
+ def test_a_first_hit_of_unknown_layout_keeps_its_place(self, tmp_path):
+ from utils import llama_cpp_path_settings as ps
+ made = self._tree(tmp_path, "linux", {"build": [], "build-cuda": ["libggml-cuda.so"]})
+ # A static build or the installer's wrapper: not proven CPU-only, so search order holds.
+ assert ps.resolve_llama_server_binary(tmp_path, platform = "linux") == made["build"]
+
+ @pytest.mark.skipif(
+ sys.platform == "win32" or os.geteuid() == 0, reason = "needs a real permission denial"
+ )
+ def test_a_denied_candidate_ahead_of_the_gpu_build_still_stops_discovery(
+ self, tmp_path, monkeypatch
+ ):
+ made = self._tree(
+ tmp_path, "linux", {"build": ["libggml-cpu.so"], "build-cuda": ["libggml-cuda.so"]}
+ )
+ monkeypatch.setattr(LlamaCppBackend, "_find_llama_server_binary", _REAL_FINDER)
+ monkeypatch.setattr(mod.sys, "platform", "linux")
+ monkeypatch.delenv("LLAMA_SERVER_PATH", raising = False)
+ monkeypatch.delenv("UNSLOTH_STUDIO_MANAGED_LLAMA_CPP_PATH", raising = False)
+ monkeypatch.setenv("UNSLOTH_LLAMA_CPP_PATH", str(tmp_path))
+ # An in-flight replace or an ACL: the pinned layout's build/ cannot be read at all.
+ (tmp_path / "build" / "bin").chmod(0)
+ try:
+ assert LlamaCppBackend._find_llama_server_binary() is None
+ finally:
+ (tmp_path / "build" / "bin").chmod(0o755)
+ assert LlamaCppBackend._find_llama_server_binary() == str(made["build-cuda"])
+
+ def test_the_runtime_finder_agrees(self, tmp_path, monkeypatch):
+ made = self._tree(
+ tmp_path,
+ "linux",
+ {"build": ["libggml-cpu.so", "libggml-base.so"], "build-cuda": ["libggml-cuda.so"]},
+ )
+ monkeypatch.setattr(LlamaCppBackend, "_find_llama_server_binary", _REAL_FINDER)
+ monkeypatch.setattr(mod.sys, "platform", "linux")
+ monkeypatch.delenv("LLAMA_SERVER_PATH", raising = False)
+ monkeypatch.delenv("UNSLOTH_STUDIO_MANAGED_LLAMA_CPP_PATH", raising = False)
+ monkeypatch.setenv("UNSLOTH_LLAMA_CPP_PATH", str(tmp_path))
+ assert LlamaCppBackend._find_llama_server_binary() == str(made["build-cuda"])
+
+ def test_the_legacy_in_tree_build_prefers_the_gpu_build_too(self, tmp_path, monkeypatch):
+ """No managed or custom runtime configured: the project_root/llama.cpp fallback."""
+ from utils import llama_cpp_path_settings as ps
+
+ made = self._tree(
+ tmp_path / "llama.cpp",
+ "linux",
+ {"build": ["libggml-cpu.so", "libggml-base.so"], "build-cuda": ["libggml-cuda.so"]},
+ )
+ monkeypatch.setattr(LlamaCppBackend, "_find_llama_server_binary", _REAL_FINDER)
+ monkeypatch.setattr(mod.sys, "platform", "linux")
+ monkeypatch.setattr(mod, "__file__", str(tmp_path / "s" / "b" / "c" / "i" / "llama_cpp.py"))
+ for var in (
+ "LLAMA_SERVER_PATH",
+ "UNSLOTH_LLAMA_CPP_PATH",
+ "UNSLOTH_STUDIO_MANAGED_LLAMA_CPP_PATH",
+ ):
+ monkeypatch.delenv(var, raising = False)
+ monkeypatch.setattr(ps, "get_stored_custom_llama_cpp_path", lambda: None)
+ monkeypatch.setattr(
+ LlamaCppBackend,
+ "_resolved_studio_root_and_is_legacy",
+ staticmethod(lambda: (tmp_path / "no-such-studio", False)),
+ )
+ monkeypatch.setattr(mod.shutil, "which", lambda name: None)
+ assert LlamaCppBackend._find_llama_server_binary() == str(made["build-cuda"])
+
+
+ sys.platform == "win32", reason = "the Linux library path is built from posix paths"
+)
+class TestTheLinuxLibrarySearchPath:
+ @pytest.mark.parametrize("prefix_name", ["Studio", "Studio[CUDA]"])
+ def test_a_bracketed_prefix_and_torch_lib_are_found(self, monkeypatch, tmp_path, prefix_name):
+ prefix = tmp_path / prefix_name
+ site = prefix / "lib" / "python3.12" / "site-packages"
+ cu = site / "nvidia" / "cu13" / "lib"
+ torch_lib = site / "torch" / "lib"
+ cu.mkdir(parents = True)
+ torch_lib.mkdir(parents = True)
+ monkeypatch.setattr(embed_mod.sys, "platform", "linux")
+ monkeypatch.setattr(embed_mod.sys, "prefix", str(prefix))
+ env: dict[str, str] = {}
+ embed_mod.LlamaServerBackend._add_linux_cuda_libs(env, "/opt/llama")
+ parts = env["LD_LIBRARY_PATH"].split(":")
+ assert str(cu) in parts and str(torch_lib) in parts
+ # The chat server's builder reads the same prefix the same way.
+ monkeypatch.setattr(mod.sys, "platform", "linux")
+ monkeypatch.setattr(mod.sys, "prefix", str(prefix))
+ monkeypatch.setattr(mod, "_llama_lib_dir", lambda binary: Path("/opt/llama"))
+ monkeypatch.setattr(mod, "_wsl_system_rocm_lib_dirs", lambda: [])
+ monkeypatch.setattr(mod, "_native_linux_system_rocm_lib_dirs", lambda binary_dir: [])
+ monkeypatch.setattr(
+ LlamaCppBackend, "_prefers_bundle_only_rocm", staticmethod(lambda b: True)
+ )
+ chat_env = LlamaCppBackend._llama_server_env_for_binary("/opt/llama/llama-server")
+ chat_parts = chat_env["LD_LIBRARY_PATH"].split(":")
+ assert str(cu) in chat_parts and str(torch_lib) in chat_parts
diff --git a/studio/backend/utils/llama_cpp_path_settings.py b/studio/backend/utils/llama_cpp_path_settings.py
index 361d65d2cb6..f875587e298 100644
--- a/studio/backend/utils/llama_cpp_path_settings.py
+++ b/studio/backend/utils/llama_cpp_path_settings.py
@@ -6,11 +6,12 @@
from __future__ import annotations
import os
+import re
import sys
import threading
from contextlib import contextmanager
from pathlib import Path
-from typing import Iterator, Optional
+from typing import Any, Callable, Iterable, Iterator, Optional
CUSTOM_LLAMA_CPP_PATH_SETTING_KEY = "custom_llama_cpp_path"
MAX_CUSTOM_LLAMA_CPP_PATH_LENGTH = 32767
@@ -59,12 +60,13 @@ def llama_server_candidates(
"""Supported llama.cpp build layouts, in the runtime's search order."""
root = Path(directory)
binary_name = llama_server_binary_name(platform)
- candidates = [
- root / binary_name,
- root / "build" / "bin" / binary_name,
- ]
- if (platform or sys.platform) == "win32":
- candidates.append(root / "build" / "bin" / "Release" / binary_name)
+ windows = (platform or sys.platform) == "win32"
+ candidates = [root / binary_name]
+ # build/ first, then the per-backend build dirs a source checkout keeps side by side (#5941).
+ for build in ("build", "build-cuda", "build-hip", "build-rocm", "build-vulkan"):
+ candidates.append(root / build / "bin" / binary_name)
+ if windows:
+ candidates.append(root / build / "bin" / "Release" / binary_name)
return tuple(candidates)
@@ -77,18 +79,157 @@ def _usable_binary(path: Path, *, platform: Optional[str] = None) -> bool:
return (platform or sys.platform) == "win32" or os.access(path, os.X_OK)
+_GPU_BACKEND_LIB_RE = re.compile(
+ r"^(?:lib)?ggml-(cann|cuda|hip|metal|musa|opencl|sycl|virtgpu|vulkan)"
+ r"(?:\.dll|\.so(?:\.\d+)*|(?:\.\d+)*\.dylib)$"
+)
+_CPU_BACKEND_LIB_RE = re.compile(r"^(?:lib)?ggml-(?:cpu|base)(?:[-.]|$)")
+# Backends bound to one vendor; vulkan, opencl and virtgpu run on any GPU.
+_BACKEND_VENDOR = {
+ "cuda": "nvidia",
+ "hip": "amd",
+ "sycl": "intel",
+ "metal": "apple",
+ "musa": "mthreads",
+ "cann": "huawei",
+}
+_DRM_VENDOR = {"0x10de": "nvidia", "0x1002": "amd", "0x8086": "intel"}
+_WINDOWS_VENDOR_DLLS = {
+ "nvidia": ("nvcuda.dll",),
+ "amd": ("amdhip64*.dll", "amdocl64.dll"),
+ "intel": ("ze_intel_gpu64.dll", "igdrcl64.dll"),
+}
+_DRM_ROOT = "/sys/class/drm"
+# WSL2 reaches the GPU through /dev/dxg (no DRM card, no /dev/kfd or /dev/nvidiactl): the
+# vendor is the runtime that drives it, librocdxg for AMD and the WSL libcuda for NVIDIA.
+_WSL_ROCM_LIB_DIRS = ("/opt/rocm/lib", "/opt/rocm/lib64")
+_WSL_CUDA_LIB_DIR = "/usr/lib/wsl/lib"
+_HOST: Any = object() # prefer_gpu_capable's default: read the vendors from this host
+
+
+def binary_gpu_backends(binary: Path | str) -> Optional[set[str]]:
+ """GPU backend libraries beside the binary; empty for a split build with only CPU
+ libraries, None when the layout says nothing (a static build or a wrapper)."""
+ try:
+ names = [p.name for p in Path(binary).resolve().parent.iterdir() if p.is_file()]
+ except OSError:
+ return None
+ gpu = {m.group(1) for m in map(_GPU_BACKEND_LIB_RE.match, names) if m}
+ if gpu or any(_CPU_BACKEND_LIB_RE.match(name) for name in names):
+ return gpu
+ return None
+
+
+def binary_gpu_verdict(binary: Path | str) -> str:
+ """``gpu``, ``cpu`` (a split-library build with no GPU backend beside it) or ``unknown``."""
+ backends = binary_gpu_backends(binary)
+ return "unknown" if backends is None else ("gpu" if backends else "cpu")
+
+
+def _mask_hides_all(*names: str) -> bool:
+ """Whether the first set mask among names (their precedence order) hides every device."""
+ mask = next((os.environ[n] for n in names if n in os.environ), "x")
+ return mask.strip() in ("", "-1")
+
+
+def _amd_mask_hides_all() -> bool:
+ """HIP_ and ROCR_VISIBLE_DEVICES stack (ROCR filters the agents HIP then indexes), so an
+ empty one at either layer hides everything; CUDA_VISIBLE_DEVICES stands in only when
+ neither is set (_active_gpu_visibility_mask)."""
+ if any(n in os.environ for n in ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES")):
+ return _mask_hides_all("HIP_VISIBLE_DEVICES") or _mask_hides_all("ROCR_VISIBLE_DEVICES")
+ return _mask_hides_all("CUDA_VISIBLE_DEVICES")
+
+
+def host_gpu_vendors() -> Optional[set[str]]:
+ """GPU vendors this process can reach: DRM sysfs vendors whose device node is present and
+ whose visibility mask is not empty on Linux, the vendors' driver DLLs on Windows. None when
+ nothing was detected at all (an unknown host filters nothing); an empty set when GPUs were
+ detected but none is reachable, so only vendor-agnostic backends fit."""
+ vendors: set[str] = set()
+ if sys.platform == "darwin":
+ return {"apple"}
+ if sys.platform == "win32":
+ system32 = Path(os.environ.get("SystemRoot", r"C:\Windows")) / "System32"
+ for vendor, patterns in _WINDOWS_VENDOR_DLLS.items():
+ if any(next(system32.glob(pattern), None) for pattern in patterns):
+ vendors.add(vendor)
+ else:
+ for vendor_file in Path(_DRM_ROOT).glob("card*/device/vendor"):
+ try:
+ vendor = _DRM_VENDOR.get(vendor_file.read_text(encoding = "utf-8").strip().lower())
+ except OSError:
+ continue
+ if vendor:
+ vendors.add(vendor)
+ if os.path.isdir("/proc/driver/nvidia"):
+ vendors.add("nvidia")
+ wsl = os.path.exists("/dev/dxg")
+ wsl_amd = wsl and any(
+ os.path.exists(os.path.join(d, n))
+ for d in _WSL_ROCM_LIB_DIRS
+ for n in ("librocdxg.so", "librocdxg.so.1")
+ )
+ wsl_nvidia = wsl and any(Path(_WSL_CUDA_LIB_DIR).glob("libcuda.so*"))
+ vendors |= {v for v, on in (("amd", wsl_amd), ("nvidia", wsl_nvidia)) if on}
+ detected = set(vendors)
+ # A container that exposes only one vendor's device nodes, or a mask hiding a vendor,
+ # must not make its backend look runnable on a hybrid box.
+ if "nvidia" in vendors and (
+ not (os.path.exists("/dev/nvidiactl") or wsl_nvidia)
+ or _mask_hides_all("CUDA_VISIBLE_DEVICES")
+ ):
+ vendors.discard("nvidia")
+ if "amd" in vendors and (
+ not (os.path.exists("/dev/kfd") or wsl_amd) or _amd_mask_hides_all()
+ ):
+ vendors.discard("amd")
+ return vendors if detected else None
+ return vendors or None
+
+
+def _fits_host(backends: set[str], vendors: Optional[set[str]]) -> bool:
+ if vendors is None:
+ return True
+ return any(_BACKEND_VENDOR.get(b) is None or _BACKEND_VENDOR[b] in vendors for b in backends)
+
+
+def prefer_gpu_capable(
+ candidates: Iterable[Path],
+ usable: Callable[[Path], bool],
+ vendors: Any = _HOST,
+) -> list[Path]:
+ """Search order, except that a first hit proven CPU-only, or proven built for a GPU vendor
+ this host lacks, yields to a later usable build shipping a GPU backend this host's vendor
+ runs (#5941: a CPU build/ beside build-cuda/; a stale build-cuda/ must not shadow
+ build-hip/ on an AMD box, whether or not a build/ precedes it). Unknown layouts stay put."""
+ ordered = list(candidates)
+ first = next((c for c in ordered if usable(c)), None)
+ if first is None:
+ return ordered
+ verdict = binary_gpu_verdict(first)
+ if verdict == "unknown":
+ return ordered
+ if vendors is _HOST:
+ vendors = host_gpu_vendors()
+ if verdict == "gpu" and _fits_host(binary_gpu_backends(first), vendors):
+ return ordered
+ for candidate in ordered:
+ if candidate == first or not usable(candidate):
+ continue
+ backends = binary_gpu_backends(candidate)
+ if backends and _fits_host(backends, vendors):
+ return [candidate] + [c for c in ordered if c != candidate]
+ return ordered
+
+
def resolve_llama_server_binary(
directory: Path | str, *, platform: Optional[str] = None
) -> Optional[Path]:
- """Return the first executable llama-server in a supported layout."""
- return next(
- (
- candidate
- for candidate in llama_server_candidates(directory, platform = platform)
- if _usable_binary(candidate, platform = platform)
- ),
- None,
- )
+ """Return the first executable llama-server in a supported layout, GPU-capable preferred."""
+ usable = lambda candidate: _usable_binary(candidate, platform = platform) # noqa: E731
+ ordered = prefer_gpu_capable(llama_server_candidates(directory, platform = platform), usable)
+ return next((candidate for candidate in ordered if usable(candidate)), None)
def get_stored_custom_llama_cpp_path() -> Optional[Path]:
diff --git a/studio/nvidia_probe.py b/studio/nvidia_probe.py
index 9496016a652..3c4663deb94 100644
--- a/studio/nvidia_probe.py
+++ b/studio/nvidia_probe.py
@@ -70,6 +70,43 @@ def _split_cuda_version(packed: int) -> tuple[int, int] | None:
return packed // 1000, (packed % 1000) // 10
+class _NvmlMemory(ctypes.Structure):
+ # nvmlMemory_t (v1): bytes, in this order.
+ _fields_ = [
+ ("total", ctypes.c_ulonglong),
+ ("free", ctypes.c_ulonglong),
+ ("used", ctypes.c_ulonglong),
+ ]
+
+
+def _mig_children(nvml, handle) -> list:
+ """(handle, uuid, memory) per MIG instance of a device; empty without MIG or on an NVML
+ too old to ask. Unused instance slots answer NOT_FOUND and are skipped."""
+ get_max = getattr(nvml, "nvmlDeviceGetMaxMigDeviceCount", None)
+ get_mig = getattr(nvml, "nvmlDeviceGetMigDeviceHandleByIndex", None)
+ if get_max is None or get_mig is None:
+ return []
+ children = []
+ try:
+ count = ctypes.c_uint(0)
+ if get_max(handle, ctypes.byref(count)) != 0:
+ return []
+ for index in range(count.value):
+ mig = ctypes.c_void_p()
+ if get_mig(handle, index, ctypes.byref(mig)) != 0:
+ continue
+ uuid = ctypes.create_string_buffer(96)
+ if nvml.nvmlDeviceGetUUID(mig, uuid, 96) != 0:
+ continue
+ memory = _NvmlMemory()
+ if nvml.nvmlDeviceGetMemoryInfo(mig, ctypes.byref(memory)) != 0:
+ memory.total = memory.free = 0
+ children.append((mig, uuid.value.decode("ascii", "replace"), memory))
+ except Exception:
+ return children
+ return children
+
+
def _probe_nvml() -> dict | None:
nvml = _load("nvml")
if nvml is None:
@@ -100,7 +137,7 @@ def _probe_nvml() -> dict | None:
for index in range(count.value):
handle = ctypes.c_void_p()
if get_handle(index, ctypes.byref(handle)) != 0:
- continue
+ return None # a GPU this reader cannot see is a GPU the selectors would miss
name = ctypes.create_string_buffer(96)
uuid = ctypes.create_string_buffer(96)
major, minor = ctypes.c_int(0), ctypes.c_int(0)
@@ -114,14 +151,33 @@ def _probe_nvml() -> dict | None:
== 0
):
cap = f"{major.value}.{minor.value}"
+ memory = _NvmlMemory()
+ if nvml.nvmlDeviceGetMemoryInfo(handle, ctypes.byref(memory)) != 0:
+ memory.total = memory.free = 0
devices.append(
{
"index": str(index),
"uuid": uuid.value.decode("ascii", "replace"),
"name": name.value.decode("utf-8", "replace"),
"compute_cap": cap,
+ "memory_total_mib": str(memory.total // (1024 * 1024)),
+ "memory_free_mib": str(memory.free // (1024 * 1024)),
}
)
+ # MIG slices, so a CUDA_VISIBLE_DEVICES=MIG-... assignment can be named: same
+ # parent index and capability, their own uuid and memory, marked "mig".
+ for mig, mig_uuid, mig_memory in _mig_children(nvml, handle):
+ devices.append(
+ {
+ "index": str(index),
+ "uuid": mig_uuid,
+ "name": name.value.decode("utf-8", "replace") + " MIG",
+ "compute_cap": cap,
+ "memory_total_mib": str(mig_memory.total // (1024 * 1024)),
+ "memory_free_mib": str(mig_memory.free // (1024 * 1024)),
+ "mig": "1",
+ }
+ )
return {
"source": "nvml",
"cuda_driver_version": list(cuda) if cuda else None,
@@ -163,6 +219,9 @@ def _probe_cuda_driver() -> dict | None:
"uuid": "",
"name": name.value.decode("utf-8", "replace"),
"compute_cap": f"{major.value}.{minor.value}" if ok else "",
+ # Memory needs a context, which this probe never creates.
+ "memory_total_mib": "0",
+ "memory_free_mib": "0",
}
)
return {
@@ -190,9 +249,11 @@ def _from_payload(payload: object) -> NvidiaLibraryInventory | None:
return None
version = payload.get("cuda_driver_version")
cuda = tuple(int(part) for part in version[:2]) if isinstance(version, list) else None
+ keys = ("index", "uuid", "name", "compute_cap", "memory_total_mib", "memory_free_mib")
devices = [
- {key: str(row.get(key, "")) for key in ("index", "uuid", "name", "compute_cap")}
+ {key: str(row.get(key, "")) for key in keys}
for row in payload.get("devices") or []
+ if not row.get("mig")
if isinstance(row, dict)
]
return NvidiaLibraryInventory(
diff --git a/tests/studio/install/test_nvidia_library_probe.py b/tests/studio/install/test_nvidia_library_probe.py
index dc5b937a3d1..af023875213 100644
--- a/tests/studio/install/test_nvidia_library_probe.py
+++ b/tests/studio/install/test_nvidia_library_probe.py
@@ -62,16 +62,49 @@ def _inventory(
class TestProbeModule:
+ def test_mig_rows_are_not_devices_to_the_installers(self):
+ payload = {
+ "source": "nvml",
+ "cuda_driver_version": [13, 0],
+ "devices": [
+ {"index": "0", "uuid": "GPU-a", "name": "H100", "compute_cap": "9.0"},
+ {
+ "index": "0",
+ "uuid": "MIG-b",
+ "name": "H100 MIG",
+ "compute_cap": "9.0",
+ "mig": "1",
+ },
+ ],
+ }
+ inv = PROBE._from_payload(payload)
+ assert [d["uuid"] for d in inv.devices] == ["GPU-a"]
+
def test_the_payload_round_trips(self):
payload = {
"source": "nvml",
"cuda_driver_version": [13, 1],
"driver_version": "590.48.01",
- "devices": [{"index": "0", "uuid": "GPU-x", "name": "B200", "compute_cap": "10.0"}],
+ "devices": [
+ {
+ "index": "0",
+ "uuid": "GPU-x",
+ "name": "B200",
+ "compute_cap": "10.0",
+ "memory_total_mib": "183359",
+ "memory_free_mib": "182630",
+ }
+ ],
}
inv = PROBE._from_payload(payload)
assert inv.cuda_driver_version == (13, 1)
assert inv.devices == payload["devices"]
+ # An older payload without the memory fields still reads; they come back empty.
+ old = {
+ **payload,
+ "devices": [{"index": "0", "uuid": "GPU-x", "name": "B200", "compute_cap": "10.0"}],
+ }
+ assert PROBE._from_payload(old).devices[0]["memory_free_mib"] == ""
assert PROBE._from_payload(None) is None
assert PROBE._from_payload({"source": "other"}) is None
@@ -199,6 +232,11 @@ class TestProbeModule:
1 for line in listing.stdout.splitlines() if line.startswith("GPU ")
)
assert inv.cuda_driver_version is not None and inv.cuda_driver_version[0] >= 11
+ # The memory reading the runtime probe needs; a busy card may legitimately have none free.
+ assert all(int(d["memory_total_mib"]) > 0 for d in inv.devices)
+ assert all(
+ 0 <= int(d["memory_free_mib"]) <= int(d["memory_total_mib"]) for d in inv.devices
+ )
# ── detect_host ──