Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: count the cards a CUDA + Vulkan llama.cpp reaches only through Vulkan (#15466)
Debojyoti Banerjeed4fa58c797cf
diff --git a/studio/backend/core/inference/_vulkan_probe.py b/studio/backend/core/inference/_vulkan_probe.pyindex 31effc54142..731be44483b 100644--- a/studio/backend/core/inference/_vulkan_probe.py+++ b/studio/backend/core/inference/_vulkan_probe.py@@ -6,33 +6,70 @@Run in a short-lived subprocess (``python _vulkan_probe.py <bindir>``) so theVulkan instance never lives in the long-running backend process. Loads thebundled ggml Vulkan backend from ``<bindir>`` and prints one-``<idx>\\t<free_bytes>\\t<is_igpu>\\t<total_bytes>\\t<name>\\t<type_known>`` line per device to+``<idx>\\t<free_bytes>\\t<is_igpu>\\t<total_bytes>\\t<name>\\t<type_known>\\t<pci_id>`` line per device tostdout. Indices are ggml's own Vulkan device ordinals, which need not matchnvidia-smi order. ``is_igpu`` (from ggml's device type) is ``1`` for anintegrated GPU sharing system RAM. ``total_bytes`` is the device-local heap;the reader uses it to reserve absolute headroom on a discrete card (paritywith the CUDA/ROCm fit) and ignores it for an iGPU, whose "VRAM" is sharedsystem RAM. ``name`` is ggml's device description (the marketing name, e.g.-"AMD Radeon RX 9070 XT"); empty when the registry lookup fails.+"AMD Radeon RX 9070 XT"); empty when the registry lookup fails. ``pci_id`` is+the lower-case PCI bus id ggml reports, empty on a build that does not.Uses only the standard library so it stays runnable as a bare script."""import ctypesimport os+import reimport sys# ggml_backend_dev_type enum (ggml-backend.h): CPU=0, GPU=1, IGPU=2, ..._GGML_BACKEND_DEVICE_TYPE_IGPU = 2+# ggml_backend_dev_props.device_id for a PCI device: "domain:bus:device.function".+_PCI_ID_RE = re.compile(r"^[0-9a-f]{4,8}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]$")# winbase.h_SEM_FAILCRITICALERRORS = 0x0001-def _igpu_flags_and_names(base, lib, count: int) -> tuple[list[bool], list[str], list[bool]]:- """Per-device iGPU flags, names, and whether each was read; an unread type makes DirectIO buffer iGPU weights. All-False on failure; reg index = ordinal"""+class _DevProps(ctypes.Structure):+ """ggml_backend_dev_props up to device_id, then slack: a ggml older than device_id writes+ less, and the zeroed or bool-sized bytes it leaves in that slot never pass as a pointer."""++ _fields_ = [+ ("name", ctypes.c_void_p),+ ("description", ctypes.c_void_p),+ ("memory_free", ctypes.c_size_t),+ ("memory_total", ctypes.c_size_t),+ ("type", ctypes.c_int),+ ("device_id", ctypes.c_void_p),+ ("_tail", ctypes.c_ubyte * 64),+ ]+++def _pci_device_id(get_props, dev) -> str:+ """Lower-case PCI bus id ("0000:c1:00.0"), which llama.cpp matches to skip a Vulkan device a+ CUDA one already drives; empty when the build does not report it."""+ props = _DevProps()+ try:+ get_props(dev, ctypes.byref(props))+ if not props.device_id or props.device_id < 0x10000:+ return ""+ raw = ctypes.cast(props.device_id, ctypes.c_char_p).value or b""+ except Exception:+ return ""+ value = raw[:64].decode("ascii", errors = "replace").strip().lower()+ return value if _PCI_ID_RE.match(value) else ""+++def _igpu_flags_and_names(+ base, lib, count: int+) -> tuple[list[bool], list[str], list[bool], list[str]]:+ """Per-device iGPU flags, names, whether each was read, and PCI ids; an unread type makes DirectIO buffer iGPU weights. All-False on failure; reg index = ordinal"""flags = [False] * countnames = [""] * countknown = [False] * count+ pci_ids = [""] * counttry:lib.ggml_backend_vk_reg.restype = ctypes.c_void_plib.ggml_backend_vk_reg.argtypes = []@@ -44,11 +81,11 @@ def _igpu_flags_and_names(base, lib, count: int) -> tuple[list[bool], list[str],base.ggml_backend_dev_type.argtypes = [ctypes.c_void_p]reg = lib.ggml_backend_vk_reg()if not reg:- return flags, names, known+ return flags, names, known, pci_idsdev_count = base.ggml_backend_reg_dev_count(reg)except Exception:# Best-effort: any failure degrades to "discrete"/"unnamed" so memory readings still get through- return flags, names, known+ return flags, names, known, pci_ids# Bound outside the type-detection try so a ggml-base lacking the description symbol still reads iGPU flags, else iGPU shared RAM counts as VRAMname_functions = []@@ -60,6 +97,12 @@ def _igpu_flags_and_names(base, lib, count: int) -> tuple[list[bool], list[str],name_functions.append(function)except Exception:pass+ try:+ get_props = base.ggml_backend_dev_get_props+ get_props.restype = None+ get_props.argtypes = [ctypes.c_void_p, ctypes.c_void_p]+ except Exception:+ get_props = Nonefor i in range(min(count, dev_count)):try:@@ -83,7 +126,9 @@ def _igpu_flags_and_names(base, lib, count: int) -> tuple[list[bool], list[str],name = raw_name.decode("utf-8", errors = "replace")names[i] = name.replace("\t", " ").replace("\r", " ").replace("\n", " ").strip()break- return flags, names, known+ if get_props is not None:+ pci_ids[i] = _pci_device_id(get_props, dev)+ return flags, names, known, pci_idsdef main() -> int:@@ -148,14 +193,14 @@ def main() -> int:]count = lib.ggml_backend_vk_get_device_count()- igpu, names, known = _igpu_flags_and_names(base, lib, count)+ igpu, names, known, pci_ids = _igpu_flags_and_names(base, lib, count)rows = []for i in range(count):free, total = ctypes.c_size_t(0), ctypes.c_size_t(0)lib.ggml_backend_vk_get_device_memory(i, ctypes.byref(free), ctypes.byref(total))rows.append(- "%d\t%d\t%d\t%d\t%s\t%d"- % (i, free.value, int(igpu[i]), total.value, names[i], int(known[i]))+ "%d\t%d\t%d\t%d\t%s\t%d\t%s"+ % (i, free.value, int(igpu[i]), total.value, names[i], int(known[i]), pci_ids[i]))sys.stdout.write("\n".join(rows))return 0diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 281611a972f..94ebbf0bbb5 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -14677,8 +14677,8 @@ class LlamaCppBackend:rows: list[dict] = []for line in result.stdout.strip().splitlines():parts = line.split("\t")- # 4 columns from an older probe (no name); 5 with the name column.- if len(parts) not in (4, 5, 6):+ # 4 columns from an older probe (no name); 5 with the name column, 7 with the PCI id.+ if len(parts) not in (4, 5, 6, 7):continuetry:free_b, total_b = int(parts[1]), int(parts[3])@@ -14694,6 +14694,7 @@ class LlamaCppBackend:"name": parts[4].strip() if len(parts) >= 5 else "",# Absent on an older staged probe, so it reads unknown and declines."type_known": len(parts) >= 6 and parts[5].strip() == "1",+ "pci_id": parts[6].strip().lower() if len(parts) >= 7 else "",})except ValueError:diff --git a/studio/backend/main.py b/studio/backend/main.pyindex 6f5d90b77fd..6a2affe266c 100644--- a/studio/backend/main.py+++ b/studio/backend/main.py@@ -2321,6 +2321,7 @@ def _get_cached_system_gpu_info(amd_driver_warning_report,get_backend_visible_gpu_info,get_cross_vendor_inference_gpu_info,+ get_mixed_vulkan_inference_gpu_info,get_visible_gpu_utilization,get_vulkan_inference_gpu_info,gpu_query,@@ -2449,6 +2450,16 @@ def _get_cached_system_gpu_info(inference_gpu_info = {**cross_vendor_info, "gguf_gpu_ids_supported": False}else:inference_gpu_info = gpu_info+ if vulkan_info is None:+ # A CUDA + Vulkan llama.cpp also runs on the cards only Vulkan reaches (#15444).+ with (+ contextlib.nullcontext()+ if refresh_memory+ else gpu_query.display_reads(max_stale = 0)+ ):+ mixed_info = get_mixed_vulkan_inference_gpu_info(inference_gpu_info)+ if mixed_info is not None:+ inference_gpu_info = mixed_infocombined_info = (gpu_info, inference_gpu_info)_system_gpu_cache = (time.monotonic(), combined_info)diff --git a/studio/backend/tests/test_llama_cpp_vulkan_probe.py b/studio/backend/tests/test_llama_cpp_vulkan_probe.pyindex 06caca27c92..39f0ae395fe 100644--- a/studio/backend/tests/test_llama_cpp_vulkan_probe.py+++ b/studio/backend/tests/test_llama_cpp_vulkan_probe.py@@ -64,11 +64,12 @@ def test_missing_description_symbol_keeps_igpu_detection():)lib = _types.SimpleNamespace(ggml_backend_vk_reg = _FakeCFunction(1))- flags, names, known = _igpu_flags_and_names(base, lib, 1)+ flags, names, known, pci_ids = _igpu_flags_and_names(base, lib, 1)assert flags == [True]assert names == ["Legacy Vulkan iGPU"]assert known == [True]+ assert pci_ids == [""]def _make_vulkan_install(tmp_path: Path) -> str:diff --git a/studio/backend/tests/test_mixed_cuda_vulkan_inference_gpu.py b/studio/backend/tests/test_mixed_cuda_vulkan_inference_gpu.pynew file mode 100644index 00000000000..5ceaaeae2cb--- /dev/null+++ b/studio/backend/tests/test_mixed_cuda_vulkan_inference_gpu.py@@ -0,0 +1,335 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""A llama.cpp built with CUDA and Vulkan also runs on the cards only Vulkan reaches (#15444): an+AMD card beside NVIDIA ones belongs in the GGUF inventory, a Vulkan view of an NVIDIA card does+not."""++import ctypes+import subprocess+from types import SimpleNamespace++import pytest++import main++GIB_MIB = 1024+++def _cuda_device(+ index,+ name,+ total_gb = 7.71,+):+ return {+ "index": index,+ "index_kind": "physical",+ "visible_ordinal": index,+ "name": name,+ "memory_total_gb": total_gb,+ "vram_used_gb": 0.3,+ "vram_free_gb": round(total_gb - 0.3, 2),+ "vram_utilization_pct": 3.9,+ }+++CUDA_INFO = {+ "available": True,+ "backend": "cuda",+ "backend_cuda_visible_devices": None,+ "parent_visible_gpu_ids": [0, 1],+ "devices": [+ _cuda_device(0, "NVIDIA GeForce RTX 3070"),+ _cuda_device(1, "NVIDIA GeForce RTX 3060"),+ ],+ "index_kind": "physical",+ "gguf_gpu_ids_supported": True,+ "vram_used_gb_aggregate": None,+}+++def _vulkan_row(index, name, total_mib, free_mib, pci_id):+ return {+ "index": index,+ "free_mib": free_mib,+ "is_igpu": False,+ "total_mib": total_mib,+ "name": name,+ "type_known": True,+ "pci_id": pci_id,+ }+++# The reporter's llama.cpp --list-devices: Vulkan0 and Vulkan2 are the CUDA cards again.+REPORTED_ROWS = [+ _vulkan_row(0, "NVIDIA GeForce RTX 3060", 8438, 7932, "0000:a1:00.0"),+ _vulkan_row(1, "AMD Radeon Pro V620 (RADV NAVI21)", 30704, 30687, "0000:04:00.0"),+ _vulkan_row(2, "NVIDIA GeForce RTX 3070", 8438, 7778, "0000:05:00.0"),+]+# nvidia-smi rows: 0 is the 3070 at 05:00.0, 1 the 3060 at a1:00.0.+SMI_BUS_IDS = {0: (0, "05:00.0"), 1: (0, "A1:00.0")}+++def mixed_build(monkeypatch):+ from core.inference.llama_cpp import LlamaCppBackend+ from utils.hardware import nvidia++ state = {+ "backends": frozenset({"base", "cpu", "cuda", "vulkan"}),+ "rows": REPORTED_ROWS,+ "bus_ids": SMI_BUS_IDS,+ "probed": 0,+ }++ def inventory(binary = None):+ state["probed"] += 1+ return [dict(row) for row in state["rows"]]++ monkeypatch.setattr(+ LlamaCppBackend,+ "_installed_ggml_backends",+ staticmethod(lambda binary = None: state["backends"]),+ )+ monkeypatch.setattr(LlamaCppBackend, "vulkan_device_inventory", staticmethod(inventory))+ monkeypatch.setattr(nvidia, "get_pci_bus_ids", lambda: dict(state["bus_ids"]))+ return state+++def _mixed(info = CUDA_INFO):+ from utils.hardware import get_mixed_vulkan_inference_gpu_info+ return get_mixed_vulkan_inference_gpu_info(info)+++def test_amd_card_beside_cuda_cards_joins_the_inference_inventory(mixed_build):+ info = _mixed()++ assert info["backend"] == "cuda+vulkan"+ names = [d["name"] for d in info["devices"]]+ assert names == [+ "NVIDIA GeForce RTX 3070",+ "NVIDIA GeForce RTX 3060",+ "AMD Radeon Pro V620 (RADV NAVI21)",+ ]+ amd = info["devices"][2]+ assert amd["index_kind"] == "vulkan"+ assert amd["index"] == 1+ assert amd["memory_total_gb"] == round(30704 / GIB_MIB, 2)+ assert amd["vram_free_gb"] == round(30687 / GIB_MIB, 2)+ # 7.71 + 7.71 + 29.98: the budget the model registry compares a GGUF against.+ assert round(sum(d["memory_total_gb"] for d in info["devices"]), 2) == 45.4+ # The CUDA rows are untouched and the input is not mutated.+ assert info["devices"][:2] == CUDA_INFO["devices"]+ assert len(CUDA_INFO["devices"]) == 2+++def test_pci_ids_match_even_when_names_differ(mixed_build):+ mixed_build["rows"] = [+ _vulkan_row(0, "NVIDIA GeForce RTX 3060 (Vulkan)", 8438, 7932, "0000:a1:00.0"),+ _vulkan_row(1, "AMD Radeon Pro V620", 30704, 30687, "0000:04:00.0"),+ _vulkan_row(2, "NVIDIA RTX 3070 (Vulkan)", 8438, 7778, "0000:05:00.0"),+ ]++ info = _mixed()++ assert [d["name"] for d in info["devices"][2:]] == ["AMD Radeon Pro V620"]+++def test_cuda_hidden_nvidia_card_is_still_counted_as_llama_cpp_uses_it(mixed_build):+ """llama.cpp only skips a Vulkan device a CUDA one already holds; a card CUDA_VISIBLE_DEVICES+ hides is not held, so Vulkan runs on it."""+ info = _mixed({**CUDA_INFO, "devices": [CUDA_INFO["devices"][1]]})++ assert [d["name"] for d in info["devices"]] == [+ "NVIDIA GeForce RTX 3060",+ "AMD Radeon Pro V620 (RADV NAVI21)",+ "NVIDIA GeForce RTX 3070",+ ]+++def test_probe_without_pci_ids_falls_back_to_names(mixed_build):+ mixed_build["rows"] = [{**row, "pci_id": ""} for row in REPORTED_ROWS]++ info = _mixed()++ assert [d["name"] for d in info["devices"][2:]] == ["AMD Radeon Pro V620 (RADV NAVI21)"]+++def test_unreadable_nvidia_bus_ids_fall_back_to_names(mixed_build):+ mixed_build["bus_ids"] = {}++ info = _mixed()++ assert [d["name"] for d in info["devices"][2:]] == ["AMD Radeon Pro V620 (RADV NAVI21)"]+++def test_vulkan_seeing_only_the_cuda_cards_changes_nothing(mixed_build):+ mixed_build["rows"] = [REPORTED_ROWS[0], REPORTED_ROWS[2]]++ assert _mixed() is None+++def test_cuda_only_build_never_runs_the_vulkan_probe(mixed_build):+ mixed_build["backends"] = frozenset({"base", "cpu", "cuda"})++ assert _mixed() is None+ assert mixed_build["probed"] == 0+++def _torch_backend(+ monkeypatch,+ label,+ smi_devices = (),+):+ import utils.hardware.hardware as hw+ monkeypatch.setattr(hw, "_backend_label", lambda _device: label)+ monkeypatch.setattr(hw, "_nvidia_inference_devices", lambda: [dict(d) for d in smi_devices])+++def test_cuda_torch_with_no_cards_is_left_alone(monkeypatch, mixed_build):+ _torch_backend(monkeypatch, "cuda", CUDA_INFO["devices"])++ assert _mixed({**CUDA_INFO, "devices": []}) is None+ assert mixed_build["probed"] == 0++[email protected]("info", [None, {"available": False, "backend": "cpu", "devices": []}])+def test_without_cuda_torch_the_nvidia_smi_rows_stand_in(monkeypatch, mixed_build, info):+ _torch_backend(monkeypatch, "cpu", CUDA_INFO["devices"])++ mixed = _mixed(info)++ assert [d["name"] for d in mixed["devices"]] == [+ "NVIDIA GeForce RTX 3070",+ "NVIDIA GeForce RTX 3060",+ "AMD Radeon Pro V620 (RADV NAVI21)",+ ]+ # nvidia-smi row numbers are not pinnable ordinals.+ assert mixed["gguf_gpu_ids_supported"] is False+++def test_without_cuda_torch_or_nvidia_smi_nothing_changes(monkeypatch, mixed_build):+ _torch_backend(monkeypatch, "cpu", ())++ assert _mixed(None) is None+ assert mixed_build["probed"] == 0+++def test_integrated_gpu_is_left_out_as_llama_cpp_skips_it_beside_discrete_cards(mixed_build):+ mixed_build["rows"] = [+ *REPORTED_ROWS,+ {**_vulkan_row(3, "AMD Radeon(TM) Graphics", 8142, 7734, ""), "is_igpu": True},+ ]++ info = _mixed()++ assert [d["name"] for d in info["devices"][2:]] == ["AMD Radeon Pro V620 (RADV NAVI21)"]+++def test_only_an_integrated_gpu_beside_cuda_changes_nothing(mixed_build):+ mixed_build["rows"] = [+ REPORTED_ROWS[0],+ {**_vulkan_row(1, "AMD Radeon(TM) Graphics", 8142, 7734, ""), "is_igpu": True},+ ]++ assert _mixed() is None+++def test_failed_vulkan_probe_keeps_the_cuda_inventory(mixed_build):+ mixed_build["rows"] = []++ assert _mixed() is None+++def _system_info(monkeypatch):+ import utils.hardware as hardware+ from core.inference.llama_cpp import LlamaCppBackend++ monkeypatch.setattr(+ hardware,+ "get_backend_visible_gpu_info",+ lambda: {k: v for k, v in CUDA_INFO.items() if k != "gguf_gpu_ids_supported"},+ )+ monkeypatch.setattr(+ hardware,+ "get_visible_gpu_utilization",+ lambda: {"backend": "cuda", "devices": []},+ )+ monkeypatch.setattr(hardware, "get_vulkan_inference_gpu_info", lambda: None)+ monkeypatch.setattr(hardware, "get_cross_vendor_inference_gpu_info", lambda: None)+ monkeypatch.setattr(LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda *a: False))+ monkeypatch.setattr(LlamaCppBackend, "_backend_lacks_gpu_lib", staticmethod(lambda *a: False))+ monkeypatch.setattr(main, "_system_gpu_cache", None)+ return main._get_cached_system_gpu_info(SimpleNamespace(debug = lambda *a: None))+++def test_system_endpoint_reports_the_amd_card_for_gguf(monkeypatch, mixed_build):+ gpu, inference_gpu = _system_info(monkeypatch)++ # Training still sees the CUDA cards only; GGUF inference sees all three.+ assert [d["name"] for d in gpu["devices"]] == [d["name"] for d in CUDA_INFO["devices"]]+ assert inference_gpu["backend"] == "cuda+vulkan"+ assert len(inference_gpu["devices"]) == 3+ assert inference_gpu["devices"][2]["name"] == "AMD Radeon Pro V620 (RADV NAVI21)"+ # CUDA pins keep working through the training inventory.+ assert inference_gpu["gguf_gpu_ids_supported"] == gpu["gguf_gpu_ids_supported"]+++def test_system_endpoint_is_unchanged_on_a_cuda_only_build(monkeypatch, mixed_build):+ mixed_build["backends"] = frozenset({"base", "cpu", "cuda"})++ gpu, inference_gpu = _system_info(monkeypatch)++ assert inference_gpu is gpu+++def test_probe_rows_carry_the_pci_id_column(monkeypatch, tmp_path):+ from core.inference import llama_cpp++ lib = "ggml-vulkan.dll" if llama_cpp.sys.platform == "win32" else "libggml-vulkan.so"+ (tmp_path / lib).write_bytes(b"stub")+ binary = tmp_path / "llama-server"+ binary.write_bytes(b"stub")+ stdout = "\n".join(+ [+ "0\t8847884288\t0\t8847884288\tNVIDIA GeForce RTX 3060\t1\t0000:a1:00.0",+ "1\t32177455104\t0\t32195477504\tAMD Radeon Pro V620 (RADV NAVI21)\t1\t0000:04:00.0",+ "2\t8155824128\t0\t8847884288\tOld Build Card\t1",+ ]+ )+ monkeypatch.setattr(+ llama_cpp.subprocess,+ "run",+ lambda cmd, *a, **k: subprocess.CompletedProcess(cmd, 0, stdout = stdout, stderr = ""),+ )++ rows = llama_cpp.LlamaCppBackend._run_vulkan_probe(str(binary))++ assert [r["pci_id"] for r in rows] == ["0000:a1:00.0", "0000:04:00.0", ""]+ assert rows[1]["name"] == "AMD Radeon Pro V620 (RADV NAVI21)"+++def _fake_get_props(device_id):+ def get_props(_dev, props_ref):+ props_ref._obj.device_id = device_id++ return get_props+++def test_probe_reads_the_pci_id_from_device_props():+ from core.inference._vulkan_probe import _pci_device_id+ raw = ctypes.create_string_buffer(b"0000:A1:00.0")+ assert _pci_device_id(_fake_get_props(ctypes.addressof(raw)), 1) == "0000:a1:00.0"++[email protected]("device_id", [None, 0, 1])+def test_probe_ignores_a_props_slot_an_older_ggml_leaves_blank(device_id):+ """A ggml without device_id writes a caps bool, or nothing, where the pointer would be."""+ from core.inference._vulkan_probe import _pci_device_id+ assert _pci_device_id(_fake_get_props(device_id), 1) == ""+++def test_probe_rejects_a_device_id_that_is_not_a_pci_address():+ from core.inference._vulkan_probe import _pci_device_id+ raw = ctypes.create_string_buffer(b"virtual-device-7")+ assert _pci_device_id(_fake_get_props(ctypes.addressof(raw)), 1) == ""diff --git a/studio/backend/utils/hardware/__init__.py b/studio/backend/utils/hardware/__init__.pyindex 8ac806fab18..779df4ffded 100644--- a/studio/backend/utils/hardware/__init__.py+++ b/studio/backend/utils/hardware/__init__.py@@ -27,6 +27,7 @@ from .hardware import (get_backend_visible_gpu_info,get_vulkan_inference_gpu_info,get_cross_vendor_inference_gpu_info,+ get_mixed_vulkan_inference_gpu_info,get_physical_gpu_count,get_visible_gpu_count,get_parent_visible_gpu_ids,@@ -111,6 +112,7 @@ __all__ = ["get_backend_visible_gpu_info","get_vulkan_inference_gpu_info","get_cross_vendor_inference_gpu_info",+ "get_mixed_vulkan_inference_gpu_info","get_physical_gpu_count","get_visible_gpu_count","get_parent_visible_gpu_ids",diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.pyindex 8ef96ed815e..44ccaaad2a8 100644--- a/studio/backend/utils/hardware/hardware.py+++ b/studio/backend/utils/hardware/hardware.py@@ -6996,28 +6996,7 @@ def get_vulkan_inference_gpu_info() -> Optional[Dict[str, Any]]:}try:for row in LlamaCppBackend.vulkan_device_inventory():- ordinal = row["index"]- shared_memory = bool(row["is_igpu"])- free_mib = _apply_igpu_host_reserve_mib(row["free_mib"], shared_memory)- total_mib = 0 if shared_memory else row["total_mib"]- budget_mib = total_mib or free_mib- used_mib = max(0, total_mib - free_mib) if total_mib else None- result["devices"].append(- {- "index": ordinal,- # ggml Vulkan ordinals are the space `--device Vulkan<i>` pins, so unlike a torch-xpu relative ordinal these are selectable.- "index_kind": "vulkan",- "visible_ordinal": ordinal,- "name": row["name"],- "memory_total_gb": round(budget_mib / 1024, 2),- "vram_used_gb": round(used_mib / 1024, 2) if used_mib is not None else None,- "vram_free_gb": round(free_mib / 1024, 2),- "vram_utilization_pct": round((used_mib / total_mib) * 100, 1)- if used_mib is not None and total_mib > 0- else None,- "shared_memory": shared_memory,- }- )+ result["devices"].append(_vulkan_inference_device(row, _apply_igpu_host_reserve_mib))except Exception as e:logger.debug("Vulkan GPU visibility query failed: %s", e)return result@@ -7026,6 +7005,107 @@ def get_vulkan_inference_gpu_info() -> Optional[Dict[str, Any]]:return result+def _vulkan_inference_device(row: Dict[str, Any], apply_igpu_host_reserve_mib) -> Dict[str, Any]:+ ordinal = row["index"]+ shared_memory = bool(row["is_igpu"])+ free_mib = apply_igpu_host_reserve_mib(row["free_mib"], shared_memory)+ total_mib = 0 if shared_memory else row["total_mib"]+ budget_mib = total_mib or free_mib+ used_mib = max(0, total_mib - free_mib) if total_mib else None+ return {+ "index": ordinal,+ # ggml Vulkan ordinals are the space `--device Vulkan<i>` pins, so unlike a torch-xpu relative ordinal these are selectable.+ "index_kind": "vulkan",+ "visible_ordinal": ordinal,+ "name": row["name"],+ "memory_total_gb": round(budget_mib / 1024, 2),+ "vram_used_gb": round(used_mib / 1024, 2) if used_mib is not None else None,+ "vram_free_gb": round(free_mib / 1024, 2),+ "vram_utilization_pct": round((used_mib / total_mib) * 100, 1)+ if used_mib is not None and total_mib > 0+ else None,+ "shared_memory": shared_memory,+ }+++def get_mixed_vulkan_inference_gpu_info(+ cuda_info: Optional[Dict[str, Any]],+) -> Optional[Dict[str, Any]]:+ """``cuda_info`` plus the Vulkan devices a llama.cpp build shipping both CUDA and Vulkan also+ runs on (an AMD card beside NVIDIA ones), or None when it has none. llama.cpp skips a Vulkan+ device whose PCI id a CUDA one already holds; a probe without ids falls back to the name.+ Without a CUDA torch, ``cuda_info`` is not CUDA's and the nvidia-smi rows stand in for it."""+ try:+ from core.inference.llama_cpp import (+ LlamaCppBackend,+ _apply_igpu_host_reserve_mib,+ )++ if not {"cuda", "vulkan"} <= LlamaCppBackend._installed_ggml_backends():+ return None+ if not cuda_info or cuda_info.get("backend") != "cuda":+ if _backend_label(get_device()) == "cuda":+ return None+ devices = _nvidia_inference_devices()+ # Capacity-less procfs rows read as 0 GB, as in get_cross_vendor_inference_gpu_info.+ if not devices or not all((d["memory_total_gb"] or 0) > 0 for d in devices):+ return None+ # SMI row numbers, not the ordinals a pin is applied in.+ cuda_info = {+ "available": True,+ "backend": "cuda",+ "backend_cuda_visible_devices": None,+ "parent_visible_gpu_ids": [],+ "devices": devices,+ "gguf_gpu_ids_supported": False,+ }+ if not cuda_info.get("devices"):+ return None+ rows = LlamaCppBackend.vulkan_device_inventory()+ except Exception as e:+ logger.debug("Could not inspect the llama.cpp Vulkan devices beside CUDA: %s", e)+ return None+ if not rows:+ return None++ from . import nvidia++ cuda_devices = cuda_info["devices"]+ bus_ids = nvidia.get_pci_bus_ids() if any(row.get("pci_id") for row in rows) else {}+ cuda_bus_ids = {+ bus_ids[d["index"]]+ for d in cuda_devices+ if d.get("index_kind") == "physical" and d.get("index") in bus_ids+ }+ # By id only when every CUDA row has one, else a card nvidia-smi missed would count twice.+ by_id = bool(cuda_bus_ids) and len(cuda_bus_ids) == len(cuda_devices)+ cuda_names = [d.get("name") for d in cuda_devices]+ extra = []+ for row in rows:+ # llama.cpp adds an integrated GPU only when no discrete one was found, and CUDA's are.+ if row["is_igpu"]:+ continue+ key = nvidia._normalize_bus_id(row.get("pci_id") or "")+ if by_id and key is not None:+ covered = key in cuda_bus_ids+ else:+ covered = row["name"] in cuda_names+ if covered:+ cuda_names.remove(row["name"])+ if not covered:+ extra.append(_vulkan_inference_device(row, _apply_igpu_host_reserve_mib))+ if not extra:+ return None+ return {+ **cuda_info,+ "backend": "cuda+vulkan",+ "devices": [*cuda_devices, *extra],+ "index_kind": "mixed",+ # A host aggregate counted on the CUDA cards does not describe the Vulkan ones.+ "vram_used_gb_aggregate": None,+ }++def _installed_llama_backend() -> Optional[str]:"""The backend the installed llama.cpp prebuilt records, as Settings shows it."""from core.inference.llama_cpp import LlamaCppBackenddiff --git a/studio/backend/utils/hardware/nvidia.py b/studio/backend/utils/hardware/nvidia.pyindex 2b73fcd97a4..776b550c890 100644--- a/studio/backend/utils/hardware/nvidia.py+++ b/studio/backend/utils/hardware/nvidia.py@@ -758,6 +758,32 @@ def get_cuda_ordinal_smi_rows() -> Optional[list[int]]:return rows or None+def get_pci_bus_ids() -> dict[int, tuple[int, str]]:+ """nvidia-smi row index -> normalized PCI bus id; {} when nvidia-smi does not answer."""+ try:+ smi = gpu_query.run_nvidia_smi(+ [_nvidia_smi_executable(), "--query-gpu=index,pci.bus_id", "--format=csv,noheader"],+ capture_output = True,+ text = True,+ encoding = "utf-8",+ errors = "replace",+ timeout = 5,+ env = child_env_without_native_path_secret(),+ **_windows_hidden_subprocess_kwargs(),+ )+ except (OSError, subprocess.TimeoutExpired):+ return {}+ if smi.returncode != 0:+ return {}+ bus_ids: dict[int, tuple[int, str]] = {}+ for line in smi.stdout.strip().splitlines():+ index, _, bus_id = (part.strip() for part in line.partition(","))+ key = _normalize_bus_id(bus_id)+ if index.isdigit() and key is not None:+ bus_ids[int(index)] = key+ return bus_ids++def get_physical_gpu_inventory() -> dict[str, Any]:"""Every NVIDIA GPU the driver enumerates, with no visibility mask and no torch. Display-only inventory: ``index`` is nvidia-smi's own row number, a physical id and NOT something a caller may pin, because the whole point of this probe is that PyTorch cannot open these devices. A failed probe comes back as a structured unavailable result, so this never raises out of an endpoint."""rows = _query_gpu_inventory("get_physical_gpu_inventory")