Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: count the cards a CUDA + Vulkan llama.cpp reaches only through Vulkan (#15466)

Debojyoti Banerjeed4fa58c797cf
diff --git a/studio/backend/core/inference/_vulkan_probe.py b/studio/backend/core/inference/_vulkan_probe.py
index 31effc54142..731be44483b 100644
--- a/studio/backend/core/inference/_vulkan_probe.py
+++ b/studio/backend/core/inference/_vulkan_probe.py
@@ -6,33 +6,70 @@
Run in a short-lived subprocess (``python _vulkan_probe.py <bindir>``) so the
Vulkan instance never lives in the long-running backend process. Loads the
bundled ggml Vulkan backend from ``<bindir>`` and prints one
-``<idx>\\t<free_bytes>\\t<is_igpu>\\t<total_bytes>\\t<name>\\t<type_known>`` line per device to
+``<idx>\\t<free_bytes>\\t<is_igpu>\\t<total_bytes>\\t<name>\\t<type_known>\\t<pci_id>`` line per device to
stdout. Indices are ggml's own Vulkan device ordinals, which need not match
nvidia-smi order. ``is_igpu`` (from ggml's device type) is ``1`` for an
integrated GPU sharing system RAM. ``total_bytes`` is the device-local heap;
the reader uses it to reserve absolute headroom on a discrete card (parity
with the CUDA/ROCm fit) and ignores it for an iGPU, whose "VRAM" is shared
system RAM. ``name`` is ggml's device description (the marketing name, e.g.
-"AMD Radeon RX 9070 XT"); empty when the registry lookup fails.
+"AMD Radeon RX 9070 XT"); empty when the registry lookup fails. ``pci_id`` is
+the lower-case PCI bus id ggml reports, empty on a build that does not.
Uses only the standard library so it stays runnable as a bare script.
"""
import ctypes
import os
+import re
import sys
# ggml_backend_dev_type enum (ggml-backend.h): CPU=0, GPU=1, IGPU=2, ...
_GGML_BACKEND_DEVICE_TYPE_IGPU = 2
+# ggml_backend_dev_props.device_id for a PCI device: "domain:bus:device.function".
+_PCI_ID_RE = re.compile(r"^[0-9a-f]{4,8}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]$")
# winbase.h
_SEM_FAILCRITICALERRORS = 0x0001
-def _igpu_flags_and_names(base, lib, count: int) -> tuple[list[bool], list[str], list[bool]]:
- """Per-device iGPU flags, names, and whether each was read; an unread type makes DirectIO buffer iGPU weights. All-False on failure; reg index = ordinal"""
+class _DevProps(ctypes.Structure):
+ """ggml_backend_dev_props up to device_id, then slack: a ggml older than device_id writes
+ less, and the zeroed or bool-sized bytes it leaves in that slot never pass as a pointer."""
+
+ _fields_ = [
+ ("name", ctypes.c_void_p),
+ ("description", ctypes.c_void_p),
+ ("memory_free", ctypes.c_size_t),
+ ("memory_total", ctypes.c_size_t),
+ ("type", ctypes.c_int),
+ ("device_id", ctypes.c_void_p),
+ ("_tail", ctypes.c_ubyte * 64),
+ ]
+
+
+def _pci_device_id(get_props, dev) -> str:
+ """Lower-case PCI bus id ("0000:c1:00.0"), which llama.cpp matches to skip a Vulkan device a
+ CUDA one already drives; empty when the build does not report it."""
+ props = _DevProps()
+ try:
+ get_props(dev, ctypes.byref(props))
+ if not props.device_id or props.device_id < 0x10000:
+ return ""
+ raw = ctypes.cast(props.device_id, ctypes.c_char_p).value or b""
+ except Exception:
+ return ""
+ value = raw[:64].decode("ascii", errors = "replace").strip().lower()
+ return value if _PCI_ID_RE.match(value) else ""
+
+
+def _igpu_flags_and_names(
+ base, lib, count: int
+) -> tuple[list[bool], list[str], list[bool], list[str]]:
+ """Per-device iGPU flags, names, whether each was read, and PCI ids; an unread type makes DirectIO buffer iGPU weights. All-False on failure; reg index = ordinal"""
flags = [False] * count
names = [""] * count
known = [False] * count
+ pci_ids = [""] * count
try:
lib.ggml_backend_vk_reg.restype = ctypes.c_void_p
lib.ggml_backend_vk_reg.argtypes = []
@@ -44,11 +81,11 @@ def _igpu_flags_and_names(base, lib, count: int) -> tuple[list[bool], list[str],
base.ggml_backend_dev_type.argtypes = [ctypes.c_void_p]
reg = lib.ggml_backend_vk_reg()
if not reg:
- return flags, names, known
+ return flags, names, known, pci_ids
dev_count = base.ggml_backend_reg_dev_count(reg)
except Exception:
# Best-effort: any failure degrades to "discrete"/"unnamed" so memory readings still get through
- return flags, names, known
+ return flags, names, known, pci_ids
# Bound outside the type-detection try so a ggml-base lacking the description symbol still reads iGPU flags, else iGPU shared RAM counts as VRAM
name_functions = []
@@ -60,6 +97,12 @@ def _igpu_flags_and_names(base, lib, count: int) -> tuple[list[bool], list[str],
name_functions.append(function)
except Exception:
pass
+ try:
+ get_props = base.ggml_backend_dev_get_props
+ get_props.restype = None
+ get_props.argtypes = [ctypes.c_void_p, ctypes.c_void_p]
+ except Exception:
+ get_props = None
for i in range(min(count, dev_count)):
try:
@@ -83,7 +126,9 @@ def _igpu_flags_and_names(base, lib, count: int) -> tuple[list[bool], list[str],
name = raw_name.decode("utf-8", errors = "replace")
names[i] = name.replace("\t", " ").replace("\r", " ").replace("\n", " ").strip()
break
- return flags, names, known
+ if get_props is not None:
+ pci_ids[i] = _pci_device_id(get_props, dev)
+ return flags, names, known, pci_ids
def main() -> int:
@@ -148,14 +193,14 @@ def main() -> int:
]
count = lib.ggml_backend_vk_get_device_count()
- igpu, names, known = _igpu_flags_and_names(base, lib, count)
+ igpu, names, known, pci_ids = _igpu_flags_and_names(base, lib, count)
rows = []
for i in range(count):
free, total = ctypes.c_size_t(0), ctypes.c_size_t(0)
lib.ggml_backend_vk_get_device_memory(i, ctypes.byref(free), ctypes.byref(total))
rows.append(
- "%d\t%d\t%d\t%d\t%s\t%d"
- % (i, free.value, int(igpu[i]), total.value, names[i], int(known[i]))
+ "%d\t%d\t%d\t%d\t%s\t%d\t%s"
+ % (i, free.value, int(igpu[i]), total.value, names[i], int(known[i]), pci_ids[i])
)
sys.stdout.write("\n".join(rows))
return 0
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 281611a972f..94ebbf0bbb5 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -14677,8 +14677,8 @@ class LlamaCppBackend:
rows: list[dict] = []
for line in result.stdout.strip().splitlines():
parts = line.split("\t")
- # 4 columns from an older probe (no name); 5 with the name column.
- if len(parts) not in (4, 5, 6):
+ # 4 columns from an older probe (no name); 5 with the name column, 7 with the PCI id.
+ if len(parts) not in (4, 5, 6, 7):
continue
try:
free_b, total_b = int(parts[1]), int(parts[3])
@@ -14694,6 +14694,7 @@ class LlamaCppBackend:
"name": parts[4].strip() if len(parts) >= 5 else "",
# Absent on an older staged probe, so it reads unknown and declines.
"type_known": len(parts) >= 6 and parts[5].strip() == "1",
+ "pci_id": parts[6].strip().lower() if len(parts) >= 7 else "",
}
)
except ValueError:
diff --git a/studio/backend/main.py b/studio/backend/main.py
index 6f5d90b77fd..6a2affe266c 100644
--- a/studio/backend/main.py
+++ b/studio/backend/main.py
@@ -2321,6 +2321,7 @@ def _get_cached_system_gpu_info(
amd_driver_warning_report,
get_backend_visible_gpu_info,
get_cross_vendor_inference_gpu_info,
+ get_mixed_vulkan_inference_gpu_info,
get_visible_gpu_utilization,
get_vulkan_inference_gpu_info,
gpu_query,
@@ -2449,6 +2450,16 @@ def _get_cached_system_gpu_info(
inference_gpu_info = {**cross_vendor_info, "gguf_gpu_ids_supported": False}
else:
inference_gpu_info = gpu_info
+ if vulkan_info is None:
+ # A CUDA + Vulkan llama.cpp also runs on the cards only Vulkan reaches (#15444).
+ with (
+ contextlib.nullcontext()
+ if refresh_memory
+ else gpu_query.display_reads(max_stale = 0)
+ ):
+ mixed_info = get_mixed_vulkan_inference_gpu_info(inference_gpu_info)
+ if mixed_info is not None:
+ inference_gpu_info = mixed_info
combined_info = (gpu_info, inference_gpu_info)
_system_gpu_cache = (time.monotonic(), combined_info)
diff --git a/studio/backend/tests/test_llama_cpp_vulkan_probe.py b/studio/backend/tests/test_llama_cpp_vulkan_probe.py
index 06caca27c92..39f0ae395fe 100644
--- a/studio/backend/tests/test_llama_cpp_vulkan_probe.py
+++ b/studio/backend/tests/test_llama_cpp_vulkan_probe.py
@@ -64,11 +64,12 @@ def test_missing_description_symbol_keeps_igpu_detection():
)
lib = _types.SimpleNamespace(ggml_backend_vk_reg = _FakeCFunction(1))
- flags, names, known = _igpu_flags_and_names(base, lib, 1)
+ flags, names, known, pci_ids = _igpu_flags_and_names(base, lib, 1)
assert flags == [True]
assert names == ["Legacy Vulkan iGPU"]
assert known == [True]
+ assert pci_ids == [""]
def _make_vulkan_install(tmp_path: Path) -> str:
diff --git a/studio/backend/tests/test_mixed_cuda_vulkan_inference_gpu.py b/studio/backend/tests/test_mixed_cuda_vulkan_inference_gpu.py
new file mode 100644
index 00000000000..5ceaaeae2cb
--- /dev/null
+++ b/studio/backend/tests/test_mixed_cuda_vulkan_inference_gpu.py
@@ -0,0 +1,335 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""A llama.cpp built with CUDA and Vulkan also runs on the cards only Vulkan reaches (#15444): an
+AMD card beside NVIDIA ones belongs in the GGUF inventory, a Vulkan view of an NVIDIA card does
+not."""
+
+import ctypes
+import subprocess
+from types import SimpleNamespace
+
+import pytest
+
+import main
+
+GIB_MIB = 1024
+
+
+def _cuda_device(
+ index,
+ name,
+ total_gb = 7.71,
+):
+ return {
+ "index": index,
+ "index_kind": "physical",
+ "visible_ordinal": index,
+ "name": name,
+ "memory_total_gb": total_gb,
+ "vram_used_gb": 0.3,
+ "vram_free_gb": round(total_gb - 0.3, 2),
+ "vram_utilization_pct": 3.9,
+ }
+
+
+CUDA_INFO = {
+ "available": True,
+ "backend": "cuda",
+ "backend_cuda_visible_devices": None,
+ "parent_visible_gpu_ids": [0, 1],
+ "devices": [
+ _cuda_device(0, "NVIDIA GeForce RTX 3070"),
+ _cuda_device(1, "NVIDIA GeForce RTX 3060"),
+ ],
+ "index_kind": "physical",
+ "gguf_gpu_ids_supported": True,
+ "vram_used_gb_aggregate": None,
+}
+
+
+def _vulkan_row(index, name, total_mib, free_mib, pci_id):
+ return {
+ "index": index,
+ "free_mib": free_mib,
+ "is_igpu": False,
+ "total_mib": total_mib,
+ "name": name,
+ "type_known": True,
+ "pci_id": pci_id,
+ }
+
+
+# The reporter's llama.cpp --list-devices: Vulkan0 and Vulkan2 are the CUDA cards again.
+REPORTED_ROWS = [
+ _vulkan_row(0, "NVIDIA GeForce RTX 3060", 8438, 7932, "0000:a1:00.0"),
+ _vulkan_row(1, "AMD Radeon Pro V620 (RADV NAVI21)", 30704, 30687, "0000:04:00.0"),
+ _vulkan_row(2, "NVIDIA GeForce RTX 3070", 8438, 7778, "0000:05:00.0"),
+]
+# nvidia-smi rows: 0 is the 3070 at 05:00.0, 1 the 3060 at a1:00.0.
+SMI_BUS_IDS = {0: (0, "05:00.0"), 1: (0, "A1:00.0")}
+
+
+def mixed_build(monkeypatch):
+ from core.inference.llama_cpp import LlamaCppBackend
+ from utils.hardware import nvidia
+
+ state = {
+ "backends": frozenset({"base", "cpu", "cuda", "vulkan"}),
+ "rows": REPORTED_ROWS,
+ "bus_ids": SMI_BUS_IDS,
+ "probed": 0,
+ }
+
+ def inventory(binary = None):
+ state["probed"] += 1
+ return [dict(row) for row in state["rows"]]
+
+ monkeypatch.setattr(
+ LlamaCppBackend,
+ "_installed_ggml_backends",
+ staticmethod(lambda binary = None: state["backends"]),
+ )
+ monkeypatch.setattr(LlamaCppBackend, "vulkan_device_inventory", staticmethod(inventory))
+ monkeypatch.setattr(nvidia, "get_pci_bus_ids", lambda: dict(state["bus_ids"]))
+ return state
+
+
+def _mixed(info = CUDA_INFO):
+ from utils.hardware import get_mixed_vulkan_inference_gpu_info
+ return get_mixed_vulkan_inference_gpu_info(info)
+
+
+def test_amd_card_beside_cuda_cards_joins_the_inference_inventory(mixed_build):
+ info = _mixed()
+
+ assert info["backend"] == "cuda+vulkan"
+ names = [d["name"] for d in info["devices"]]
+ assert names == [
+ "NVIDIA GeForce RTX 3070",
+ "NVIDIA GeForce RTX 3060",
+ "AMD Radeon Pro V620 (RADV NAVI21)",
+ ]
+ amd = info["devices"][2]
+ assert amd["index_kind"] == "vulkan"
+ assert amd["index"] == 1
+ assert amd["memory_total_gb"] == round(30704 / GIB_MIB, 2)
+ assert amd["vram_free_gb"] == round(30687 / GIB_MIB, 2)
+ # 7.71 + 7.71 + 29.98: the budget the model registry compares a GGUF against.
+ assert round(sum(d["memory_total_gb"] for d in info["devices"]), 2) == 45.4
+ # The CUDA rows are untouched and the input is not mutated.
+ assert info["devices"][:2] == CUDA_INFO["devices"]
+ assert len(CUDA_INFO["devices"]) == 2
+
+
+def test_pci_ids_match_even_when_names_differ(mixed_build):
+ mixed_build["rows"] = [
+ _vulkan_row(0, "NVIDIA GeForce RTX 3060 (Vulkan)", 8438, 7932, "0000:a1:00.0"),
+ _vulkan_row(1, "AMD Radeon Pro V620", 30704, 30687, "0000:04:00.0"),
+ _vulkan_row(2, "NVIDIA RTX 3070 (Vulkan)", 8438, 7778, "0000:05:00.0"),
+ ]
+
+ info = _mixed()
+
+ assert [d["name"] for d in info["devices"][2:]] == ["AMD Radeon Pro V620"]
+
+
+def test_cuda_hidden_nvidia_card_is_still_counted_as_llama_cpp_uses_it(mixed_build):
+ """llama.cpp only skips a Vulkan device a CUDA one already holds; a card CUDA_VISIBLE_DEVICES
+ hides is not held, so Vulkan runs on it."""
+ info = _mixed({**CUDA_INFO, "devices": [CUDA_INFO["devices"][1]]})
+
+ assert [d["name"] for d in info["devices"]] == [
+ "NVIDIA GeForce RTX 3060",
+ "AMD Radeon Pro V620 (RADV NAVI21)",
+ "NVIDIA GeForce RTX 3070",
+ ]
+
+
+def test_probe_without_pci_ids_falls_back_to_names(mixed_build):
+ mixed_build["rows"] = [{**row, "pci_id": ""} for row in REPORTED_ROWS]
+
+ info = _mixed()
+
+ assert [d["name"] for d in info["devices"][2:]] == ["AMD Radeon Pro V620 (RADV NAVI21)"]
+
+
+def test_unreadable_nvidia_bus_ids_fall_back_to_names(mixed_build):
+ mixed_build["bus_ids"] = {}
+
+ info = _mixed()
+
+ assert [d["name"] for d in info["devices"][2:]] == ["AMD Radeon Pro V620 (RADV NAVI21)"]
+
+
+def test_vulkan_seeing_only_the_cuda_cards_changes_nothing(mixed_build):
+ mixed_build["rows"] = [REPORTED_ROWS[0], REPORTED_ROWS[2]]
+
+ assert _mixed() is None
+
+
+def test_cuda_only_build_never_runs_the_vulkan_probe(mixed_build):
+ mixed_build["backends"] = frozenset({"base", "cpu", "cuda"})
+
+ assert _mixed() is None
+ assert mixed_build["probed"] == 0
+
+
+def _torch_backend(
+ monkeypatch,
+ label,
+ smi_devices = (),
+):
+ import utils.hardware.hardware as hw
+ monkeypatch.setattr(hw, "_backend_label", lambda _device: label)
+ monkeypatch.setattr(hw, "_nvidia_inference_devices", lambda: [dict(d) for d in smi_devices])
+
+
+def test_cuda_torch_with_no_cards_is_left_alone(monkeypatch, mixed_build):
+ _torch_backend(monkeypatch, "cuda", CUDA_INFO["devices"])
+
+ assert _mixed({**CUDA_INFO, "devices": []}) is None
+ assert mixed_build["probed"] == 0
+
+
[email protected]("info", [None, {"available": False, "backend": "cpu", "devices": []}])
+def test_without_cuda_torch_the_nvidia_smi_rows_stand_in(monkeypatch, mixed_build, info):
+ _torch_backend(monkeypatch, "cpu", CUDA_INFO["devices"])
+
+ mixed = _mixed(info)
+
+ assert [d["name"] for d in mixed["devices"]] == [
+ "NVIDIA GeForce RTX 3070",
+ "NVIDIA GeForce RTX 3060",
+ "AMD Radeon Pro V620 (RADV NAVI21)",
+ ]
+ # nvidia-smi row numbers are not pinnable ordinals.
+ assert mixed["gguf_gpu_ids_supported"] is False
+
+
+def test_without_cuda_torch_or_nvidia_smi_nothing_changes(monkeypatch, mixed_build):
+ _torch_backend(monkeypatch, "cpu", ())
+
+ assert _mixed(None) is None
+ assert mixed_build["probed"] == 0
+
+
+def test_integrated_gpu_is_left_out_as_llama_cpp_skips_it_beside_discrete_cards(mixed_build):
+ mixed_build["rows"] = [
+ *REPORTED_ROWS,
+ {**_vulkan_row(3, "AMD Radeon(TM) Graphics", 8142, 7734, ""), "is_igpu": True},
+ ]
+
+ info = _mixed()
+
+ assert [d["name"] for d in info["devices"][2:]] == ["AMD Radeon Pro V620 (RADV NAVI21)"]
+
+
+def test_only_an_integrated_gpu_beside_cuda_changes_nothing(mixed_build):
+ mixed_build["rows"] = [
+ REPORTED_ROWS[0],
+ {**_vulkan_row(1, "AMD Radeon(TM) Graphics", 8142, 7734, ""), "is_igpu": True},
+ ]
+
+ assert _mixed() is None
+
+
+def test_failed_vulkan_probe_keeps_the_cuda_inventory(mixed_build):
+ mixed_build["rows"] = []
+
+ assert _mixed() is None
+
+
+def _system_info(monkeypatch):
+ import utils.hardware as hardware
+ from core.inference.llama_cpp import LlamaCppBackend
+
+ monkeypatch.setattr(
+ hardware,
+ "get_backend_visible_gpu_info",
+ lambda: {k: v for k, v in CUDA_INFO.items() if k != "gguf_gpu_ids_supported"},
+ )
+ monkeypatch.setattr(
+ hardware,
+ "get_visible_gpu_utilization",
+ lambda: {"backend": "cuda", "devices": []},
+ )
+ monkeypatch.setattr(hardware, "get_vulkan_inference_gpu_info", lambda: None)
+ monkeypatch.setattr(hardware, "get_cross_vendor_inference_gpu_info", lambda: None)
+ monkeypatch.setattr(LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda *a: False))
+ monkeypatch.setattr(LlamaCppBackend, "_backend_lacks_gpu_lib", staticmethod(lambda *a: False))
+ monkeypatch.setattr(main, "_system_gpu_cache", None)
+ return main._get_cached_system_gpu_info(SimpleNamespace(debug = lambda *a: None))
+
+
+def test_system_endpoint_reports_the_amd_card_for_gguf(monkeypatch, mixed_build):
+ gpu, inference_gpu = _system_info(monkeypatch)
+
+ # Training still sees the CUDA cards only; GGUF inference sees all three.
+ assert [d["name"] for d in gpu["devices"]] == [d["name"] for d in CUDA_INFO["devices"]]
+ assert inference_gpu["backend"] == "cuda+vulkan"
+ assert len(inference_gpu["devices"]) == 3
+ assert inference_gpu["devices"][2]["name"] == "AMD Radeon Pro V620 (RADV NAVI21)"
+ # CUDA pins keep working through the training inventory.
+ assert inference_gpu["gguf_gpu_ids_supported"] == gpu["gguf_gpu_ids_supported"]
+
+
+def test_system_endpoint_is_unchanged_on_a_cuda_only_build(monkeypatch, mixed_build):
+ mixed_build["backends"] = frozenset({"base", "cpu", "cuda"})
+
+ gpu, inference_gpu = _system_info(monkeypatch)
+
+ assert inference_gpu is gpu
+
+
+def test_probe_rows_carry_the_pci_id_column(monkeypatch, tmp_path):
+ from core.inference import llama_cpp
+
+ lib = "ggml-vulkan.dll" if llama_cpp.sys.platform == "win32" else "libggml-vulkan.so"
+ (tmp_path / lib).write_bytes(b"stub")
+ binary = tmp_path / "llama-server"
+ binary.write_bytes(b"stub")
+ stdout = "\n".join(
+ [
+ "0\t8847884288\t0\t8847884288\tNVIDIA GeForce RTX 3060\t1\t0000:a1:00.0",
+ "1\t32177455104\t0\t32195477504\tAMD Radeon Pro V620 (RADV NAVI21)\t1\t0000:04:00.0",
+ "2\t8155824128\t0\t8847884288\tOld Build Card\t1",
+ ]
+ )
+ monkeypatch.setattr(
+ llama_cpp.subprocess,
+ "run",
+ lambda cmd, *a, **k: subprocess.CompletedProcess(cmd, 0, stdout = stdout, stderr = ""),
+ )
+
+ rows = llama_cpp.LlamaCppBackend._run_vulkan_probe(str(binary))
+
+ assert [r["pci_id"] for r in rows] == ["0000:a1:00.0", "0000:04:00.0", ""]
+ assert rows[1]["name"] == "AMD Radeon Pro V620 (RADV NAVI21)"
+
+
+def _fake_get_props(device_id):
+ def get_props(_dev, props_ref):
+ props_ref._obj.device_id = device_id
+
+ return get_props
+
+
+def test_probe_reads_the_pci_id_from_device_props():
+ from core.inference._vulkan_probe import _pci_device_id
+ raw = ctypes.create_string_buffer(b"0000:A1:00.0")
+ assert _pci_device_id(_fake_get_props(ctypes.addressof(raw)), 1) == "0000:a1:00.0"
+
+
[email protected]("device_id", [None, 0, 1])
+def test_probe_ignores_a_props_slot_an_older_ggml_leaves_blank(device_id):
+ """A ggml without device_id writes a caps bool, or nothing, where the pointer would be."""
+ from core.inference._vulkan_probe import _pci_device_id
+ assert _pci_device_id(_fake_get_props(device_id), 1) == ""
+
+
+def test_probe_rejects_a_device_id_that_is_not_a_pci_address():
+ from core.inference._vulkan_probe import _pci_device_id
+ raw = ctypes.create_string_buffer(b"virtual-device-7")
+ assert _pci_device_id(_fake_get_props(ctypes.addressof(raw)), 1) == ""
diff --git a/studio/backend/utils/hardware/__init__.py b/studio/backend/utils/hardware/__init__.py
index 8ac806fab18..779df4ffded 100644
--- a/studio/backend/utils/hardware/__init__.py
+++ b/studio/backend/utils/hardware/__init__.py
@@ -27,6 +27,7 @@ from .hardware import (
get_backend_visible_gpu_info,
get_vulkan_inference_gpu_info,
get_cross_vendor_inference_gpu_info,
+ get_mixed_vulkan_inference_gpu_info,
get_physical_gpu_count,
get_visible_gpu_count,
get_parent_visible_gpu_ids,
@@ -111,6 +112,7 @@ __all__ = [
"get_backend_visible_gpu_info",
"get_vulkan_inference_gpu_info",
"get_cross_vendor_inference_gpu_info",
+ "get_mixed_vulkan_inference_gpu_info",
"get_physical_gpu_count",
"get_visible_gpu_count",
"get_parent_visible_gpu_ids",
diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.py
index 8ef96ed815e..44ccaaad2a8 100644
--- a/studio/backend/utils/hardware/hardware.py
+++ b/studio/backend/utils/hardware/hardware.py
@@ -6996,28 +6996,7 @@ def get_vulkan_inference_gpu_info() -> Optional[Dict[str, Any]]:
}
try:
for row in LlamaCppBackend.vulkan_device_inventory():
- ordinal = row["index"]
- shared_memory = bool(row["is_igpu"])
- free_mib = _apply_igpu_host_reserve_mib(row["free_mib"], shared_memory)
- total_mib = 0 if shared_memory else row["total_mib"]
- budget_mib = total_mib or free_mib
- used_mib = max(0, total_mib - free_mib) if total_mib else None
- result["devices"].append(
- {
- "index": ordinal,
- # ggml Vulkan ordinals are the space `--device Vulkan<i>` pins, so unlike a torch-xpu relative ordinal these are selectable.
- "index_kind": "vulkan",
- "visible_ordinal": ordinal,
- "name": row["name"],
- "memory_total_gb": round(budget_mib / 1024, 2),
- "vram_used_gb": round(used_mib / 1024, 2) if used_mib is not None else None,
- "vram_free_gb": round(free_mib / 1024, 2),
- "vram_utilization_pct": round((used_mib / total_mib) * 100, 1)
- if used_mib is not None and total_mib > 0
- else None,
- "shared_memory": shared_memory,
- }
- )
+ result["devices"].append(_vulkan_inference_device(row, _apply_igpu_host_reserve_mib))
except Exception as e:
logger.debug("Vulkan GPU visibility query failed: %s", e)
return result
@@ -7026,6 +7005,107 @@ def get_vulkan_inference_gpu_info() -> Optional[Dict[str, Any]]:
return result
+def _vulkan_inference_device(row: Dict[str, Any], apply_igpu_host_reserve_mib) -> Dict[str, Any]:
+ ordinal = row["index"]
+ shared_memory = bool(row["is_igpu"])
+ free_mib = apply_igpu_host_reserve_mib(row["free_mib"], shared_memory)
+ total_mib = 0 if shared_memory else row["total_mib"]
+ budget_mib = total_mib or free_mib
+ used_mib = max(0, total_mib - free_mib) if total_mib else None
+ return {
+ "index": ordinal,
+ # ggml Vulkan ordinals are the space `--device Vulkan<i>` pins, so unlike a torch-xpu relative ordinal these are selectable.
+ "index_kind": "vulkan",
+ "visible_ordinal": ordinal,
+ "name": row["name"],
+ "memory_total_gb": round(budget_mib / 1024, 2),
+ "vram_used_gb": round(used_mib / 1024, 2) if used_mib is not None else None,
+ "vram_free_gb": round(free_mib / 1024, 2),
+ "vram_utilization_pct": round((used_mib / total_mib) * 100, 1)
+ if used_mib is not None and total_mib > 0
+ else None,
+ "shared_memory": shared_memory,
+ }
+
+
+def get_mixed_vulkan_inference_gpu_info(
+ cuda_info: Optional[Dict[str, Any]],
+) -> Optional[Dict[str, Any]]:
+ """``cuda_info`` plus the Vulkan devices a llama.cpp build shipping both CUDA and Vulkan also
+ runs on (an AMD card beside NVIDIA ones), or None when it has none. llama.cpp skips a Vulkan
+ device whose PCI id a CUDA one already holds; a probe without ids falls back to the name.
+ Without a CUDA torch, ``cuda_info`` is not CUDA's and the nvidia-smi rows stand in for it."""
+ try:
+ from core.inference.llama_cpp import (
+ LlamaCppBackend,
+ _apply_igpu_host_reserve_mib,
+ )
+
+ if not {"cuda", "vulkan"} <= LlamaCppBackend._installed_ggml_backends():
+ return None
+ if not cuda_info or cuda_info.get("backend") != "cuda":
+ if _backend_label(get_device()) == "cuda":
+ return None
+ devices = _nvidia_inference_devices()
+ # Capacity-less procfs rows read as 0 GB, as in get_cross_vendor_inference_gpu_info.
+ if not devices or not all((d["memory_total_gb"] or 0) > 0 for d in devices):
+ return None
+ # SMI row numbers, not the ordinals a pin is applied in.
+ cuda_info = {
+ "available": True,
+ "backend": "cuda",
+ "backend_cuda_visible_devices": None,
+ "parent_visible_gpu_ids": [],
+ "devices": devices,
+ "gguf_gpu_ids_supported": False,
+ }
+ if not cuda_info.get("devices"):
+ return None
+ rows = LlamaCppBackend.vulkan_device_inventory()
+ except Exception as e:
+ logger.debug("Could not inspect the llama.cpp Vulkan devices beside CUDA: %s", e)
+ return None
+ if not rows:
+ return None
+
+ from . import nvidia
+
+ cuda_devices = cuda_info["devices"]
+ bus_ids = nvidia.get_pci_bus_ids() if any(row.get("pci_id") for row in rows) else {}
+ cuda_bus_ids = {
+ bus_ids[d["index"]]
+ for d in cuda_devices
+ if d.get("index_kind") == "physical" and d.get("index") in bus_ids
+ }
+ # By id only when every CUDA row has one, else a card nvidia-smi missed would count twice.
+ by_id = bool(cuda_bus_ids) and len(cuda_bus_ids) == len(cuda_devices)
+ cuda_names = [d.get("name") for d in cuda_devices]
+ extra = []
+ for row in rows:
+ # llama.cpp adds an integrated GPU only when no discrete one was found, and CUDA's are.
+ if row["is_igpu"]:
+ continue
+ key = nvidia._normalize_bus_id(row.get("pci_id") or "")
+ if by_id and key is not None:
+ covered = key in cuda_bus_ids
+ else:
+ covered = row["name"] in cuda_names
+ if covered:
+ cuda_names.remove(row["name"])
+ if not covered:
+ extra.append(_vulkan_inference_device(row, _apply_igpu_host_reserve_mib))
+ if not extra:
+ return None
+ return {
+ **cuda_info,
+ "backend": "cuda+vulkan",
+ "devices": [*cuda_devices, *extra],
+ "index_kind": "mixed",
+ # A host aggregate counted on the CUDA cards does not describe the Vulkan ones.
+ "vram_used_gb_aggregate": None,
+ }
+
+
def _installed_llama_backend() -> Optional[str]:
"""The backend the installed llama.cpp prebuilt records, as Settings shows it."""
from core.inference.llama_cpp import LlamaCppBackend
diff --git a/studio/backend/utils/hardware/nvidia.py b/studio/backend/utils/hardware/nvidia.py
index 2b73fcd97a4..776b550c890 100644
--- a/studio/backend/utils/hardware/nvidia.py
+++ b/studio/backend/utils/hardware/nvidia.py
@@ -758,6 +758,32 @@ def get_cuda_ordinal_smi_rows() -> Optional[list[int]]:
return rows or None
+def get_pci_bus_ids() -> dict[int, tuple[int, str]]:
+ """nvidia-smi row index -> normalized PCI bus id; {} when nvidia-smi does not answer."""
+ try:
+ smi = gpu_query.run_nvidia_smi(
+ [_nvidia_smi_executable(), "--query-gpu=index,pci.bus_id", "--format=csv,noheader"],
+ capture_output = True,
+ text = True,
+ encoding = "utf-8",
+ errors = "replace",
+ timeout = 5,
+ env = child_env_without_native_path_secret(),
+ **_windows_hidden_subprocess_kwargs(),
+ )
+ except (OSError, subprocess.TimeoutExpired):
+ return {}
+ if smi.returncode != 0:
+ return {}
+ bus_ids: dict[int, tuple[int, str]] = {}
+ for line in smi.stdout.strip().splitlines():
+ index, _, bus_id = (part.strip() for part in line.partition(","))
+ key = _normalize_bus_id(bus_id)
+ if index.isdigit() and key is not None:
+ bus_ids[int(index)] = key
+ return bus_ids
+
+
def get_physical_gpu_inventory() -> dict[str, Any]:
"""Every NVIDIA GPU the driver enumerates, with no visibility mask and no torch. Display-only inventory: ``index`` is nvidia-smi's own row number, a physical id and NOT something a caller may pin, because the whole point of this probe is that PyTorch cannot open these devices. A failed probe comes back as a structured unavailable result, so this never raises out of an endpoint."""
rows = _query_gpu_inventory("get_physical_gpu_inventory")