Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Vulkan free-VRAM reader tests on a fake probe; GGML_VK_VISIBLE_DEVICES is left for ggml to apply."""
from __future__ import annotations
import subprocess
import sys
import types as _types
from pathlib import Path
from unittest import mock
import pytest
_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
if _BACKEND_DIR not in sys.path:
sys.path.insert(0, _BACKEND_DIR)
import importlib as _importlib # noqa: E402
def _maybe_stub(name: str, builder):
try:
_importlib.import_module(name)
except ImportError:
sys.modules[name] = builder()
def _build_loggers_stub():
m = _types.ModuleType("loggers")
m.get_logger = lambda name: __import__("logging").getLogger(name)
return m
_maybe_stub("loggers", _build_loggers_stub)
_maybe_stub("structlog", lambda: _types.ModuleType("structlog"))
from core.inference import llama_cpp as _llama_mod # noqa: E402
from core.inference._vulkan_probe import _igpu_flags_and_names # noqa: E402
from core.inference.llama_cpp import ( # noqa: E402
LlamaCppBackend,
_llama_lib_dir,
)
MIB = 1024 * 1024
GIB = 1024 * MIB
class _FakeCFunction:
def __init__(self, result):
self.result = result
def __call__(self, *_args):
return self.result
def test_missing_description_symbol_keeps_igpu_detection():
base = _types.SimpleNamespace(
ggml_backend_reg_dev_count = _FakeCFunction(1),
ggml_backend_reg_dev_get = _FakeCFunction(1),
ggml_backend_dev_type = _FakeCFunction(2),
ggml_backend_dev_name = _FakeCFunction(b"Legacy Vulkan iGPU"),
)
lib = _types.SimpleNamespace(ggml_backend_vk_reg = _FakeCFunction(1))
flags, names, known, pci_ids = _igpu_flags_and_names(base, lib, 1)
assert flags == [True]
assert names == ["Legacy Vulkan iGPU"]
assert known == [True]
assert pci_ids == [""]
def _make_vulkan_install(tmp_path: Path) -> str:
"""A binary whose sibling dir holds the Vulkan ggml lib, so the
reader's ``is_vulkan_backend`` sibling-file check passes."""
bindir = tmp_path / "build" / "bin"
bindir.mkdir(parents = True)
binary = bindir / ("llama-server.exe" if sys.platform == "win32" else "llama-server")
binary.write_bytes(b"stub")
vulkan_lib = "ggml-vulkan.dll" if sys.platform == "win32" else "libggml-vulkan.so"
(bindir / vulkan_lib).write_bytes(b"stub")
return str(binary)
def _mock_probe(rows: list[str], captured_env: dict | None = None):
"""Patch subprocess.run so the _vulkan_probe.py call returns ``rows``
(already tab-formatted), recording the env it was launched with."""
real_run = subprocess.run
def fake_run(cmd, *args, **kwargs):
if isinstance(cmd, list) and any("_vulkan_probe" in str(c) for c in cmd):
if captured_env is not None:
captured_env.clear()
captured_env.update(kwargs.get("env") or {})
return subprocess.CompletedProcess(
args = cmd, returncode = 0, stdout = "\n".join(rows), stderr = ""
)
return real_run(cmd, *args, **kwargs)
return mock.patch("subprocess.run", side_effect = fake_run)
def _row(
idx: int,
free_bytes: int,
is_igpu: int,
total_bytes: int = 0,
name: str | None = None,
) -> str:
row = f"{idx}\t{free_bytes}\t{is_igpu}\t{total_bytes}"
return f"{row}\t{name}" if name is not None else row
def _host_memory(monkeypatch, *, available_mib, total_mib):
"""Set deterministic host-memory bounds for iGPU tests."""
monkeypatch.setattr(
LlamaCppBackend, "_available_system_memory_mib", staticmethod(lambda: available_mib)
)
monkeypatch.setattr(
LlamaCppBackend, "_total_system_memory_mib", staticmethod(lambda: total_mib)
)
def test_integrated_gpu_leaves_host_margin(tmp_path, monkeypatch):
binary = _make_vulkan_install(tmp_path)
# Flat 1024 MiB reserve (llama.cpp --fit-target); total 0: shared RAM is no VRAM budget.
_host_memory(monkeypatch, available_mib = 31 * 1024, total_mib = 32 * 1024)
rows = [_row(0, 30 * GIB, is_igpu = 1, total_bytes = 32 * GIB)]
with _mock_probe(rows):
gpus = LlamaCppBackend._get_gpu_free_memory_vulkan(binary)
assert gpus == [(0, 30 * 1024 - 1024, 0)], gpus
def test_integrated_gpu_host_bound_does_not_replace_planner_reading(tmp_path, monkeypatch):
"""Placement keeps Vulkan free while admission caps its host-backed share."""
binary = _make_vulkan_install(tmp_path)
_host_memory(monkeypatch, available_mib = 4000, total_mib = 31000)
rows = [_row(0, 15900 * MIB, is_igpu = 1, total_bytes = 16012 * MIB)]
with _mock_probe(rows):
gpus = LlamaCppBackend._get_gpu_free_memory_vulkan(binary)
assert gpus == [(0, 15900 - 1024, 0)], gpus
assert LlamaCppBackend._igpu_backed_usable_mib(gpus[0][1]) == 4000 - 2048
def test_integrated_gpu_keeps_a_heap_the_os_cannot_see(tmp_path, monkeypatch):
"""Free memory above MemTotal is a conservative carve-out floor."""
binary = _make_vulkan_install(tmp_path)
_host_memory(monkeypatch, available_mib = 13312, total_mib = 32154)
rows = [_row(0, 108782 * MIB, is_igpu = 1, total_bytes = 114507 * MIB)]
with _mock_probe(rows):
gpus = LlamaCppBackend._get_gpu_free_memory_vulkan(binary)
assert gpus == [(0, 108782 - 1024, 0)], gpus
expected = (108782 - 32154) + 13312 - 2048
assert LlamaCppBackend._igpu_backed_usable_mib(gpus[0][1]) == expected
def test_a_carve_out_another_process_holds_is_not_credited(tmp_path, monkeypatch):
"""Use free, not total, to exclude carve-out memory held elsewhere."""
binary = _make_vulkan_install(tmp_path)
_host_memory(monkeypatch, available_mib = 4096, total_mib = 32768)
rows = [_row(0, 22528 * MIB, is_igpu = 1, total_bytes = 114688 * MIB)]
with _mock_probe(rows):
gpus = LlamaCppBackend._get_gpu_free_memory_vulkan(binary)
assert gpus == [(0, 22528 - 1024, 0)], gpus
assert LlamaCppBackend._igpu_backed_usable_mib(gpus[0][1]) == 4096 - 2048
def test_wsl_credits_no_heap_beyond_its_own_memtotal(tmp_path, monkeypatch):
"""WSL MemTotal does not describe the adapter's host pool."""
binary = _make_vulkan_install(tmp_path)
_host_memory(monkeypatch, available_mib = 6000, total_mib = 8192)
monkeypatch.setattr(_llama_mod, "_is_wsl", lambda: True)
rows = [_row(0, 11000 * MIB, is_igpu = 1, total_bytes = 12000 * MIB)]
with _mock_probe(rows):
gpus = LlamaCppBackend._get_gpu_free_memory_vulkan(binary)
assert gpus == [(0, 11000 - 1024, 0)], gpus
assert LlamaCppBackend._igpu_backed_usable_mib(gpus[0][1]) == 6000 - 2048
def test_unreadable_host_memory_leaves_the_integrated_reading_alone(tmp_path, monkeypatch):
"""Missing host readings leave the Vulkan value unchanged."""
binary = _make_vulkan_install(tmp_path)
_host_memory(monkeypatch, available_mib = None, total_mib = None)
rows = [_row(0, 30 * GIB, is_igpu = 1, total_bytes = 32 * GIB)]
with _mock_probe(rows):
gpus = LlamaCppBackend._get_gpu_free_memory_vulkan(binary)
assert gpus == [(0, 30 * 1024 - 1024, 0)], gpus
assert LlamaCppBackend._igpu_backed_usable_mib(gpus[0][1]) == 30 * 1024 - 1024
def test_discrete_gpu_free_is_untouched_and_total_passed_through(tmp_path):
binary = _make_vulkan_install(tmp_path)
rows = [_row(0, 6 * GIB, is_igpu = 0, total_bytes = 24 * GIB)]
with _mock_probe(rows):
gpus = LlamaCppBackend._get_gpu_free_memory_vulkan(binary)
assert gpus == [(0, 6 * 1024, 24 * 1024)], gpus
def test_large_discrete_gpu_is_untouched(tmp_path):
binary = _make_vulkan_install(tmp_path)
rows = [_row(0, 47 * GIB, is_igpu = 0, total_bytes = 48 * GIB)]
with _mock_probe(rows):
gpus = LlamaCppBackend._get_gpu_free_memory_vulkan(binary)
assert gpus == [(0, 47 * 1024, 48 * 1024)], gpus
@pytest.mark.parametrize(
"free_mib, total_mib, is_igpu, expected_free",
[
# Intel's Windows driver reports a heap budget above the heap size.
(18 * 1024, 16 * 1024, 0, 16 * 1024),
(40 * 1024, 32 * 1024, 1, 32 * 1024),
(6 * 1024, 24 * 1024, 0, 6 * 1024),
(24 * 1024, 24 * 1024, 0, 24 * 1024),
(8 * 1024, 0, 1, 8 * 1024),
],
ids = ["dgpu_over", "igpu_over", "dgpu_under", "dgpu_equal", "total_unknown"],
)
def test_vulkan_free_never_exceeds_total(tmp_path, free_mib, total_mib, is_igpu, expected_free):
binary = _make_vulkan_install(tmp_path)
rows = [_row(0, free_mib * MIB, is_igpu = is_igpu, total_bytes = total_mib * MIB, name = "GPU")]
with _mock_probe(rows):
(row,) = LlamaCppBackend._run_vulkan_probe(binary)
assert (row["free_mib"], row["total_mib"]) == (expected_free, total_mib)
def test_inherited_visible_devices_mask_is_passed_through_to_probe(tmp_path, monkeypatch):
# Mask passes through unchanged: ggml parses it in raw device space, not compact ordinals.
binary = _make_vulkan_install(tmp_path)
monkeypatch.setenv("GGML_VK_VISIBLE_DEVICES", "1")
captured: dict = {}
rows = [_row(0, 23 * GIB, is_igpu = 0, total_bytes = 24 * GIB)]
with _mock_probe(rows, captured_env = captured):
LlamaCppBackend._get_gpu_free_memory_vulkan(binary)
assert captured.get("GGML_VK_VISIBLE_DEVICES") == "1", captured
def test_vulkan_pin_args_uses_device_names_not_env_mask():
assert LlamaCppBackend._vulkan_pin_args([0]) == ["--device", "Vulkan0"]
assert LlamaCppBackend._vulkan_pin_args([1, 2]) == ["--device", "Vulkan1,Vulkan2"]
assert LlamaCppBackend._vulkan_pin_args(None) == []
assert LlamaCppBackend._vulkan_pin_args([]) == []
def test_vulkan_only_build_is_detected(tmp_path):
binary = _make_vulkan_install(tmp_path)
assert LlamaCppBackend._is_vulkan_backend(binary) is True
def test_multi_backend_build_is_not_vulkan_only(tmp_path):
# A build shipping CUDA/HIP alongside Vulkan must not be treated as Vulkan-only.
binary = _make_vulkan_install(tmp_path)
cuda = "ggml-cuda.dll" if sys.platform == "win32" else "libggml-cuda.so"
(_llama_lib_dir(binary) / cuda).write_bytes(b"stub")
assert LlamaCppBackend._is_vulkan_backend(binary) is False
@pytest.mark.skipif(sys.platform == "win32", reason = "shell wrapper fallback is POSIX")
def test_shell_wrapper_entrypoint_resolves_to_real_lib_dir(tmp_path):
# _llama_lib_dir must follow the #!/bin/sh wrapper's exec target to build/bin.
import os
binary = _make_vulkan_install(tmp_path)
bindir = Path(binary).parent
wrapper = tmp_path / "llama-server"
wrapper.write_text('#!/bin/sh\nexec "$(dirname "$0")/build/bin/llama-server" "$@"\n')
os.chmod(wrapper, 0o755)
assert _llama_lib_dir(str(wrapper)) == bindir
assert LlamaCppBackend._is_vulkan_backend(str(wrapper)) is True
@pytest.mark.skipif(sys.platform == "win32", reason = "soname versioning is POSIX")
def test_versioned_only_vulkan_soname_is_probed(tmp_path):
bindir = tmp_path / "build" / "bin"
bindir.mkdir(parents = True)
binary = bindir / "llama-server"
binary.write_bytes(b"stub")
(bindir / "libggml-vulkan.so.0").write_bytes(b"stub")
rows = [_row(0, 23 * GIB, is_igpu = 0, total_bytes = 24 * GIB)]
with _mock_probe(rows):
gpus = LlamaCppBackend._get_gpu_free_memory_vulkan(str(binary))
assert gpus == [(0, 23 * 1024, 24 * 1024)], gpus
def test_windows_probe_sets_the_error_mode_before_loading_ggml(tmp_path, monkeypatch, capsys):
"""SEM_FAILCRITICALERRORS is OR-ed into the inherited mode before either ggml DLL loads."""
from core.inference import _vulkan_probe as probe
for name in ("ggml-base.dll", "ggml-vulkan.dll"):
(tmp_path / name).write_bytes(b"stub")
error_mode = {"value": 0x8000}
class _Kernel32:
def GetErrorMode(self):
return error_mode["value"]
def SetErrorMode(self, value):
previous, error_mode["value"] = error_mode["value"], value
return previous
loads = []
def fake_cdll(path, mode = 0):
loads.append((Path(path).name, error_mode["value"]))
if Path(path).name == "ggml-vulkan.dll":
raise OSError(127, "The specified procedure could not be found")
return object()
monkeypatch.setattr(probe.sys, "argv", ["_vulkan_probe.py", str(tmp_path)])
monkeypatch.setattr(probe.os, "add_dll_directory", lambda _path: None, raising = False)
monkeypatch.setattr(probe.ctypes, "WinDLL", lambda _name: _Kernel32(), raising = False)
monkeypatch.setattr(probe.ctypes, "CDLL", fake_cdll)
monkeypatch.setattr(probe.sys, "platform", "win32")
assert probe.main() == 1
assert loads == [("ggml-base.dll", 0x8001), ("ggml-vulkan.dll", 0x8001)]
assert "ggml-vulkan load failed" in capsys.readouterr().err
@pytest.mark.skipif(sys.platform != "win32", reason = "real kernel32 error mode")
def test_windows_probe_sets_the_real_process_error_mode(tmp_path):
probe_script = Path(__file__).resolve().parents[1] / "core" / "inference" / "_vulkan_probe.py"
code = (
"import ctypes, runpy, sys\n"
"k = ctypes.WinDLL('kernel32'); k.SetErrorMode(0)\n"
f"sys.argv = ['_vulkan_probe.py', {str(tmp_path)!r}]\n"
f"g = runpy.run_path({str(probe_script)!r})\n"
"rc = g['main']()\n"
"print(rc, k.GetErrorMode() & 1)\n"
)
result = subprocess.run(
[sys.executable, "-c", code], capture_output = True, text = True, timeout = 60
)
assert result.stdout.split() == ["1", "1"], (result.stdout, result.stderr)
if __name__ == "__main__":
raise SystemExit(pytest.main([__file__, "-v"]))