Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""nvidia-smi is mocked (no GPU in CI); resolver, PATH builder and install layout run for real."""
from __future__ import annotations
import os
import subprocess
import sys
import types as _types
import zipfile
from pathlib import Path
from unittest import mock
import pytest
_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
if _BACKEND_DIR not in sys.path:
sys.path.insert(0, _BACKEND_DIR)
# Stub only on failure (unconditional stubs shadow sibling tests); try-import, not find_spec.
import importlib as _importlib # noqa: E402
def _maybe_stub(name: str, builder):
try:
_importlib.import_module(name)
except ImportError:
sys.modules[name] = builder()
def _build_loggers_stub():
m = _types.ModuleType("loggers")
m.get_logger = lambda name: __import__("logging").getLogger(name)
return m
def _build_structlog_stub():
return _types.ModuleType("structlog")
def _build_httpx_stub():
m = _types.ModuleType("httpx")
for _exc_name in (
"ConnectError",
"TimeoutException",
"ReadTimeout",
"ReadError",
"RemoteProtocolError",
"CloseError",
"HTTPError",
):
setattr(m, _exc_name, type(_exc_name, (Exception,), {}))
m.Response = type("Response", (), {})
class _FakeTimeout:
def __init__(self, *a, **kw):
pass
m.Timeout = _FakeTimeout
m.Client = type(
"Client",
(),
{
"__init__": lambda self, **kw: None,
"__enter__": lambda self: self,
"__exit__": lambda self, *a: None,
},
)
return m
_maybe_stub("loggers", _build_loggers_stub)
_maybe_stub("structlog", _build_structlog_stub)
_maybe_stub("httpx", _build_httpx_stub)
from core.inference.llama_cpp import LlamaCppBackend # noqa: E402
REAL_UPSTREAM_CUDART_BUNDLE = {
"12.4": ("cudart64_12.dll", "cublas64_12.dll", "cublasLt64_12.dll"),
"13.1": ("cudart64_13.dll", "cublas64_13.dll", "cublasLt64_13.dll"),
}
REAL_PIP_NVIDIA_WHEEL_LAYOUTS = {
"nvidia/cuda_runtime/bin": ["cudart64_12.dll"],
"nvidia/cublas/bin": [
"cublas64_12.dll",
"cublasLt64_12.dll",
"nvblas64_12.dll",
],
"nvidia/cudnn/bin": [
"cudnn64_9.dll",
"cudnn_adv64_9.dll",
"cudnn_ops64_9.dll",
],
"nvidia/cu13/bin/x86_64": [
"cudart64_13.dll",
"cublas64_13.dll",
"cublasLt64_13.dll",
"nvblas64_13.dll",
],
}
def _populate_studio_venv(prefix: Path) -> None:
"""Lay out fake nvidia + torch wheels matching real win_amd64 layouts (stub bytes)."""
site = prefix / "Lib" / "site-packages"
for rel, dlls in REAL_PIP_NVIDIA_WHEEL_LAYOUTS.items():
d = site / Path(rel)
d.mkdir(parents = True, exist_ok = True)
for name in dlls:
(d / name).write_bytes(b"PE-stub")
(site / "torch" / "lib").mkdir(parents = True, exist_ok = True)
for fn in ("c10.dll", "torch.dll", "torch_cpu.dll", "torch_python.dll"):
(site / "torch" / "lib" / fn).write_bytes(b"PE-stub")
def _populate_studio_install(install_dir: Path, runtime: str = "13.1") -> None:
"""Lay out install_dir/build/bin/Release/ as #5322 leaves it: payload + cudart overlay."""
rel = install_dir / "build" / "bin" / "Release"
rel.mkdir(parents = True, exist_ok = True)
for fn in (
"llama-server.exe",
"llama-quantize.exe",
"llama-cli.exe",
"llama.dll",
"ggml.dll",
"ggml-base.dll",
"ggml-cuda.dll",
"mtmd.dll",
):
(rel / fn).write_bytes(b"PE-stub")
for fn in REAL_UPSTREAM_CUDART_BUNDLE[runtime]:
(rel / fn).write_bytes(b"PE-stub")
def _build_path_dirs_like_start_llama_server(
binary_dir: Path,
prefix: Path,
cuda_path: str = "",
) -> list[str]:
"""Wrapper around the real _build_windows_path_dirs staticmethod."""
return LlamaCppBackend._build_windows_path_dirs(str(binary_dir), str(prefix), cuda_path)
def _mock_nvidia_smi_run(fake_output: str, returncode: int = 0) -> "mock._patch":
"""Patch subprocess.run so the nvidia-smi probe returns fake_output;
other calls pass through."""
real_run = subprocess.run
def fake_run(cmd, *args, **kwargs):
if isinstance(cmd, list) and cmd and "nvidia-smi" in cmd[0]:
return subprocess.CompletedProcess(
args = cmd, returncode = returncode, stdout = fake_output, stderr = ""
)
return real_run(cmd, *args, **kwargs)
return mock.patch("subprocess.run", side_effect = fake_run)
class TestWindowsGpuDetectionAfter5106Fix:
"""End-to-end #5106 fix on a synthetic Windows layout; only nvidia-smi is mocked."""
def test_nvidia_smi_probe_reports_synthetic_gpu(self, monkeypatch):
"""Probe parses CSV output and returns (index, free_mib)."""
monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)
monkeypatch.delenv("NVIDIA_VISIBLE_DEVICES", raising = False)
fake_csv = "0, 22805\n"
with _mock_nvidia_smi_run(fake_csv):
gpus = LlamaCppBackend._get_gpu_free_memory()
assert gpus == [(0, 22805)], f"GPU probe failed to parse mocked nvidia-smi output: {gpus}"
def test_nvidia_smi_probe_respects_cuda_visible_devices(self, monkeypatch):
"""CUDA_VISIBLE_DEVICES=1 -> only GPU 1 visible."""
fake_csv = "0, 22805\n1, 24576\n2, 16384\n"
monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "1")
with _mock_nvidia_smi_run(fake_csv):
gpus = LlamaCppBackend._get_gpu_free_memory()
assert gpus == [(1, 24576)], gpus
def test_get_gpu_memory_parses_three_and_two_column(self, monkeypatch):
"""Total is parsed when present; a legacy two-column line or non-integer total (N/A) gives
total 0 for back-compat instead of dropping the GPU, which would spill to CPU."""
monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)
with _mock_nvidia_smi_run("0, 22805, 24576\n"):
assert LlamaCppBackend._get_gpu_memory() == [(0, 22805, 24576)]
with _mock_nvidia_smi_run("0, 22805\n"):
assert LlamaCppBackend._get_gpu_memory() == [(0, 22805, 0)]
with _mock_nvidia_smi_run("0, 22805, N/A\n"):
assert LlamaCppBackend._get_gpu_memory() == [(0, 22805, 0)]
with _mock_nvidia_smi_run("0, N/A, 24576\n1, 22805, 24576\n"):
assert LlamaCppBackend._get_gpu_memory() == [(1, 22805, 24576)]
def test_windows_install_dir_has_all_three_cudart_dlls(self, tmp_path):
"""All three DLLs must be in install_dir/build/bin/Release, or ggml-cuda.dll breaks."""
install = tmp_path / "studio_install"
_populate_studio_install(install, runtime = "13.1")
rel = install / "build" / "bin" / "Release"
for fn in REAL_UPSTREAM_CUDART_BUNDLE["13.1"]:
assert (rel / fn).exists(), f"missing {fn} in {rel}"
assert (rel / "llama-server.exe").exists()
assert (rel / "ggml-cuda.dll").exists()
def test_resolver_finds_real_pypi_wheel_layouts(self, tmp_path):
"""Resolver must find nvidia/<pkg>/bin, nvidia/<pkg>/bin/x86_64 and torch/lib layouts."""
prefix = tmp_path / "studio_venv"
_populate_studio_venv(prefix)
out = LlamaCppBackend._windows_pip_nvidia_dll_dirs(str(prefix))
site = prefix / "Lib" / "site-packages"
for expected in (
site / "nvidia" / "cuda_runtime" / "bin",
site / "nvidia" / "cublas" / "bin",
site / "nvidia" / "cudnn" / "bin",
site / "nvidia" / "cu13" / "bin" / "x86_64",
site / "torch" / "lib",
):
assert str(expected) in out, f"resolver missed {expected.relative_to(prefix)}: {out}"
def test_path_assembly_makes_cudart_reachable_without_toolkit(self, tmp_path):
"""The #5106 scenario (GPU detected, pip nvidia wheels, no system CUDA toolkit): cudart must
be reachable from PATH via both binary_dir (#5322) and a pip nvidia dir (#5324)."""
prefix = tmp_path / "studio_venv"
install = tmp_path / "studio_install"
_populate_studio_venv(prefix)
_populate_studio_install(install, runtime = "13.1")
binary_dir = install / "build" / "bin" / "Release"
path_dirs = _build_path_dirs_like_start_llama_server(binary_dir, prefix, cuda_path = "")
assert path_dirs[0] == str(
binary_dir
), f"binary_dir must be first in PATH; got {path_dirs[0]}"
cudart_locations = []
for entry in path_dirs:
for cudart_name in ("cudart64_12.dll", "cudart64_13.dll"):
if (Path(entry) / cudart_name).exists():
cudart_locations.append((entry, cudart_name))
assert cudart_locations, (
f"cudart unreachable from any PATH entry -- #5106 not fixed.\n"
f"PATH entries searched: {path_dirs}"
)
sources = {Path(e).relative_to(tmp_path).parts[0] for e, _ in cudart_locations}
assert "studio_install" in sources, f"#5322's cudart drop not reachable: {cudart_locations}"
assert (
"studio_venv" in sources
), f"#5324's pip nvidia dir not contributing cudart: {cudart_locations}"
def test_cublas_and_cublasLt_also_reachable(self, tmp_path):
"""ggml-cuda chains cublas64 to cublasLt64; all 3 must resolve or LoadLibrary fails."""
prefix = tmp_path / "studio_venv"
install = tmp_path / "studio_install"
_populate_studio_venv(prefix)
_populate_studio_install(install, runtime = "13.1")
binary_dir = install / "build" / "bin" / "Release"
path_dirs = _build_path_dirs_like_start_llama_server(binary_dir, prefix)
for required in REAL_UPSTREAM_CUDART_BUNDLE["13.1"]:
reachable = any((Path(d) / required).exists() for d in path_dirs)
assert reachable, (
f"{required} unreachable from PATH; #5106 not fixed.\n" f"PATH entries: {path_dirs}"
)
def test_no_pip_nvidia_wheels_still_works_via_install_dir(self, tmp_path):
"""No pip nvidia wheels (CPU-only torch): cudart still resolves via #5322's binary_dir."""
prefix = tmp_path / "bare_venv"
prefix.mkdir()
install = tmp_path / "studio_install"
_populate_studio_install(install, runtime = "13.1")
binary_dir = install / "build" / "bin" / "Release"
path_dirs = _build_path_dirs_like_start_llama_server(binary_dir, prefix)
assert path_dirs == [str(binary_dir)], f"bare venv produced unexpected PATH: {path_dirs}"
for required in REAL_UPSTREAM_CUDART_BUNDLE["13.1"]:
assert (
binary_dir / required
).exists(), f"{required} missing from binary_dir on bare venv install"
def test_no_install_dir_still_works_via_pip_wheels(self, tmp_path):
"""Pre-#5322 binary_dir lacks cudart; #5324's pip wheel dirs on PATH still resolve it."""
prefix = tmp_path / "studio_venv"
_populate_studio_venv(prefix)
install = tmp_path / "studio_install_pre5322"
rel = install / "build" / "bin" / "Release"
rel.mkdir(parents = True)
for fn in (
"llama-server.exe",
"llama.dll",
"ggml-cuda.dll",
"ggml-base.dll",
):
(rel / fn).write_bytes(b"PE-stub")
path_dirs = _build_path_dirs_like_start_llama_server(rel, prefix)
cudart_reachable = any(
(Path(d) / "cudart64_12.dll").exists() or (Path(d) / "cudart64_13.dll").exists()
for d in path_dirs
)
assert cudart_reachable, (
"#5324 pip wheel fallback failed: cudart unreachable from PATH "
f"on cudart-less install. PATH entries: {path_dirs}"
)
cublas_reachable = any(
(Path(d) / "cublas64_12.dll").exists() or (Path(d) / "cublas64_13.dll").exists()
for d in path_dirs
)
assert cublas_reachable, "cublas unreachable on cudart-less install"
def test_pre_pr_scenario_would_have_failed(self, tmp_path):
"""Negative control: pre-#5322 and pre-#5324 leave cudart unreachable, the original failure
mode; confirms the test catches regressions."""
prefix = tmp_path / "studio_venv"
_populate_studio_venv(prefix)
install = tmp_path / "pre_pr_install"
rel = install / "build" / "bin" / "Release"
rel.mkdir(parents = True)
for fn in ("llama-server.exe", "llama.dll", "ggml-cuda.dll"):
(rel / fn).write_bytes(b"PE-stub")
pre_pr_path_dirs = [str(rel)]
cudart_reachable_pre = any(
(Path(d) / "cudart64_12.dll").exists() or (Path(d) / "cudart64_13.dll").exists()
for d in pre_pr_path_dirs
)
assert not cudart_reachable_pre, (
"Test self-check failed: pre-PR scenario unexpectedly had "
f"cudart reachable. {pre_pr_path_dirs}"
)
class TestWindowsSysPlatformMocked:
"""Tests the win32 branch of start_llama_server, not the linux fallback, by patching
sys.platform and re-running the branch-selecting helper."""
def test_sys_platform_win32_uses_pip_nvidia_resolver(self, monkeypatch, tmp_path):
monkeypatch.setattr(sys, "platform", "win32")
prefix = tmp_path / "studio_venv"
_populate_studio_venv(prefix)
out = LlamaCppBackend._windows_pip_nvidia_dll_dirs(str(prefix))
assert out, f"resolver returned empty under sys.platform=win32: {out}"
cu13_arch = prefix / "Lib" / "site-packages" / "nvidia" / "cu13" / "bin" / "x86_64"
assert str(cu13_arch) in out