Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

13.6 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""nvidia-smi is mocked (no GPU in CI); resolver, PATH builder and install layout run for real."""

from __future__ import annotations

import os
import subprocess
import sys
import types as _types
import zipfile
from pathlib import Path
from unittest import mock

import pytest

_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
if _BACKEND_DIR not in sys.path:
    sys.path.insert(0, _BACKEND_DIR)

# Stub only on failure (unconditional stubs shadow sibling tests); try-import, not find_spec.
import importlib as _importlib  # noqa: E402


def _maybe_stub(name: str, builder):
    try:
        _importlib.import_module(name)
    except ImportError:
        sys.modules[name] = builder()


def _build_loggers_stub():
    m = _types.ModuleType("loggers")
    m.get_logger = lambda name: __import__("logging").getLogger(name)
    return m


def _build_structlog_stub():
    return _types.ModuleType("structlog")


def _build_httpx_stub():
    m = _types.ModuleType("httpx")
    for _exc_name in (
        "ConnectError",
        "TimeoutException",
        "ReadTimeout",
        "ReadError",
        "RemoteProtocolError",
        "CloseError",
        "HTTPError",
    ):
        setattr(m, _exc_name, type(_exc_name, (Exception,), {}))
    m.Response = type("Response", (), {})

    class _FakeTimeout:
        def __init__(self, *a, **kw):
            pass

    m.Timeout = _FakeTimeout
    m.Client = type(
        "Client",
        (),
        {
            "__init__": lambda self, **kw: None,
            "__enter__": lambda self: self,
            "__exit__": lambda self, *a: None,
        },
    )
    return m


_maybe_stub("loggers", _build_loggers_stub)
_maybe_stub("structlog", _build_structlog_stub)
_maybe_stub("httpx", _build_httpx_stub)

from core.inference.llama_cpp import LlamaCppBackend  # noqa: E402


REAL_UPSTREAM_CUDART_BUNDLE = {
    "12.4": ("cudart64_12.dll", "cublas64_12.dll", "cublasLt64_12.dll"),
    "13.1": ("cudart64_13.dll", "cublas64_13.dll", "cublasLt64_13.dll"),
}

REAL_PIP_NVIDIA_WHEEL_LAYOUTS = {
    "nvidia/cuda_runtime/bin": ["cudart64_12.dll"],
    "nvidia/cublas/bin": [
        "cublas64_12.dll",
        "cublasLt64_12.dll",
        "nvblas64_12.dll",
    ],
    "nvidia/cudnn/bin": [
        "cudnn64_9.dll",
        "cudnn_adv64_9.dll",
        "cudnn_ops64_9.dll",
    ],
    "nvidia/cu13/bin/x86_64": [
        "cudart64_13.dll",
        "cublas64_13.dll",
        "cublasLt64_13.dll",
        "nvblas64_13.dll",
    ],
}


def _populate_studio_venv(prefix: Path) -> None:
    """Lay out fake nvidia + torch wheels matching real win_amd64 layouts (stub bytes)."""
    site = prefix / "Lib" / "site-packages"
    for rel, dlls in REAL_PIP_NVIDIA_WHEEL_LAYOUTS.items():
        d = site / Path(rel)
        d.mkdir(parents = True, exist_ok = True)
        for name in dlls:
            (d / name).write_bytes(b"PE-stub")
    (site / "torch" / "lib").mkdir(parents = True, exist_ok = True)
    for fn in ("c10.dll", "torch.dll", "torch_cpu.dll", "torch_python.dll"):
        (site / "torch" / "lib" / fn).write_bytes(b"PE-stub")


def _populate_studio_install(install_dir: Path, runtime: str = "13.1") -> None:
    """Lay out install_dir/build/bin/Release/ as #5322 leaves it: payload + cudart overlay."""
    rel = install_dir / "build" / "bin" / "Release"
    rel.mkdir(parents = True, exist_ok = True)
    for fn in (
        "llama-server.exe",
        "llama-quantize.exe",
        "llama-cli.exe",
        "llama.dll",
        "ggml.dll",
        "ggml-base.dll",
        "ggml-cuda.dll",
        "mtmd.dll",
    ):
        (rel / fn).write_bytes(b"PE-stub")
    for fn in REAL_UPSTREAM_CUDART_BUNDLE[runtime]:
        (rel / fn).write_bytes(b"PE-stub")


def _build_path_dirs_like_start_llama_server(
    binary_dir: Path,
    prefix: Path,
    cuda_path: str = "",
) -> list[str]:
    """Wrapper around the real _build_windows_path_dirs staticmethod."""
    return LlamaCppBackend._build_windows_path_dirs(str(binary_dir), str(prefix), cuda_path)


def _mock_nvidia_smi_run(fake_output: str, returncode: int = 0) -> "mock._patch":
    """Patch subprocess.run so the nvidia-smi probe returns fake_output;
    other calls pass through."""
    real_run = subprocess.run

    def fake_run(cmd, *args, **kwargs):
        if isinstance(cmd, list) and cmd and "nvidia-smi" in cmd[0]:
            return subprocess.CompletedProcess(
                args = cmd, returncode = returncode, stdout = fake_output, stderr = ""
            )
        return real_run(cmd, *args, **kwargs)

    return mock.patch("subprocess.run", side_effect = fake_run)


class TestWindowsGpuDetectionAfter5106Fix:
    """End-to-end #5106 fix on a synthetic Windows layout; only nvidia-smi is mocked."""

    def test_nvidia_smi_probe_reports_synthetic_gpu(self, monkeypatch):
        """Probe parses CSV output and returns (index, free_mib)."""
        monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)
        monkeypatch.delenv("NVIDIA_VISIBLE_DEVICES", raising = False)
        fake_csv = "0, 22805\n"
        with _mock_nvidia_smi_run(fake_csv):
            gpus = LlamaCppBackend._get_gpu_free_memory()
        assert gpus == [(0, 22805)], f"GPU probe failed to parse mocked nvidia-smi output: {gpus}"

    def test_nvidia_smi_probe_respects_cuda_visible_devices(self, monkeypatch):
        """CUDA_VISIBLE_DEVICES=1 -> only GPU 1 visible."""
        fake_csv = "0, 22805\n1, 24576\n2, 16384\n"
        monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "1")
        with _mock_nvidia_smi_run(fake_csv):
            gpus = LlamaCppBackend._get_gpu_free_memory()
        assert gpus == [(1, 24576)], gpus

    def test_get_gpu_memory_parses_three_and_two_column(self, monkeypatch):
        """Total is parsed when present; a legacy two-column line or non-integer total (N/A) gives
        total 0 for back-compat instead of dropping the GPU, which would spill to CPU."""
        monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)
        with _mock_nvidia_smi_run("0, 22805, 24576\n"):
            assert LlamaCppBackend._get_gpu_memory() == [(0, 22805, 24576)]
        with _mock_nvidia_smi_run("0, 22805\n"):
            assert LlamaCppBackend._get_gpu_memory() == [(0, 22805, 0)]
        with _mock_nvidia_smi_run("0, 22805, N/A\n"):
            assert LlamaCppBackend._get_gpu_memory() == [(0, 22805, 0)]
        with _mock_nvidia_smi_run("0, N/A, 24576\n1, 22805, 24576\n"):
            assert LlamaCppBackend._get_gpu_memory() == [(1, 22805, 24576)]

    def test_windows_install_dir_has_all_three_cudart_dlls(self, tmp_path):
        """All three DLLs must be in install_dir/build/bin/Release, or ggml-cuda.dll breaks."""
        install = tmp_path / "studio_install"
        _populate_studio_install(install, runtime = "13.1")
        rel = install / "build" / "bin" / "Release"
        for fn in REAL_UPSTREAM_CUDART_BUNDLE["13.1"]:
            assert (rel / fn).exists(), f"missing {fn} in {rel}"
        assert (rel / "llama-server.exe").exists()
        assert (rel / "ggml-cuda.dll").exists()

    def test_resolver_finds_real_pypi_wheel_layouts(self, tmp_path):
        """Resolver must find nvidia/<pkg>/bin, nvidia/<pkg>/bin/x86_64 and torch/lib layouts."""
        prefix = tmp_path / "studio_venv"
        _populate_studio_venv(prefix)
        out = LlamaCppBackend._windows_pip_nvidia_dll_dirs(str(prefix))
        site = prefix / "Lib" / "site-packages"
        for expected in (
            site / "nvidia" / "cuda_runtime" / "bin",
            site / "nvidia" / "cublas" / "bin",
            site / "nvidia" / "cudnn" / "bin",
            site / "nvidia" / "cu13" / "bin" / "x86_64",
            site / "torch" / "lib",
        ):
            assert str(expected) in out, f"resolver missed {expected.relative_to(prefix)}: {out}"

    def test_path_assembly_makes_cudart_reachable_without_toolkit(self, tmp_path):
        """The #5106 scenario (GPU detected, pip nvidia wheels, no system CUDA toolkit): cudart must
        be reachable from PATH via both binary_dir (#5322) and a pip nvidia dir (#5324)."""
        prefix = tmp_path / "studio_venv"
        install = tmp_path / "studio_install"
        _populate_studio_venv(prefix)
        _populate_studio_install(install, runtime = "13.1")
        binary_dir = install / "build" / "bin" / "Release"
        path_dirs = _build_path_dirs_like_start_llama_server(binary_dir, prefix, cuda_path = "")
        assert path_dirs[0] == str(
            binary_dir
        ), f"binary_dir must be first in PATH; got {path_dirs[0]}"
        cudart_locations = []
        for entry in path_dirs:
            for cudart_name in ("cudart64_12.dll", "cudart64_13.dll"):
                if (Path(entry) / cudart_name).exists():
                    cudart_locations.append((entry, cudart_name))
        assert cudart_locations, (
            f"cudart unreachable from any PATH entry -- #5106 not fixed.\n"
            f"PATH entries searched: {path_dirs}"
        )
        sources = {Path(e).relative_to(tmp_path).parts[0] for e, _ in cudart_locations}
        assert "studio_install" in sources, f"#5322's cudart drop not reachable: {cudart_locations}"
        assert (
            "studio_venv" in sources
        ), f"#5324's pip nvidia dir not contributing cudart: {cudart_locations}"

    def test_cublas_and_cublasLt_also_reachable(self, tmp_path):
        """ggml-cuda chains cublas64 to cublasLt64; all 3 must resolve or LoadLibrary fails."""
        prefix = tmp_path / "studio_venv"
        install = tmp_path / "studio_install"
        _populate_studio_venv(prefix)
        _populate_studio_install(install, runtime = "13.1")
        binary_dir = install / "build" / "bin" / "Release"
        path_dirs = _build_path_dirs_like_start_llama_server(binary_dir, prefix)
        for required in REAL_UPSTREAM_CUDART_BUNDLE["13.1"]:
            reachable = any((Path(d) / required).exists() for d in path_dirs)
            assert reachable, (
                f"{required} unreachable from PATH; #5106 not fixed.\n" f"PATH entries: {path_dirs}"
            )

    def test_no_pip_nvidia_wheels_still_works_via_install_dir(self, tmp_path):
        """No pip nvidia wheels (CPU-only torch): cudart still resolves via #5322's binary_dir."""
        prefix = tmp_path / "bare_venv"
        prefix.mkdir()
        install = tmp_path / "studio_install"
        _populate_studio_install(install, runtime = "13.1")
        binary_dir = install / "build" / "bin" / "Release"
        path_dirs = _build_path_dirs_like_start_llama_server(binary_dir, prefix)
        assert path_dirs == [str(binary_dir)], f"bare venv produced unexpected PATH: {path_dirs}"
        for required in REAL_UPSTREAM_CUDART_BUNDLE["13.1"]:
            assert (
                binary_dir / required
            ).exists(), f"{required} missing from binary_dir on bare venv install"

    def test_no_install_dir_still_works_via_pip_wheels(self, tmp_path):
        """Pre-#5322 binary_dir lacks cudart; #5324's pip wheel dirs on PATH still resolve it."""
        prefix = tmp_path / "studio_venv"
        _populate_studio_venv(prefix)
        install = tmp_path / "studio_install_pre5322"
        rel = install / "build" / "bin" / "Release"
        rel.mkdir(parents = True)
        for fn in (
            "llama-server.exe",
            "llama.dll",
            "ggml-cuda.dll",
            "ggml-base.dll",
        ):
            (rel / fn).write_bytes(b"PE-stub")
        path_dirs = _build_path_dirs_like_start_llama_server(rel, prefix)
        cudart_reachable = any(
            (Path(d) / "cudart64_12.dll").exists() or (Path(d) / "cudart64_13.dll").exists()
            for d in path_dirs
        )
        assert cudart_reachable, (
            "#5324 pip wheel fallback failed: cudart unreachable from PATH "
            f"on cudart-less install. PATH entries: {path_dirs}"
        )
        cublas_reachable = any(
            (Path(d) / "cublas64_12.dll").exists() or (Path(d) / "cublas64_13.dll").exists()
            for d in path_dirs
        )
        assert cublas_reachable, "cublas unreachable on cudart-less install"

    def test_pre_pr_scenario_would_have_failed(self, tmp_path):
        """Negative control: pre-#5322 and pre-#5324 leave cudart unreachable, the original failure
        mode; confirms the test catches regressions."""
        prefix = tmp_path / "studio_venv"
        _populate_studio_venv(prefix)
        install = tmp_path / "pre_pr_install"
        rel = install / "build" / "bin" / "Release"
        rel.mkdir(parents = True)
        for fn in ("llama-server.exe", "llama.dll", "ggml-cuda.dll"):
            (rel / fn).write_bytes(b"PE-stub")
        pre_pr_path_dirs = [str(rel)]
        cudart_reachable_pre = any(
            (Path(d) / "cudart64_12.dll").exists() or (Path(d) / "cudart64_13.dll").exists()
            for d in pre_pr_path_dirs
        )
        assert not cudart_reachable_pre, (
            "Test self-check failed: pre-PR scenario unexpectedly had "
            f"cudart reachable. {pre_pr_path_dirs}"
        )


class TestWindowsSysPlatformMocked:
    """Tests the win32 branch of start_llama_server, not the linux fallback, by patching
    sys.platform and re-running the branch-selecting helper."""

    def test_sys_platform_win32_uses_pip_nvidia_resolver(self, monkeypatch, tmp_path):
        monkeypatch.setattr(sys, "platform", "win32")
        prefix = tmp_path / "studio_venv"
        _populate_studio_venv(prefix)
        out = LlamaCppBackend._windows_pip_nvidia_dll_dirs(str(prefix))
        assert out, f"resolver returned empty under sys.platform=win32: {out}"
        cu13_arch = prefix / "Lib" / "site-packages" / "nvidia" / "cu13" / "bin" / "x86_64"
        assert str(cu13_arch) in out