Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

15.1 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""A bundle whose oldest arch is newer than every GPU fails fast; an older bundle JITs PTX and runs."""

import json
import os
import struct
import subprocess
import types
from pathlib import Path
from unittest.mock import patch

import pytest

from core.inference.llama_cpp import GgufLoadIntent, LlamaCppBackend


def _binary_with_marker(tmp_path, payload):
    (tmp_path / "UNSLOTH_PREBUILT_INFO.json").write_text(json.dumps(payload), encoding = "utf-8")
    return str(tmp_path / "build" / "bin" / "llama-server")


class TestInstalledLlamaCudaSms:
    def test_reads_supported_sms(self, tmp_path):
        binary = _binary_with_marker(tmp_path, {"supported_sms": ["75", "80", 86, " 89 "]})
        assert LlamaCppBackend._installed_llama_cuda_sms(binary) == frozenset({75, 80, 86, 89})

    def test_no_marker_is_unknown(self, tmp_path):
        assert LlamaCppBackend._installed_llama_cuda_sms(str(tmp_path / "llama-server")) is None

    def test_no_binary_is_unknown(self, monkeypatch):
        monkeypatch.setattr(
            LlamaCppBackend, "_find_llama_server_binary", staticmethod(lambda: None)
        )
        assert LlamaCppBackend._installed_llama_cuda_sms() is None

    @pytest.mark.parametrize("sms", [None, [], ["gfx1100"], ["86", "abc"], "86"])
    def test_missing_or_malformed_is_unknown(self, tmp_path, sms):
        binary = _binary_with_marker(
            tmp_path, {"supported_sms": sms} if sms is not None else {"asset": "x.tar.gz"}
        )
        assert LlamaCppBackend._installed_llama_cuda_sms(binary) is None

    def test_unreadable_marker_is_unknown(self, tmp_path, monkeypatch):
        import utils.llama_cpp_freshness as freshness

        def _boom(_binary):
            raise OSError("marker read failed")

        monkeypatch.setattr(freshness, "read_install_marker", _boom)
        assert LlamaCppBackend._installed_llama_cuda_sms(str(tmp_path / "llama-server")) is None


def _fake_smi(
    monkeypatch,
    stdout,
    returncode = 0,
):
    def _run(cmd, **_kwargs):
        assert cmd[0] == "nvidia-smi"
        return types.SimpleNamespace(returncode = returncode, stdout = stdout)

    monkeypatch.setattr(subprocess, "run", _run)


class TestCudaComputeCaps:
    def test_parses_index_and_cap(self, monkeypatch):
        monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)
        _fake_smi(monkeypatch, "0, 9.0\n1, 12.0\n")
        assert LlamaCppBackend._cuda_compute_caps() == {0: 90, 1: 120}

    def test_honors_visible_devices_mask(self, monkeypatch):
        monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "1")
        monkeypatch.setenv("CUDA_DEVICE_ORDER", "PCI_BUS_ID")
        _fake_smi(monkeypatch, "0, 7.5\n1, 9.0\n")
        assert LlamaCppBackend._cuda_compute_caps() == {1: 90}

    def test_numeric_mask_with_non_physical_order_fails_open(self, monkeypatch):
        # Under FASTEST_FIRST, CUDA ordinal 0 can name physical GPU 1.
        monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0")
        monkeypatch.setenv("CUDA_DEVICE_ORDER", "FASTEST_FIRST")
        _fake_smi(monkeypatch, "0, 7.5\n1, 9.0\n")
        assert LlamaCppBackend._cuda_compute_caps() == {}

    def test_bad_lines_are_skipped(self, monkeypatch):
        monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)
        _fake_smi(monkeypatch, "0, 9.0\nno-cap-line\n1, N/A\n")
        assert LlamaCppBackend._cuda_compute_caps() == {0: 90}

    def test_probe_failure_is_empty(self, monkeypatch):
        _fake_smi(monkeypatch, "", returncode = 1)
        assert LlamaCppBackend._cuda_compute_caps() == {}

        def _raise(*_args, **_kwargs):
            raise OSError("no nvidia-smi")

        monkeypatch.setattr(subprocess, "run", _raise)
        assert LlamaCppBackend._cuda_compute_caps() == {}


class TestCudaSmGateError:
    def _caps(self, monkeypatch, caps):
        monkeypatch.setattr(LlamaCppBackend, "_cuda_compute_caps", staticmethod(lambda: caps))

    def _managed(self, monkeypatch, managed):
        monkeypatch.setattr(
            LlamaCppBackend, "_is_unsloth_managed_binary", staticmethod(lambda _binary: managed)
        )

    def test_a_gpu_older_than_every_compiled_arch_errors(self, tmp_path, monkeypatch):
        self._caps(monkeypatch, {0: 75})
        self._managed(monkeypatch, True)
        binary = _binary_with_marker(tmp_path, {"supported_sms": ["86", "89", "120"]})
        error = LlamaCppBackend._cuda_sm_gate_error(binary)
        assert error is not None
        assert "sm_86 and newer" in error
        assert "GPU 0 is sm_75" in error
        assert "unsloth studio update" in error

    def test_a_custom_binary_is_told_to_rebuild_not_to_update(self, tmp_path, monkeypatch):
        self._caps(monkeypatch, {0: 75})
        self._managed(monkeypatch, False)
        binary = _binary_with_marker(tmp_path, {"supported_sms": ["86", "89", "120"]})
        error = LlamaCppBackend._cuda_sm_gate_error(binary)
        assert error is not None
        assert "GPU 0 is sm_75" in error
        assert "reinstall or rebuild that custom llama.cpp" in error
        assert "unsloth studio update" not in error

    def test_covered_gpu_passes(self, tmp_path, monkeypatch):
        self._caps(monkeypatch, {0: 90})
        binary = _binary_with_marker(tmp_path, {"supported_sms": ["86", "89", "90", "120"]})
        assert LlamaCppBackend._cuda_sm_gate_error(binary) is None

    def test_fastest_first_numeric_mask_does_not_reject_a_supported_gpu(
        self, tmp_path, monkeypatch
    ):
        monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0")
        monkeypatch.setenv("CUDA_DEVICE_ORDER", "FASTEST_FIRST")
        _fake_smi(monkeypatch, "0, 7.5\n1, 9.0\n")
        binary = _binary_with_marker(tmp_path, {"supported_sms": ["90"]})
        assert LlamaCppBackend._cuda_sm_gate_error(binary) is None

    def test_a_newer_gpu_jits_the_bundles_ptx(self, tmp_path, monkeypatch):
        self._caps(monkeypatch, {0: 90})
        binary = _binary_with_marker(tmp_path, {"supported_sms": ["75", "80", "86", "89"]})
        assert LlamaCppBackend._cuda_sm_gate_error(binary) is None

    def test_a_ptx_only_legacy_bundle_runs_on_modern_cards(self, tmp_path, monkeypatch):
        self._caps(monkeypatch, {0: 89, 1: 86})
        binary = _binary_with_marker(tmp_path, {"supported_sms": ["50", "52", "60", "61"]})
        assert LlamaCppBackend._cuda_sm_gate_error(binary) is None

    def test_any_gpu_at_or_above_the_floor_passes_a_mixed_host(self, tmp_path, monkeypatch):
        self._caps(monkeypatch, {0: 90, 1: 61})
        binary = _binary_with_marker(tmp_path, {"supported_sms": ["86", "89", "90"]})
        assert LlamaCppBackend._cuda_sm_gate_error(binary) is None

    def test_unknown_coverage_fails_open(self, tmp_path, monkeypatch):
        self._caps(monkeypatch, {0: 90})
        binary = _binary_with_marker(tmp_path, {"asset": "x.tar.gz"})
        assert LlamaCppBackend._cuda_sm_gate_error(binary) is None

    def test_unknown_caps_fail_open(self, tmp_path, monkeypatch):
        self._caps(monkeypatch, {})
        binary = _binary_with_marker(tmp_path, {"supported_sms": ["75", "80"]})
        assert LlamaCppBackend._cuda_sm_gate_error(binary) is None


def _gated_backend(
    tmp_path,
    monkeypatch,
    *,
    supported_sms = ("86", "89", "120"),
):
    """A compute_86 bundle on an sm_75 T4 has no cubin or PTX, so the gate must refuse the launch."""
    install = tmp_path / "llama.cpp"
    (install / "build" / "bin").mkdir(parents = True)
    binary = _binary_with_marker(install, {"supported_sms": list(supported_sms)})
    Path(binary).write_text("", encoding = "utf-8")
    os.chmod(binary, 0o755)

    gguf = tmp_path / "model.gguf"

    def _string(value):
        data = value.encode()
        return struct.pack("<Q", len(data)) + data

    gguf.write_bytes(
        struct.pack("<IIQQ", 0x46554747, 3, 0, 1)
        + _string("general.architecture")
        + struct.pack("<I", 8)
        + _string("llama")
    )

    monkeypatch.setattr(LlamaCppBackend, "_cuda_compute_caps", staticmethod(lambda: {0: 75}))
    monkeypatch.setattr(
        LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda *_a, **_kw: False)
    )
    # Pinned so the refusal wording does not depend on where the runner puts tmp_path.
    monkeypatch.setattr(
        LlamaCppBackend, "_is_unsloth_managed_binary", staticmethod(lambda _binary: True)
    )
    # The build probe runs `--version`, which the Popen patch below would count as a launch.
    monkeypatch.setattr(
        LlamaCppBackend, "probe_build_number", classmethod(lambda cls, binary = None: None)
    )
    backend = LlamaCppBackend()
    backend._get_gpu_memory = lambda _binary = None, **_kw: []
    backend._get_gpu_free_memory = lambda _binary = None, **_kw: []
    backend._read_gguf_metadata = lambda _path: None
    backend._can_estimate_kv = lambda: False
    backend._get_gguf_size_bytes = lambda _path: 1024
    backend._mmproj_vram_bytes = lambda _path: 0
    backend._resolve_launch_mmproj_path = lambda **_kw: None
    backend._apu_ram_shortfall_message = lambda *_a, **_kw: None
    backend._amd_apu_wants_unified_memory = lambda *_a, **_kw: False
    backend._find_llama_server_binary = lambda include_denied = False: binary
    backend._fit_off_retry_eligible = lambda *_a, **_kw: False
    backend.probe_server_capabilities = lambda _binary: {"found": True}
    backend._record_server_pid = lambda _pid: None
    backend._clear_server_pid = lambda: None
    backend._prepare_cpu_fallback_launch = lambda *_a, **_kw: None
    backend._detect_audio_type_strict = lambda: None
    backend._apply_detected_audio = lambda _detected: True
    backend._wait_for_health = lambda timeout, **_kw: True
    backend._llama_server_env_for_binary = lambda _binary: {"PATH": os.environ.get("PATH", "")}
    return backend, gguf


def _drive_load(backend, gguf, **intent_kwargs):
    """Returns (launches, error): a refusal has no launch, so it is returned, not raised."""
    launches = []

    class _Process:
        pid = 123
        stdout = ()
        returncode = None

        def poll(self):
            return None

        def terminate(self):
            return None

        def wait(self, timeout = None):
            return 0

        def kill(self):
            return None

    def _popen(cmd, **kwargs):
        launches.append((list(cmd), dict(kwargs.get("env") or {})))
        return _Process()

    error = None
    with patch.object(subprocess, "Popen", side_effect = _popen):
        try:
            backend.load_model(
                GgufLoadIntent(
                    gguf_path = str(gguf),
                    model_identifier = "owner/model",
                    **intent_kwargs,
                )
            )
        except Exception as exc:
            error = exc
    return launches, error


class TestTheGateSparesADeliberateCpuOnlyLoad:
    """A manual zero-offload load launches with CUDA_VISIBLE_DEVICES=-1, so the
    child never initialises CUDA and the missing kernels cannot reach it. Gating
    it turned a load that works on the mismatched host into a hard refusal."""

    def test_manual_zero_offload_still_launches_on_cpu(self, tmp_path, monkeypatch):
        backend, gguf = _gated_backend(tmp_path, monkeypatch)
        launches, error = _drive_load(backend, gguf, gpu_memory_mode = "manual", gpu_layers = 0)
        assert error is None, f"the SM gate refused a CPU-only load: {error}"
        assert len(launches) == 1
        _cmd, env = launches[0]
        assert env.get("CUDA_VISIBLE_DEVICES") == "-1"

    def test_a_gpu_offload_request_is_still_refused(self, tmp_path, monkeypatch):
        backend, gguf = _gated_backend(tmp_path, monkeypatch)
        launches, error = _drive_load(backend, gguf, gpu_memory_mode = "auto")
        assert isinstance(error, RuntimeError)
        assert "unsloth studio update" in str(error)
        assert launches == []

    @pytest.mark.parametrize(
        "companion",
        [
            {"extra_args": ["--device", "CUDA0"]},
            {"extra_args": ["--model-draft", "/tmp/draft.gguf"]},
        ],
        ids = ["device_pin", "gpu_drafter"],
    )
    def test_a_gpu_companion_keeps_the_gate(self, tmp_path, monkeypatch, companion):
        backend, gguf = _gated_backend(tmp_path, monkeypatch)
        launches, error = _drive_load(
            backend, gguf, gpu_memory_mode = "manual", gpu_layers = 0, **companion
        )
        assert isinstance(error, RuntimeError)
        assert launches == []

    def test_the_uis_default_speculative_auto_still_launches_on_cpu(self, tmp_path, monkeypatch):
        backend, gguf = _gated_backend(tmp_path, monkeypatch)
        launches, error = _drive_load(
            backend,
            gguf,
            gpu_memory_mode = "manual",
            gpu_layers = 0,
            speculative_type = "auto",
        )
        assert error is None, f"the SM gate refused the default CPU-only load: {error}"
        assert len(launches) == 1
        _cmd, env = launches[0]
        assert env.get("CUDA_VISIBLE_DEVICES") == "-1"

    def test_a_cpu_pinned_drafter_still_launches_on_cpu(self, tmp_path, monkeypatch):
        backend, gguf = _gated_backend(tmp_path, monkeypatch)
        launches, error = _drive_load(
            backend,
            gguf,
            gpu_memory_mode = "manual",
            gpu_layers = 0,
            speculative_type = "auto",
            extra_args = ["--spec-draft-ngl", "0"],
        )
        assert error is None, f"the SM gate refused a CPU-pinned drafter: {error}"
        assert len(launches) == 1
        _cmd, env = launches[0]
        assert env.get("CUDA_VISIBLE_DEVICES") == "-1"

    def test_a_cpu_pinned_projector_still_launches_on_cpu(self, tmp_path, monkeypatch):
        backend, gguf = _gated_backend(tmp_path, monkeypatch)
        mmproj = tmp_path / "mmproj.gguf"
        mmproj.write_bytes(b"")
        backend._resolve_launch_mmproj_path = lambda **_kw: str(mmproj)
        launches, error = _drive_load(
            backend,
            gguf,
            gpu_memory_mode = "manual",
            gpu_layers = 0,
            is_vision = True,
            extra_args = ["--no-mmproj-offload"],
        )
        assert error is None, f"the SM gate refused a CPU-pinned projector: {error}"
        assert len(launches) == 1
        cmd, env = launches[0]
        assert "--no-mmproj-offload" in cmd
        assert env.get("CUDA_VISIBLE_DEVICES") == "-1"

    def test_a_gpu_projector_keeps_the_gate(self, tmp_path, monkeypatch):
        backend, gguf = _gated_backend(tmp_path, monkeypatch)
        mmproj = tmp_path / "mmproj.gguf"
        mmproj.write_bytes(b"")
        backend._resolve_launch_mmproj_path = lambda **_kw: str(mmproj)
        launches, error = _drive_load(
            backend, gguf, gpu_memory_mode = "manual", gpu_layers = 0, is_vision = True
        )
        assert isinstance(error, RuntimeError)
        assert launches == []

    def test_an_auto_placement_device_none_keeps_the_gate(self, tmp_path, monkeypatch):
        backend, gguf = _gated_backend(tmp_path, monkeypatch)
        launches, error = _drive_load(
            backend, gguf, gpu_memory_mode = "auto", extra_args = ["--device", "none"]
        )
        assert isinstance(error, RuntimeError)
        assert launches == []