Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

13.9 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""A Vulkan pin mixing a discrete card with a shared-memory iGPU: the card fills first.
Reported on an RX 7700 XT + Ryzen iGPU (27B: 8.3 GB on the iGPU, 1.05 t/s)."""

from __future__ import annotations

import inspect
import sys
import types as _types
from pathlib import Path

import pytest

_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
if _BACKEND_DIR not in sys.path:
    sys.path.insert(0, _BACKEND_DIR)

import importlib as _importlib  # noqa: E402


def _maybe_stub(name: str, builder):
    try:
        _importlib.import_module(name)
    except ImportError:
        sys.modules[name] = builder()


def _build_loggers_stub():
    m = _types.ModuleType("loggers")
    m.get_logger = lambda name: __import__("logging").getLogger(name)
    return m


_maybe_stub("loggers", _build_loggers_stub)
_maybe_stub("structlog", lambda: _types.ModuleType("structlog"))

from core.inference.llama_cpp import LlamaCppBackend  # noqa: E402

MIB = 1024 * 1024

# The reporter's probe: VK0 the RX 7700 XT, VK1 the iGPU after its host reserve.
DGPU, IGPU = 0, 1
GPUS = [(DGPU, 11313), (IGPU, 14352)]
SHARED = {IGPU}


def test_a_model_either_device_holds_lands_on_the_discrete_card():
    picked, use_fit = LlamaCppBackend._select_gpus(
        6000 * MIB, GPUS, usable_fraction = 0.9, shared_gpu_ids = SHARED
    )
    assert (picked, use_fit) == ([DGPU], False)


def test_without_shared_ids_the_larger_shared_pool_still_wins():
    # CUDA / ROCm callers pass no shared set and keep the old ranking.
    picked, _ = LlamaCppBackend._select_gpus(6000 * MIB, GPUS, usable_fraction = 0.9)
    assert picked == [IGPU]


def test_a_model_that_needs_both_still_pins_both():
    picked, use_fit = LlamaCppBackend._select_gpus(
        15000 * MIB, GPUS, usable_fraction = 0.9, shared_gpu_ids = SHARED
    )
    assert (picked, use_fit) == ([DGPU, IGPU], False)


def test_split_aware_passes_the_shared_set_through():
    picked, _ = LlamaCppBackend._select_gpus_split_aware(
        6000 * MIB,
        GPUS,
        usable_fraction = 0.9,
        split_extra_bytes = 256 * MIB,
        shared_gpu_ids = SHARED,
    )
    assert picked == [DGPU]


def test_a_card_too_full_for_its_split_buffers_does_not_outrank_the_igpu():
    picked, use_fit = LlamaCppBackend._select_gpus(
        10000 * MIB,
        [(DGPU, 900), (IGPU, 14352)],
        usable_fraction = 0.9,
        per_device_overhead_bytes = 1024 * MIB,
        shared_gpu_ids = SHARED,
    )
    assert (picked, use_fit) == ([IGPU], False)


def test_every_auto_placement_sort_ranks_discrete_first():
    src = inspect.getsource(LlamaCppBackend.load_model)
    assert "key = lambda g: _gpu_usable(" not in src
    assert src.count("key = lambda g: _gpu_rank(") == 3
    assert "_gpu_rank(g, pin_fraction, _rank_floor_mib)" in src


def test_a_busy_card_too_small_alone_does_not_pull_in_the_igpu():
    # Another model runs on the card; the iGPU holds this one alone, so it goes there.
    picked, use_fit = LlamaCppBackend._select_gpus(
        12000 * MIB,
        GPUS,
        usable_fraction = 0.9,
        shared = frozenset({DGPU}),
        shared_gpu_ids = SHARED,
    )
    assert (picked, use_fit) == ([IGPU], False)


def test_the_fit_on_retry_takes_the_generated_split_back_out():
    src = inspect.getsource(LlamaCppBackend.load_model)
    retry = src[src.index("with forced --fit off; the fit estimate was optimistic") :]
    retry = retry[: retry.index("self._fit_load_mode_flags")]
    assert "_without_subsequence(_run, self._mixed_split_flags)" in retry
    assert "self._auto_tensor_split_emitted = None" in retry


split = LlamaCppBackend._discrete_first_split


def test_the_card_takes_the_layers_its_room_holds_and_the_igpu_the_rest():
    # 10 blocks of 1000 MiB + a 500 MiB output layer; the card keeps 300 for itself.
    shares = split(
        [DGPU, IGPU], {DGPU: 10180.0, IGPU: 12917.0}, SHARED, [1000.0] * 10 + [500.0], 300.0
    )
    # 9 layers then 2; each boundary half a layer early against float rounding.
    assert shares == [8.5, 2.5]


def test_the_heaviest_contiguous_run_decides_not_the_average():
    # A full-attention layer among window layers: an even byte share would give 7.
    layers = [100.0] * 4 + [3000.0] + [100.0] * 4
    assert split([DGPU, IGPU], {DGPU: 3500.0, IGPU: 9000.0}, SHARED, layers) == [5.5, 3.5]


def test_nothing_left_over_gives_the_igpu_nothing():
    assert split([DGPU, IGPU], {DGPU: 10000.0, IGPU: 12000.0}, SHARED, [1000.0] * 5 + [100.0]) == [
        5.5,
        0.5,
    ]


def test_the_split_is_positional_over_the_pin_order():
    shares = split([IGPU, DGPU], {DGPU: 10000.0, IGPU: 12000.0}, SHARED, [1000.0] * 12)
    assert shares == [1.5, 10.5]


def test_device_zero_carries_the_one_time_reserve():
    # iGPU first: the flat buffer lands there, so the card keeps its whole budget.
    shares = split(
        [IGPU, DGPU], {DGPU: 10000.0, IGPU: 12000.0}, SHARED, [1000.0] * 12, main_reserve_mib = 1500.0
    )
    assert shares == [1.5, 10.5]
    shares = split(
        [DGPU, IGPU], {DGPU: 10000.0, IGPU: 12000.0}, SHARED, [1000.0] * 12, main_reserve_mib = 1500.0
    )
    assert shares == [7.5, 4.5]


def test_overflow_is_shared_across_igpus_by_their_own_room():
    shares = split(
        [0, 1, 2],
        {0: 4000.0, 1: 3300.0, 2: 1300.0},
        {1, 2},
        [1000.0] * 7,
        per_device_mib = 100.0,
        pipeline_mib = 200.0,
    )
    assert shares == [2.5, 3.0, 1.5]


def test_only_a_mixed_pin_gets_a_split():
    usable = {0: 10000.0, 1: 12000.0}
    assert split([0, 1], usable, set(), [1000.0] * 15) is None
    assert split([0, 1], usable, {0, 1}, [1000.0] * 15) is None
    assert split([0, 1], usable, {1}, []) is None
    # A card that cannot hold even one layer: llama.cpp's split, as before.
    assert split([0, 1], {0: 500.0, 1: 12000.0}, {1}, [1000.0] * 15) is None


def test_the_launch_hands_placement_back_wherever_it_cannot_price_it():
    src = inspect.getsource(LlamaCppBackend._mixed_pin_split)
    for needle in (
        "tensor_parallel",
        "layer_min_gpus > 1",
        'spill_inputs["separate_draft_on_gpu"]',
        '"env_mmproj_unsized"',
        '"host_mmproj_bytes"',
        "_kv_offload_from_args(extra_args, env)",
        "_extra_args_have_tensor_split(extra_args, env)",
        "_extra_args_main_device(extra_args) is not None",
        "not pin_owns_devices",
        '"LLAMA_ARG_DEVICE"',
        "_sidecar_adapter_paths(extra_args)",
        '"kv_layer_weights"',
        '"compute_buffer_flat"',
        'spill_inputs["ctx_compute_per_device"] + spill_inputs["soft_overhead"]',
        '"extra_gpu_bytes"',
        '"env_mmproj_bytes"',
    ):
        assert needle in src, needle


def test_the_launch_records_what_it_emitted():
    src = inspect.getsource(LlamaCppBackend.load_model)
    arm = src[
        src.index("_mixed_split = self._mixed_pin_split(") : src.index(
            "# Expose Prometheus /metrics"
        )
    ]
    assert "self._mixed_split_flags = [" in arm
    assert "pin_owns_devices = _gpu_ids_own_device_flags" in arm
    assert "self._auto_tensor_split_emitted = self._auto_split_fingerprint(" in arm


def test_an_igpu_run_that_overflows_its_room_leaves_the_split_off():
    # The pooled fit passes (10 of 10), but the 9 MiB layer would land on the 1 MiB iGPU.
    assert split([DGPU, IGPU], {DGPU: 9.0, IGPU: 1.0}, SHARED, [1.0, 9.0]) is None


def test_few_rows_over_many_igpus_never_go_negative():
    shares = split(
        [0, 1, 2, 3, 4],
        {0: 3.0, 1: 30.0, 2: 30.0, 3: 30.0, 4: 10.0},
        {1, 2, 3, 4},
        [1.0] * 5,
    )
    assert shares is not None and min(shares) >= 0 and sum(shares) == 5
    assert shares[0] == 2.5  # three rows on the card


def test_trailing_mtp_rows_keep_their_place_in_the_split():
    src = inspect.getsource(LlamaCppBackend._mixed_pin_split)
    assert "self._nextn_predict_layers" in src
    assert "layer_bytes.extend([mtp_row] * n_mtp)" in src


def test_a_row_or_tensor_split_mode_leaves_the_split_to_llama_cpp():
    class Priced(Exception):
        pass

    backend = LlamaCppBackend.__new__(LlamaCppBackend)

    def _priced(*_a, **_k):
        raise Priced

    backend._tensor_spill_layout = _priced
    inputs = {"separate_draft_on_gpu": False, "model_path": "m.gguf"}

    def call(args, env):
        return backend._mixed_pin_split(
            [DGPU, IGPU],
            inputs,
            SHARED,
            args,
            env,
            layer_min_gpus = 1,
            tensor_parallel = False,
            pin_owns_devices = True,
        )

    for args in (
        ["-sm", "row"],
        ["--split-mode=tensor"],
        ["--rpc", "10.0.0.2:50052"],
        ["--fit", "on"],
        ["--fit=on"],
    ):
        assert call(args, {}) is None, args
    assert call([], {"LLAMA_ARG_RPC": "10.0.0.2:50052"}) is None
    # The launch pops an inherited non-layer mode, so the child runs layer split.
    with pytest.raises(Priced):
        call([], {"LLAMA_ARG_SPLIT_MODE": "row"})


def test_an_unpriced_uneven_kv_cache_leaves_the_split_to_llama_cpp():
    backend = LlamaCppBackend.__new__(LlamaCppBackend)
    block = _types.SimpleNamespace(spillable_bytes = 1, resident_bytes = 1)
    layout = _types.SimpleNamespace(
        complete = True,
        blocks = [block] * 4,
        has_excluded_blocks = False,
        lm_head_bytes = 0,
        other_resident_bytes = 0,
    )
    backend._tensor_spill_layout = lambda _path: layout
    # No compute_buffer_flat: a load that passes the KV gate stops on the KeyError.
    inputs = {
        "separate_draft_on_gpu": False,
        "model_path": "m.gguf",
        "kv_cache_bytes": 4 * MIB,
        "kv_layer_weights": [],
    }

    def call():
        return backend._mixed_pin_split(
            [DGPU, IGPU],
            inputs,
            SHARED,
            [],
            {},
            layer_min_gpus = 1,
            tensor_parallel = False,
            pin_owns_devices = True,
        )

    for mla, ssm, pattern in ((512, None, None), (None, 1024, None), (None, None, [1, 0, 1, 0])):
        backend._kv_lora_rank, backend._ssm_inner_size = mla, ssm
        backend._sliding_window_pattern = pattern
        assert call() is None, (mla, ssm, pattern)
    # A declared window whose pattern did not resolve is still uneven.
    backend._kv_lora_rank = backend._ssm_inner_size = backend._sliding_window_pattern = None
    backend._sliding_window, backend._context_length = 4096, 1048576
    assert call() is None
    # A plain cache is uniform, so it is priced.
    backend._sliding_window = None
    with pytest.raises(KeyError):
        call()


def _priced_backend(layout):
    backend = LlamaCppBackend.__new__(LlamaCppBackend)
    backend._tensor_spill_layout = lambda _path: layout
    backend._kv_lora_rank = backend._ssm_inner_size = backend._sliding_window_pattern = None
    return backend


def _mixed(
    backend,
    inputs,
    args = (),
    env = None,
):
    return backend._mixed_pin_split(
        [DGPU, IGPU],
        inputs,
        SHARED,
        list(args),
        env or {},
        layer_min_gpus = 1,
        tensor_parallel = False,
        pin_owns_devices = True,
    )


class _Priced(Exception):
    pass


def _pricing_backend():
    backend = LlamaCppBackend.__new__(LlamaCppBackend)

    def _priced(*_a, **_k):
        raise _Priced

    backend._tensor_spill_layout = _priced
    return backend


def test_a_projector_pinned_to_another_device_leaves_the_split_to_llama_cpp():
    backend = _pricing_backend()
    inputs = {"separate_draft_on_gpu": False, "model_path": "m.gguf"}
    assert _mixed(backend, inputs, ["--mmproj-device", "Vulkan1"]) is None
    assert _mixed(backend, inputs, ["-mmdev=Vulkan1"]) is None
    assert _mixed(backend, inputs, env = {"MTMD_BACKEND_DEVICE": "Vulkan1"}) is None
    with pytest.raises(_Priced):
        _mixed(backend, inputs)


def test_a_larger_pass_through_parallel_leaves_the_split_to_llama_cpp():
    backend = _pricing_backend()
    inputs = {"separate_draft_on_gpu": False, "model_path": "m.gguf", "n_parallel": 1}
    assert _mixed(backend, inputs, ["--parallel", "4"]) is None
    assert _mixed(backend, inputs, ["-np=2"]) is None
    # Fewer slots than priced leaves the plan over-reserved, the safe direction.
    with pytest.raises(_Priced):
        _mixed(backend, {**inputs, "n_parallel": 4}, ["--parallel", "2"])


def test_trailing_mtp_rows_carry_a_layer_of_cache():
    block = _types.SimpleNamespace(spillable_bytes = 0, resident_bytes = 0)
    layout = _types.SimpleNamespace(
        complete = True,
        blocks = [block] * 4,
        has_excluded_blocks = True,
        excluded_block_bytes = 0,
        n_attention_layers = 4,
        lm_head_bytes = 0,
        other_resident_bytes = 0,
    )
    backend = _priced_backend(layout)
    backend._nextn_predict_layers = 1
    seen = {}

    def _record(gpu_indices, usable, shared, layer_mib, *_a):
        seen["rows"] = layer_mib
        return None

    backend._discrete_first_split = _record
    _mixed(
        backend,
        {
            "separate_draft_on_gpu": False,
            "model_path": "m.gguf",
            "kv_cache_bytes": 8 * MIB,
            "kv_layer_weights": [],
            "mtp_will_engage": False,
            "compute_buffer_flat": 0,
            "extra_gpu_bytes": 0,
            "ctx_compute_per_device": 0,
            "soft_overhead": 0,
            "gpu_usable_mib": {DGPU: 100.0, IGPU: 100.0},
        },
    )
    # 4 blocks + 1 nextn row + output: the nextn row holds one block's 2 MiB of cache.
    assert seen["rows"] == [2.0, 2.0, 2.0, 2.0, 2.0, 0.0]


def test_a_tensor_override_leaves_the_split_to_llama_cpp():
    backend = _pricing_backend()
    inputs = {"separate_draft_on_gpu": False, "model_path": "m.gguf"}
    assert _mixed(backend, inputs, ["-ot", "blk\\.1[2-5]\\.ffn_.*=Vulkan0"]) is None
    assert _mixed(backend, inputs, ["--n-cpu-moe", "4"]) is None
    assert _mixed(backend, inputs, env = {"LLAMA_ARG_OVERRIDE_TENSOR": "ffn=Vulkan0"}) is None
    with pytest.raises(_Priced):
        _mixed(backend, inputs, ["--n-cpu-moe", "0"])