Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

18.8 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Auto tensor-split fallback driven through LlamaCppBackend against faked GPU inventories."""

from __future__ import annotations

import math
import sys
from pathlib import Path

_TESTS_DIR = str(Path(__file__).resolve().parent)
if _TESTS_DIR not in sys.path:
    sys.path.insert(0, _TESTS_DIR)

import pytest  # noqa: E402

from test_llama_cpp_placement import _backend, _launch  # noqa: E402

from core.inference.llama_cpp import GgufLoadIntent  # noqa: E402

GIB = 1024**3


def _tp_backend(
    tmp_path: Path,
    *,
    memory,
    model_gib: float = 1.0,
    vulkan: bool = False,
    reserve_mib: int = 256,
):
    """KV and context compute buffers are zeroed, so only model size against the cards is weighed."""
    backend, gguf = _backend(tmp_path, vulkan = vulkan, memory = memory)
    backend._can_estimate_kv = lambda: True
    backend._estimate_kv_cache_bytes = lambda *a, **k: 0
    backend._compute_buffer_ctx_bytes = lambda *a, **k: 0
    backend._get_gguf_size_bytes = lambda _path: int(model_gib * GIB)
    backend._TENSOR_PARALLEL_BUFFER_RESERVE_MIB = reserve_mib
    return backend, gguf


def _auto_tp(backend, gguf, **kwargs):
    """An auto-mode tensor-parallel load, the shape the PR is about."""
    params = dict(
        gpu_memory_mode = "auto",
        tensor_parallel = True,
        n_ctx = 4096,
    )
    params.update(kwargs)
    return _launch(backend, gguf, **params)["cmd"]


def _flag(cmd, name):
    """The value of ``name`` in an argv, or None. Refuses a duplicate."""
    hits = [i for i, tok in enumerate(cmd) if tok == name]
    assert len(hits) <= 1, f"{name} appears {len(hits)} times in {cmd}"
    if not hits:
        return None
    return cmd[hits[0] + 1]


def _split_values(cmd):
    raw = _flag(cmd, "--tensor-split")
    if raw is None:
        return None
    return [float(part) for part in raw.split(",")]


def _normalized(values):
    total = sum(values)
    return tuple(round(v / total, 9) for v in values)


@pytest.mark.parametrize("vulkan", [False, True], ids = ["cuda", "vulkan"])
@pytest.mark.parametrize(
    "memory, ratio",
    [
        ([(0, 24_000, 24_000), (1, 24_000, 24_000)], [3, 1]),
        ([(0, 15_360, 15_360), (1, 15_360, 15_360)], [3, 1]),
        ([(0, 24_000, 24_000), (1, 24_000, 24_000)], [1, 3]),
        ([(0, 24_000, 24_000), (1, 24_000, 24_000)], [1, 1]),
        ([(i, 24_000, 24_000) for i in range(3)], [2, 1, 1]),
        ([(i, 24_000, 24_000) for i in range(4)], [4, 3, 2, 1]),
        ([(0, 11_000, 11_000), (1, 24_000, 24_000)], [1, 3]),
    ],
)
def test_the_user_ratio_reaches_llama_server_in_auto_mode(tmp_path, memory, ratio, vulkan):
    """The regression in #10355: auto mode dropped the user's ratio."""
    backend, gguf = _tp_backend(tmp_path, memory = memory, vulkan = vulkan)
    cmd = _auto_tp(
        backend,
        gguf,
        tensor_split = list(ratio),
        gpu_ids = [idx for idx, *_ in memory],
    )

    assert _flag(cmd, "--split-mode") == "tensor"
    assert _split_values(cmd) is not None, "the user ratio was dropped again"
    assert _normalized(_split_values(cmd)) == pytest.approx(
        _normalized([float(x) for x in ratio]), abs = 1e-6
    )


def test_the_emitted_token_is_plain_decimal(tmp_path):
    """llama.cpp splits --tensor-split on commas and parses each field. A token
    in scientific notation, or one carrying a locale decimal comma, would either
    be read as a different number or split into extra fields."""
    backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)])
    cmd = _auto_tp(backend, gguf, tensor_split = [0.75, 0.25], gpu_ids = [0, 1])
    raw = _flag(cmd, "--tensor-split")
    assert raw is not None
    assert raw.count(",") == 1, f"{raw!r} would parse as more fields than there are GPUs"
    for part in raw.split(","):
        assert "e" not in part.lower(), f"{part!r} is scientific notation"
        float(part)


@pytest.mark.parametrize(
    "ratio",
    [
        [1_000_000, 1],  # a ratio is relative; this is legal and means 1e6:1
        [0.0000004, 0.0000001],  # ...and so is this, which means 4:1
        [1, 3],
    ],
)
def test_an_extreme_ratio_survives_formatting(tmp_path, ratio):
    """%g renders 1000000 as 1e+06, and fixed-point rounds tiny ratios to 0,0, a division by zero."""
    backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)])
    cmd = _auto_tp(backend, gguf, tensor_split = list(ratio), gpu_ids = [0, 1])
    raw = _flag(cmd, "--tensor-split")
    if raw is None:
        pytest.skip("the budget check declined this ratio; formatting is not reached")

    assert raw.count(",") == len(ratio) - 1, f"{raw!r} parses as the wrong field count"
    values = [float(part) for part in raw.split(",")]
    assert sum(values) > 0, f"{raw!r} normalizes to a division by zero"
    assert _normalized(values) == pytest.approx(
        _normalized([float(x) for x in ratio]), abs = 1e-6
    ), f"{raw!r} is not the proportion that was asked for"


@pytest.mark.parametrize(
    "ratio, why",
    [
        ([3, 1, 1], "three weights for two cards"),
        ([3], "one weight for two cards"),
        ([0, 0], "a total of zero"),
        ([-1, -1], "negatives, which sanitize to zero"),
        ([float("nan"), 1], "a NaN weight"),
        ([float("inf"), 1], "an infinite weight"),
    ],
)
def test_a_degenerate_ratio_is_dropped_not_forwarded(tmp_path, ratio, why):
    """A bad ratio must leave llama.cpp on its own default, never reach it."""
    backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)])
    cmd = _auto_tp(backend, gguf, tensor_split = list(ratio), gpu_ids = [0, 1])
    assert _flag(cmd, "--split-mode") == "tensor", why
    raw = _flag(cmd, "--tensor-split")
    if raw is not None:
        for part in raw.split(","):
            value = float(part)
            assert math.isfinite(value) and value >= 0.0, f"{raw!r} ({why})"


def test_a_ratio_that_overshoots_one_card_is_dropped(tmp_path):
    """The planner's own budget, applied to the user's ratio. A 25GB model at
    3:1 puts ~19GB on a 16GB card."""
    backend, gguf = _tp_backend(
        tmp_path,
        memory = [(0, 16_000, 16_000), (1, 16_000, 16_000)],
        model_gib = 25,
    )
    cmd = _auto_tp(backend, gguf, tensor_split = [3, 1], gpu_ids = [0, 1])
    assert _flag(cmd, "--split-mode") == "tensor"
    assert _flag(cmd, "--tensor-split") is None


def test_the_budget_check_agrees_with_the_planner_on_an_even_share(tmp_path):
    """An even user ratio must be accepted exactly when the planner's even-share rule says the load fits."""
    backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)])
    mib = 1024 * 1024
    cc_per_device_mib = 1024
    backend._compute_buffer_ctx_bytes = lambda *a, **k: cc_per_device_mib * mib

    gpus = [(0, 24_000), (1, 24_000)]
    total_by_idx = {0: 24_000, 1: 24_000}
    # Planner ceiling is 23024 MiB a card; the even share must sit in the gap below it.
    model_mib = 43_452
    even_share = (model_mib + 2 * cc_per_device_mib) / 2
    assert 22_000 < even_share < 23_024, "the case has drifted out of the gap it tests"

    assert (
        backend._tensor_split_fits_budget(
            [1.0, 1.0],
            gpus,
            [0, 1],
            model_mib * mib,
            4096,
            n_ubatch = 512,
            total_by_idx = total_by_idx,
            vram_fraction = 0.97,
        )
        is True
    )


def test_the_context_buffer_is_charged_flat_not_by_the_ratio(tmp_path):
    """The context compute buffer is replicated on every device, so it is charged flat, not by ratio."""
    backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)])
    mib = 1024 * 1024
    cc_per_device_mib = 6 * 1024
    backend._compute_buffer_ctx_bytes = lambda *a, **k: cc_per_device_mib * mib
    model_mib = 14 * 1024

    ceiling_mib = 23_024
    assert model_mib / 2 + cc_per_device_mib < ceiling_mib, "the planner would not have"
    assert 0.9 * model_mib < ceiling_mib - cc_per_device_mib, "charged flat, 1:9 fits"
    assert 0.9 * (model_mib + 2 * cc_per_device_mib) > ceiling_mib, "charged by ratio, it does not"

    assert (
        backend._tensor_split_fits_budget(
            [1.0, 9.0],
            [(0, 24_000), (1, 24_000)],
            [0, 1],
            model_mib * mib,
            4096,
            n_ubatch = 512,
            total_by_idx = {0: 24_000, 1: 24_000},
            vram_fraction = 0.97,
        )
        is True
    )


def test_the_budget_check_still_refuses_what_does_not_fit(tmp_path):
    """Relaxing the double charge must not make the budget check accept anything."""
    backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)])
    mib = 1024 * 1024
    backend._compute_buffer_ctx_bytes = lambda *a, **k: 0

    assert (
        backend._tensor_split_fits_budget(
            [9.0, 1.0],
            [(0, 24_000), (1, 24_000)],
            [0, 1],
            40_000 * mib,
            4096,
            n_ubatch = 512,
            total_by_idx = {0: 24_000, 1: 24_000},
            vram_fraction = 0.97,
        )
        is False
    )


def test_the_budget_check_fails_closed_on_an_unsurveyed_card(tmp_path):
    """A card missing from the tensor-parallel survey must fail closed, not raise KeyError."""
    backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)])
    assert (
        backend._tensor_split_fits_budget(
            [1.0, 1.0],
            [(0, 24_000)],
            [0, 1],
            1024,
            4096,
            total_by_idx = {0: 24_000},
        )
        is False
    )


def test_manual_mode_is_untouched_by_the_new_branch(tmp_path):
    """Manual mode has always emitted the ratio through its own path. The new
    elif is guarded on `gpu_memory_mode != "manual"`, so this must be
    byte-identical to its pre-PR behaviour."""
    backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)])
    cmd = _launch(
        backend,
        gguf,
        gpu_memory_mode = "manual",
        gpu_layers = 99,
        tensor_parallel = True,
        tensor_split = [3, 1],
        gpu_ids = [0, 1],
        n_ctx = 4096,
    )["cmd"]
    assert _flag(cmd, "--split-mode") == "tensor"
    assert _flag(cmd, "--tensor-split") == "3,1"
    assert backend._auto_tensor_split is None, "manual mode must not record an auto ratio"


def test_a_single_gpu_never_gets_a_split(tmp_path):
    backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000)])
    cmd = _auto_tp(backend, gguf, tensor_split = [3, 1], gpu_ids = [0])
    assert _flag(cmd, "--tensor-split") is None


def test_no_gpu_at_all_does_not_raise(tmp_path):
    """CPU-only: the new code divides by len(gpu_indices) and sum(split); neither may be zero."""
    backend, gguf = _tp_backend(tmp_path, memory = [])
    cmd = _auto_tp(backend, gguf, tensor_split = [3, 1])
    assert _flag(cmd, "--tensor-split") is None


def test_cards_whose_driver_reports_no_total(tmp_path):
    """Some Vulkan drivers report free memory without a total. The new budget
    check reads `total_by_idx.get(idx, 0)`, so this is the shape that exercises
    the zero-total branch of _vram_usable_mib."""
    backend, gguf = _tp_backend(
        tmp_path,
        memory = [(0, 24_000, 0), (1, 24_000, 0)],
        vulkan = True,
    )
    cmd = _auto_tp(backend, gguf, tensor_split = [3, 1], gpu_ids = [0, 1])
    assert _flag(cmd, "--split-mode") == "tensor"


def test_paravirtual_metal_never_launches_a_tensor_split(tmp_path, monkeypatch):
    """Paravirtual Metal pins to CPU and overrides --split-mode, so no tensor split may launch."""
    import core.inference.llama_cpp as llama_cpp

    monkeypatch.setattr(llama_cpp, "_metal_device_is_paravirtual", lambda: True)
    backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)])
    cmd = _auto_tp(backend, gguf, tensor_split = [3, 1], gpu_ids = [0, 1])
    assert _flag(cmd, "--tensor-split") is None


# The except arm drops the plan but keeps tensor_parallel, so gpu_indices can outrun tp_gpus.


def _planner_raises(backend):
    def _boom(*args, **kwargs):
        raise RuntimeError("simulated GPU selection failure")

    backend._plan_tensor_parallel = _boom


def test_a_failed_plan_still_launches_on_vulkan(tmp_path):
    """Vulkan rebuilds gpu_indices from all detected cards, so it can name a dropped card."""
    backend, gguf = _tp_backend(
        tmp_path,
        memory = [(0, 24_000, 24_000), (1, 24_000, 24_000), (2, 200, 24_000)],
        vulkan = True,
    )
    _planner_raises(backend)
    cmd = _auto_tp(backend, gguf, tensor_split = [2, 1, 1])
    assert cmd, "the load must still launch, as it did before the fallback existed"


def test_a_failed_plan_still_launches_when_gpu_ids_were_pinned(tmp_path):
    """CUDA and ROCm return gpu_indices as sorted(gpu_ids), which can include an excluded card."""
    backend, gguf = _tp_backend(
        tmp_path,
        memory = [(0, 24_000, 24_000), (1, 24_000, 24_000), (2, 200, 24_000)],
    )
    _planner_raises(backend)
    cmd = _auto_tp(backend, gguf, tensor_split = [2, 1, 1], gpu_ids = [0, 1, 2])
    assert cmd


def test_a_plan_that_fails_before_the_gpu_survey_still_launches(tmp_path):
    """The earliest throw: nothing the fallback reads has been bound yet."""
    backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)])

    def _boom(_path):
        raise RuntimeError("simulated GGUF read failure")

    backend._get_gguf_size_bytes = _boom
    cmd = _auto_tp(backend, gguf, tensor_split = [3, 1], gpu_ids = [0, 1])
    assert cmd


def _intent(gguf, **kwargs):
    params = dict(
        gguf_path = str(gguf),
        model_identifier = "test",
        gpu_memory_mode = "auto",
        tensor_parallel = True,
        n_ctx = 4096,
    )
    params.update(kwargs)
    return GgufLoadIntent(**params)


def _reuses(backend, gguf, **kwargs):
    """Whether a second identical /load would reuse the running server."""
    return backend.adopt_load_intent_if_matched(_intent(gguf, **kwargs))


def test_an_identical_auto_request_with_a_ratio_reuses_the_server(tmp_path):
    backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)])
    _auto_tp(backend, gguf, tensor_split = [3, 1], gpu_ids = [0, 1])
    assert _reuses(backend, gguf, tensor_split = [3, 1], gpu_ids = [0, 1]) is True


def test_a_changed_ratio_does_not_reuse_the_server(tmp_path):
    backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)])
    _auto_tp(backend, gguf, tensor_split = [3, 1], gpu_ids = [0, 1])
    assert _reuses(backend, gguf, tensor_split = [1, 3], gpu_ids = [0, 1]) is False


def test_an_identical_auto_request_with_no_ratio_reuses_the_server(tmp_path):
    """The plain auto tensor-parallel load: the user set no ratio at all."""
    backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)])
    _auto_tp(backend, gguf, gpu_ids = [0, 1])
    assert _reuses(backend, gguf, gpu_ids = [0, 1]) is True


def test_a_planner_weighted_split_still_reuses_the_server(tmp_path):
    """A planner-weighted split must reuse the server; comparing it to the user's ratio reloads forever."""
    backend, gguf = _tp_backend(
        tmp_path,
        memory = [(0, 24_000, 24_000), (1, 16_000, 16_000)],
        model_gib = 30,
    )
    cmd = _auto_tp(backend, gguf, gpu_ids = [0, 1])
    assert (
        _flag(cmd, "--tensor-split") is not None
    ), "this case is only meaningful when the planner emitted its own split"
    assert _reuses(backend, gguf, gpu_ids = [0, 1]) is True


def test_a_rejected_user_ratio_still_reuses_the_server(tmp_path):
    """A declined ratio emits nothing, so an identical repeat request must reuse the server, not reload."""
    backend, gguf = _tp_backend(
        tmp_path,
        memory = [(0, 16_000, 16_000), (1, 16_000, 16_000)],
        model_gib = 25,
    )
    cmd = _auto_tp(backend, gguf, tensor_split = [3, 1], gpu_ids = [0, 1])
    assert _flag(cmd, "--tensor-split") is None
    assert _reuses(backend, gguf, tensor_split = [3, 1], gpu_ids = [0, 1]) is True


def test_an_equivalent_ratio_reuses_the_server(tmp_path):
    """3:1 and 6:2 are the same instruction to llama.cpp."""
    backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)])
    _auto_tp(backend, gguf, tensor_split = [3, 1], gpu_ids = [0, 1])
    assert _reuses(backend, gguf, tensor_split = [6, 2], gpu_ids = [0, 1]) is True


def test_unload_clears_the_recorded_ratio(tmp_path):
    backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)])
    _auto_tp(backend, gguf, tensor_split = [3, 1], gpu_ids = [0, 1])
    assert backend._auto_tensor_split is not None
    backend.unload_model()
    assert backend._auto_tensor_split is None


def test_a_layer_split_load_clears_the_recorded_ratio(tmp_path):
    """A layer-split load must clear the recorded ratio, or the next auto load compares to it."""
    backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)])
    _auto_tp(backend, gguf, tensor_split = [3, 1], gpu_ids = [0, 1])
    _launch(
        backend,
        gguf,
        gpu_memory_mode = "auto",
        tensor_parallel = False,
        gpu_ids = [0, 1],
        n_ctx = 4096,
    )
    assert backend._auto_tensor_split is None


def _crash_once_with_an_arch_error(backend):
    """The first spawn dies with an arch error, so the arch-crash retry runs and succeeds."""
    calls: list[int] = []

    def _health(timeout, **_kw):
        calls.append(1)
        return len(calls) > 1

    backend._wait_for_health = _health
    backend._kernel_image_invalid = lambda _text: True
    return calls


def test_the_arch_crash_retry_stops_reporting_the_ratio_it_dropped(tmp_path):
    """After an arch-crash retry narrows devices, the recorded tensor_split ratio must be dropped too."""
    backend, gguf = _tp_backend(
        tmp_path,
        memory = [(0, 24_000, 24_000), (1, 24_000, 24_000), (2, 400, 24_000)],
    )
    _crash_once_with_an_arch_error(backend)

    cmd = _auto_tp(backend, gguf, tensor_split = [3, 1])

    assert _flag(cmd, "--tensor-split") is None, "the retry kept a split it re-indexed"
    assert backend.tensor_parallel is False
    assert backend.tensor_split is None, (
        "/status still reports the dropped ratio: " f"{backend.tensor_split}"
    )


def test_a_launch_that_does_not_crash_still_reports_its_ratio(tmp_path):
    """Recovery-arm clears must not reach the ordinary launch, or its ratio report is lost."""
    backend, gguf = _tp_backend(
        tmp_path,
        memory = [(0, 24_000, 24_000), (1, 24_000, 24_000), (2, 400, 24_000)],
    )
    cmd = _auto_tp(backend, gguf, tensor_split = [3, 1])
    assert _flag(cmd, "--tensor-split") == "3,1"
    assert backend.tensor_split == [0.75, 0.25]