Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""tensor_parallel must round-trip as a bool, and --split-mode tensor is emitted only behind it."""
from __future__ import annotations
import asyncio
import inspect
import re
import socket
import sys
import threading
import time
import types as _types
from dataclasses import replace
from pathlib import Path
import pytest
def _shared_setup_1(monkeypatch):
b = _recovery_backend()
proc = _ToggleProcess()
b._process = proc
fired = threading.Event()
monkeypatch.setattr(b, "_maybe_recover_from_mtp_crash", lambda *a, **k: fired.set() or True)
b._start_mtp_crash_watchdog()
return b, fired, proc
def _shared_setup_2(b, monkeypatch):
loads: list[GgufLoadIntent] = []
monkeypatch.setattr(b, "load_model", lambda intent: loads.append(intent) or True)
assert b._respawn_if_dead() is False
return loads
_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
if _BACKEND_DIR not in sys.path:
sys.path.insert(0, _BACKEND_DIR)
_loggers_stub = _types.ModuleType("loggers")
_loggers_stub.get_logger = lambda name: __import__("logging").getLogger(name)
sys.modules.setdefault("loggers", _loggers_stub)
_structlog_stub = _types.ModuleType("structlog")
_structlog_stub.get_logger = lambda *a, **k: __import__("logging").getLogger("stub")
sys.modules.setdefault("structlog", _structlog_stub)
_httpx_stub = _types.ModuleType("httpx")
for _exc in (
"ConnectError",
"TimeoutException",
"ReadTimeout",
"ReadError",
"RemoteProtocolError",
"CloseError",
):
setattr(_httpx_stub, _exc, type(_exc, (Exception,), {}))
_httpx_stub.Timeout = type("T", (), {"__init__": lambda s, *a, **k: None})
_httpx_stub.Client = type(
"C",
(),
{
"__init__": lambda s, **kw: None,
"__enter__": lambda s: s,
"__exit__": lambda s, *a: None,
},
)
# Stub httpx only if missing: this stub lacks Response, which starlette.testclient needs.
try:
import httpx # noqa: F401
except ImportError:
sys.modules.setdefault("httpx", _httpx_stub)
from core.inference import llama_cpp as llama_cpp_module
from core.inference.llama_cpp import (
_CTX_FIT_VRAM_FRACTION,
GgufLoadIntent,
LlamaCppBackend,
)
from core.inference.llama_server_args import (
_effective_tensor_parallel,
resolve_tensor_parallel,
)
from core.inference.tensor_fallback import load_with_tensor_fallback
from models.inference import (
InferenceStatusResponse,
LoadRequest,
LoadResponse,
)
def test_load_request_defaults_tensor_parallel_false():
req = LoadRequest(model_path = "owner/repo")
assert req.tensor_parallel is False
def test_load_request_accepts_tensor_parallel():
req = LoadRequest(model_path = "owner/repo", tensor_parallel = True)
assert req.tensor_parallel is True
def test_load_request_round_trips_json_key():
req = LoadRequest.model_validate({"model_path": "owner/repo", "tensor_parallel": True})
assert req.tensor_parallel is True
assert req.model_dump()["tensor_parallel"] is True
@pytest.mark.parametrize("model_cls", [LoadResponse, InferenceStatusResponse])
def test_response_models_emit_tensor_parallel(model_cls):
if model_cls is LoadResponse:
default = model_cls(
status = "loaded",
model = "owner/repo",
display_name = "repo",
inference = {},
)
on = model_cls(
status = "loaded",
model = "owner/repo",
display_name = "repo",
inference = {},
tensor_parallel = True,
)
else:
default = model_cls()
on = model_cls(tensor_parallel = True)
assert default.model_dump()["tensor_parallel"] is False
assert on.model_dump()["tensor_parallel"] is True
class _FakeProcess:
"""Stand-in for subprocess.Popen so _kill_process is a no-op."""
def terminate(self):
pass
def wait(self, timeout = None):
return 0
def kill(self):
pass
def poll(self):
return 0
def test_tensor_parallel_property_defaults_false():
assert LlamaCppBackend().tensor_parallel is False
def test_tensor_parallel_property_reflects_field():
backend = LlamaCppBackend()
backend._tensor_parallel = True
assert backend.tensor_parallel is True
def test_unload_resets_tensor_parallel():
backend = LlamaCppBackend()
backend._process = _FakeProcess()
backend._tensor_parallel = True
backend.unload_model()
assert backend.tensor_parallel is False
def _loaded_backend(tensor_parallel: bool) -> LlamaCppBackend:
backend = LlamaCppBackend()
backend._process = _FakeProcess()
backend._healthy = True
backend._model_identifier = "owner/repo"
backend._hf_variant = "Q4_K_M"
backend._requested_n_ctx = 8192
backend._cache_type_kv = None
backend._requested_spec_mode = "auto"
backend._chat_template_override = None
backend._is_vision = False
backend._extra_args = None
backend._gguf_path = None
backend._tensor_parallel = tensor_parallel
return backend
def _target_state(backend: LlamaCppBackend, tensor_parallel: bool) -> bool:
return backend.adopt_load_intent_if_matched(
GgufLoadIntent(
gguf_path = None,
model_identifier = "owner/repo",
hf_variant = "Q4_K_M",
n_ctx = 8192,
cache_type_kv = None,
speculative_type = "auto",
chat_template_override = None,
extra_args = None,
is_vision = False,
tensor_parallel = tensor_parallel,
)
)
@pytest.mark.parametrize("flag", [True, False])
def test_already_in_target_state_matches_same_tensor_parallel(flag):
assert _target_state(_loaded_backend(flag), flag) is True
@pytest.mark.parametrize(
"loaded,requested",
[(False, True), (True, False)],
)
def test_already_in_target_state_reloads_on_tensor_parallel_change(loaded, requested):
assert _target_state(_loaded_backend(loaded), requested) is False
def test_already_in_target_state_reloads_when_swa_full_env_changes(monkeypatch):
backend = _loaded_backend(False)
backend._swa_full = False
monkeypatch.setenv("LLAMA_ARG_SWA_FULL", "1")
assert _target_state(backend, False) is False
def test_already_in_target_state_reconciles_split_mode_extras():
# A --split-mode tensor in extras must match a running tensor server: no spurious reload.
backend = _loaded_backend(tensor_parallel = True)
backend._extra_args = ["--split-mode", "tensor"]
assert (
backend.adopt_load_intent_if_matched(
GgufLoadIntent(
gguf_path = None,
model_identifier = "owner/repo",
hf_variant = "Q4_K_M",
n_ctx = 8192,
cache_type_kv = None,
speculative_type = "auto",
chat_template_override = None,
extra_args = ["--split-mode", "tensor"],
is_vision = False,
tensor_parallel = False,
)
)
is True
)
def _load_model_source() -> str:
return inspect.getsource(llama_cpp_module.LlamaCppBackend.load_model)
def test_split_mode_tensor_is_gated_on_the_toggle():
src = _load_model_source()
assert (
'cmd.extend(["--split-mode", "tensor"])' in src
), "the tensor-parallel flag emission must be present in load_model"
# The emission lives behind `if tensor_parallel:` -- it must never be
# part of the unconditional base cmd list.
base_start = src.find("cmd = [")
base_end = src.find("\n ]", base_start)
base_block = src[base_start:base_end] if base_end > base_start else ""
assert (
"--split-mode" not in base_block
), "--split-mode must be conditional, not in the base cmd list"
gate = src.find("if tensor_parallel:")
emit = src.find('cmd.extend(["--split-mode", "tensor"])')
assert 0 <= gate < emit, "emission must sit under `if tensor_parallel:`"
def test_proportional_tensor_split_is_emitted_in_tensor_mode():
# Asymmetric GPUs OOM the smaller card under the even default split.
src = _load_model_source()
assert '"--tensor-split"' in src
gate = src.find("if tensor_parallel:")
# Find the TP block's --tensor-split after the gate; manual mode emits one earlier.
ts = src.find('"--tensor-split"', gate)
nxt_else = src.find("self._tensor_parallel = False")
assert 0 <= gate < ts < nxt_else, "--tensor-split must be emitted under `if tensor_parallel:`"
assert "tp_tensor_split" in src[gate:nxt_else]
def test_mtp_decode_probe_wired_under_tensor_parallel():
# MTP-draft can crash the CUDA FA kernel on the first decode under --split-mode tensor,
# so load_model probes a decode and falls back (FA off, then drop MTP).
src = _load_model_source()
probe = src.find("_probe_mtp_decode()")
assert probe != -1, "load_model must decode-probe MTP under tensor parallelism"
guard = src[max(0, probe - 400) : probe]
assert "self._tensor_parallel" in guard and "_spec_requested_mtp" in guard
after = src[probe : probe + 900]
assert "_with_flash_attn_off" in after and "healthy = False" in after
# The drafter-drop fallback is `if not healthy and (<any drafter requested>) ...`. Matched on
# its shape, not its exact spelling: #13806 added eagle3 to that list and wrapped it over
# several lines, and a one-line literal then found nothing.
match = re.search(
r"not healthy\s+and\s*\(\s*_spec_requested_mtp\s+or\s+_spec_requested_dspark\b"
r"[^)]*?_spec_requested_dflash\b",
src,
)
assert match is not None, "load_model no longer has a drafter-drop fallback for MTP"
assert probe < match.start(), "the probe must precede the drafter-drop fallback"
def test_probe_mtp_decode_returns_false_on_crash(monkeypatch):
backend = LlamaCppBackend()
backend._port = 0
class _Resp:
def __init__(self, code):
self.status_code = code
backend._process = None
monkeypatch.setattr(llama_cpp_module.httpx, "post", lambda *a, **k: _Resp(200), raising = False)
assert backend._probe_mtp_decode(timeout = 1.0) is True
def _drop(*a, **k):
raise llama_cpp_module.httpx.RemoteProtocolError("peer closed connection")
monkeypatch.setattr(llama_cpp_module.httpx, "post", _drop, raising = False)
assert backend._probe_mtp_decode(timeout = 1.0) is False
monkeypatch.setattr(llama_cpp_module.httpx, "post", lambda *a, **k: _Resp(500), raising = False)
assert backend._probe_mtp_decode(timeout = 1.0) is False
backend._process = _FakeProcess()
monkeypatch.setattr(llama_cpp_module.httpx, "post", lambda *a, **k: _Resp(200), raising = False)
assert backend._probe_mtp_decode(timeout = 1.0) is False
def _recovery_backend() -> LlamaCppBackend:
b = LlamaCppBackend()
b._tensor_parallel = True
b._speculative_type = "draft-mtp"
b._mtp_runtime_fallback_active = True
b._process = _FakeProcess()
b._last_load_intent = GgufLoadIntent(
model_identifier = "owner/repo",
tensor_parallel = True,
speculative_type = "auto",
n_parallel = 4,
)
return b
def test_generate_chat_completion_wires_runtime_recovery():
# The non-tool generation path must route a mid-stream death to the recovery helper.
src = inspect.getsource(LlamaCppBackend.generate_chat_completion)
assert "_maybe_recover_from_mtp_crash" in src
def test_runtime_recovery_reloads_without_mtp(monkeypatch):
b = _recovery_backend()
done = threading.Event()
captured = {}
def _fake_load_model(intent):
captured.update(vars(intent))
done.set()
return True
monkeypatch.setattr(b, "load_model", _fake_load_model)
assert b._maybe_recover_from_mtp_crash(RuntimeError("peer closed")) is True
assert done.wait(timeout = 5)
assert captured["speculative_type"] == "off"
assert captured["model_identifier"] == "owner/repo"
assert captured["n_parallel"] == 4
deadline = time.monotonic() + 2
while b._spec_fallback_reason != "runtime_error" and time.monotonic() < deadline:
time.sleep(0.02)
assert b._spec_fallback_reason == "runtime_error"
# The reload thread clears the flag a beat after setting the reason; wait for it.
deadline = time.monotonic() + 2
while b._mtp_runtime_fallback_in_progress and time.monotonic() < deadline:
time.sleep(0.02)
assert b._mtp_runtime_fallback_in_progress is False
@pytest.mark.parametrize(
"mutate",
[
lambda b: setattr(b, "_mtp_runtime_fallback_active", False),
lambda b: setattr(b, "_last_load_intent", None),
lambda b: setattr(b, "_process", None),
lambda b: b._cancel_event.set(),
],
)
def test_runtime_recovery_skips_when_not_applicable(monkeypatch, mutate):
b = _recovery_backend()
mutate(b)
calls = []
monkeypatch.setattr(b, "load_model", lambda intent: calls.append(intent))
assert b._maybe_recover_from_mtp_crash(RuntimeError()) is False
assert calls == []
class _BlockingDeadProc:
def __init__(self):
self._dead = threading.Event()
def poll(self):
return 0 if self._dead.is_set() else None
def terminate(self):
self._dead.set()
def kill(self):
self._dead.set()
def wait(self, timeout = None):
self._dead.set()
return 0
def release(self):
self._dead.set()
def test_runtime_recovery_fires_for_user_env_mtp(monkeypatch):
# MTP driven by user extra_args / LLAMA_ARG_SPEC_TYPE leaves _speculative_type
# unset, but the launch flag still gates recovery on (pass-through MTP).
b = _recovery_backend()
b._speculative_type = None
done = threading.Event()
captured = {}
def _fake_load_model(intent):
captured.update(vars(intent))
done.set()
return True
monkeypatch.setattr(b, "load_model", _fake_load_model)
assert b._maybe_recover_from_mtp_crash(RuntimeError()) is True
assert done.wait(timeout = 5)
assert captured["speculative_type"] == "off"
def test_runtime_recovery_strips_user_mtp_extra_args(monkeypatch):
# llama.cpp appends spec types rather than replacing, so the user flag must be removed.
b = _recovery_backend()
b._last_load_intent = replace(
b._last_load_intent,
extra_args = ["--spec-type", "draft-mtp"],
)
done = threading.Event()
captured = {}
def _fake_load_model(intent):
captured.update(vars(intent))
done.set()
return True
monkeypatch.setattr(b, "load_model", _fake_load_model)
assert b._maybe_recover_from_mtp_crash(RuntimeError()) is True
assert done.wait(timeout = 5)
assert captured["speculative_type"] == "off"
assert "--spec-type" not in captured["extra_args"]
assert "draft-mtp" not in captured["extra_args"]
def test_runtime_recovery_restores_requested_mode(monkeypatch):
b = _recovery_backend()
b._last_load_intent = replace(b._last_load_intent, speculative_type = "mtp")
done = threading.Event()
def _fake_load_model(intent):
b._requested_spec_mode = "off"
done.set()
return True
monkeypatch.setattr(b, "load_model", _fake_load_model)
assert b._maybe_recover_from_mtp_crash(RuntimeError()) is True
assert done.wait(timeout = 5)
deadline = time.monotonic() + 2
while b._requested_spec_mode != "mtp" and time.monotonic() < deadline:
time.sleep(0.02)
assert b._requested_spec_mode == "mtp"
assert b._spec_fallback_reason == "runtime_error"
def test_runtime_recovery_does_not_publish_state_when_reload_is_cancelled(monkeypatch):
b = _recovery_backend()
b._requested_spec_mode = "auto"
b._spec_fallback_reason = "existing"
done = threading.Event()
def _cancelled_load(intent):
done.set()
return False
monkeypatch.setattr(b, "load_model", _cancelled_load)
assert b._maybe_recover_from_mtp_crash(RuntimeError()) is True
assert done.wait(timeout = 5)
deadline = time.monotonic() + 2
while b._mtp_runtime_fallback_in_progress and time.monotonic() < deadline:
time.sleep(0.02)
assert b._requested_spec_mode == "auto"
assert b._spec_fallback_reason == "existing"
def test_runtime_recovery_undoes_an_unload_during_reload(monkeypatch):
b = _recovery_backend()
b._requested_spec_mode = "auto"
b._spec_fallback_reason = "existing"
done = threading.Event()
unloads = []
def _racing_load(intent):
b._unload_epoch += 1
done.set()
return True
monkeypatch.setattr(b, "load_model", _racing_load)
monkeypatch.setattr(b, "unload_model", lambda: unloads.append(1) or True)
assert b._maybe_recover_from_mtp_crash(RuntimeError()) is True
assert done.wait(timeout = 5)
deadline = time.monotonic() + 2
while b._mtp_runtime_fallback_in_progress and time.monotonic() < deadline:
time.sleep(0.02)
assert unloads == [1]
assert b._requested_spec_mode == "auto"
assert b._spec_fallback_reason == "existing"
def test_runtime_recovery_publishes_state_atomically_with_unload(monkeypatch):
b = _recovery_backend()
b._last_load_intent = replace(
b._last_load_intent,
extra_args = ("--spec-type", "draft-mtp"),
gpu_ids = (0,),
)
b._requested_extra_args = ["old"]
b._requested_spec_mode = "auto"
b._spec_fallback_reason = "existing"
load_finished = threading.Event()
unload_entered = threading.Event()
unload_done = threading.Event()
unload_threads = []
def _fake_load(intent):
load_finished.set()
return True
def _unload_state():
unload_entered.set()
b._cancel_event.set()
with b._lock:
b._unload_epoch += 1
b._requested_extra_args = None
b._requested_spec_mode = None
b._spec_fallback_reason = None
unload_done.set()
def _race_after_epoch_check(extra_args):
thread = threading.Thread(target = _unload_state)
unload_threads.append(thread)
thread.start()
assert unload_entered.wait(timeout = 2)
unload_done.wait(timeout = 0.2)
return list(extra_args)
monkeypatch.setattr(b, "load_model", _fake_load)
monkeypatch.setattr(b, "_strip_device_extra_args", _race_after_epoch_check)
assert b._maybe_recover_from_mtp_crash(RuntimeError()) is True
assert load_finished.wait(timeout = 5)
assert unload_done.wait(timeout = 5)
for thread in unload_threads:
thread.join(timeout = 2)
deadline = time.monotonic() + 2
while b._mtp_runtime_fallback_in_progress and time.monotonic() < deadline:
time.sleep(0.02)
assert b._requested_extra_args is None
assert b._requested_spec_mode is None
assert b._spec_fallback_reason is None
def test_runtime_recovery_skips_when_process_replaced(monkeypatch):
# A newer load that replaces the process mid-poll must not be clobbered by stale recovery.
b = _recovery_backend()
p1 = _BlockingDeadProc()
b._process = p1
calls = []
monkeypatch.setattr(b, "load_model", lambda intent: calls.append(intent))
assert b._maybe_recover_from_mtp_crash(RuntimeError()) is True
b._process = _FakeProcess()
p1.release()
time.sleep(0.6)
assert calls == [], "stale recovery replayed over a newer load"
def test_runtime_recovery_skips_when_snapshot_changed(monkeypatch):
b = _recovery_backend()
p1 = _BlockingDeadProc()
b._process = p1
calls = []
monkeypatch.setattr(b, "load_model", lambda intent: calls.append(intent))
assert b._maybe_recover_from_mtp_crash(RuntimeError()) is True
b._last_load_intent = replace(b._last_load_intent, model_identifier = "other/model")
p1.release()
time.sleep(0.6)
assert calls == []
def test_runtime_recovery_is_single_flight(monkeypatch):
b = _recovery_backend()
started = threading.Event()
release = threading.Event()
def _slow_load(intent):
started.set()
release.wait(timeout = 5)
return True
monkeypatch.setattr(b, "load_model", _slow_load)
assert b._maybe_recover_from_mtp_crash(RuntimeError()) is True
assert started.wait(timeout = 5)
assert b._maybe_recover_from_mtp_crash(RuntimeError()) is False
release.set()
def test_single_flight_claim_is_released_when_the_reload_cannot_start(monkeypatch):
# Only the reload thread clears the claim, so a failed thread start must not latch it.
b = _recovery_backend()
class _NoThread:
def __init__(self, *args, **kwargs):
pass
def start(self):
raise RuntimeError("can't start new thread")
monkeypatch.setattr(llama_cpp_module.threading, "Thread", _NoThread)
assert b._maybe_recover_from_mtp_crash(RuntimeError()) is False
assert b._mtp_runtime_fallback_in_progress is False
def test_load_intent_is_read_once_before_the_claim(monkeypatch):
# Gate and snapshot must share one read, or an unload can strand the flag.
b = _recovery_backend()
class _CountingIntent:
def __init__(self, value):
self.value = value
self.reads = 0
def __get__(self, obj, owner):
if obj is None:
return self
self.reads += 1
return self.value
def __set__(self, obj, value):
self.value = value
counter = _CountingIntent(GgufLoadIntent(model_identifier = "owner/repo"))
monkeypatch.setattr(type(b), "_last_load_intent", counter, raising = False)
class _UnstartedThread:
def __init__(self, *args, **kwargs):
pass
def start(self):
pass
monkeypatch.setattr(llama_cpp_module.threading, "Thread", _UnstartedThread)
assert b._maybe_recover_from_mtp_crash(RuntimeError()) is True
assert counter.reads == 1, f"read {counter.reads} times; an unload can race the claim"
def test_respawn_defers_to_an_inflight_mtp_reload(monkeypatch):
# "Already recovering" must not read as "not an MTP crash".
b = _recovery_backend()
b._mtp_runtime_fallback_in_progress = True
loads = _shared_setup_2(b, monkeypatch)
assert loads == []
b._mtp_runtime_fallback_in_progress = False
b._process.returncode = -9
assert b._respawn_if_dead() is True
assert [intent.speculative_type for intent in loads] == ["auto"]
def test_respawn_does_not_wait_out_the_grace_on_a_replacement(monkeypatch):
# Lock waiters that find a healthy replacement must not each wait out the reap grace.
class _LiveProcess(_FakeProcess):
returncode = None
def __init__(self):
self.polls = 0
def poll(self):
self.polls += 1
return None
workers = 4
b = _recovery_backend()
b._healthy = True
b._process.returncode = -9
live = _LiveProcess()
loads: list[GgufLoadIntent] = []
guard = threading.Lock()
all_in_flight = threading.Event()
# Subclass this instance: a class descriptor would redirect _process for every backend.
state = {"proc": b._process, "readers": set()}
class _Tracked(type(b)):
@property
def _process(self):
"""Reports when every worker has taken its pre-lock look at the child."""
with guard:
state["readers"].add(threading.get_ident())
everyone = len(state["readers"]) >= workers
if everyone:
all_in_flight.set()
return state["proc"]
@_process.setter
def _process(self, value):
state["proc"] = value
b.__class__ = _Tracked
def _load(intent):
# Timeout keeps the pre-fix build failing instead of hanging.
all_in_flight.wait(timeout = 2)
with guard:
loads.append(intent)
b._process = live
b._healthy = True
return True
monkeypatch.setattr(b, "load_model", _load)
results: list[bool] = []
def _respawn():
outcome = b._respawn_if_dead()
with guard:
results.append(outcome)
threads = [threading.Thread(target = _respawn) for _ in range(workers)]
started = time.monotonic()
for thread in threads:
thread.start()
for thread in threads:
thread.join(timeout = 30)
elapsed = time.monotonic() - started
assert results == [True] * workers, results
assert len(loads) == 1, f"{len(loads)} reloads, expected one"
assert live.polls == 0, "queued caller waited out the grace on a healthy server"
assert elapsed < llama_cpp_module._RESPAWN_REAP_GRACE_S * (workers - 1)
class _DyingChild(_FakeProcess):
"""Alive for the first polls, then reapable: what a terminate() looks like."""
def __init__(
self,
code = -15,
alive_polls = 2,
on_death = None,
):
self.polls = 0
self.returncode = None
self._code = code
self._alive_polls = alive_polls
self._on_death = on_death
def poll(self):
self.polls += 1
if self.polls <= self._alive_polls:
return None
if self.returncode is None:
self.returncode = self._code
if self._on_death is not None:
self._on_death()
return self._code
def test_respawn_does_not_resurrect_a_deliberate_unload(monkeypatch):
# unload_model sets _cancel_event before killing; that exit is not a crash.
b = _recovery_backend()
b._healthy = True
b._process = _DyingChild()
b._cancel_event.set()
loads = _shared_setup_2(b, monkeypatch)
assert loads == [], "resurrected a model the user unloaded"
def test_respawn_rechecks_the_cancel_flag_after_the_grace_wait(monkeypatch):
b = _recovery_backend()
b._healthy = True
b._process = _DyingChild(on_death = b._cancel_event.set)
loads = _shared_setup_2(b, monkeypatch)
assert loads == [], "checked the cancel flag only before the wait"
def test_respawn_does_not_revert_a_newer_load(monkeypatch):
# A model switch that lands during the wait must win; replaying old intent undoes it.
b = _recovery_backend()
b._healthy = True
replacement = _DyingChild(alive_polls = 10**6)
b._process = _DyingChild(on_death = lambda: setattr(b, "_process", replacement))
loads: list[GgufLoadIntent] = []
monkeypatch.setattr(b, "load_model", lambda intent: loads.append(intent) or True)
b._respawn_if_dead()
assert loads == [], "replayed stale kwargs over a newer load"
assert b._process is replacement
def test_respawn_still_recovers_an_ordinary_crash(monkeypatch):
b = _recovery_backend()
b._healthy = True
b._process = _DyingChild(code = -9)
loads: list[GgufLoadIntent] = []
monkeypatch.setattr(b, "load_model", lambda intent: loads.append(intent) or True)
assert b._respawn_if_dead() is True
assert len(loads) == 1
class _NeverReapable(_FakeProcess):
"""A child that stays unreapable, so only the port can tell alive from dead."""
returncode = None
def poll(self):
return None
def test_a_transient_error_against_a_live_server_costs_nothing(monkeypatch):
# The grace sleep is held under _respawn_lock, so it must not hit a live server.
listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
listener.bind(("127.0.0.1", 0))
listener.listen(16)
try:
b = _recovery_backend()
b._healthy = True
b._process = _NeverReapable()
b._port = listener.getsockname()[1]
loads: list[GgufLoadIntent] = []
monkeypatch.setattr(b, "load_model", lambda intent: loads.append(intent) or True)
started = time.monotonic()
assert b._respawn_if_dead() is True
elapsed = time.monotonic() - started
assert loads == [], "a live server must not be reloaded"
assert (
elapsed < llama_cpp_module._RESPAWN_REAP_GRACE_S / 2
), f"waited {elapsed:.2f}s on a server that is still accepting"
finally:
listener.close()
def test_a_closed_port_still_waits_for_the_child_to_be_reapable(monkeypatch):
probe = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
probe.bind(("127.0.0.1", 0))
dead_port = probe.getsockname()[1]
probe.close()
b = _recovery_backend()
b._healthy = True
b._process = _DyingChild(code = -9)
b._port = dead_port
loads: list[GgufLoadIntent] = []
monkeypatch.setattr(b, "load_model", lambda intent: loads.append(intent) or True)
assert b._respawn_if_dead() is True
assert len(loads) == 1
def test_socket_fast_path_honours_a_pending_unload(monkeypatch):
# unload_model sets _cancel_event before killing, so the child still accepts here.
listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
listener.bind(("127.0.0.1", 0))
listener.listen(8)
try:
b = _recovery_backend()
b._healthy = True
b._process = _NeverReapable()
b._port = listener.getsockname()[1]
b._cancel_event.set()
loads: list[GgufLoadIntent] = []
monkeypatch.setattr(b, "load_model", lambda intent: loads.append(intent) or True)
assert b._respawn_if_dead() is False
assert loads == []
finally:
listener.close()
def test_an_unload_landing_during_the_reload_is_undone(monkeypatch):
# unload_model never takes _serial_load_lock and load_model clears _cancel_event,
# so _unload_epoch is the surviving evidence of a cancel.
b = _recovery_backend()
b._healthy = True
b._process = _FakeProcess()
b._process.returncode = -9
loads: list[GgufLoadIntent] = []
monkeypatch.setattr(b, "load_model", lambda intent: loads.append(intent) or True)
unloads: list[int] = []
real_unload = b.unload_model
monkeypatch.setattr(b, "unload_model", lambda: unloads.append(1) or real_unload())
real_warning = llama_cpp_module.logger.warning
fired: list[int] = []
def racing_warning(*args, **kwargs):
if not fired:
fired.append(1)
real_unload()
return real_warning(*args, **kwargs)
monkeypatch.setattr(llama_cpp_module.logger, "warning", racing_warning)
assert b._respawn_if_dead() is False
assert unloads, "the racing unload was not honoured"
def test_socket_probe_is_false_without_a_port():
b = _recovery_backend()
b._port = None
assert b._server_socket_is_open() is False
def test_runtime_recovery_rechecks_cancel_after_the_death_poll(monkeypatch):
b = _recovery_backend()
process = _BlockingDeadProc()
b._process = process
loads = []
monkeypatch.setattr(b, "load_model", lambda intent: loads.append(intent) or True)
assert b._maybe_recover_from_mtp_crash(RuntimeError()) is True
b._cancel_event.set()
process.release()
deadline = time.monotonic() + 2
while b._mtp_runtime_fallback_in_progress and time.monotonic() < deadline:
time.sleep(0.02)
assert loads == []
def test_probe_mtp_decode_uses_api_key_auth(monkeypatch):
# Direct-stream mode uses --api-key; the probe must send bearer auth.
backend = LlamaCppBackend()
backend._port = 0
backend._process = None
captured = {}
class _Resp:
status_code = 200
def _capture(*a, **k):
captured.clear()
captured.update(k)
return _Resp()
monkeypatch.setattr(llama_cpp_module.httpx, "post", _capture, raising = False)
backend._api_key = "secret"
backend._probe_mtp_decode(timeout = 1.0)
assert captured["headers"] == {"Authorization": "Bearer secret"}
assert captured["trust_env"] is False
backend._api_key = None
backend._probe_mtp_decode(timeout = 1.0)
assert captured["headers"] is None
class _ToggleProcess:
"""A subprocess stand-in whose liveness can be flipped at runtime."""
def __init__(self):
self._alive = True
def poll(self):
return None if self._alive else 0
def terminate(self):
self._alive = False
def kill(self):
self._alive = False
def wait(self, timeout = None):
self._alive = False
return 0
def die(self):
self._alive = False
def test_crash_watchdog_triggers_recovery_on_death(monkeypatch):
b, fired, proc = _shared_setup_1(monkeypatch)
assert b._mtp_watchdog_thread is not None
proc.die()
assert fired.wait(timeout = 3)
def test_crash_watchdog_ignores_intentional_termination(monkeypatch):
# The watchdog stops before a planned kill, so that exit is not mistaken for a crash.
b, fired, proc = _shared_setup_1(monkeypatch)
b._stop_mtp_crash_watchdog()
proc.die()
assert not fired.wait(timeout = 2)
assert b._mtp_watchdog_thread is None
@pytest.mark.parametrize(
"mutate",
[
lambda b: setattr(b, "_process", None),
],
)
def test_crash_watchdog_not_armed_when_inapplicable(mutate):
b = _recovery_backend()
b._process = _ToggleProcess()
mutate(b)
b._start_mtp_crash_watchdog()
assert b._mtp_watchdog_thread is None
def test_kill_process_stops_crash_watchdog(monkeypatch):
b, fired, proc = _shared_setup_1(monkeypatch)
b._kill_process()
assert b._mtp_watchdog_thread is None
assert b._process is None
assert not fired.wait(timeout = 2)
def test_kill_process_stops_watchdog_before_terminate():
# Stop the watchdog before terminating (now inside _kill_process_body).
src = inspect.getsource(LlamaCppBackend._kill_process) + inspect.getsource(
LlamaCppBackend._kill_process_body
)
stop = src.find("_stop_mtp_crash_watchdog()")
term = src.find(".terminate(")
assert 0 <= stop < term, "must stop the watchdog before terminating"
def test_crash_watchdog_rechecks_stop_before_recovery():
# After an exit, re-check the stop flag: a kill racing the poll must not fire recovery.
src = inspect.getsource(LlamaCppBackend._start_mtp_crash_watchdog)
check = src.find("stop.is_set()")
recover = src.find("_maybe_recover_from_mtp_crash")
assert 0 <= check < recover, "must re-check stop before recovering"
def test_load_model_arms_crash_watchdog():
src = inspect.getsource(LlamaCppBackend.load_model)
assert "_start_mtp_crash_watchdog" in src
def _kv_seeded_backend() -> LlamaCppBackend:
backend = LlamaCppBackend()
backend._n_layers = 32
backend._embedding_length = 4096
backend._n_heads = 32
backend._n_kv_heads = 8
backend._context_length = 131072
return backend
def test_fit_context_budget_frac_override_is_tighter():
backend = _kv_seeded_backend()
model_size = 8 * 1024**3
pool_mib = 24 * 1024
fit_default = backend._fit_context_to_vram(131072, pool_mib, model_size, "f16")
fit_tp = backend._fit_context_to_vram(131072, pool_mib, model_size, "f16", budget_frac = 0.80)
assert fit_tp < 131072, "expected the context to be capped at this VRAM tier"
assert fit_tp <= fit_default, "a tighter budget must not allow MORE context"
assert backend._fit_context_to_vram(131072, pool_mib, model_size, "f16") == fit_default
def test_fit_context_follows_the_user_vram_budget(monkeypatch):
"""A VRAM budget setting moves the fitted context both ways; unset keeps the historical default."""
import utils.vram_budget_settings as vb
backend = _kv_seeded_backend()
model_size = 8 * 1024**3
pool_mib = 24 * 1024
total_mib = 24 * 1024
def _fit():
return backend._fit_context_to_vram(
131072, pool_mib, model_size, "f16", total_mib = total_mib
)
monkeypatch.delenv(vb.VRAM_FRACTION_ENV_VAR, raising = False)
monkeypatch.setattr(vb, "_cached_setting", lambda _key: None)
fit_default = _fit()
assert fit_default < 131072, "expected the context to be capped at this VRAM tier"
monkeypatch.setattr(vb, "_cached_setting", lambda _key: 1.0)
fit_max = _fit()
# 0.80, not VRAM_FRACTION_MIN: at 0.10 the 8 GiB weights alone exceed the budget, which hands the load to the offload path.
monkeypatch.setattr(vb, "_cached_setting", lambda _key: 0.80)
fit_min = _fit()
assert fit_max > fit_default, "claiming the whole card must buy context back"
assert fit_min < fit_default, "a smaller budget must give context up"
monkeypatch.setattr(vb, "_cached_setting", lambda _key: None)
assert _fit() == fit_default
def test_tensor_plan_leaves_the_floor_reserve_at_a_full_budget():
"""At 100% a tensor plan must keep the 512 MiB floor reserve, as the layer path does."""
import core.inference.llama_cpp as lc
backend = _kv_seeded_backend()
total_mib = 24 * 1024
gpus = [(0, total_mib), (1, total_mib)]
total_by_idx = {0: total_mib, 1: total_mib}
def _plan(frac: float, model_gb: int):
return backend._plan_tensor_parallel(
list(gpus),
model_gb * 1024**3,
131072,
cache_type_kv = "f16",
total_by_idx = dict(total_by_idx),
vram_fraction = frac,
)
# A fraction whose reserve equals the floor on this card plans the same as a floored 100%.
floor_frac = 1.0 - lc._VRAM_FLOOR_RESERVE_MIB / total_mib
assert floor_frac > lc._CTX_FIT_VRAM_FRACTION, "24 GiB: 1% is under the floor"
*_, split_full = _plan(1.0, 40)
*_, split_floor = _plan(floor_frac, 40)
*_, split_default = _plan(lc._CTX_FIT_VRAM_FRACTION, 40)
assert split_full is not None, "expected a weighted split at this model size"
assert (
split_full == split_floor
), f"100% spent {split_full} per card where the floor allows {split_floor}"
# The default reserves its full 3% (737 MiB > floor).
assert all(d < f for d, f in zip(split_default, split_full))
ctx_full, *_ = _plan(1.0, 36)
ctx_floor, *_ = _plan(floor_frac, 36)
assert ctx_full < 131072, "expected the context to be capped at this VRAM tier"
assert ctx_full == ctx_floor
def test_split_mode_tensor_arch_failure_message():
msg = LlamaCppBackend._classify_llama_start_failure(
"llama_model_create: LLAMA_SPLIT_MODE_TENSOR not implemented for architecture 'deepseek2'",
None,
"unsloth/DeepSeek-V3-GGUF",
)
assert "Tensor parallelism is not supported" in msg
def test_unrelated_arch_failure_not_hijacked_by_tensor_message():
msg = LlamaCppBackend._classify_llama_start_failure(
"unknown model architecture: 'flux'", "/models/flux.gguf", None
)
assert "Tensor parallelism" not in msg
# Seeded KV (~128 KiB/token); assert relationships, not magic numbers.
_GB = 1024**3
_ASYM = [(0, 48000), (1, 24000)]
_SYM = [(0, 24000), (1, 24000)]
def _plan(
model_gb,
target = 131072,
gpus = _ASYM,
mtp = False,
):
b = _kv_seeded_backend()
return b, b._plan_tensor_parallel(gpus, int(model_gb * _GB), target, mtp_engaged = mtp)
def _kv_budget_b(model_gb, gpus = _ASYM):
reserve = LlamaCppBackend._TENSOR_PARALLEL_BUFFER_RESERVE_MIB
usable = sum(f * _CTX_FIT_VRAM_FRACTION for _, f in gpus)
return (usable - len(gpus) * reserve) * 1024 * 1024 - int(model_gb * _GB)
def test_tp_plan_weighted_split_on_asymmetric_big_model():
b, (ec, mac, gi, ts) = _plan(50)
reserve = b._TENSOR_PARALLEL_BUFFER_RESERVE_MIB
assert gi == [0, 1]
# Split weighted by usable - flat buffer - per-device context compute.
cc_per_dev = b._compute_buffer_ctx_bytes(ec, None, None) // (1024 * 1024)
assert cc_per_dev > 0
assert ts == [
int(48000 * _CTX_FIT_VRAM_FRACTION - reserve - cc_per_dev),
int(24000 * _CTX_FIT_VRAM_FRACTION - reserve - cc_per_dev),
]
assert ec < 131072
def test_tp_plan_even_split_when_model_fits():
# Even share fitting the smallest GPU keeps None; some archs crash on weighted splits.
_, (ec, mac, gi, ts) = _plan(4)
assert ts is None
def test_tp_plan_symmetric_gpus_use_even_split():
_, (ec, mac, gi, ts) = _plan(8, gpus = _SYM)
assert ts is None
def test_tp_plan_context_fits_pool_budget_no_oom():
b, (ec, mac, gi, ts) = _plan(50)
assert b._estimate_kv_cache_bytes(ec) <= _kv_budget_b(50)
def test_tp_plan_uses_available_vram_not_wasteful():
b, (ec, mac, gi, ts) = _plan(50)
assert b._estimate_kv_cache_bytes(ec) >= 0.9 * _kv_budget_b(50)
def test_tp_plan_weights_exceed_pool_floors_context():
# 70 GB > pool minus per-GPU reserves -> floor (triggers layer fallback)
_, (ec, mac, gi, ts) = _plan(70)
assert ec == 2048
def test_tp_plan_floor_never_exceeds_explicit_small_context():
# An explicit context below the 2048 floor must not be raised.
_, (ec, mac, gi, ts) = _plan(70, target = 1024) # weights exceed pool -> floor path
assert ec == 1024
_, (ec2, *_rest) = _plan(50, target = 1024)
assert ec2 <= 1024
def test_tp_plan_explicit_context_honored_when_it_fits():
_, (ec, mac, gi, ts) = _plan(50, target = 8192)
assert ec == 8192
def test_tp_plan_explicit_context_capped_when_too_large():
_, (ec, mac, gi, ts) = _plan(50, target = 131072)
assert 2048 <= ec < 131072
def test_tp_plan_max_available_ctx_reports_native_not_explicit_ctx():
# max_available_ctx must reflect the hardware cap, not the request.
b = _kv_seeded_backend()
ec, mac, _gi, _ts = b._plan_tensor_parallel(_ASYM, int(50 * _GB), 8192, max_target_ctx = 131072)
_, native_mac, *_ = b._plan_tensor_parallel(_ASYM, int(50 * _GB), 131072)
assert ec == 8192
assert mac == native_mac > ec
def test_tp_plan_mtp_reserves_extra_and_shrinks_context():
_, (ec_no, *_rest) = _plan(50)
_, (ec_mtp, *_rest) = _plan(50, mtp = True)
assert ec_mtp < ec_no
def test_tp_plan_reserves_context_linear_compute_buffer():
# Tensor mode replicates the compute graph per device (~n_ubatch*2 B/token each);
# the fit must reserve n_dev x that.
b, (ec, mac, gi, ts) = _plan(50)
cc = len(gi) * b._compute_buffer_ctx_bytes(ec, None, "f16")
assert cc > 0
assert b._estimate_kv_cache_bytes(ec) + cc <= _kv_budget_b(50)
def test_tp_plan_context_shrinks_vs_compute_unaware():
b, (ec, *_r) = _plan(50)
b2 = _kv_seeded_backend()
b2._embedding_length = 0 # kills the context-linear compute term (returns 0)
ec_naive, *_r2 = b2._plan_tensor_parallel(_ASYM, int(50 * _GB), 131072)
assert ec < ec_naive
def test_tp_plan_soft_overhead_shrinks_context():
# Tensor mode has no --fit valve, so soft_overhead must shrink the context.
b = _kv_seeded_backend()
ec_no, *_r = b._plan_tensor_parallel(_ASYM, int(50 * _GB), 131072)
ec_soft, *_r2 = b._plan_tensor_parallel(
_ASYM, int(50 * _GB), 131072, soft_overhead_bytes = 2 * _GB
)
assert 2048 < ec_soft < ec_no
def test_tp_plan_soft_overhead_reserved_against_budget():
b = _kv_seeded_backend()
soft = 2 * _GB
ec, *_r = b._plan_tensor_parallel(_ASYM, int(50 * _GB), 131072, soft_overhead_bytes = soft)
cc = len(_ASYM) * b._compute_buffer_ctx_bytes(ec, None, None)
assert b._estimate_kv_cache_bytes(ec) + cc + soft <= _kv_budget_b(50)
def test_tp_plan_weighted_split_keeps_small_gpu_within_budget():
# Subtract replicated compute from each card, or the smaller one is weighted past budget.
b, (ec, mac, gi, ts) = _plan(50)
assert ts is not None and len(ts) == len(gi) == 2
reserve = b._TENSOR_PARALLEL_BUFFER_RESERVE_MIB
cc_per_dev = b._compute_buffer_ctx_bytes(ec, None, None) // (1024 * 1024)
free_by_idx = {0: 48000, 1: 24000}
split_content_mib = (int(50 * _GB) + b._estimate_kv_cache_bytes(ec)) / (1024 * 1024)
total_weight = sum(ts)
for w, idx in zip(ts, gi):
placed = split_content_mib * w / total_weight
usable = free_by_idx[idx] * _CTX_FIT_VRAM_FRACTION
assert placed + reserve + cc_per_dev <= usable + 1 # +1 MiB for int rounding
old_adj = [int(free_by_idx[i] * _CTX_FIT_VRAM_FRACTION - reserve) for i in gi]
old_small_placed = split_content_mib * old_adj[1] / sum(old_adj)
assert old_small_placed + reserve + cc_per_dev > free_by_idx[1] * _CTX_FIT_VRAM_FRACTION
def test_tp_plan_no_kv_metadata_floors_context():
b = LlamaCppBackend()
ec, mac, gi, ts = b._plan_tensor_parallel(_ASYM, int(50 * _GB), 131072)
assert ec <= 4096
def test_tp_plan_single_gpu_never_splits():
# Tensor mode is a no-op below 2 GPUs; a single-GPU plan must never emit a tensor split.
b = _kv_seeded_backend()
ec, mac, gi, ts = b._plan_tensor_parallel([(0, 24000)], int(8 * _GB), 8192)
assert ts is None
assert gi == [0]
def test_tp_plan_zero_gpus_never_splits():
b = _kv_seeded_backend()
ec, mac, gi, ts = b._plan_tensor_parallel([], int(8 * _GB), 8192)
assert ts is None
assert gi == []
def test_tp_plan_drops_gpu_below_buffer_reserve():
# A GPU with less free VRAM than the per-device compute-buffer reserve
# can't host tensor mode; it's excluded, which here leaves <2 usable -> no
# split (and gpu_indices reflects only the usable device).
b = _kv_seeded_backend()
reserve = LlamaCppBackend._TENSOR_PARALLEL_BUFFER_RESERVE_MIB
ec, mac, gi, ts = b._plan_tensor_parallel([(0, 48000), (1, reserve - 1)], int(8 * _GB), 8192)
assert gi == [0]
assert ts is None
# A tensor-incompatible model makes load_model raise; /load must retry with layer split.
class _RecordingLoader:
"""Fake attempt_load crashes under tensor mode (bool or --split-mode) and succeeds on layer split."""
def __init__(self):
self.calls: list[tuple] = []
async def __call__(self, tensor_parallel, extra_args):
self.calls.append((tensor_parallel, list(extra_args) if extra_args else extra_args))
if resolve_tensor_parallel(extra_args, tensor_parallel):
raise RuntimeError("llama-server failed to start")
return True
def test_tensor_fallback_retries_layer_on_crash():
loader = _RecordingLoader()
ok = asyncio.run(
load_with_tensor_fallback(loader, requested_tensor = True, extra_args = None, label = "m")
)
assert ok is True
assert [c[0] for c in loader.calls] == [True, False]
def test_tensor_fallback_no_retry_on_success():
calls: list[bool] = []
async def _ok(tensor_parallel, extra_args):
calls.append(tensor_parallel)
return True
ok = asyncio.run(
load_with_tensor_fallback(_ok, requested_tensor = True, extra_args = None, label = "m")
)
assert ok is True
assert calls == [True]
def test_tensor_fallback_retries_when_tensor_returns_false():
# A False return from load_model must trigger the layer-split retry, just as a raise does.
calls: list[bool] = []
async def _false_on_tensor(tensor_parallel, extra_args):
calls.append(tensor_parallel)
return not resolve_tensor_parallel(extra_args, tensor_parallel)
ok = asyncio.run(
load_with_tensor_fallback(
_false_on_tensor, requested_tensor = True, extra_args = None, label = "m"
)
)
assert ok is True
assert calls == [True, False]
def test_tensor_fallback_returns_false_when_both_attempts_fail():
calls: list[bool] = []
async def _always_false(tensor_parallel, extra_args):
calls.append(tensor_parallel)
return False
ok = asyncio.run(
load_with_tensor_fallback(_always_false, requested_tensor = True, extra_args = None, label = "m")
)
assert ok is False
assert calls == [True, False]
def test_tensor_fallback_skips_layer_retry_when_cancelled():
# A cancellation also returns False and must not be retried.
calls: list[bool] = []
async def _false_on_tensor(tensor_parallel, extra_args):
calls.append(tensor_parallel)
return False
ok = asyncio.run(
load_with_tensor_fallback(
_false_on_tensor,
requested_tensor = True,
extra_args = None,
label = "m",
cancelled = lambda: True,
)
)
assert ok is False
assert calls == [True]
@pytest.mark.parametrize(
"extras",
[
["--split-mode", "tensor", "-c", "4096"],
["-sm", "tensor", "-c", "4096"],
["--split-mode=tensor", "-c", "4096"],
["-sm=tensor", "-c", "4096"],
],
)
def test_tensor_fallback_strips_split_mode_from_extras_on_retry(extras):
# The retry must drop every --split-mode form and force layer.
loader = _RecordingLoader()
ok = asyncio.run(
load_with_tensor_fallback(loader, requested_tensor = False, extra_args = extras, label = "m")
)
assert ok is True
assert len(loader.calls) == 2
assert loader.calls[1][1] == ["-c", "4096", "--split-mode", "layer"]
def test_tensor_fallback_env_tensor_retry_forces_layer(monkeypatch):
# Env-only tensor (LLAMA_ARG_SPLIT_MODE): retry must force --split-mode layer.
monkeypatch.setenv("LLAMA_ARG_SPLIT_MODE", "tensor")
calls: list = []
async def _crash_when_effectively_tensor(tensor_parallel, extra_args):
calls.append(list(extra_args) if extra_args else extra_args)
if _effective_tensor_parallel(extra_args, tensor_parallel):
raise RuntimeError("llama-server failed to start (tensor)")
return True
ok = asyncio.run(
load_with_tensor_fallback(
_crash_when_effectively_tensor,
requested_tensor = False,
extra_args = None,
label = "m",
)
)
assert ok is True
assert len(calls) == 2
assert calls[1] == ["--split-mode", "layer"]
def test_tensor_fallback_propagates_non_tensor_crash():
async def _always_raise(tensor_parallel, extra_args):
raise RuntimeError("bad model")
with pytest.raises(RuntimeError, match = "bad model"):
asyncio.run(
load_with_tensor_fallback(
_always_raise, requested_tensor = False, extra_args = None, label = "m"
)
)
def test_tensor_caps_context_to_total_vram_budget():
# With total_by_idx, cap occupancy at 0.95*total.
b = _kv_seeded_backend()
gpus = [(0, 20000), (1, 20000)]
totals = {0: 81920, 1: 81920}
model = int(18 * _GB)
with_total, *_ = b._plan_tensor_parallel(gpus, model, 131072, total_by_idx = totals)
without, *_ = b._plan_tensor_parallel(gpus, model, 131072)
assert with_total < without
MIB = 1024 * 1024
reserve = LlamaCppBackend._TENSOR_PARALLEL_BUFFER_RESERVE_MIB
pool_usable = sum(f - (1.0 - _CTX_FIT_VRAM_FRACTION) * totals[i] for i, f in gpus)
foot_total = (model + b._estimate_kv_cache_bytes(with_total, None)) / MIB + len(gpus) * reserve
foot_free = (model + b._estimate_kv_cache_bytes(without, None)) / MIB + len(gpus) * reserve
assert foot_total <= pool_usable + 2
assert foot_free > pool_usable
def test_tensor_unknown_total_keeps_fraction_cushion():
# Total 0 (two-column nvidia-smi probe) must fall back to free*frac, not raw free.
b = _kv_seeded_backend()
gpus = [(0, 20000), (1, 20000)]
MIB = 1024 * 1024
reserve = LlamaCppBackend._TENSOR_PARALLEL_BUFFER_RESERVE_MIB
model = int(18 * _GB)
ec_zero, *_ = b._plan_tensor_parallel(gpus, model, 131072, total_by_idx = {0: 0, 1: 0})
ec_none, *_ = b._plan_tensor_parallel(gpus, model, 131072)
assert ec_zero == ec_none
pool_free = sum(f for _, f in gpus)
foot = (model + b._estimate_kv_cache_bytes(ec_zero, None)) / MIB + len(gpus) * reserve
assert foot <= pool_free * _CTX_FIT_VRAM_FRACTION + 2
def test_tensor_reserve_scales_with_ubatch():
b = _kv_seeded_backend()
b._vocab_size = 152064
gpus = [(0, 16000), (1, 16000)]
model = int(18 * _GB)
small_ub, *_ = b._plan_tensor_parallel(gpus, model, 131072, n_ubatch = 512)
big_ub, *_ = b._plan_tensor_parallel(gpus, model, 131072, n_ubatch = 4096)
assert big_ub < small_ub
def test_plan_tensor_carries_unsized_mtp_flat_reserve():
# A weights-only MTP reserve must still subtract the flat unsized-KV cushion.
b = _kv_seeded_backend()
gpus = [(0, 14000), (1, 14000)]
model = int(8 * _GB)
weights_only = lambda c: 3 * _GB # noqa: E731 -- constant drafter weights, no KV term
ctx_no_flat, *_ = b._plan_tensor_parallel(
gpus,
model,
131072,
mtp_engaged = True,
mtp_overhead_fn = weights_only,
mtp_flat_reserve_bytes = 0,
)
ctx_flat, *_ = b._plan_tensor_parallel(
gpus,
model,
131072,
mtp_engaged = True,
mtp_overhead_fn = weights_only,
mtp_flat_reserve_bytes = 2 * _GB,
)
assert 0 < ctx_flat < ctx_no_flat
def test_tensor_admission_drops_gpu_below_usable_budget():
# Admit by usable budget (free - 0.05*total), not raw free.
b = _kv_seeded_backend()
gpus = [(0, 6000), (1, 40000)]
totals = {0: 81920, 1: 81920}
_ec, _mac, gi, ts = b._plan_tensor_parallel(gpus, int(8 * _GB), 8192, total_by_idx = totals)
assert gi == [1] and ts is None
_ec2, _mac2, gi_raw, _ts2 = b._plan_tensor_parallel(gpus, int(8 * _GB), 8192)
assert gi_raw == [0, 1]
def test_load_model_tensor_admission_and_capacity_gate_use_usable_budget():
# load_model is too entangled to drive end-to-end, so assert at source level.
src = inspect.getsource(LlamaCppBackend.load_model)
assert "_gpu_usable(g) >= reserve_mib" in src
assert "g[1] >= reserve_mib" not in src
assert "_tp_weight_budget_mib" in src
assert "falling back to layer split" in src
# The gate's required footprint must include the non-shrinkable MTP reserve,
# not weights alone, or a separate-drafter MTP load can still overcommit.
assert "_tp_mtp_floor" in src
assert "model_size - _host_input_embd + _tp_mtp_floor" in src
def test_load_model_tensor_floor_keeps_flat_reserve_for_weights_only():
# Tensor mode has no --fit valve, so a weights-only drafter keeps the flat reserve.
compact = "".join(inspect.getsource(LlamaCppBackend.load_model).split())
assert "mtp_overhead_fnisnotNoneandnot_mtp_kv_unsized" in compact
assert "_tp_mtp_floor=max(" in compact
def test_load_model_reserves_pipeline_per_device_overhead():
# Layer split reserves fixed overhead per EXTRA device; k=1 adds nothing.
assert LlamaCppBackend._PIPELINE_PER_DEVICE_OVERHEAD_MIB > 0
compact = "".join(inspect.getsource(LlamaCppBackend.load_model).split())
assert "def_subset_model_size(n_gpus:int)->int:" in compact
assert "max(0,n_gpus-1)*_pipeline_overhead_bytes" in compact
assert "_subset_model_size(n_gpus)" in compact
def test_load_model_does_not_gate_the_kv_cache_on_tensor_mode():
# llama.cpp runs quantized KV under --split-mode tensor (ggml-org/llama.cpp#23792).
assert not hasattr(LlamaCppBackend, "_TENSOR_PARALLEL_KV_TYPES")
class TestLegacyBuildQuantizedKvInTensorMode:
"""Legacy binaries refuse quantized KV under tensor split; that must skip --fit and name the remedy."""
_REJECTION = (
"llama_init_from_model: simultaneous use of SPLIT_MODE_TENSOR and "
"KV cache quantization not implemented"
)
_QUANTIZED = ("q8_0", "q4_0", "q4_1", "q5_0", "q5_1", "iq4_nl")
@classmethod
def _doomed(cls, cmd) -> bool:
"""What a pre-#23792 llama-server rejects: tensor split + a quantized axis."""
if "--split-mode" not in cmd or cmd[cmd.index("--split-mode") + 1] != "tensor":
return False
return any(
flag in ("--cache-type-k", "--cache-type-v") and cmd[i + 1] in cls._QUANTIZED
for i, flag in enumerate(cmd[:-1])
)
def _run(self, tmp_path, monkeypatch):
"""One load_model call against a legacy child; returns the spawned commands."""
import subprocess
from unittest.mock import patch
import importlib.util
from pathlib import Path as _Path
spec = importlib.util.spec_from_file_location(
"_placement_harness_legacy_build",
_Path(__file__).resolve().parent / "test_llama_cpp_placement.py",
)
placement = importlib.util.module_from_spec(spec)
spec.loader.exec_module(placement)
backend, gguf = placement._backend(
tmp_path, vulkan = False, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)]
)
real_popen = subprocess.Popen
spawns: list[list[str]] = []
def fake_popen(cmd, **kwargs):
if not cmd or str(cmd[0]) != "/fake/llama-server":
return real_popen(cmd, **kwargs)
cmd = list(cmd)
spawns.append(cmd)
doomed = self._doomed(cmd)
if doomed:
backend._stdout_lines = [self._REJECTION, "srv load_model: failed to load model"]
return type(
"Process",
(),
{
"pid": 123,
"stdout": (),
"returncode": 1 if doomed else None,
"poll": lambda _self, _d = doomed: 1 if _d else None,
"terminate": lambda _self: None,
"wait": lambda _self, timeout = None: 1 if doomed else 0,
"kill": lambda _self: None,
},
)()
backend._wait_for_health = lambda timeout, **_kw: not self._doomed(spawns[-1])
error: list[BaseException] = []
with patch.object(subprocess, "Popen", side_effect = fake_popen):
try:
backend.load_model(
GgufLoadIntent(
gguf_path = str(gguf),
model_identifier = "test",
tensor_parallel = True,
cache_type_kv = "q8_0",
)
)
except BaseException as exc: # noqa: BLE001
error.append(exc)
return backend, gguf, spawns, (error[0] if error else None)
@pytest.fixture(autouse = True)
def _clear_latch(self):
"""Both latches are process-wide: a leak would let a later cell pass for an earlier cell's
reason."""
LlamaCppBackend._tensor_split_abort_keys.clear()
LlamaCppBackend._tensor_quant_kv_unsupported_binaries.clear()
yield
LlamaCppBackend._tensor_split_abort_keys.clear()
LlamaCppBackend._tensor_quant_kv_unsupported_binaries.clear()
def test_the_refusal_costs_exactly_one_spawn(self, tmp_path, monkeypatch):
"""Not two: a --fit retry cannot fix a missing capability; each attempt is a full load."""
_backend, _gguf, spawns, error = self._run(tmp_path, monkeypatch)
assert len(spawns) == 1, [c[c.index("--split-mode") + 1] for c in spawns]
assert isinstance(error, RuntimeError), error
def test_it_raises_to_the_route_fallback_naming_the_build(self, tmp_path, monkeypatch):
"""The re-raise to the route fallback must name the build; that text reaches the user."""
_backend, _gguf, _spawns, error = self._run(tmp_path, monkeypatch)
assert "b9455" in str(error)
assert "layer split" in str(error)
def test_the_binary_is_latched_so_the_next_load_skips_tensor(self, tmp_path, monkeypatch):
"""Without this the doomed attempt repeats on every load, forever."""
backend, gguf, _spawns, _error = self._run(tmp_path, monkeypatch)
# Keyed by binary alone: this build lacks quantized-KV tensor splits for every model.
assert LlamaCppBackend._tensor_quant_kv_unsupported_binary(
"/fake/llama-server", ("q8_0", "q8_0")
)
assert LlamaCppBackend._tensor_quant_kv_unsupported_binary(
"/fake/llama-server", ("q4_0", "q4_0")
)
# f16 still works under tensor split there; the latch must not drop a working config.
assert not LlamaCppBackend._tensor_quant_kv_unsupported_binary(
"/fake/llama-server", ("f16", "f16")
)
assert not LlamaCppBackend._tensor_split_abort_keys
def test_a_current_build_is_untouched(self, tmp_path, monkeypatch):
"""The guard is free on a supporting binary: one spawn, tensor kept, quantized cache."""
import importlib.util
from pathlib import Path as _Path
spec = importlib.util.spec_from_file_location(
"_placement_harness_current_build",
_Path(__file__).resolve().parent / "test_llama_cpp_placement.py",
)
placement = importlib.util.module_from_spec(spec)
spec.loader.exec_module(placement)
backend, gguf = placement._backend(
tmp_path, vulkan = False, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)]
)
backend._tensor_split_aborts = lambda *a, **k: False
cmd = placement._launch(backend, gguf, tensor_parallel = True, cache_type_kv = "q8_0")["cmd"]
assert cmd[cmd.index("--split-mode") + 1] == "tensor"
assert cmd[cmd.index("--cache-type-k") + 1] == "q8_0"
assert not LlamaCppBackend._tensor_split_abort_keys
class TestLegacyBuildLatchIsBinaryWide:
"""The pre-b9455 refusal is a binary capability, so the latch is binary-wide, not per model."""
@pytest.fixture(autouse = True)
def _clear(self):
LlamaCppBackend._tensor_quant_kv_unsupported_binaries.clear()
yield
LlamaCppBackend._tensor_quant_kv_unsupported_binaries.clear()
def test_one_detection_covers_other_models_and_other_types(self, tmp_path):
binary = str(tmp_path / "llama-server")
Path(binary).write_text("x")
LlamaCppBackend._record_tensor_quant_kv_unsupported(binary)
assert LlamaCppBackend._tensor_quant_kv_unsupported_binary(binary, ("q4_0", "q4_0"))
assert LlamaCppBackend._tensor_quant_kv_unsupported_binary(binary, ("q8_0", "q8_0"))
# One quantized axis is enough, since llama.cpp checks both.
assert LlamaCppBackend._tensor_quant_kv_unsupported_binary(binary, ("q4_0", "f16"))
assert LlamaCppBackend._tensor_quant_kv_unsupported_binary(binary, ("f16", "q4_0"))
def test_a_non_quantized_pair_still_gets_a_tensor_split(self, tmp_path):
"""The refusal targets quantized KV only; latching it must not drop f16 tensor splits."""
binary = str(tmp_path / "llama-server")
Path(binary).write_text("x")
LlamaCppBackend._record_tensor_quant_kv_unsupported(binary)
assert not LlamaCppBackend._tensor_quant_kv_unsupported_binary(binary, ("f16", "f16"))
assert not LlamaCppBackend._tensor_quant_kv_unsupported_binary(binary, ("bf16", "f32"))
def test_a_replaced_binary_is_re_probed(self, tmp_path):
"""An update is exactly what fixes this, so the mtime must invalidate it."""
import os
import time
binary = str(tmp_path / "llama-server")
Path(binary).write_text("old")
LlamaCppBackend._record_tensor_quant_kv_unsupported(binary)
assert LlamaCppBackend._tensor_quant_kv_unsupported_binary(binary, ("q8_0", "q8_0"))
time.sleep(0.01)
Path(binary).write_text("new")
os.utime(binary, ns = (time.time_ns(), time.time_ns()))
assert not LlamaCppBackend._tensor_quant_kv_unsupported_binary(binary, ("q8_0", "q8_0"))
def test_an_unseen_binary_is_not_latched(self, tmp_path):
assert not LlamaCppBackend._tensor_quant_kv_unsupported_binary(
str(tmp_path / "never-seen"), ("q8_0", "q8_0")
)
assert not LlamaCppBackend._tensor_quant_kv_unsupported_binary(None, ("q8_0", "q8_0"))
def test_the_skip_keeps_the_multi_gpu_request(self):
"""Missing capability is not capacity; the layer fallback still spreads across GPUs."""
load = "".join(inspect.getsource(LlamaCppBackend.load_model).split())
gate = load.find("self._tensor_quant_kv_unsupported_binary(")
assert gate != -1
nxt = load.find("self._tensor_split_aborts(", gate)
assert nxt != -1
assert "_layer_min_gpus=max(_layer_min_gpus,len(gpus))" in load[gate:nxt]
def test_the_loader_consults_it_before_spawning(self):
"""Source-pinned: the binary check must sit at the up-front gate, before any spawn."""
load = "".join(inspect.getsource(LlamaCppBackend.load_model).split())
gate = load.find("self._tensor_quant_kv_unsupported_binary(")
assert gate != -1, "the binary-wide gate is gone"
assert gate < load.find("healthy=_spawn_and_wait(cmd)")
assert "extra_args=strip_split_mode_only(extra_args)" in load