Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

63.4 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""tensor_parallel must round-trip as a bool, and --split-mode tensor is emitted only behind it."""

from __future__ import annotations

import asyncio
import inspect
import re
import socket
import sys
import threading
import time
import types as _types
from dataclasses import replace
from pathlib import Path

import pytest


def _shared_setup_1(monkeypatch):
    b = _recovery_backend()
    proc = _ToggleProcess()
    b._process = proc
    fired = threading.Event()
    monkeypatch.setattr(b, "_maybe_recover_from_mtp_crash", lambda *a, **k: fired.set() or True)
    b._start_mtp_crash_watchdog()
    return b, fired, proc


def _shared_setup_2(b, monkeypatch):
    loads: list[GgufLoadIntent] = []
    monkeypatch.setattr(b, "load_model", lambda intent: loads.append(intent) or True)

    assert b._respawn_if_dead() is False
    return loads


_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
if _BACKEND_DIR not in sys.path:
    sys.path.insert(0, _BACKEND_DIR)

_loggers_stub = _types.ModuleType("loggers")
_loggers_stub.get_logger = lambda name: __import__("logging").getLogger(name)
sys.modules.setdefault("loggers", _loggers_stub)

_structlog_stub = _types.ModuleType("structlog")
_structlog_stub.get_logger = lambda *a, **k: __import__("logging").getLogger("stub")
sys.modules.setdefault("structlog", _structlog_stub)

_httpx_stub = _types.ModuleType("httpx")
for _exc in (
    "ConnectError",
    "TimeoutException",
    "ReadTimeout",
    "ReadError",
    "RemoteProtocolError",
    "CloseError",
):
    setattr(_httpx_stub, _exc, type(_exc, (Exception,), {}))
_httpx_stub.Timeout = type("T", (), {"__init__": lambda s, *a, **k: None})
_httpx_stub.Client = type(
    "C",
    (),
    {
        "__init__": lambda s, **kw: None,
        "__enter__": lambda s: s,
        "__exit__": lambda s, *a: None,
    },
)
# Stub httpx only if missing: this stub lacks Response, which starlette.testclient needs.
try:
    import httpx  # noqa: F401
except ImportError:
    sys.modules.setdefault("httpx", _httpx_stub)

from core.inference import llama_cpp as llama_cpp_module
from core.inference.llama_cpp import (
    _CTX_FIT_VRAM_FRACTION,
    GgufLoadIntent,
    LlamaCppBackend,
)
from core.inference.llama_server_args import (
    _effective_tensor_parallel,
    resolve_tensor_parallel,
)
from core.inference.tensor_fallback import load_with_tensor_fallback
from models.inference import (
    InferenceStatusResponse,
    LoadRequest,
    LoadResponse,
)


def test_load_request_defaults_tensor_parallel_false():
    req = LoadRequest(model_path = "owner/repo")
    assert req.tensor_parallel is False


def test_load_request_accepts_tensor_parallel():
    req = LoadRequest(model_path = "owner/repo", tensor_parallel = True)
    assert req.tensor_parallel is True


def test_load_request_round_trips_json_key():
    req = LoadRequest.model_validate({"model_path": "owner/repo", "tensor_parallel": True})
    assert req.tensor_parallel is True
    assert req.model_dump()["tensor_parallel"] is True


@pytest.mark.parametrize("model_cls", [LoadResponse, InferenceStatusResponse])
def test_response_models_emit_tensor_parallel(model_cls):
    if model_cls is LoadResponse:
        default = model_cls(
            status = "loaded",
            model = "owner/repo",
            display_name = "repo",
            inference = {},
        )
        on = model_cls(
            status = "loaded",
            model = "owner/repo",
            display_name = "repo",
            inference = {},
            tensor_parallel = True,
        )
    else:
        default = model_cls()
        on = model_cls(tensor_parallel = True)
    assert default.model_dump()["tensor_parallel"] is False
    assert on.model_dump()["tensor_parallel"] is True


class _FakeProcess:
    """Stand-in for subprocess.Popen so _kill_process is a no-op."""

    def terminate(self):
        pass

    def wait(self, timeout = None):
        return 0

    def kill(self):
        pass

    def poll(self):
        return 0


def test_tensor_parallel_property_defaults_false():
    assert LlamaCppBackend().tensor_parallel is False


def test_tensor_parallel_property_reflects_field():
    backend = LlamaCppBackend()
    backend._tensor_parallel = True
    assert backend.tensor_parallel is True


def test_unload_resets_tensor_parallel():
    backend = LlamaCppBackend()
    backend._process = _FakeProcess()
    backend._tensor_parallel = True
    backend.unload_model()
    assert backend.tensor_parallel is False


def _loaded_backend(tensor_parallel: bool) -> LlamaCppBackend:
    backend = LlamaCppBackend()
    backend._process = _FakeProcess()
    backend._healthy = True
    backend._model_identifier = "owner/repo"
    backend._hf_variant = "Q4_K_M"
    backend._requested_n_ctx = 8192
    backend._cache_type_kv = None
    backend._requested_spec_mode = "auto"
    backend._chat_template_override = None
    backend._is_vision = False
    backend._extra_args = None
    backend._gguf_path = None
    backend._tensor_parallel = tensor_parallel
    return backend


def _target_state(backend: LlamaCppBackend, tensor_parallel: bool) -> bool:
    return backend.adopt_load_intent_if_matched(
        GgufLoadIntent(
            gguf_path = None,
            model_identifier = "owner/repo",
            hf_variant = "Q4_K_M",
            n_ctx = 8192,
            cache_type_kv = None,
            speculative_type = "auto",
            chat_template_override = None,
            extra_args = None,
            is_vision = False,
            tensor_parallel = tensor_parallel,
        )
    )


@pytest.mark.parametrize("flag", [True, False])
def test_already_in_target_state_matches_same_tensor_parallel(flag):
    assert _target_state(_loaded_backend(flag), flag) is True


@pytest.mark.parametrize(
    "loaded,requested",
    [(False, True), (True, False)],
)
def test_already_in_target_state_reloads_on_tensor_parallel_change(loaded, requested):
    assert _target_state(_loaded_backend(loaded), requested) is False


def test_already_in_target_state_reloads_when_swa_full_env_changes(monkeypatch):
    backend = _loaded_backend(False)
    backend._swa_full = False
    monkeypatch.setenv("LLAMA_ARG_SWA_FULL", "1")
    assert _target_state(backend, False) is False


def test_already_in_target_state_reconciles_split_mode_extras():
    # A --split-mode tensor in extras must match a running tensor server: no spurious reload.
    backend = _loaded_backend(tensor_parallel = True)
    backend._extra_args = ["--split-mode", "tensor"]
    assert (
        backend.adopt_load_intent_if_matched(
            GgufLoadIntent(
                gguf_path = None,
                model_identifier = "owner/repo",
                hf_variant = "Q4_K_M",
                n_ctx = 8192,
                cache_type_kv = None,
                speculative_type = "auto",
                chat_template_override = None,
                extra_args = ["--split-mode", "tensor"],
                is_vision = False,
                tensor_parallel = False,
            )
        )
        is True
    )


def _load_model_source() -> str:
    return inspect.getsource(llama_cpp_module.LlamaCppBackend.load_model)


def test_split_mode_tensor_is_gated_on_the_toggle():
    src = _load_model_source()
    assert (
        'cmd.extend(["--split-mode", "tensor"])' in src
    ), "the tensor-parallel flag emission must be present in load_model"
    # The emission lives behind `if tensor_parallel:` -- it must never be
    # part of the unconditional base cmd list.
    base_start = src.find("cmd = [")
    base_end = src.find("\n                ]", base_start)
    base_block = src[base_start:base_end] if base_end > base_start else ""
    assert (
        "--split-mode" not in base_block
    ), "--split-mode must be conditional, not in the base cmd list"
    gate = src.find("if tensor_parallel:")
    emit = src.find('cmd.extend(["--split-mode", "tensor"])')
    assert 0 <= gate < emit, "emission must sit under `if tensor_parallel:`"


def test_proportional_tensor_split_is_emitted_in_tensor_mode():
    # Asymmetric GPUs OOM the smaller card under the even default split.
    src = _load_model_source()
    assert '"--tensor-split"' in src
    gate = src.find("if tensor_parallel:")
    # Find the TP block's --tensor-split after the gate; manual mode emits one earlier.
    ts = src.find('"--tensor-split"', gate)
    nxt_else = src.find("self._tensor_parallel = False")
    assert 0 <= gate < ts < nxt_else, "--tensor-split must be emitted under `if tensor_parallel:`"
    assert "tp_tensor_split" in src[gate:nxt_else]


def test_mtp_decode_probe_wired_under_tensor_parallel():
    # MTP-draft can crash the CUDA FA kernel on the first decode under --split-mode tensor,
    # so load_model probes a decode and falls back (FA off, then drop MTP).
    src = _load_model_source()
    probe = src.find("_probe_mtp_decode()")
    assert probe != -1, "load_model must decode-probe MTP under tensor parallelism"
    guard = src[max(0, probe - 400) : probe]
    assert "self._tensor_parallel" in guard and "_spec_requested_mtp" in guard
    after = src[probe : probe + 900]
    assert "_with_flash_attn_off" in after and "healthy = False" in after
    # The drafter-drop fallback is `if not healthy and (<any drafter requested>) ...`. Matched on
    # its shape, not its exact spelling: #13806 added eagle3 to that list and wrapped it over
    # several lines, and a one-line literal then found nothing.
    match = re.search(
        r"not healthy\s+and\s*\(\s*_spec_requested_mtp\s+or\s+_spec_requested_dspark\b"
        r"[^)]*?_spec_requested_dflash\b",
        src,
    )
    assert match is not None, "load_model no longer has a drafter-drop fallback for MTP"
    assert probe < match.start(), "the probe must precede the drafter-drop fallback"


def test_probe_mtp_decode_returns_false_on_crash(monkeypatch):
    backend = LlamaCppBackend()
    backend._port = 0

    class _Resp:
        def __init__(self, code):
            self.status_code = code

    backend._process = None
    monkeypatch.setattr(llama_cpp_module.httpx, "post", lambda *a, **k: _Resp(200), raising = False)
    assert backend._probe_mtp_decode(timeout = 1.0) is True

    def _drop(*a, **k):
        raise llama_cpp_module.httpx.RemoteProtocolError("peer closed connection")

    monkeypatch.setattr(llama_cpp_module.httpx, "post", _drop, raising = False)
    assert backend._probe_mtp_decode(timeout = 1.0) is False

    monkeypatch.setattr(llama_cpp_module.httpx, "post", lambda *a, **k: _Resp(500), raising = False)
    assert backend._probe_mtp_decode(timeout = 1.0) is False

    backend._process = _FakeProcess()
    monkeypatch.setattr(llama_cpp_module.httpx, "post", lambda *a, **k: _Resp(200), raising = False)
    assert backend._probe_mtp_decode(timeout = 1.0) is False


def _recovery_backend() -> LlamaCppBackend:
    b = LlamaCppBackend()
    b._tensor_parallel = True
    b._speculative_type = "draft-mtp"
    b._mtp_runtime_fallback_active = True
    b._process = _FakeProcess()
    b._last_load_intent = GgufLoadIntent(
        model_identifier = "owner/repo",
        tensor_parallel = True,
        speculative_type = "auto",
        n_parallel = 4,
    )
    return b


def test_generate_chat_completion_wires_runtime_recovery():
    # The non-tool generation path must route a mid-stream death to the recovery helper.
    src = inspect.getsource(LlamaCppBackend.generate_chat_completion)
    assert "_maybe_recover_from_mtp_crash" in src


def test_runtime_recovery_reloads_without_mtp(monkeypatch):
    b = _recovery_backend()
    done = threading.Event()
    captured = {}

    def _fake_load_model(intent):
        captured.update(vars(intent))
        done.set()
        return True

    monkeypatch.setattr(b, "load_model", _fake_load_model)
    assert b._maybe_recover_from_mtp_crash(RuntimeError("peer closed")) is True
    assert done.wait(timeout = 5)
    assert captured["speculative_type"] == "off"
    assert captured["model_identifier"] == "owner/repo"
    assert captured["n_parallel"] == 4
    deadline = time.monotonic() + 2
    while b._spec_fallback_reason != "runtime_error" and time.monotonic() < deadline:
        time.sleep(0.02)
    assert b._spec_fallback_reason == "runtime_error"
    # The reload thread clears the flag a beat after setting the reason; wait for it.
    deadline = time.monotonic() + 2
    while b._mtp_runtime_fallback_in_progress and time.monotonic() < deadline:
        time.sleep(0.02)
    assert b._mtp_runtime_fallback_in_progress is False


@pytest.mark.parametrize(
    "mutate",
    [
        lambda b: setattr(b, "_mtp_runtime_fallback_active", False),
        lambda b: setattr(b, "_last_load_intent", None),
        lambda b: setattr(b, "_process", None),
        lambda b: b._cancel_event.set(),
    ],
)
def test_runtime_recovery_skips_when_not_applicable(monkeypatch, mutate):
    b = _recovery_backend()
    mutate(b)
    calls = []
    monkeypatch.setattr(b, "load_model", lambda intent: calls.append(intent))
    assert b._maybe_recover_from_mtp_crash(RuntimeError()) is False
    assert calls == []


class _BlockingDeadProc:
    def __init__(self):
        self._dead = threading.Event()

    def poll(self):
        return 0 if self._dead.is_set() else None

    def terminate(self):
        self._dead.set()

    def kill(self):
        self._dead.set()

    def wait(self, timeout = None):
        self._dead.set()
        return 0

    def release(self):
        self._dead.set()


def test_runtime_recovery_fires_for_user_env_mtp(monkeypatch):
    # MTP driven by user extra_args / LLAMA_ARG_SPEC_TYPE leaves _speculative_type
    # unset, but the launch flag still gates recovery on (pass-through MTP).
    b = _recovery_backend()
    b._speculative_type = None
    done = threading.Event()
    captured = {}

    def _fake_load_model(intent):
        captured.update(vars(intent))
        done.set()
        return True

    monkeypatch.setattr(b, "load_model", _fake_load_model)
    assert b._maybe_recover_from_mtp_crash(RuntimeError()) is True
    assert done.wait(timeout = 5)
    assert captured["speculative_type"] == "off"


def test_runtime_recovery_strips_user_mtp_extra_args(monkeypatch):
    # llama.cpp appends spec types rather than replacing, so the user flag must be removed.
    b = _recovery_backend()
    b._last_load_intent = replace(
        b._last_load_intent,
        extra_args = ["--spec-type", "draft-mtp"],
    )
    done = threading.Event()
    captured = {}

    def _fake_load_model(intent):
        captured.update(vars(intent))
        done.set()
        return True

    monkeypatch.setattr(b, "load_model", _fake_load_model)
    assert b._maybe_recover_from_mtp_crash(RuntimeError()) is True
    assert done.wait(timeout = 5)
    assert captured["speculative_type"] == "off"
    assert "--spec-type" not in captured["extra_args"]
    assert "draft-mtp" not in captured["extra_args"]


def test_runtime_recovery_restores_requested_mode(monkeypatch):
    b = _recovery_backend()
    b._last_load_intent = replace(b._last_load_intent, speculative_type = "mtp")
    done = threading.Event()

    def _fake_load_model(intent):
        b._requested_spec_mode = "off"
        done.set()
        return True

    monkeypatch.setattr(b, "load_model", _fake_load_model)
    assert b._maybe_recover_from_mtp_crash(RuntimeError()) is True
    assert done.wait(timeout = 5)
    deadline = time.monotonic() + 2
    while b._requested_spec_mode != "mtp" and time.monotonic() < deadline:
        time.sleep(0.02)
    assert b._requested_spec_mode == "mtp"
    assert b._spec_fallback_reason == "runtime_error"


def test_runtime_recovery_does_not_publish_state_when_reload_is_cancelled(monkeypatch):
    b = _recovery_backend()
    b._requested_spec_mode = "auto"
    b._spec_fallback_reason = "existing"
    done = threading.Event()

    def _cancelled_load(intent):
        done.set()
        return False

    monkeypatch.setattr(b, "load_model", _cancelled_load)
    assert b._maybe_recover_from_mtp_crash(RuntimeError()) is True
    assert done.wait(timeout = 5)
    deadline = time.monotonic() + 2
    while b._mtp_runtime_fallback_in_progress and time.monotonic() < deadline:
        time.sleep(0.02)
    assert b._requested_spec_mode == "auto"
    assert b._spec_fallback_reason == "existing"


def test_runtime_recovery_undoes_an_unload_during_reload(monkeypatch):
    b = _recovery_backend()
    b._requested_spec_mode = "auto"
    b._spec_fallback_reason = "existing"
    done = threading.Event()
    unloads = []

    def _racing_load(intent):
        b._unload_epoch += 1
        done.set()
        return True

    monkeypatch.setattr(b, "load_model", _racing_load)
    monkeypatch.setattr(b, "unload_model", lambda: unloads.append(1) or True)
    assert b._maybe_recover_from_mtp_crash(RuntimeError()) is True
    assert done.wait(timeout = 5)
    deadline = time.monotonic() + 2
    while b._mtp_runtime_fallback_in_progress and time.monotonic() < deadline:
        time.sleep(0.02)
    assert unloads == [1]
    assert b._requested_spec_mode == "auto"
    assert b._spec_fallback_reason == "existing"


def test_runtime_recovery_publishes_state_atomically_with_unload(monkeypatch):
    b = _recovery_backend()
    b._last_load_intent = replace(
        b._last_load_intent,
        extra_args = ("--spec-type", "draft-mtp"),
        gpu_ids = (0,),
    )
    b._requested_extra_args = ["old"]
    b._requested_spec_mode = "auto"
    b._spec_fallback_reason = "existing"
    load_finished = threading.Event()
    unload_entered = threading.Event()
    unload_done = threading.Event()
    unload_threads = []

    def _fake_load(intent):
        load_finished.set()
        return True

    def _unload_state():
        unload_entered.set()
        b._cancel_event.set()
        with b._lock:
            b._unload_epoch += 1
            b._requested_extra_args = None
            b._requested_spec_mode = None
            b._spec_fallback_reason = None
        unload_done.set()

    def _race_after_epoch_check(extra_args):
        thread = threading.Thread(target = _unload_state)
        unload_threads.append(thread)
        thread.start()
        assert unload_entered.wait(timeout = 2)
        unload_done.wait(timeout = 0.2)
        return list(extra_args)

    monkeypatch.setattr(b, "load_model", _fake_load)
    monkeypatch.setattr(b, "_strip_device_extra_args", _race_after_epoch_check)
    assert b._maybe_recover_from_mtp_crash(RuntimeError()) is True
    assert load_finished.wait(timeout = 5)
    assert unload_done.wait(timeout = 5)
    for thread in unload_threads:
        thread.join(timeout = 2)
    deadline = time.monotonic() + 2
    while b._mtp_runtime_fallback_in_progress and time.monotonic() < deadline:
        time.sleep(0.02)
    assert b._requested_extra_args is None
    assert b._requested_spec_mode is None
    assert b._spec_fallback_reason is None


def test_runtime_recovery_skips_when_process_replaced(monkeypatch):
    # A newer load that replaces the process mid-poll must not be clobbered by stale recovery.
    b = _recovery_backend()
    p1 = _BlockingDeadProc()
    b._process = p1
    calls = []
    monkeypatch.setattr(b, "load_model", lambda intent: calls.append(intent))
    assert b._maybe_recover_from_mtp_crash(RuntimeError()) is True
    b._process = _FakeProcess()
    p1.release()
    time.sleep(0.6)
    assert calls == [], "stale recovery replayed over a newer load"


def test_runtime_recovery_skips_when_snapshot_changed(monkeypatch):
    b = _recovery_backend()
    p1 = _BlockingDeadProc()
    b._process = p1
    calls = []
    monkeypatch.setattr(b, "load_model", lambda intent: calls.append(intent))
    assert b._maybe_recover_from_mtp_crash(RuntimeError()) is True
    b._last_load_intent = replace(b._last_load_intent, model_identifier = "other/model")
    p1.release()
    time.sleep(0.6)
    assert calls == []


def test_runtime_recovery_is_single_flight(monkeypatch):
    b = _recovery_backend()
    started = threading.Event()
    release = threading.Event()

    def _slow_load(intent):
        started.set()
        release.wait(timeout = 5)
        return True

    monkeypatch.setattr(b, "load_model", _slow_load)
    assert b._maybe_recover_from_mtp_crash(RuntimeError()) is True
    assert started.wait(timeout = 5)
    assert b._maybe_recover_from_mtp_crash(RuntimeError()) is False
    release.set()


def test_single_flight_claim_is_released_when_the_reload_cannot_start(monkeypatch):
    # Only the reload thread clears the claim, so a failed thread start must not latch it.
    b = _recovery_backend()

    class _NoThread:
        def __init__(self, *args, **kwargs):
            pass

        def start(self):
            raise RuntimeError("can't start new thread")

    monkeypatch.setattr(llama_cpp_module.threading, "Thread", _NoThread)

    assert b._maybe_recover_from_mtp_crash(RuntimeError()) is False
    assert b._mtp_runtime_fallback_in_progress is False


def test_load_intent_is_read_once_before_the_claim(monkeypatch):
    # Gate and snapshot must share one read, or an unload can strand the flag.
    b = _recovery_backend()

    class _CountingIntent:
        def __init__(self, value):
            self.value = value
            self.reads = 0

        def __get__(self, obj, owner):
            if obj is None:
                return self
            self.reads += 1
            return self.value

        def __set__(self, obj, value):
            self.value = value

    counter = _CountingIntent(GgufLoadIntent(model_identifier = "owner/repo"))
    monkeypatch.setattr(type(b), "_last_load_intent", counter, raising = False)

    class _UnstartedThread:
        def __init__(self, *args, **kwargs):
            pass

        def start(self):
            pass

    monkeypatch.setattr(llama_cpp_module.threading, "Thread", _UnstartedThread)

    assert b._maybe_recover_from_mtp_crash(RuntimeError()) is True
    assert counter.reads == 1, f"read {counter.reads} times; an unload can race the claim"


def test_respawn_defers_to_an_inflight_mtp_reload(monkeypatch):
    # "Already recovering" must not read as "not an MTP crash".
    b = _recovery_backend()
    b._mtp_runtime_fallback_in_progress = True
    loads = _shared_setup_2(b, monkeypatch)
    assert loads == []

    b._mtp_runtime_fallback_in_progress = False
    b._process.returncode = -9
    assert b._respawn_if_dead() is True
    assert [intent.speculative_type for intent in loads] == ["auto"]


def test_respawn_does_not_wait_out_the_grace_on_a_replacement(monkeypatch):
    # Lock waiters that find a healthy replacement must not each wait out the reap grace.
    class _LiveProcess(_FakeProcess):
        returncode = None

        def __init__(self):
            self.polls = 0

        def poll(self):
            self.polls += 1
            return None

    workers = 4
    b = _recovery_backend()
    b._healthy = True
    b._process.returncode = -9
    live = _LiveProcess()
    loads: list[GgufLoadIntent] = []
    guard = threading.Lock()
    all_in_flight = threading.Event()

    # Subclass this instance: a class descriptor would redirect _process for every backend.
    state = {"proc": b._process, "readers": set()}

    class _Tracked(type(b)):
        @property
        def _process(self):
            """Reports when every worker has taken its pre-lock look at the child."""
            with guard:
                state["readers"].add(threading.get_ident())
                everyone = len(state["readers"]) >= workers
            if everyone:
                all_in_flight.set()
            return state["proc"]

        @_process.setter
        def _process(self, value):
            state["proc"] = value

    b.__class__ = _Tracked

    def _load(intent):
        # Timeout keeps the pre-fix build failing instead of hanging.
        all_in_flight.wait(timeout = 2)
        with guard:
            loads.append(intent)
        b._process = live
        b._healthy = True
        return True

    monkeypatch.setattr(b, "load_model", _load)
    results: list[bool] = []

    def _respawn():
        outcome = b._respawn_if_dead()
        with guard:
            results.append(outcome)

    threads = [threading.Thread(target = _respawn) for _ in range(workers)]
    started = time.monotonic()
    for thread in threads:
        thread.start()
    for thread in threads:
        thread.join(timeout = 30)
    elapsed = time.monotonic() - started

    assert results == [True] * workers, results
    assert len(loads) == 1, f"{len(loads)} reloads, expected one"
    assert live.polls == 0, "queued caller waited out the grace on a healthy server"
    assert elapsed < llama_cpp_module._RESPAWN_REAP_GRACE_S * (workers - 1)


class _DyingChild(_FakeProcess):
    """Alive for the first polls, then reapable: what a terminate() looks like."""

    def __init__(
        self,
        code = -15,
        alive_polls = 2,
        on_death = None,
    ):
        self.polls = 0
        self.returncode = None
        self._code = code
        self._alive_polls = alive_polls
        self._on_death = on_death

    def poll(self):
        self.polls += 1
        if self.polls <= self._alive_polls:
            return None
        if self.returncode is None:
            self.returncode = self._code
            if self._on_death is not None:
                self._on_death()
        return self._code


def test_respawn_does_not_resurrect_a_deliberate_unload(monkeypatch):
    # unload_model sets _cancel_event before killing; that exit is not a crash.
    b = _recovery_backend()
    b._healthy = True
    b._process = _DyingChild()
    b._cancel_event.set()
    loads = _shared_setup_2(b, monkeypatch)
    assert loads == [], "resurrected a model the user unloaded"


def test_respawn_rechecks_the_cancel_flag_after_the_grace_wait(monkeypatch):
    b = _recovery_backend()
    b._healthy = True
    b._process = _DyingChild(on_death = b._cancel_event.set)
    loads = _shared_setup_2(b, monkeypatch)
    assert loads == [], "checked the cancel flag only before the wait"


def test_respawn_does_not_revert_a_newer_load(monkeypatch):
    # A model switch that lands during the wait must win; replaying old intent undoes it.
    b = _recovery_backend()
    b._healthy = True
    replacement = _DyingChild(alive_polls = 10**6)
    b._process = _DyingChild(on_death = lambda: setattr(b, "_process", replacement))
    loads: list[GgufLoadIntent] = []
    monkeypatch.setattr(b, "load_model", lambda intent: loads.append(intent) or True)

    b._respawn_if_dead()
    assert loads == [], "replayed stale kwargs over a newer load"
    assert b._process is replacement


def test_respawn_still_recovers_an_ordinary_crash(monkeypatch):
    b = _recovery_backend()
    b._healthy = True
    b._process = _DyingChild(code = -9)
    loads: list[GgufLoadIntent] = []
    monkeypatch.setattr(b, "load_model", lambda intent: loads.append(intent) or True)

    assert b._respawn_if_dead() is True
    assert len(loads) == 1


class _NeverReapable(_FakeProcess):
    """A child that stays unreapable, so only the port can tell alive from dead."""

    returncode = None

    def poll(self):
        return None


def test_a_transient_error_against_a_live_server_costs_nothing(monkeypatch):
    # The grace sleep is held under _respawn_lock, so it must not hit a live server.
    listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    listener.bind(("127.0.0.1", 0))
    listener.listen(16)
    try:
        b = _recovery_backend()
        b._healthy = True
        b._process = _NeverReapable()
        b._port = listener.getsockname()[1]
        loads: list[GgufLoadIntent] = []
        monkeypatch.setattr(b, "load_model", lambda intent: loads.append(intent) or True)

        started = time.monotonic()
        assert b._respawn_if_dead() is True
        elapsed = time.monotonic() - started

        assert loads == [], "a live server must not be reloaded"
        assert (
            elapsed < llama_cpp_module._RESPAWN_REAP_GRACE_S / 2
        ), f"waited {elapsed:.2f}s on a server that is still accepting"
    finally:
        listener.close()


def test_a_closed_port_still_waits_for_the_child_to_be_reapable(monkeypatch):
    probe = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    probe.bind(("127.0.0.1", 0))
    dead_port = probe.getsockname()[1]
    probe.close()

    b = _recovery_backend()
    b._healthy = True
    b._process = _DyingChild(code = -9)
    b._port = dead_port
    loads: list[GgufLoadIntent] = []
    monkeypatch.setattr(b, "load_model", lambda intent: loads.append(intent) or True)

    assert b._respawn_if_dead() is True
    assert len(loads) == 1


def test_socket_fast_path_honours_a_pending_unload(monkeypatch):
    # unload_model sets _cancel_event before killing, so the child still accepts here.
    listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    listener.bind(("127.0.0.1", 0))
    listener.listen(8)
    try:
        b = _recovery_backend()
        b._healthy = True
        b._process = _NeverReapable()
        b._port = listener.getsockname()[1]
        b._cancel_event.set()
        loads: list[GgufLoadIntent] = []
        monkeypatch.setattr(b, "load_model", lambda intent: loads.append(intent) or True)

        assert b._respawn_if_dead() is False
        assert loads == []
    finally:
        listener.close()


def test_an_unload_landing_during_the_reload_is_undone(monkeypatch):
    # unload_model never takes _serial_load_lock and load_model clears _cancel_event,
    # so _unload_epoch is the surviving evidence of a cancel.
    b = _recovery_backend()
    b._healthy = True
    b._process = _FakeProcess()
    b._process.returncode = -9
    loads: list[GgufLoadIntent] = []
    monkeypatch.setattr(b, "load_model", lambda intent: loads.append(intent) or True)

    unloads: list[int] = []
    real_unload = b.unload_model
    monkeypatch.setattr(b, "unload_model", lambda: unloads.append(1) or real_unload())

    real_warning = llama_cpp_module.logger.warning
    fired: list[int] = []

    def racing_warning(*args, **kwargs):
        if not fired:
            fired.append(1)
            real_unload()
        return real_warning(*args, **kwargs)

    monkeypatch.setattr(llama_cpp_module.logger, "warning", racing_warning)

    assert b._respawn_if_dead() is False
    assert unloads, "the racing unload was not honoured"


def test_socket_probe_is_false_without_a_port():
    b = _recovery_backend()
    b._port = None
    assert b._server_socket_is_open() is False


def test_runtime_recovery_rechecks_cancel_after_the_death_poll(monkeypatch):
    b = _recovery_backend()
    process = _BlockingDeadProc()
    b._process = process
    loads = []
    monkeypatch.setattr(b, "load_model", lambda intent: loads.append(intent) or True)

    assert b._maybe_recover_from_mtp_crash(RuntimeError()) is True
    b._cancel_event.set()
    process.release()
    deadline = time.monotonic() + 2
    while b._mtp_runtime_fallback_in_progress and time.monotonic() < deadline:
        time.sleep(0.02)
    assert loads == []


def test_probe_mtp_decode_uses_api_key_auth(monkeypatch):
    # Direct-stream mode uses --api-key; the probe must send bearer auth.
    backend = LlamaCppBackend()
    backend._port = 0
    backend._process = None
    captured = {}

    class _Resp:
        status_code = 200

    def _capture(*a, **k):
        captured.clear()
        captured.update(k)
        return _Resp()

    monkeypatch.setattr(llama_cpp_module.httpx, "post", _capture, raising = False)
    backend._api_key = "secret"
    backend._probe_mtp_decode(timeout = 1.0)
    assert captured["headers"] == {"Authorization": "Bearer secret"}

    assert captured["trust_env"] is False
    backend._api_key = None
    backend._probe_mtp_decode(timeout = 1.0)
    assert captured["headers"] is None


class _ToggleProcess:
    """A subprocess stand-in whose liveness can be flipped at runtime."""

    def __init__(self):
        self._alive = True

    def poll(self):
        return None if self._alive else 0

    def terminate(self):
        self._alive = False

    def kill(self):
        self._alive = False

    def wait(self, timeout = None):
        self._alive = False
        return 0

    def die(self):
        self._alive = False


def test_crash_watchdog_triggers_recovery_on_death(monkeypatch):
    b, fired, proc = _shared_setup_1(monkeypatch)
    assert b._mtp_watchdog_thread is not None
    proc.die()
    assert fired.wait(timeout = 3)


def test_crash_watchdog_ignores_intentional_termination(monkeypatch):
    # The watchdog stops before a planned kill, so that exit is not mistaken for a crash.
    b, fired, proc = _shared_setup_1(monkeypatch)
    b._stop_mtp_crash_watchdog()
    proc.die()
    assert not fired.wait(timeout = 2)
    assert b._mtp_watchdog_thread is None


@pytest.mark.parametrize(
    "mutate",
    [
        lambda b: setattr(b, "_process", None),
    ],
)
def test_crash_watchdog_not_armed_when_inapplicable(mutate):
    b = _recovery_backend()
    b._process = _ToggleProcess()
    mutate(b)
    b._start_mtp_crash_watchdog()
    assert b._mtp_watchdog_thread is None


def test_kill_process_stops_crash_watchdog(monkeypatch):
    b, fired, proc = _shared_setup_1(monkeypatch)
    b._kill_process()
    assert b._mtp_watchdog_thread is None
    assert b._process is None
    assert not fired.wait(timeout = 2)


def test_kill_process_stops_watchdog_before_terminate():
    # Stop the watchdog before terminating (now inside _kill_process_body).
    src = inspect.getsource(LlamaCppBackend._kill_process) + inspect.getsource(
        LlamaCppBackend._kill_process_body
    )
    stop = src.find("_stop_mtp_crash_watchdog()")
    term = src.find(".terminate(")
    assert 0 <= stop < term, "must stop the watchdog before terminating"


def test_crash_watchdog_rechecks_stop_before_recovery():
    # After an exit, re-check the stop flag: a kill racing the poll must not fire recovery.
    src = inspect.getsource(LlamaCppBackend._start_mtp_crash_watchdog)
    check = src.find("stop.is_set()")
    recover = src.find("_maybe_recover_from_mtp_crash")
    assert 0 <= check < recover, "must re-check stop before recovering"


def test_load_model_arms_crash_watchdog():
    src = inspect.getsource(LlamaCppBackend.load_model)
    assert "_start_mtp_crash_watchdog" in src


def _kv_seeded_backend() -> LlamaCppBackend:
    backend = LlamaCppBackend()
    backend._n_layers = 32
    backend._embedding_length = 4096
    backend._n_heads = 32
    backend._n_kv_heads = 8
    backend._context_length = 131072
    return backend


def test_fit_context_budget_frac_override_is_tighter():
    backend = _kv_seeded_backend()
    model_size = 8 * 1024**3
    pool_mib = 24 * 1024

    fit_default = backend._fit_context_to_vram(131072, pool_mib, model_size, "f16")
    fit_tp = backend._fit_context_to_vram(131072, pool_mib, model_size, "f16", budget_frac = 0.80)
    assert fit_tp < 131072, "expected the context to be capped at this VRAM tier"
    assert fit_tp <= fit_default, "a tighter budget must not allow MORE context"
    assert backend._fit_context_to_vram(131072, pool_mib, model_size, "f16") == fit_default


def test_fit_context_follows_the_user_vram_budget(monkeypatch):
    """A VRAM budget setting moves the fitted context both ways; unset keeps the historical default."""
    import utils.vram_budget_settings as vb

    backend = _kv_seeded_backend()
    model_size = 8 * 1024**3
    pool_mib = 24 * 1024
    total_mib = 24 * 1024

    def _fit():
        return backend._fit_context_to_vram(
            131072, pool_mib, model_size, "f16", total_mib = total_mib
        )

    monkeypatch.delenv(vb.VRAM_FRACTION_ENV_VAR, raising = False)
    monkeypatch.setattr(vb, "_cached_setting", lambda _key: None)
    fit_default = _fit()
    assert fit_default < 131072, "expected the context to be capped at this VRAM tier"

    monkeypatch.setattr(vb, "_cached_setting", lambda _key: 1.0)
    fit_max = _fit()

    # 0.80, not VRAM_FRACTION_MIN: at 0.10 the 8 GiB weights alone exceed the budget, which hands the load to the offload path.
    monkeypatch.setattr(vb, "_cached_setting", lambda _key: 0.80)
    fit_min = _fit()

    assert fit_max > fit_default, "claiming the whole card must buy context back"
    assert fit_min < fit_default, "a smaller budget must give context up"

    monkeypatch.setattr(vb, "_cached_setting", lambda _key: None)
    assert _fit() == fit_default


def test_tensor_plan_leaves_the_floor_reserve_at_a_full_budget():
    """At 100% a tensor plan must keep the 512 MiB floor reserve, as the layer path does."""
    import core.inference.llama_cpp as lc

    backend = _kv_seeded_backend()
    total_mib = 24 * 1024
    gpus = [(0, total_mib), (1, total_mib)]
    total_by_idx = {0: total_mib, 1: total_mib}

    def _plan(frac: float, model_gb: int):
        return backend._plan_tensor_parallel(
            list(gpus),
            model_gb * 1024**3,
            131072,
            cache_type_kv = "f16",
            total_by_idx = dict(total_by_idx),
            vram_fraction = frac,
        )

    # A fraction whose reserve equals the floor on this card plans the same as a floored 100%.
    floor_frac = 1.0 - lc._VRAM_FLOOR_RESERVE_MIB / total_mib
    assert floor_frac > lc._CTX_FIT_VRAM_FRACTION, "24 GiB: 1% is under the floor"

    *_, split_full = _plan(1.0, 40)
    *_, split_floor = _plan(floor_frac, 40)
    *_, split_default = _plan(lc._CTX_FIT_VRAM_FRACTION, 40)
    assert split_full is not None, "expected a weighted split at this model size"
    assert (
        split_full == split_floor
    ), f"100% spent {split_full} per card where the floor allows {split_floor}"
    # The default reserves its full 3% (737 MiB > floor).
    assert all(d < f for d, f in zip(split_default, split_full))

    ctx_full, *_ = _plan(1.0, 36)
    ctx_floor, *_ = _plan(floor_frac, 36)
    assert ctx_full < 131072, "expected the context to be capped at this VRAM tier"
    assert ctx_full == ctx_floor


def test_split_mode_tensor_arch_failure_message():
    msg = LlamaCppBackend._classify_llama_start_failure(
        "llama_model_create: LLAMA_SPLIT_MODE_TENSOR not implemented for architecture 'deepseek2'",
        None,
        "unsloth/DeepSeek-V3-GGUF",
    )
    assert "Tensor parallelism is not supported" in msg


def test_unrelated_arch_failure_not_hijacked_by_tensor_message():
    msg = LlamaCppBackend._classify_llama_start_failure(
        "unknown model architecture: 'flux'", "/models/flux.gguf", None
    )
    assert "Tensor parallelism" not in msg


# Seeded KV (~128 KiB/token); assert relationships, not magic numbers.

_GB = 1024**3
_ASYM = [(0, 48000), (1, 24000)]
_SYM = [(0, 24000), (1, 24000)]


def _plan(
    model_gb,
    target = 131072,
    gpus = _ASYM,
    mtp = False,
):
    b = _kv_seeded_backend()
    return b, b._plan_tensor_parallel(gpus, int(model_gb * _GB), target, mtp_engaged = mtp)


def _kv_budget_b(model_gb, gpus = _ASYM):
    reserve = LlamaCppBackend._TENSOR_PARALLEL_BUFFER_RESERVE_MIB
    usable = sum(f * _CTX_FIT_VRAM_FRACTION for _, f in gpus)
    return (usable - len(gpus) * reserve) * 1024 * 1024 - int(model_gb * _GB)


def test_tp_plan_weighted_split_on_asymmetric_big_model():
    b, (ec, mac, gi, ts) = _plan(50)
    reserve = b._TENSOR_PARALLEL_BUFFER_RESERVE_MIB
    assert gi == [0, 1]
    # Split weighted by usable - flat buffer - per-device context compute.
    cc_per_dev = b._compute_buffer_ctx_bytes(ec, None, None) // (1024 * 1024)
    assert cc_per_dev > 0
    assert ts == [
        int(48000 * _CTX_FIT_VRAM_FRACTION - reserve - cc_per_dev),
        int(24000 * _CTX_FIT_VRAM_FRACTION - reserve - cc_per_dev),
    ]
    assert ec < 131072


def test_tp_plan_even_split_when_model_fits():
    # Even share fitting the smallest GPU keeps None; some archs crash on weighted splits.
    _, (ec, mac, gi, ts) = _plan(4)
    assert ts is None


def test_tp_plan_symmetric_gpus_use_even_split():
    _, (ec, mac, gi, ts) = _plan(8, gpus = _SYM)
    assert ts is None


def test_tp_plan_context_fits_pool_budget_no_oom():
    b, (ec, mac, gi, ts) = _plan(50)
    assert b._estimate_kv_cache_bytes(ec) <= _kv_budget_b(50)


def test_tp_plan_uses_available_vram_not_wasteful():
    b, (ec, mac, gi, ts) = _plan(50)
    assert b._estimate_kv_cache_bytes(ec) >= 0.9 * _kv_budget_b(50)


def test_tp_plan_weights_exceed_pool_floors_context():
    # 70 GB > pool minus per-GPU reserves -> floor (triggers layer fallback)
    _, (ec, mac, gi, ts) = _plan(70)
    assert ec == 2048


def test_tp_plan_floor_never_exceeds_explicit_small_context():
    # An explicit context below the 2048 floor must not be raised.
    _, (ec, mac, gi, ts) = _plan(70, target = 1024)  # weights exceed pool -> floor path
    assert ec == 1024
    _, (ec2, *_rest) = _plan(50, target = 1024)
    assert ec2 <= 1024


def test_tp_plan_explicit_context_honored_when_it_fits():
    _, (ec, mac, gi, ts) = _plan(50, target = 8192)
    assert ec == 8192


def test_tp_plan_explicit_context_capped_when_too_large():
    _, (ec, mac, gi, ts) = _plan(50, target = 131072)
    assert 2048 <= ec < 131072


def test_tp_plan_max_available_ctx_reports_native_not_explicit_ctx():
    # max_available_ctx must reflect the hardware cap, not the request.
    b = _kv_seeded_backend()
    ec, mac, _gi, _ts = b._plan_tensor_parallel(_ASYM, int(50 * _GB), 8192, max_target_ctx = 131072)
    _, native_mac, *_ = b._plan_tensor_parallel(_ASYM, int(50 * _GB), 131072)
    assert ec == 8192
    assert mac == native_mac > ec


def test_tp_plan_mtp_reserves_extra_and_shrinks_context():
    _, (ec_no, *_rest) = _plan(50)
    _, (ec_mtp, *_rest) = _plan(50, mtp = True)
    assert ec_mtp < ec_no


def test_tp_plan_reserves_context_linear_compute_buffer():
    # Tensor mode replicates the compute graph per device (~n_ubatch*2 B/token each);
    # the fit must reserve n_dev x that.
    b, (ec, mac, gi, ts) = _plan(50)
    cc = len(gi) * b._compute_buffer_ctx_bytes(ec, None, "f16")
    assert cc > 0
    assert b._estimate_kv_cache_bytes(ec) + cc <= _kv_budget_b(50)


def test_tp_plan_context_shrinks_vs_compute_unaware():
    b, (ec, *_r) = _plan(50)
    b2 = _kv_seeded_backend()
    b2._embedding_length = 0  # kills the context-linear compute term (returns 0)
    ec_naive, *_r2 = b2._plan_tensor_parallel(_ASYM, int(50 * _GB), 131072)
    assert ec < ec_naive


def test_tp_plan_soft_overhead_shrinks_context():
    # Tensor mode has no --fit valve, so soft_overhead must shrink the context.
    b = _kv_seeded_backend()
    ec_no, *_r = b._plan_tensor_parallel(_ASYM, int(50 * _GB), 131072)
    ec_soft, *_r2 = b._plan_tensor_parallel(
        _ASYM, int(50 * _GB), 131072, soft_overhead_bytes = 2 * _GB
    )
    assert 2048 < ec_soft < ec_no


def test_tp_plan_soft_overhead_reserved_against_budget():
    b = _kv_seeded_backend()
    soft = 2 * _GB
    ec, *_r = b._plan_tensor_parallel(_ASYM, int(50 * _GB), 131072, soft_overhead_bytes = soft)
    cc = len(_ASYM) * b._compute_buffer_ctx_bytes(ec, None, None)
    assert b._estimate_kv_cache_bytes(ec) + cc + soft <= _kv_budget_b(50)


def test_tp_plan_weighted_split_keeps_small_gpu_within_budget():
    # Subtract replicated compute from each card, or the smaller one is weighted past budget.
    b, (ec, mac, gi, ts) = _plan(50)
    assert ts is not None and len(ts) == len(gi) == 2
    reserve = b._TENSOR_PARALLEL_BUFFER_RESERVE_MIB
    cc_per_dev = b._compute_buffer_ctx_bytes(ec, None, None) // (1024 * 1024)
    free_by_idx = {0: 48000, 1: 24000}
    split_content_mib = (int(50 * _GB) + b._estimate_kv_cache_bytes(ec)) / (1024 * 1024)
    total_weight = sum(ts)
    for w, idx in zip(ts, gi):
        placed = split_content_mib * w / total_weight
        usable = free_by_idx[idx] * _CTX_FIT_VRAM_FRACTION
        assert placed + reserve + cc_per_dev <= usable + 1  # +1 MiB for int rounding

    old_adj = [int(free_by_idx[i] * _CTX_FIT_VRAM_FRACTION - reserve) for i in gi]
    old_small_placed = split_content_mib * old_adj[1] / sum(old_adj)
    assert old_small_placed + reserve + cc_per_dev > free_by_idx[1] * _CTX_FIT_VRAM_FRACTION


def test_tp_plan_no_kv_metadata_floors_context():
    b = LlamaCppBackend()
    ec, mac, gi, ts = b._plan_tensor_parallel(_ASYM, int(50 * _GB), 131072)
    assert ec <= 4096


def test_tp_plan_single_gpu_never_splits():
    # Tensor mode is a no-op below 2 GPUs; a single-GPU plan must never emit a tensor split.
    b = _kv_seeded_backend()
    ec, mac, gi, ts = b._plan_tensor_parallel([(0, 24000)], int(8 * _GB), 8192)
    assert ts is None
    assert gi == [0]


def test_tp_plan_zero_gpus_never_splits():
    b = _kv_seeded_backend()
    ec, mac, gi, ts = b._plan_tensor_parallel([], int(8 * _GB), 8192)
    assert ts is None
    assert gi == []


def test_tp_plan_drops_gpu_below_buffer_reserve():
    # A GPU with less free VRAM than the per-device compute-buffer reserve
    # can't host tensor mode; it's excluded, which here leaves <2 usable -> no
    # split (and gpu_indices reflects only the usable device).
    b = _kv_seeded_backend()
    reserve = LlamaCppBackend._TENSOR_PARALLEL_BUFFER_RESERVE_MIB
    ec, mac, gi, ts = b._plan_tensor_parallel([(0, 48000), (1, reserve - 1)], int(8 * _GB), 8192)
    assert gi == [0]
    assert ts is None


# A tensor-incompatible model makes load_model raise; /load must retry with layer split.


class _RecordingLoader:
    """Fake attempt_load crashes under tensor mode (bool or --split-mode) and succeeds on layer split."""

    def __init__(self):
        self.calls: list[tuple] = []

    async def __call__(self, tensor_parallel, extra_args):
        self.calls.append((tensor_parallel, list(extra_args) if extra_args else extra_args))
        if resolve_tensor_parallel(extra_args, tensor_parallel):
            raise RuntimeError("llama-server failed to start")
        return True


def test_tensor_fallback_retries_layer_on_crash():
    loader = _RecordingLoader()
    ok = asyncio.run(
        load_with_tensor_fallback(loader, requested_tensor = True, extra_args = None, label = "m")
    )
    assert ok is True
    assert [c[0] for c in loader.calls] == [True, False]


def test_tensor_fallback_no_retry_on_success():
    calls: list[bool] = []

    async def _ok(tensor_parallel, extra_args):
        calls.append(tensor_parallel)
        return True

    ok = asyncio.run(
        load_with_tensor_fallback(_ok, requested_tensor = True, extra_args = None, label = "m")
    )
    assert ok is True
    assert calls == [True]


def test_tensor_fallback_retries_when_tensor_returns_false():
    # A False return from load_model must trigger the layer-split retry, just as a raise does.
    calls: list[bool] = []

    async def _false_on_tensor(tensor_parallel, extra_args):
        calls.append(tensor_parallel)
        return not resolve_tensor_parallel(extra_args, tensor_parallel)

    ok = asyncio.run(
        load_with_tensor_fallback(
            _false_on_tensor, requested_tensor = True, extra_args = None, label = "m"
        )
    )
    assert ok is True
    assert calls == [True, False]


def test_tensor_fallback_returns_false_when_both_attempts_fail():
    calls: list[bool] = []

    async def _always_false(tensor_parallel, extra_args):
        calls.append(tensor_parallel)
        return False

    ok = asyncio.run(
        load_with_tensor_fallback(_always_false, requested_tensor = True, extra_args = None, label = "m")
    )
    assert ok is False
    assert calls == [True, False]


def test_tensor_fallback_skips_layer_retry_when_cancelled():
    # A cancellation also returns False and must not be retried.
    calls: list[bool] = []

    async def _false_on_tensor(tensor_parallel, extra_args):
        calls.append(tensor_parallel)
        return False

    ok = asyncio.run(
        load_with_tensor_fallback(
            _false_on_tensor,
            requested_tensor = True,
            extra_args = None,
            label = "m",
            cancelled = lambda: True,
        )
    )
    assert ok is False
    assert calls == [True]


@pytest.mark.parametrize(
    "extras",
    [
        ["--split-mode", "tensor", "-c", "4096"],
        ["-sm", "tensor", "-c", "4096"],
        ["--split-mode=tensor", "-c", "4096"],
        ["-sm=tensor", "-c", "4096"],
    ],
)
def test_tensor_fallback_strips_split_mode_from_extras_on_retry(extras):
    # The retry must drop every --split-mode form and force layer.
    loader = _RecordingLoader()
    ok = asyncio.run(
        load_with_tensor_fallback(loader, requested_tensor = False, extra_args = extras, label = "m")
    )
    assert ok is True
    assert len(loader.calls) == 2
    assert loader.calls[1][1] == ["-c", "4096", "--split-mode", "layer"]


def test_tensor_fallback_env_tensor_retry_forces_layer(monkeypatch):
    # Env-only tensor (LLAMA_ARG_SPLIT_MODE): retry must force --split-mode layer.
    monkeypatch.setenv("LLAMA_ARG_SPLIT_MODE", "tensor")
    calls: list = []

    async def _crash_when_effectively_tensor(tensor_parallel, extra_args):
        calls.append(list(extra_args) if extra_args else extra_args)
        if _effective_tensor_parallel(extra_args, tensor_parallel):
            raise RuntimeError("llama-server failed to start (tensor)")
        return True

    ok = asyncio.run(
        load_with_tensor_fallback(
            _crash_when_effectively_tensor,
            requested_tensor = False,
            extra_args = None,
            label = "m",
        )
    )
    assert ok is True
    assert len(calls) == 2
    assert calls[1] == ["--split-mode", "layer"]


def test_tensor_fallback_propagates_non_tensor_crash():
    async def _always_raise(tensor_parallel, extra_args):
        raise RuntimeError("bad model")

    with pytest.raises(RuntimeError, match = "bad model"):
        asyncio.run(
            load_with_tensor_fallback(
                _always_raise, requested_tensor = False, extra_args = None, label = "m"
            )
        )


def test_tensor_caps_context_to_total_vram_budget():
    # With total_by_idx, cap occupancy at 0.95*total.
    b = _kv_seeded_backend()
    gpus = [(0, 20000), (1, 20000)]
    totals = {0: 81920, 1: 81920}
    model = int(18 * _GB)
    with_total, *_ = b._plan_tensor_parallel(gpus, model, 131072, total_by_idx = totals)
    without, *_ = b._plan_tensor_parallel(gpus, model, 131072)
    assert with_total < without

    MIB = 1024 * 1024
    reserve = LlamaCppBackend._TENSOR_PARALLEL_BUFFER_RESERVE_MIB
    pool_usable = sum(f - (1.0 - _CTX_FIT_VRAM_FRACTION) * totals[i] for i, f in gpus)
    foot_total = (model + b._estimate_kv_cache_bytes(with_total, None)) / MIB + len(gpus) * reserve
    foot_free = (model + b._estimate_kv_cache_bytes(without, None)) / MIB + len(gpus) * reserve
    assert foot_total <= pool_usable + 2
    assert foot_free > pool_usable


def test_tensor_unknown_total_keeps_fraction_cushion():
    # Total 0 (two-column nvidia-smi probe) must fall back to free*frac, not raw free.
    b = _kv_seeded_backend()
    gpus = [(0, 20000), (1, 20000)]
    MIB = 1024 * 1024
    reserve = LlamaCppBackend._TENSOR_PARALLEL_BUFFER_RESERVE_MIB
    model = int(18 * _GB)
    ec_zero, *_ = b._plan_tensor_parallel(gpus, model, 131072, total_by_idx = {0: 0, 1: 0})
    ec_none, *_ = b._plan_tensor_parallel(gpus, model, 131072)
    assert ec_zero == ec_none
    pool_free = sum(f for _, f in gpus)
    foot = (model + b._estimate_kv_cache_bytes(ec_zero, None)) / MIB + len(gpus) * reserve
    assert foot <= pool_free * _CTX_FIT_VRAM_FRACTION + 2


def test_tensor_reserve_scales_with_ubatch():
    b = _kv_seeded_backend()
    b._vocab_size = 152064
    gpus = [(0, 16000), (1, 16000)]
    model = int(18 * _GB)
    small_ub, *_ = b._plan_tensor_parallel(gpus, model, 131072, n_ubatch = 512)
    big_ub, *_ = b._plan_tensor_parallel(gpus, model, 131072, n_ubatch = 4096)
    assert big_ub < small_ub


def test_plan_tensor_carries_unsized_mtp_flat_reserve():
    # A weights-only MTP reserve must still subtract the flat unsized-KV cushion.
    b = _kv_seeded_backend()
    gpus = [(0, 14000), (1, 14000)]
    model = int(8 * _GB)
    weights_only = lambda c: 3 * _GB  # noqa: E731 -- constant drafter weights, no KV term
    ctx_no_flat, *_ = b._plan_tensor_parallel(
        gpus,
        model,
        131072,
        mtp_engaged = True,
        mtp_overhead_fn = weights_only,
        mtp_flat_reserve_bytes = 0,
    )
    ctx_flat, *_ = b._plan_tensor_parallel(
        gpus,
        model,
        131072,
        mtp_engaged = True,
        mtp_overhead_fn = weights_only,
        mtp_flat_reserve_bytes = 2 * _GB,
    )
    assert 0 < ctx_flat < ctx_no_flat


def test_tensor_admission_drops_gpu_below_usable_budget():
    # Admit by usable budget (free - 0.05*total), not raw free.
    b = _kv_seeded_backend()
    gpus = [(0, 6000), (1, 40000)]
    totals = {0: 81920, 1: 81920}
    _ec, _mac, gi, ts = b._plan_tensor_parallel(gpus, int(8 * _GB), 8192, total_by_idx = totals)
    assert gi == [1] and ts is None
    _ec2, _mac2, gi_raw, _ts2 = b._plan_tensor_parallel(gpus, int(8 * _GB), 8192)
    assert gi_raw == [0, 1]


def test_load_model_tensor_admission_and_capacity_gate_use_usable_budget():
    # load_model is too entangled to drive end-to-end, so assert at source level.
    src = inspect.getsource(LlamaCppBackend.load_model)
    assert "_gpu_usable(g) >= reserve_mib" in src
    assert "g[1] >= reserve_mib" not in src
    assert "_tp_weight_budget_mib" in src
    assert "falling back to layer split" in src
    # The gate's required footprint must include the non-shrinkable MTP reserve,
    # not weights alone, or a separate-drafter MTP load can still overcommit.
    assert "_tp_mtp_floor" in src
    assert "model_size - _host_input_embd + _tp_mtp_floor" in src


def test_load_model_tensor_floor_keeps_flat_reserve_for_weights_only():
    # Tensor mode has no --fit valve, so a weights-only drafter keeps the flat reserve.
    compact = "".join(inspect.getsource(LlamaCppBackend.load_model).split())
    assert "mtp_overhead_fnisnotNoneandnot_mtp_kv_unsized" in compact
    assert "_tp_mtp_floor=max(" in compact


def test_load_model_reserves_pipeline_per_device_overhead():
    # Layer split reserves fixed overhead per EXTRA device; k=1 adds nothing.
    assert LlamaCppBackend._PIPELINE_PER_DEVICE_OVERHEAD_MIB > 0
    compact = "".join(inspect.getsource(LlamaCppBackend.load_model).split())
    assert "def_subset_model_size(n_gpus:int)->int:" in compact
    assert "max(0,n_gpus-1)*_pipeline_overhead_bytes" in compact
    assert "_subset_model_size(n_gpus)" in compact


def test_load_model_does_not_gate_the_kv_cache_on_tensor_mode():
    # llama.cpp runs quantized KV under --split-mode tensor (ggml-org/llama.cpp#23792).
    assert not hasattr(LlamaCppBackend, "_TENSOR_PARALLEL_KV_TYPES")


class TestLegacyBuildQuantizedKvInTensorMode:
    """Legacy binaries refuse quantized KV under tensor split; that must skip --fit and name the remedy."""

    _REJECTION = (
        "llama_init_from_model: simultaneous use of SPLIT_MODE_TENSOR and "
        "KV cache quantization not implemented"
    )
    _QUANTIZED = ("q8_0", "q4_0", "q4_1", "q5_0", "q5_1", "iq4_nl")

    @classmethod
    def _doomed(cls, cmd) -> bool:
        """What a pre-#23792 llama-server rejects: tensor split + a quantized axis."""
        if "--split-mode" not in cmd or cmd[cmd.index("--split-mode") + 1] != "tensor":
            return False
        return any(
            flag in ("--cache-type-k", "--cache-type-v") and cmd[i + 1] in cls._QUANTIZED
            for i, flag in enumerate(cmd[:-1])
        )

    def _run(self, tmp_path, monkeypatch):
        """One load_model call against a legacy child; returns the spawned commands."""
        import subprocess
        from unittest.mock import patch

        import importlib.util
        from pathlib import Path as _Path

        spec = importlib.util.spec_from_file_location(
            "_placement_harness_legacy_build",
            _Path(__file__).resolve().parent / "test_llama_cpp_placement.py",
        )
        placement = importlib.util.module_from_spec(spec)
        spec.loader.exec_module(placement)

        backend, gguf = placement._backend(
            tmp_path, vulkan = False, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)]
        )
        real_popen = subprocess.Popen
        spawns: list[list[str]] = []

        def fake_popen(cmd, **kwargs):
            if not cmd or str(cmd[0]) != "/fake/llama-server":
                return real_popen(cmd, **kwargs)
            cmd = list(cmd)
            spawns.append(cmd)
            doomed = self._doomed(cmd)
            if doomed:
                backend._stdout_lines = [self._REJECTION, "srv load_model: failed to load model"]
            return type(
                "Process",
                (),
                {
                    "pid": 123,
                    "stdout": (),
                    "returncode": 1 if doomed else None,
                    "poll": lambda _self, _d = doomed: 1 if _d else None,
                    "terminate": lambda _self: None,
                    "wait": lambda _self, timeout = None: 1 if doomed else 0,
                    "kill": lambda _self: None,
                },
            )()

        backend._wait_for_health = lambda timeout, **_kw: not self._doomed(spawns[-1])
        error: list[BaseException] = []
        with patch.object(subprocess, "Popen", side_effect = fake_popen):
            try:
                backend.load_model(
                    GgufLoadIntent(
                        gguf_path = str(gguf),
                        model_identifier = "test",
                        tensor_parallel = True,
                        cache_type_kv = "q8_0",
                    )
                )
            except BaseException as exc:  # noqa: BLE001
                error.append(exc)
        return backend, gguf, spawns, (error[0] if error else None)

    @pytest.fixture(autouse = True)
    def _clear_latch(self):
        """Both latches are process-wide: a leak would let a later cell pass for an earlier cell's
        reason."""
        LlamaCppBackend._tensor_split_abort_keys.clear()
        LlamaCppBackend._tensor_quant_kv_unsupported_binaries.clear()
        yield
        LlamaCppBackend._tensor_split_abort_keys.clear()
        LlamaCppBackend._tensor_quant_kv_unsupported_binaries.clear()

    def test_the_refusal_costs_exactly_one_spawn(self, tmp_path, monkeypatch):
        """Not two: a --fit retry cannot fix a missing capability; each attempt is a full load."""
        _backend, _gguf, spawns, error = self._run(tmp_path, monkeypatch)

        assert len(spawns) == 1, [c[c.index("--split-mode") + 1] for c in spawns]
        assert isinstance(error, RuntimeError), error

    def test_it_raises_to_the_route_fallback_naming_the_build(self, tmp_path, monkeypatch):
        """The re-raise to the route fallback must name the build; that text reaches the user."""
        _backend, _gguf, _spawns, error = self._run(tmp_path, monkeypatch)

        assert "b9455" in str(error)
        assert "layer split" in str(error)

    def test_the_binary_is_latched_so_the_next_load_skips_tensor(self, tmp_path, monkeypatch):
        """Without this the doomed attempt repeats on every load, forever."""
        backend, gguf, _spawns, _error = self._run(tmp_path, monkeypatch)

        # Keyed by binary alone: this build lacks quantized-KV tensor splits for every model.
        assert LlamaCppBackend._tensor_quant_kv_unsupported_binary(
            "/fake/llama-server", ("q8_0", "q8_0")
        )
        assert LlamaCppBackend._tensor_quant_kv_unsupported_binary(
            "/fake/llama-server", ("q4_0", "q4_0")
        )
        # f16 still works under tensor split there; the latch must not drop a working config.
        assert not LlamaCppBackend._tensor_quant_kv_unsupported_binary(
            "/fake/llama-server", ("f16", "f16")
        )
        assert not LlamaCppBackend._tensor_split_abort_keys

    def test_a_current_build_is_untouched(self, tmp_path, monkeypatch):
        """The guard is free on a supporting binary: one spawn, tensor kept, quantized cache."""
        import importlib.util
        from pathlib import Path as _Path

        spec = importlib.util.spec_from_file_location(
            "_placement_harness_current_build",
            _Path(__file__).resolve().parent / "test_llama_cpp_placement.py",
        )
        placement = importlib.util.module_from_spec(spec)
        spec.loader.exec_module(placement)

        backend, gguf = placement._backend(
            tmp_path, vulkan = False, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)]
        )
        backend._tensor_split_aborts = lambda *a, **k: False
        cmd = placement._launch(backend, gguf, tensor_parallel = True, cache_type_kv = "q8_0")["cmd"]

        assert cmd[cmd.index("--split-mode") + 1] == "tensor"
        assert cmd[cmd.index("--cache-type-k") + 1] == "q8_0"
        assert not LlamaCppBackend._tensor_split_abort_keys


class TestLegacyBuildLatchIsBinaryWide:
    """The pre-b9455 refusal is a binary capability, so the latch is binary-wide, not per model."""

    @pytest.fixture(autouse = True)
    def _clear(self):
        LlamaCppBackend._tensor_quant_kv_unsupported_binaries.clear()
        yield
        LlamaCppBackend._tensor_quant_kv_unsupported_binaries.clear()

    def test_one_detection_covers_other_models_and_other_types(self, tmp_path):
        binary = str(tmp_path / "llama-server")
        Path(binary).write_text("x")
        LlamaCppBackend._record_tensor_quant_kv_unsupported(binary)

        assert LlamaCppBackend._tensor_quant_kv_unsupported_binary(binary, ("q4_0", "q4_0"))
        assert LlamaCppBackend._tensor_quant_kv_unsupported_binary(binary, ("q8_0", "q8_0"))
        # One quantized axis is enough, since llama.cpp checks both.
        assert LlamaCppBackend._tensor_quant_kv_unsupported_binary(binary, ("q4_0", "f16"))
        assert LlamaCppBackend._tensor_quant_kv_unsupported_binary(binary, ("f16", "q4_0"))

    def test_a_non_quantized_pair_still_gets_a_tensor_split(self, tmp_path):
        """The refusal targets quantized KV only; latching it must not drop f16 tensor splits."""
        binary = str(tmp_path / "llama-server")
        Path(binary).write_text("x")
        LlamaCppBackend._record_tensor_quant_kv_unsupported(binary)

        assert not LlamaCppBackend._tensor_quant_kv_unsupported_binary(binary, ("f16", "f16"))
        assert not LlamaCppBackend._tensor_quant_kv_unsupported_binary(binary, ("bf16", "f32"))

    def test_a_replaced_binary_is_re_probed(self, tmp_path):
        """An update is exactly what fixes this, so the mtime must invalidate it."""
        import os
        import time

        binary = str(tmp_path / "llama-server")
        Path(binary).write_text("old")
        LlamaCppBackend._record_tensor_quant_kv_unsupported(binary)
        assert LlamaCppBackend._tensor_quant_kv_unsupported_binary(binary, ("q8_0", "q8_0"))

        time.sleep(0.01)
        Path(binary).write_text("new")
        os.utime(binary, ns = (time.time_ns(), time.time_ns()))

        assert not LlamaCppBackend._tensor_quant_kv_unsupported_binary(binary, ("q8_0", "q8_0"))

    def test_an_unseen_binary_is_not_latched(self, tmp_path):
        assert not LlamaCppBackend._tensor_quant_kv_unsupported_binary(
            str(tmp_path / "never-seen"), ("q8_0", "q8_0")
        )
        assert not LlamaCppBackend._tensor_quant_kv_unsupported_binary(None, ("q8_0", "q8_0"))

    def test_the_skip_keeps_the_multi_gpu_request(self):
        """Missing capability is not capacity; the layer fallback still spreads across GPUs."""
        load = "".join(inspect.getsource(LlamaCppBackend.load_model).split())
        gate = load.find("self._tensor_quant_kv_unsupported_binary(")
        assert gate != -1
        nxt = load.find("self._tensor_split_aborts(", gate)
        assert nxt != -1
        assert "_layer_min_gpus=max(_layer_min_gpus,len(gpus))" in load[gate:nxt]

    def test_the_loader_consults_it_before_spawning(self):
        """Source-pinned: the binary check must sit at the up-front gate, before any spawn."""
        load = "".join(inspect.getsource(LlamaCppBackend.load_model).split())
        gate = load.find("self._tensor_quant_kv_unsupported_binary(")
        assert gate != -1, "the binary-wide gate is gone"
        assert gate < load.find("healthy=_spawn_and_wait(cmd)")
        assert "extra_args=strip_split_mode_only(extra_args)" in load