Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

82.6 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Models loaded alongside the primary one are routed to by request model name."""

import asyncio
import inspect
import threading
from types import SimpleNamespace
from unittest.mock import AsyncMock

import pytest
from fastapi import FastAPI, HTTPException
from fastapi.testclient import TestClient

import core.inference.llama_cpp as llama_cpp
import core.inference.llama_keepwarm as keepwarm
import core.inference.model_slots as model_slots
import core.inference.npu_backend as npu_backend
import core.inference.orchestrator as orchestrator
import routes.inference as inf
from auth import policy
from auth.authentication import get_current_subject
from core.inference import gpu_arbiter
from core.inference.llama_cpp import GpuMemoryShortError
from hub.services.models import deletion
from models.inference import (
    ChatCountTokensRequest,
    InferenceStatusResponse,
    LoadRequest,
    LoadResponse,
    UnloadRequest,
)
from routes import training_vram
from state import active_generations
from utils import multi_model_settings
from utils.account_context import AccountContext, run_as

ALICE = AccountContext("a" * 32, "alice")
BOB = AccountContext("b" * 32, "bob")


class FakeLlama:
    holds_no_vram = False

    def __init__(
        self,
        identifier = None,
        variant = None,
    ):
        self.model_identifier = identifier
        self.hf_variant = variant
        self.is_loaded = self.is_active = identifier is not None
        self.is_vision = False
        self.context_length = 4096

    def unload_model(self):
        self.is_loaded = self.is_active = False

    def _cleanup(self):
        pass


class FakeOrchestrator:
    def __init__(self, active = None):
        self.active_model_name = active
        self.models = {active: {}} if active else {}

    def unload_model(self, name):
        self.active_model_name = None

    def _cleanup(self):
        pass


@pytest.fixture
def backends(monkeypatch):
    primary = FakeLlama("org/A-GGUF", "Q4_K_M")
    extra = model_slots.ExtraSlot(FakeLlama("org/B-GGUF", "Q8_0"), FakeOrchestrator(), "owner")
    monkeypatch.setattr(inf, "_llama_cpp_backend", primary)
    monkeypatch.setattr(orchestrator, "_inference_backend", FakeOrchestrator())
    monkeypatch.setattr(model_slots, "slots", [extra])
    monkeypatch.setattr(inf, "LlamaCppBackend", FakeLlama)
    monkeypatch.setattr(inf, "InferenceOrchestrator", FakeOrchestrator)
    monkeypatch.setattr(inf, "_raise_if_sidecar_swap_in_progress", lambda: None)
    monkeypatch.setattr(multi_model_settings, "get_multi_model_enabled", lambda: True)
    monkeypatch.setattr(model_slots, "stuck", [])
    return primary, extra


def _routed(requested):
    async def run():
        slot = await inf._route_to_extra_slot(requested)
        return slot, inf.get_llama_cpp_backend()

    return asyncio.run(run())


def test_request_model_name_picks_the_backend(backends):
    primary, extra = backends
    assert _routed("org/B-GGUF") == (extra, extra.llama)
    assert _routed("org/B-GGUF:Q8_0") == (extra, extra.llama)
    assert _routed("org/B-GGUF:Q4_K_M") == (None, primary)
    assert _routed("org/A-GGUF") == (None, primary)
    assert _routed(None) == (None, primary)
    assert extra.last_used > 0.0
    model_slots.slots.append(
        model_slots.ExtraSlot(FakeLlama("org/A-GGUF", "Q8_0"), FakeOrchestrator(), "owner")
    )
    assert _routed("org/A-GGUF") == (None, primary)
    assert _routed("org/A-GGUF:Q8_0")[0] is model_slots.slots[-1]


def test_a_safetensors_slot_is_served_by_its_own_orchestrator(backends):
    safetensors = model_slots.ExtraSlot(FakeLlama(), FakeOrchestrator("org/C"), "owner")
    model_slots.slots.append(safetensors)

    async def run():
        slot = await inf._route_to_extra_slot("org/C")
        return slot, inf.get_llama_cpp_backend().is_loaded, inf.get_inference_backend()

    assert asyncio.run(run()) == (safetensors, False, safetensors.orchestrator)


def _serving(info):
    return SimpleNamespace(active_model_name = "org/Audio", models = {"org/Audio": info})


_MUSIC, _SPEECH = {"audio_music": {"modes": []}}, {"is_audio": True, "audio_type": "csm"}


@pytest.mark.parametrize(
    "workflow, primary, kept",
    [
        ("separate", None, [_SPEECH, {"is_audio": True, "audio_workflows": ["separate"]}]),
        ("music", None, [_SPEECH, _MUSIC]),
        ("music", _MUSIC, [_MUSIC]),  # the primary runs it: stays there
    ],
)
def test_omitted_audio_model_routes_to_the_kept_one(backends, monkeypatch, workflow, primary, kept):
    if primary:
        monkeypatch.setattr(orchestrator, "_inference_backend", _serving(primary))
    model_slots.slots[:] = [model_slots.ExtraSlot(FakeLlama(), _serving(i), "owner") for i in kept]

    async def run():
        await inf._route_to_kept_audio_model(workflow)
        return inf.get_inference_backend()

    want = orchestrator._inference_backend if primary else model_slots.slots[-1].orchestrator
    assert asyncio.run(run()) is want


def test_loaded_models_lists_every_backend(backends):
    app = FastAPI()
    app.include_router(inf.studio_router, prefix = "/api/inference")
    app.dependency_overrides[get_current_subject] = lambda: "test-subject"
    with TestClient(app) as client:
        data = client.get("/api/inference/loaded-models").json()["data"]
    assert {entry["id"] for entry in data} == {"org/A-GGUF", "org/B-GGUF"}
    model_slots.slots.append(model_slots.ExtraSlot(FakeLlama(), FakeOrchestrator("org/C"), "owner"))
    with TestClient(app) as client:
        data = client.get("/api/inference/loaded-models").json()["data"]
    assert [entry["id"] for entry in data] == ["org/A-GGUF", "org/B-GGUF", "org/C"]


def _selected(request):
    async def run():
        slot = await inf._select_load_slot(request)
        assert inf.routed_slot.get() is slot
        return slot

    return asyncio.run(run())


def test_a_load_picks_its_slot(backends):
    primary, extra = backends
    alongside = LoadRequest(model_path = "org/C-GGUF", alongside = True)
    assert _selected(alongside) not in (None, extra)
    assert _selected(LoadRequest(model_path = "org/C-GGUF")) is None
    served = LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q8_0")
    assert _selected(served) is extra
    npu = LoadRequest(model_path = "lemonade:qwen3-0.6b-FLM", alongside = True)
    assert _selected(npu) is None
    assert _selected(alongside.model_copy(update = {"engine": "vllm"})) is None
    primary.unload_model()
    assert _selected(alongside) is None


def test_alongside_is_off_until_settings_turns_it_on(backends, monkeypatch):
    _, extra = backends
    monkeypatch.setattr(multi_model_settings, "get_multi_model_enabled", lambda: False)
    served = LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q8_0", alongside = True)
    assert _selected(served) is extra
    extra.refs += 1
    assert _selected(LoadRequest(model_path = "org/C-GGUF", alongside = True)) is None
    assert model_slots.slots == [extra]
    extra.refs -= 1
    assert _selected(LoadRequest(model_path = "org/D-GGUF")) is None
    assert model_slots.slots == [] and not extra.llama.is_active


def test_a_kept_model_with_no_gpu_layers_does_not_hold_the_gpu(backends):
    _, extra = backends
    assert model_slots.holds_vram()
    extra.llama._gpu_offload_active = False
    assert not model_slots.holds_vram() and model_slots.busy()


def test_the_multi_model_setting_defaults_off(monkeypatch):
    stored = {}
    import storage.studio_db as studio_db

    monkeypatch.setattr(
        studio_db, "get_app_settings", lambda keys: {k: stored[k] for k in keys if k in stored}
    )
    monkeypatch.setattr(studio_db, "upsert_app_settings", stored.update)
    multi_model_settings.invalidate_multi_model_cache()
    assert multi_model_settings.get_multi_model_enabled() is False
    assert multi_model_settings.set_multi_model_enabled(True) is True
    assert multi_model_settings.get_multi_model_enabled() is True
    with pytest.raises(ValueError):
        multi_model_settings.set_multi_model_enabled("yes")


def test_turning_the_setting_off_saves_first_and_unloads_after_the_reply(backends, monkeypatch):
    from fastapi import BackgroundTasks
    import routes.settings as settings_routes

    _, extra = backends
    monkeypatch.setattr(settings_routes, "set_multi_model_mode", lambda mode: True)

    def fail():
        raise RuntimeError("llama-server would not exit")

    monkeypatch.setattr(extra.llama, "unload_model", fail)
    tasks = BackgroundTasks()
    response = settings_routes.update_multi_model(
        settings_routes.MultiModelPayload(enabled = False), tasks, "s"
    )
    assert response.enabled is False and model_slots.slots == [extra]
    asyncio.run(tasks())
    assert model_slots.slots == [] and model_slots.stuck == [extra] and model_slots.holds_vram()


def test_unload_drops_only_the_named_extra_slot(backends, monkeypatch):
    primary, extra = backends
    _hold_chat_claim(monkeypatch)
    released, release = [], inf.release_chat_gpu_claim
    monkeypatch.setattr(inf, "release_chat_gpu_claim", lambda: released.append(release()))
    response = asyncio.run(inf._unload_model_impl(UnloadRequest(model_path = "org/B-GGUF"), "s"))
    assert response.status == "unloaded"
    assert primary.is_loaded and not extra.llama.is_loaded and model_slots.slots == []
    assert released == [False] and gpu_arbiter.current_owner() == gpu_arbiter.CHAT


def test_a_failed_llama_unload_still_cleans_the_orchestrator(backends):
    _, extra = backends
    cleaned = []
    extra.llama.unload_model = lambda: 1 / 0
    extra.orchestrator._cleanup = lambda: cleaned.append(True)
    with pytest.raises(ZeroDivisionError):
        model_slots.drop(extra)
    assert model_slots.slots == [] and cleaned == [True]


def test_a_managed_account_sees_only_its_own_slots(backends, monkeypatch):
    primary, extra = backends
    monkeypatch.setattr(inf.account_access, "managed_account", lambda: True)
    monkeypatch.setattr(model_slots, "current_account_id", lambda: "someone-else")
    assert _routed("org/B-GGUF") == (None, primary)
    filling = model_slots.ExtraSlot(FakeLlama(), FakeOrchestrator(), "owner")
    monkeypatch.setattr(model_slots, "loading", (filling, "org/D-GGUF"))
    model_slots.slots.append(filling)
    assert model_slots.visible_loading() is None
    monkeypatch.setattr(model_slots, "current_account_id", lambda: "owner")
    assert _routed("org/B-GGUF") == (extra, extra.llama)


def test_idle_unload_spares_pinned_and_filling_slots(backends, monkeypatch):
    _, extra = backends
    assert model_slots.unload_extra_models(keep = lambda llama: True) == 0
    monkeypatch.setattr(model_slots, "loading", (extra, "org/B-GGUF"))
    assert model_slots.unload_extra_models(keep = lambda llama: False, spare_filling = True) == 0
    assert model_slots.slots == [extra] and extra.llama.is_active
    assert model_slots.unload_extra_models(keep = lambda llama: False) == 1
    assert model_slots.slots == []


def test_status_describes_the_named_slot_and_lists_the_rest(backends, monkeypatch):
    async def slot_status(subject):
        return InferenceStatusResponse(active_model = inf.get_llama_cpp_backend().model_identifier)

    monkeypatch.setattr(inf, "_slot_status", slot_status)
    named = asyncio.run(inf.get_status("s", model = "org/B-GGUF"))
    assert (named.active_model, named.loaded) == ("org/B-GGUF", ["org/A-GGUF"])
    primary = asyncio.run(inf.get_status("s"))
    assert (primary.active_model, primary.loaded) == ("org/A-GGUF", ["org/B-GGUF"])

    async def held_behind(subject):
        return InferenceStatusResponse(
            active_model = "org/A-GGUF", loaded = ["org/A-GGUF", "org/held-hf"]
        )

    monkeypatch.setattr(inf, "_slot_status", held_behind)
    status = asyncio.run(inf.get_status("s"))
    assert status.serving == ["org/A-GGUF", "org/B-GGUF"] and "org/held-hf" in status.loaded
    monkeypatch.setattr(model_slots, "slots", [])
    assert asyncio.run(inf.get_status("s")).serving == ["org/A-GGUF"]


def test_a_status_read_does_not_count_as_use_for_eviction(backends, monkeypatch):
    # The Loaded models poll sizes every kept model by name; that must not reorder the LRU.
    _, extra = backends
    extra.last_used = 1.0

    async def slot_status(subject):
        return InferenceStatusResponse(active_model = inf.get_llama_cpp_backend().model_identifier)

    monkeypatch.setattr(inf, "_slot_status", slot_status)
    named = asyncio.run(inf.get_status("s", model = "org/B-GGUF"))
    assert named.active_model == "org/B-GGUF"
    assert extra.last_used == 1.0
    assert _routed("org/B-GGUF")[0] is extra and extra.last_used > 1.0


def test_status_pairs_each_serving_model_with_the_checkpoint_to_select_it_by(backends, monkeypatch):
    _, extra = backends
    local, twin = "/home/alice/models/B-local.gguf", "/home/alice/other/B-local.gguf"
    extra.llama = FakeLlama(local, "Q8_0")
    model_slots.slots.append(
        model_slots.ExtraSlot(FakeLlama(twin, "Q8_0"), FakeOrchestrator(), "owner")
    )
    public = inf._llama_public_model_id
    monkeypatch.setattr(
        inf,
        "_llama_public_model_id",
        lambda llama: (
            "B-local" if str(llama.model_identifier).endswith("B-local.gguf") else public(llama)
        ),
    )

    async def slot_status(subject):
        return InferenceStatusResponse(active_model = "org/A-GGUF", model_identifier = "org/A-GGUF")

    monkeypatch.setattr(inf, "_slot_status", slot_status)
    status = asyncio.run(inf.get_status("s"))
    assert status.serving == ["org/A-GGUF", "B-local", "B-local"]
    assert status.serving_checkpoints == ["org/A-GGUF", local, twin]


def test_stop_loading_reaches_the_slot_being_filled(backends, monkeypatch):
    primary, _ = backends
    filling = model_slots.ExtraSlot(FakeLlama(), FakeOrchestrator(), "owner")
    filling.llama.model_identifier = "org/D-GGUF"
    filling.llama.is_active = True
    model_slots.slots.append(filling)
    monkeypatch.setattr(model_slots, "loading", (filling, "org/D-GGUF"))
    response = asyncio.run(inf._unload_model_impl(UnloadRequest(model_path = "org/D-GGUF"), "s"))
    assert response.status == "unloaded"
    assert primary.is_loaded and not filling.llama.is_active


def test_a_slot_evicted_while_it_loads_is_torn_down(backends, monkeypatch):
    spawned = []

    async def load_after_eviction(request, *args, **kwargs):
        slot = model_slots.slots[-1]
        model_slots.unload_extra_models()
        slot.llama.is_loaded = slot.llama.is_active = True
        spawned.append(slot)
        return "loaded"

    monkeypatch.setattr(inf, "_run_tracked_load_model_impl", load_after_eviction)
    monkeypatch.setattr(inf, "release_chat_gpu_claim", lambda: None)
    request = LoadRequest(model_path = "org/C-GGUF", alongside = True)
    assert asyncio.run(inf.load_model_gated(request, None, "s")) == "loaded"
    assert model_slots.slots == [] and not spawned[0].llama.is_active


def test_only_a_new_slot_skips_the_running_chat_check(backends, monkeypatch):
    seen = []

    async def load(request, *args, on_reload_confirmed, **kwargs):
        seen.append(on_reload_confirmed is None)
        llama = inf.get_llama_cpp_backend()
        llama.model_identifier = request.model_path
        llama.is_loaded = llama.is_active = True
        return "loaded"

    monkeypatch.setattr(inf, "_run_tracked_load_model_impl", load)
    for request in (
        LoadRequest(model_path = "org/C-GGUF", alongside = True),
        LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q8_0", force_reload = True),
        LoadRequest(model_path = "org/D-GGUF"),
    ):
        asyncio.run(inf.load_model_gated(request, None, "s"))
    assert seen == [True, False, False]


def test_loading_alongside_leaves_the_primary_with_the_account_that_loaded_it(monkeypatch):
    monkeypatch.setattr(gpu_arbiter, "_owner", None)
    monkeypatch.setattr(gpu_arbiter, "_owner_account", None)
    gpu_arbiter.acquire_for(gpu_arbiter.CHAT, lambda: None, account_id = ALICE.account_id)
    gpu_arbiter.acquire_for(
        gpu_arbiter.CHAT, lambda: None, account_id = BOB.account_id, alongside = True
    )
    assert gpu_arbiter.owner_account() == ALICE.account_id
    gpu_arbiter.acquire_for(gpu_arbiter.CHAT, lambda: None, account_id = BOB.account_id)
    assert gpu_arbiter.owner_account() == BOB.account_id


def test_an_account_lists_its_own_slot_but_not_a_foreign_primary(backends, monkeypatch):
    primary, extra = backends
    monkeypatch.setattr(policy, "installation_is_multi_user", lambda: True)
    monkeypatch.setattr(gpu_arbiter, "_owner", gpu_arbiter.CHAT)
    monkeypatch.setattr(gpu_arbiter, "_owner_account", ALICE.account_id)
    extra.account = BOB.account_id

    def listed():
        return [entry["id"] for entry in inf._openai_model_objects()]

    assert run_as(BOB, listed) == ["org/B-GGUF"]
    assert run_as(ALICE, listed) == ["org/A-GGUF"]


def _slot(
    model,
    variant = None,
    last_used = 0.0,
):
    return model_slots.ExtraSlot(
        FakeLlama(model, variant),
        FakeOrchestrator(),
        "owner",
        LoadRequest(model_path = model, gguf_variant = variant, alongside = True),
        last_used,
    )


def _gated_load_fakes(
    monkeypatch,
    short_fits,
    response = "loaded",
):
    monkeypatch.setattr(inf, "release_chat_gpu_claim", lambda: None)

    async def load(request, *args, **kwargs):
        if short_fits and not request.force_alongside:
            kind = short_fits.pop()
            raise GpuMemoryShortError(
                "needs 13 GB, 8 GB free", capped = kind == "capped", short_mib = 5000
            )
        llama = inf.get_llama_cpp_backend()
        llama.model_identifier, llama.hf_variant = request.model_path, request.gguf_variant
        llama.is_loaded = llama.is_active = True
        return response

    monkeypatch.setattr(inf, "_run_tracked_load_model_impl", load)


def test_a_short_fit_evicts_the_least_recently_used_slot_and_names_it(backends, monkeypatch):
    _, extra = backends
    extra.request, extra.last_used = LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q8_0"), 5.0
    older = _slot("org/D-GGUF", "Q4_K_M", last_used = 1.0)
    model_slots.slots.append(older)
    loaded = LoadResponse.model_construct(status = "loaded", model = "org/C-GGUF", display_name = "C")
    _gated_load_fakes(monkeypatch, short_fits = [1], response = loaded)
    request = LoadRequest(model_path = "org/C-GGUF", alongside = True)
    assert asyncio.run(inf.load_model_gated(request, None, "s")).evicted == ["org/D-GGUF:Q4_K_M"]
    assert [s.llama.model_identifier for s in model_slots.slots] == ["org/B-GGUF", "org/C-GGUF"]
    assert not older.llama.is_active
    assert model_slots.slots[-1].request.model_path == "org/C-GGUF"


def test_a_load_cancelled_after_a_short_fit_evicts_nothing(backends, monkeypatch):
    _, extra = backends
    extra.request, extra.last_used = LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q8_0"), 5.0
    older = _slot("org/D-GGUF", "Q4_K_M", last_used = 1.0)
    model_slots.slots.append(older)
    cancel = threading.Event()
    _gated_load_fakes(monkeypatch, short_fits = [1])
    short = inf._run_tracked_load_model_impl

    async def load(request, *args, **kwargs):
        cancel.set()  # the API request is cancelled while the first attempt reports its shortfall
        return await short(request, *args, **kwargs)

    monkeypatch.setattr(inf, "_run_tracked_load_model_impl", load)
    request = LoadRequest(model_path = "org/C-GGUF", alongside = True)
    with pytest.raises(HTTPException) as exc:
        asyncio.run(
            inf.load_model_gated(
                request, None, "s", background = True, impl_kwargs = {"load_cancel_event": cancel}
            )
        )
    assert exc.value.status_code == 409 and older in model_slots.slots and older.llama.is_active


def test_eviction_takes_as_many_lru_slots_as_the_shortfall_needs(backends, monkeypatch):
    _, extra = backends
    extra.request, extra.last_used = LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q8_0"), 3.0
    extra.llama._planned_vram_mib = {0: 9000}
    small_old = _slot("org/D-GGUF", last_used = 1.0)
    small_old.llama._planned_vram_mib = {0: 500}
    mid = _slot("org/E-GGUF", last_used = 2.0)
    mid.llama._planned_vram_mib = {0: 6000}
    model_slots.slots += [small_old, mid]
    assert model_slots.eviction_victims(None, 5000) == [small_old, mid]
    _gated_load_fakes(monkeypatch, short_fits = [1])
    request = LoadRequest(model_path = "org/C-GGUF", alongside = True)
    assert asyncio.run(inf.load_model_gated(request, None, "s")) == "loaded"
    assert [s.llama.model_identifier for s in model_slots.slots] == ["org/B-GGUF", "org/C-GGUF"]


def test_a_model_that_fits_nowhere_beside_the_others_replaces_the_active_one(backends, monkeypatch):
    primary, extra = backends
    extra.request = LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q8_0")
    loaded = LoadResponse.model_construct(status = "loaded", model = "org/C-GGUF", display_name = "C")
    _gated_load_fakes(monkeypatch, short_fits = [1, 1], response = loaded)
    request = LoadRequest(model_path = "org/C-GGUF", alongside = True)
    response = asyncio.run(inf.load_model_gated(request, None, "s"))
    assert response.evicted == ["org/B-GGUF:Q8_0", "org/A-GGUF"]
    assert model_slots.slots == [] and not extra.llama.is_active
    assert primary.model_identifier == "org/C-GGUF" and primary.is_loaded


BESIDE_ONLY = AccountContext("owner", "o", "owner", via_api_key = True, load_policy = "beside_only")


@pytest.mark.parametrize("alongside, short_fits", [(True, [1, 1]), (False, [])])
def test_a_beside_only_load_unloads_nothing(backends, monkeypatch, alongside, short_fits):
    a, b = backends
    b.request = LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q8_0")
    monkeypatch.setattr(multi_model_settings, "get_multi_model_mode", lambda: "fit")
    _gated_load_fakes(monkeypatch, short_fits = short_fits)
    request = LoadRequest(model_path = "org/C-GGUF", alongside = alongside)
    with pytest.raises(HTTPException) as exc:
        run_as(BESIDE_ONLY, asyncio.run, inf.load_model_gated(request, None, "s"))
    assert exc.value.headers["X-Unsloth-Problem"].startswith("needs_user_load kind=llm")
    assert a.is_loaded and a.model_identifier == "org/A-GGUF" and model_slots.slots == [b]


def test_a_beside_only_load_into_an_empty_primary_frees_only_media(backends, monkeypatch):
    backends[0].unload_model()
    monkeypatch.setattr(multi_model_settings, "get_multi_model_mode", lambda: "fit")
    _gated_load_fakes(monkeypatch, short_fits = [1])
    monkeypatch.setattr(inf, "_free_for_chat", free := AsyncMock(return_value = []))
    request = LoadRequest(model_path = "org/C-GGUF", alongside = True)
    run_as(BESIDE_ONLY, asyncio.run, inf.load_model_gated(request, None, "s", background = True))
    assert [call.kwargs["media_only"] for call in free.await_args_list] == [True]


def test_beside_only_and_measured_always_refusals_unload_nothing_and_name_no_model():
    # The beside-only refusal comes before vLLM or SGLang retires every kept model.
    source = inspect.getsource(inf._load_model_impl)
    refuse = source.index("_require_load_keeps_chat(fastapi_request, request)")
    assert refuse < source.index("_gate_kept_models(request, cancel = False)")
    # No kind: it would read as the model to unload, but an image or video model may hold the GPU.
    source = inspect.getsource(inf.load_model_gated)
    assert "raise MultiModelAlwaysRefusal(str(exc)) from exc" in source


def _short_load_fakes(monkeypatch, errors):
    monkeypatch.setattr(inf, "release_chat_gpu_claim", lambda: None)
    seen = []

    async def load(request, *args, **kwargs):
        seen.append(request)
        if errors and not request.force_alongside:
            raise errors.pop(0)
        llama = inf.get_llama_cpp_backend()
        llama.model_identifier, llama.hf_variant = request.model_path, request.gguf_variant
        llama.is_loaded = llama.is_active = True
        return LoadResponse.model_construct(status = "loaded", model = request.model_path)

    monkeypatch.setattr(inf, "_run_tracked_load_model_impl", load)
    return seen


def test_idle_kept_models_go_with_the_active_one_when_only_together_they_make_room(
    backends, monkeypatch
):
    # B and C alone free 12 of the 14 GB D is short; with A they cover it.
    primary, extra = backends
    primary._planned_vram_mib = {0: 6000}
    extra.request, extra.last_used = LoadRequest(model_path = "org/B-GGUF"), 1.0
    extra.llama._planned_vram_mib = {0: 6000}
    other = _slot("org/C-GGUF", last_used = 2.0)
    other.llama._planned_vram_mib = {0: 6000}
    model_slots.slots.append(other)
    monkeypatch.setattr(inf, "_unload_primary", lambda: primary.unload_model())
    _short_load_fakes(monkeypatch, [GpuMemoryShortError("short", short_mib = 14000)])
    request = LoadRequest(model_path = "org/D-GGUF", alongside = True)
    response = asyncio.run(inf.load_model_gated(request, None, "s"))
    assert response.evicted == ["org/B-GGUF", "org/C-GGUF", "org/A-GGUF"]
    assert model_slots.slots == [] and not extra.llama.is_active and not other.llama.is_active
    assert primary.model_identifier == "org/D-GGUF"


def test_idle_kept_models_stay_when_the_active_one_alone_makes_room(backends, monkeypatch):
    primary, extra = backends
    primary._planned_vram_mib = {0: 16000}
    extra.request = LoadRequest(model_path = "org/B-GGUF")
    extra.llama._planned_vram_mib = {0: 6000}
    monkeypatch.setattr(inf, "_unload_primary", lambda: primary.unload_model())
    _short_load_fakes(monkeypatch, [GpuMemoryShortError("short", short_mib = 14000)])
    request = LoadRequest(model_path = "org/D-GGUF", alongside = True)
    assert asyncio.run(inf.load_model_gated(request, None, "s")).evicted == ["org/A-GGUF"]
    assert model_slots.slots == [extra] and extra.llama.is_active


def test_a_host_ram_spill_evicts_an_idle_kept_model_under_when_they_fit(backends, monkeypatch):
    _, extra = backends
    extra.request = LoadRequest(model_path = "org/B-GGUF")
    monkeypatch.setattr(multi_model_settings, "get_multi_model_mode", lambda: "fit")
    _short_load_fakes(monkeypatch, [GpuMemoryShortError("spills", host = True)])
    request = LoadRequest(model_path = "org/C-GGUF", alongside = True)
    assert asyncio.run(inf.load_model_gated(request, None, "s")).evicted == ["org/B-GGUF"]
    assert [s.llama.model_identifier for s in model_slots.slots] == ["org/C-GGUF"]


def test_a_host_ram_spill_loads_beside_the_others_under_always(backends, monkeypatch):
    _, extra = backends
    monkeypatch.setattr(multi_model_settings, "get_multi_model_mode", lambda: "always")
    seen = _short_load_fakes(monkeypatch, [GpuMemoryShortError("spills", host = True)])
    request = LoadRequest(model_path = "org/C-GGUF", alongside = True)
    assert not asyncio.run(inf.load_model_gated(request, None, "s")).evicted
    assert seen[-1].force_alongside and extra.llama.is_active


def test_a_capped_context_is_taken_only_once_nothing_is_left_to_evict(backends, monkeypatch):
    _, extra = backends
    extra.request = LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q8_0")
    _gated_load_fakes(monkeypatch, short_fits = ["capped", "capped"])
    request = LoadRequest(model_path = "org/C-GGUF", alongside = True)
    assert asyncio.run(inf.load_model_gated(request, None, "s")) == "loaded"
    assert not extra.llama.is_active
    assert [s.llama.model_identifier for s in model_slots.slots] == ["org/C-GGUF"]


def test_a_slot_load_drops_no_claim_and_a_load_that_tears_nothing_down_stops_no_chat():
    source = inspect.getsource(inf._load_model_impl)
    assert source.count("if replacing and not chat_load_needs_gpu:") == 2
    assert "if not chat_load_needs_gpu:" not in source
    assert source.count("if serving and on_reload_confirmed is not None:") == 3
    assert source.count("if replacing and serving:") == 2
    assert "if on_reload_confirmed is not None:" not in source


def test_a_load_prices_the_vram_the_other_servers_hold():
    from core.inference.llama_cpp import LlamaCppBackend

    a, b, c, stray = (LlamaCppBackend(manages_processes = False) for _ in range(4))
    for backend in (a, b, c):
        llama_cpp.register_serving_backend(backend)
    try:
        a._process = b._process = stray._process = object()
        a._planned_vram_mib = {0: 12000}
        b._planned_vram_mib = {0: 1000, 1: 500}
        stray._planned_vram_mib = {0: 7000}
        assert c._other_planned_vram_mib() == {0: 13000, 1: 500}
        assert a._other_planned_vram_mib() == {0: 1000, 1: 500}
        assert stray._other_planned_vram_mib() == {}
        a._kill_process()
        assert a._planned_vram_mib == {} and c._other_planned_vram_mib() == {0: 1000, 1: 500}
        b._process = None
        assert c._other_planned_vram_mib() == {}
    finally:
        for backend in (a, b, c):
            llama_cpp.unregister_serving_backend(backend)


def test_a_kept_server_that_died_no_longer_holds_vram(monkeypatch):
    from core.inference import gpu_residency
    from core.inference.llama_cpp import LlamaCppBackend

    monkeypatch.setattr(gpu_residency, "media_held_mib", lambda: {})
    a, c = (LlamaCppBackend(manages_processes = False) for _ in range(2))
    for backend in (a, c):
        llama_cpp.register_serving_backend(backend)
    try:
        a._planned_vram_mib = {0: 12000}
        a._process = SimpleNamespace(poll = lambda: None)
        assert c._other_planned_vram_mib() == {0: 12000} and c._other_servers_running()
        a._process = SimpleNamespace(poll = lambda: -9)
        assert c._other_planned_vram_mib() == {} and not c._other_servers_running()
        monkeypatch.setattr(gpu_residency, "media_held_mib", lambda: {0: 2048})
        assert c._other_servers_running()
    finally:
        a._process = None
        for backend in (a, c):
            llama_cpp.unregister_serving_backend(backend)


def _hold_chat_claim(monkeypatch):
    monkeypatch.setattr(gpu_arbiter, "_owner", gpu_arbiter.CHAT)
    monkeypatch.setattr(llama_cpp, "chat_load_active", lambda: False)


def test_a_failed_slot_load_drops_the_slot_but_keeps_the_primarys_claim(backends, monkeypatch):
    primary, extra = backends
    _hold_chat_claim(monkeypatch)
    released = []
    release = inf.release_chat_gpu_claim
    monkeypatch.setattr(inf, "release_chat_gpu_claim", lambda: released.append(release()))

    async def failing_load(*args, **kwargs):
        raise RuntimeError("no such repo")

    monkeypatch.setattr(inf, "_run_tracked_load_model_impl", failing_load)
    request = LoadRequest(model_path = "org/missing-GGUF", alongside = True)
    with pytest.raises(RuntimeError):
        asyncio.run(inf.load_model_gated(request, None, "s"))
    assert model_slots.slots == [extra] and model_slots.loading is None and released == [False]
    assert primary.is_active and gpu_arbiter.current_owner() == gpu_arbiter.CHAT


def test_a_failed_slot_load_ends_its_attempt_even_when_the_slot_will_not_stop(
    backends, monkeypatch
):
    async def failing_load(*args, **kwargs):
        raise RuntimeError("no such repo")

    def refuse(slot):
        raise RuntimeError("llama-server ignored SIGKILL")

    monkeypatch.setattr(inf, "_run_tracked_load_model_impl", failing_load)
    monkeypatch.setattr(model_slots, "drop", refuse)
    request = LoadRequest(model_path = "org/missing-GGUF", alongside = True, load_request_id = "r1")
    with pytest.raises(RuntimeError):
        asyncio.run(inf.load_model_gated(request, None, "s"))
    assert inf._pending_load_attempts == {}
    assert not any(key[1] == "r1" for key in inf._scoped_load_attempts)


def test_a_generation_is_tracked_on_the_slot_serving_it(backends):
    _, extra = backends
    event = threading.Event()

    async def run():
        await inf._route_to_extra_slot("org/B-GGUF")
        with inf._TrackedCancel(event, "k"):
            inside = set(extra.generations)
        return inside

    assert asyncio.run(run()) == {event} and extra.generations == set()


def test_a_slot_still_generating_is_never_evicted(backends):
    _, extra = backends
    older = _slot("org/D-GGUF", last_used = 1.0)
    model_slots.slots.append(older)
    older.generations.add(threading.Event())
    assert model_slots.eviction_victims(None, 5000) == [extra]
    extra.generations.add(threading.Event())
    assert model_slots.eviction_victims(None, 5000) == []


def test_unloading_a_generating_slot_is_refused_unless_forced(backends, monkeypatch):
    _, extra = backends
    monkeypatch.setattr(inf, "release_chat_gpu_claim", lambda: True)
    event = threading.Event()
    extra.generations.add(event)
    with (
        active_generations.ActiveGeneration(event, thread_id = "t1"),
        pytest.raises(HTTPException) as excinfo,
    ):
        asyncio.run(inf._unload_model_impl(UnloadRequest(model_path = "org/B-GGUF"), "s"))
    assert excinfo.value.status_code == 409 and excinfo.value.detail["thread_ids"] == ["t1"]
    assert extra.llama.is_loaded and not event.is_set()

    def finish_on_cancel():
        event.wait(5)
        extra.generations.discard(event)

    threading.Thread(target = finish_on_cancel, daemon = True).start()
    forced = UnloadRequest(model_path = "org/B-GGUF", force_cancel_active = True)
    asyncio.run(inf._unload_model_impl(forced, "s"))
    assert event.is_set() and not extra.llama.is_loaded and model_slots.slots == []


def test_unloading_a_slot_waits_for_a_request_already_routed_to_it(backends, monkeypatch):
    _, extra = backends
    monkeypatch.setattr(inf, "release_chat_gpu_claim", lambda: True)
    event = threading.Event()

    async def run(starts_generating):
        extra.refs += 1
        seen = {}

        async def routed_request():
            await asyncio.sleep(0.2)
            seen["loaded_while_routed"] = extra.llama.is_loaded
            if starts_generating:
                extra.generations.add(event)
            extra.refs -= 1

        task = asyncio.create_task(routed_request())
        try:
            await inf._unload_model_impl(UnloadRequest(model_path = "org/B-GGUF"), "s")
        finally:
            await task
        return seen

    with active_generations.ActiveGeneration(event, thread_id = "t1"):
        with pytest.raises(HTTPException) as excinfo:
            asyncio.run(run(starts_generating = True))
    assert excinfo.value.status_code == 409 and extra.llama.is_loaded
    extra.generations.discard(event)
    assert asyncio.run(run(starts_generating = False)) == {"loaded_while_routed": True}
    assert model_slots.slots == [] and not extra.llama.is_loaded


def test_unloading_a_slot_waits_for_a_reload_of_it_queued_on_the_gate(backends, monkeypatch):
    from core.inference.llama_keepwarm import inference_lifecycle_gate, model_load_gate

    _, extra = backends
    monkeypatch.setattr(inf, "release_chat_after_kept_models", lambda: True)
    monkeypatch.setattr(inf, "_POST_CANCEL_DRAIN_TIMEOUT_S", 0.5)
    seen = {}

    async def reload():
        async with model_load_gate():
            extra.refs += 1
            await asyncio.sleep(0.3)
            async with inference_lifecycle_gate():
                seen["still_kept"] = extra in model_slots.slots
            extra.refs -= 1

    async def run():
        task = asyncio.create_task(reload())
        await asyncio.sleep(0.1)
        await inf._unload_model_impl(UnloadRequest(model_path = "org/B-GGUF"), "s")
        await task

    asyncio.run(run())
    assert seen == {"still_kept": True}
    assert model_slots.slots == [] and not extra.llama.is_loaded


def test_a_slot_server_leaves_the_primary_pidfile_alone(monkeypatch):
    import utils.process_lifetime as process_lifetime
    from core.inference.llama_cpp import LlamaCppBackend

    written, cleared, adopted = [], [], []
    monkeypatch.setattr(
        LlamaCppBackend, "_record_server_pid", classmethod(lambda cls, pid: written.append(pid))
    )
    monkeypatch.setattr(
        LlamaCppBackend, "_clear_server_pid", classmethod(lambda cls: cleared.append(True))
    )
    monkeypatch.setattr(process_lifetime, "adopt_pid", adopted.append)
    slot = LlamaCppBackend(manages_processes = False)
    slot._owns_pidfile = False
    slot._note_server_pid(4242)
    slot._process = object()
    slot._kill_process()
    assert written == [] and cleared == [] and adopted == [4242]
    primary = LlamaCppBackend(manages_processes = False)
    primary._note_server_pid(4343)
    assert written == [4343]


def test_a_primary_swap_neither_refuses_on_nor_stops_another_models_chats(backends):
    _, extra = backends
    on_slot, on_primary = threading.Event(), threading.Event()
    extra.generations.add(on_slot)
    with active_generations.ActiveGeneration(on_slot, thread_id = "slot-chat"):
        assert inf._raise_or_cancel_active_generations(force = False, action = "Loading a model") == 0
        with active_generations.ActiveGeneration(on_primary, thread_id = "primary-chat"):
            with pytest.raises(HTTPException) as excinfo:
                inf._raise_or_cancel_active_generations(force = False, action = "Loading a model")
            assert excinfo.value.detail["running"] == 1
            assert excinfo.value.detail["thread_ids"] == ["primary-chat"]
            assert (
                inf._raise_or_cancel_active_generations(force = True, action = "Loading a model") == 1
            )
    assert on_primary.is_set() and not on_slot.is_set()


def test_reloading_a_kept_model_stops_only_its_own_chats(backends, monkeypatch):
    _, extra = backends
    monkeypatch.setattr(inf, "release_chat_gpu_claim", lambda: None)

    async def load(request, *args, on_reload_confirmed, **kwargs):
        assert inf.get_llama_cpp_backend() is extra.llama
        on_reload_confirmed(cancel = False)
        on_reload_confirmed(cancel = True)
        return "loaded"

    monkeypatch.setattr(inf, "_run_tracked_load_model_impl", load)
    reload = LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q8_0", force_reload = True)
    on_primary = threading.Event()
    with active_generations.ActiveGeneration(on_primary, thread_id = "chat-on-A"):
        assert asyncio.run(inf.load_model_gated(reload, None, "s")) == "loaded"
        on_slot = threading.Event()
        extra.generations.add(on_slot)
        forced = reload.model_copy(update = {"force_cancel_active": True})
        assert asyncio.run(inf.load_model_gated(forced, None, "s")) == "loaded"
    assert on_slot.is_set() and not on_primary.is_set()


def test_training_sizes_and_frees_the_models_kept_alongside(backends, monkeypatch):
    primary, extra = backends
    primary.unload_model()
    summary = training_vram.summarize_resident_chat()
    assert summary["any"] and not summary["loading"]
    monkeypatch.setattr(model_slots, "loading", (extra, "org/D-GGUF"))
    assert training_vram.summarize_resident_chat()["loading"]
    monkeypatch.setattr(model_slots, "loading", None)
    extra.orchestrator.loading_models = {"org/C"}
    assert training_vram.summarize_resident_chat()["loading"]
    extra.orchestrator.loading_models = set()
    assert training_vram.free_chat_models_for_training("test") == ["kept:org/B-GGUF"]
    assert model_slots.slots == [] and not extra.llama.is_active


def test_training_frees_the_kept_models_before_the_active_one(backends, monkeypatch):
    primary, extra = backends
    verdicts = [False, True]
    monkeypatch.setattr(training_vram, "summarize_resident_stt", lambda: {"any": False})
    freed = training_vram.coordinate_models_for_training(lambda: (verdicts.pop(0), {}))
    assert freed == ["kept:org/B-GGUF"] and verdicts == []
    assert model_slots.slots == [] and primary.is_active


def test_another_quant_of_a_loaded_model_replaces_it_in_place(backends):
    _, extra = backends
    primary_quant = LoadRequest(model_path = "org/A-GGUF", gguf_variant = "Q8_0", alongside = True)
    assert _selected(primary_quant) is None
    slot_quant = LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q4_K_M", alongside = True)
    assert _selected(slot_quant) is extra
    assert _selected(slot_quant.model_copy(update = {"alongside": False})) is extra


def test_an_alongside_load_counts_as_activity(backends, monkeypatch):
    stamped = []
    monkeypatch.setattr(keepwarm, "_note_activity", lambda: stamped.append(True))
    _gated_load_fakes(monkeypatch, short_fits = [])
    request = LoadRequest(model_path = "org/C-GGUF", alongside = True)
    assert asyncio.run(inf.load_model_gated(request, None, "s")) == "loaded"
    assert stamped == [True]


def test_a_chat_starting_on_a_victim_during_eviction_spares_it(backends, monkeypatch):
    _, extra = backends
    extra.last_used = 9.0
    older = _slot("org/D-GGUF", last_used = 1.0)
    older.llama._planned_vram_mib = {0: 3000}
    mid = _slot("org/E-GGUF", last_used = 2.0)
    mid.llama._planned_vram_mib = {0: 3000}
    model_slots.slots += [older, mid]
    unload = older.llama.unload_model

    def chat_starts_on_mid():
        mid.generations.add(threading.Event())
        unload()

    older.llama.unload_model = chat_starts_on_mid
    _gated_load_fakes(monkeypatch, short_fits = [1])
    request = LoadRequest(model_path = "org/C-GGUF", alongside = True)
    assert asyncio.run(inf.load_model_gated(request, None, "s")) == "loaded"
    assert mid in model_slots.slots and mid.llama.is_active


def test_a_routed_request_holds_its_slot_until_it_ends(backends):
    _, extra = backends
    routed, ended = asyncio.Event(), asyncio.Event()
    seen = {}

    async def request():
        await inf._route_to_extra_slot("org/B-GGUF")
        routed.set()
        await ended.wait()

    async def main():
        task = asyncio.create_task(request())
        await routed.wait()
        seen["refs"] = extra.refs
        seen["victims"] = model_slots.eviction_victims(None, 5000)
        seen["claimed"] = model_slots.claim_victim(extra)
        ended.set()
        await task

    asyncio.run(main())
    assert seen == {"refs": 1, "victims": [], "claimed": False}
    assert extra.refs == 0 and model_slots.eviction_victims(None, 5000) == [extra]


def test_a_chat_run_holds_its_slot_after_its_post_has_answered(backends):
    # POST /chat-runs answers 202 before routing, so the slot must stay held past the POST.

    _, extra = backends
    seen = {}

    async def run():
        await asyncio.sleep(0)
        await inf._route_to_extra_slot("org/B-GGUF")
        await asyncio.sleep(0)
        seen["refs"] = extra.refs
        seen["victims"] = model_slots.eviction_victims(None, 5000)

    async def post_chat_run():
        keepwarm.set_current_response_scope({})
        task = asyncio.create_task(run())
        keepwarm.set_current_response_scope(None)
        await task

    asyncio.run(post_chat_run())
    assert seen == {"refs": 1, "victims": []}
    assert extra.refs == 0


def test_a_slot_evicted_while_a_request_routes_is_not_served(backends, monkeypatch):
    _, extra = backends
    probe = model_slots.serving_slot

    def evicted_mid_probe(requested, slots, satisfies):
        found = probe(requested, slots, satisfies)
        assert model_slots.claim_victim(extra)
        return found

    monkeypatch.setattr(model_slots, "serving_slot", evicted_mid_probe)
    assert _routed("org/B-GGUF") == (None, backends[0])


def test_eviction_skips_other_gpus_and_victims_that_cannot_make_room(backends):
    _, extra = backends
    extra.llama._planned_vram_mib = {1: 9000}
    other = _slot("org/D-GGUF", last_used = 5.0)
    other.llama._planned_vram_mib = {0: 6000}
    model_slots.slots.append(other)
    assert model_slots.eviction_victims(None, 5000, (0,)) == [other]
    assert model_slots.eviction_victims(None, 5000) == [extra]
    assert model_slots.eviction_victims(None, 20000) == []


def test_a_token_count_waits_only_on_its_own_models_chats(backends):
    _, extra = backends

    async def count_on(model):
        await inf._route_to_extra_slot(model)
        return model_slots.routed_generation_count()

    on_primary = threading.Event()
    with active_generations.ActiveGeneration(on_primary, thread_id = "chat-on-A"):
        assert asyncio.run(count_on("org/B-GGUF")) == 0
        assert asyncio.run(count_on("org/A-GGUF")) == 1
    on_slot = threading.Event()
    with active_generations.ActiveGeneration(on_slot, thread_id = "chat-on-B"):
        extra.generations.add(on_slot)
        assert asyncio.run(count_on("org/B-GGUF")) == 1
        assert asyncio.run(count_on("org/A-GGUF")) == 0


def test_counting_for_an_idle_kept_model_ignores_the_primarys_chat(backends):
    payload = ChatCountTokensRequest(
        model = "org/B-GGUF",
        messages = [{"role": "user", "content": "hi"}],
    )
    with active_generations.ActiveGeneration(threading.Event(), thread_id = "chat-on-A"):
        try:
            asyncio.run(inf.chat_count_tokens(payload, "s", None))
        except HTTPException as exc:
            assert exc.detail != "Cannot count tokens while a generation is in progress."
        except Exception:
            pass


def test_a_model_still_loading_alongside_keeps_the_chat_claim(backends, monkeypatch):
    primary, extra = backends
    _hold_chat_claim(monkeypatch)
    primary.unload_model()
    extra.llama.unload_model()
    extra.orchestrator.loading_models = {"org/C"}
    inf.release_chat_gpu_claim()
    assert gpu_arbiter.current_owner() == gpu_arbiter.CHAT
    extra.orchestrator.loading_models = set()
    monkeypatch.setattr(model_slots, "loading", (extra, "org/C"))
    inf.release_chat_gpu_claim()
    assert gpu_arbiter.current_owner() == gpu_arbiter.CHAT
    monkeypatch.setattr(model_slots, "loading", None)
    inf.release_chat_gpu_claim()
    assert gpu_arbiter.current_owner() != gpu_arbiter.CHAT


def test_a_llama_update_stops_every_llama_slot_and_only_those(backends, monkeypatch):
    _, extra = backends
    safetensors = model_slots.ExtraSlot(FakeLlama(), FakeOrchestrator("org/C"), "owner")
    starting = model_slots.ExtraSlot(FakeLlama(), FakeOrchestrator(), "owner")
    model_slots.slots += [safetensors, starting]
    monkeypatch.setattr(model_slots, "loading", (starting, "org/D-GGUF"))
    assert model_slots.unload_llama_slots() == 2
    assert model_slots.slots == [safetensors]
    loading = model_slots.ExtraSlot(FakeLlama(), FakeOrchestrator(), "owner")
    loading.orchestrator.loading_models = {"org/E"}
    model_slots.slots.append(loading)
    monkeypatch.setattr(model_slots, "loading", (loading, "org/E"))
    assert model_slots.unload_llama_slots() == 0


def test_deleting_a_model_another_slot_serves_or_fills_is_refused(backends, monkeypatch):
    _, extra = backends
    assert deletion._llama_cpp_blocks_delete("org/B-GGUF", None)
    assert deletion._llama_cpp_blocks_delete("org/B-GGUF", "Q8_0")
    assert not deletion._llama_cpp_blocks_delete("org/B-GGUF", "Q4_K_M")
    assert not deletion._llama_cpp_blocks_delete("org/Z-GGUF", None)
    model_slots.slots.append(model_slots.ExtraSlot(FakeLlama(), FakeOrchestrator("org/C"), "owner"))
    assert deletion._inference_backend_blocks_delete("org/C")
    assert not deletion._inference_backend_blocks_delete("org/Z")
    starting = model_slots.ExtraSlot(FakeLlama(), FakeOrchestrator(), "owner")
    model_slots.slots.append(starting)
    monkeypatch.setattr(model_slots, "loading", (starting, "org/D-GGUF"))
    assert deletion._llama_cpp_blocks_delete("org/D-GGUF", "Q4_K_M")
    monkeypatch.setattr(model_slots, "loading", None)
    assert not deletion._llama_cpp_blocks_delete("org/D-GGUF", "Q4_K_M")
    starting.orchestrator.loading_models = {"org/E"}
    assert deletion._inference_backend_blocks_delete("org/E")


def test_clearing_the_cache_waits_for_models_kept_alongside(backends, monkeypatch):
    primary, extra = backends
    monkeypatch.setattr(llama_cpp, "chat_load_active", lambda: False)
    primary.unload_model()
    assert deletion.any_model_load_blocks_cache_clear() == (
        "Unload the model before clearing the model cache"
    )
    extra.llama.unload_model()
    monkeypatch.setattr(model_slots, "loading", (extra, "org/B-GGUF"))
    assert "load" in deletion.any_model_load_blocks_cache_clear()


def test_the_model_list_marks_kept_models_loaded(backends, monkeypatch):
    import routes.models as models_routes

    _, extra = backends
    fake = FakeOrchestrator()
    fake.default_models = []
    monkeypatch.setattr(models_routes, "peek_inference_backend", lambda: fake)
    model_slots.slots.append(model_slots.ExtraSlot(FakeLlama(), FakeOrchestrator("org/C"), "owner"))
    app = FastAPI()
    app.include_router(models_routes.router, prefix = "/api/models")
    app.dependency_overrides[get_current_subject] = lambda: "test-subject"
    with TestClient(app) as client:
        ids = {m["id"] for m in client.get("/api/models/list").json()["models"]}
    assert {"org/A-GGUF", "org/B-GGUF", "org/C"} <= ids


def test_a_model_that_does_not_fit_says_so_without_api_flags():
    from core.inference.llama_cpp import _gpu_short_message

    spill = _gpu_short_message(13.2, 8.4, 32768, 32768, False)
    assert "13.2 GB" in spill and "8.4 GB" in spill and "Keep multiple models loaded" in spill
    capped = _gpu_short_message(13.2, 8.4, 32768, 8192, True)
    assert "8192 context" in capped
    assert "force_alongside" not in spill + capped


def test_a_slot_load_during_a_llama_update_is_refused(backends, monkeypatch):
    primary, extra = backends
    _gated_load_fakes(monkeypatch, short_fits = [])
    seen = []
    load = inf._run_tracked_load_model_impl

    async def noting_update(request, *args, **kwargs):
        seen.append(inf.get_llama_cpp_backend()._llama_update_in_progress)
        return await load(request, *args, **kwargs)

    monkeypatch.setattr(inf, "_run_tracked_load_model_impl", noting_update)
    reload = LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q8_0", force_reload = True)
    primary._llama_update_in_progress = True
    for request in (LoadRequest(model_path = "org/C-GGUF", alongside = True), reload):
        asyncio.run(inf.load_model_gated(request, None, "s"))
    primary._llama_update_in_progress = False
    asyncio.run(inf.load_model_gated(reload, None, "s"))
    assert seen == [True, True, False]


def test_a_resident_npu_model_keeps_to_the_primarys_seat(backends, monkeypatch):
    primary, extra = backends
    primary.unload_model()
    npu_model = SimpleNamespace(model_path = "lemonade:qwen3-0.6b-FLM", id = "qwen3-0.6b-FLM")
    npu = SimpleNamespace(is_loaded = True, loaded_model = npu_model, resident = lambda: None)
    monkeypatch.setattr(npu_backend, "peek_npu_backend", lambda: npu)

    async def route(model):
        slot = await inf._route_to_extra_slot(model)
        return slot, inf._resident_npu_model(), inf._loaded_slot_ident()

    assert asyncio.run(route("org/B-GGUF")) == (extra, None, "org/B-GGUF")
    assert asyncio.run(route("qwen3-0.6b-FLM")) == (None, npu_model, npu_model.model_path)


def test_the_loaded_models_list_names_the_npu_model_once(backends, monkeypatch):
    primary, _ = backends
    primary.unload_model()
    model_slots.slots.append(
        model_slots.ExtraSlot(FakeLlama("org/D-GGUF"), FakeOrchestrator(), "owner")
    )
    npu_model = SimpleNamespace(
        model_path = "lemonade:qwen3-0.6b-FLM",
        id = "qwen3-0.6b-FLM",
        vision = False,
        max_context_length = 4096,
        reasoning = False,
        tools = False,
    )
    resident = SimpleNamespace(model = npu_model, context_length = 4096)
    npu = SimpleNamespace(is_loaded = True, loaded_model = npu_model, resident = lambda: resident)
    monkeypatch.setattr(npu_backend, "peek_npu_backend", lambda: npu)
    app = FastAPI()
    app.include_router(inf.studio_router, prefix = "/api/inference")
    app.dependency_overrides[get_current_subject] = lambda: "test-subject"
    with TestClient(app) as client:
        ids = [entry["id"] for entry in client.get("/api/inference/loaded-models").json()["data"]]
    assert ids.count("lemonade:qwen3-0.6b-FLM") == 1 and "org/B-GGUF" in ids and "org/D-GGUF" in ids


def test_a_load_over_a_streaming_npu_model_asks_before_stopping_it(monkeypatch, tmp_path):
    import struct

    npu = SimpleNamespace(
        is_loaded = True,
        loaded_model = SimpleNamespace(model_path = "lemonade:qwen3-0.6b-FLM", id = "qwen3-0.6b-FLM"),
        resident = lambda: None,
        unload = lambda: None,
    )
    monkeypatch.setattr(orchestrator, "_inference_backend", FakeOrchestrator())
    monkeypatch.setattr(npu_backend, "peek_npu_backend", lambda: npu)
    monkeypatch.setattr(inf, "_raise_if_sidecar_swap_in_progress", lambda: None)
    torn_down = []

    async def _teardown():
        torn_down.append(True)
        raise RuntimeError("NPU unloaded while its chat is still streaming")

    monkeypatch.setattr(inf, "_unload_npu_before_local_load", _teardown)
    asked = []

    def _gate(*, cancel):
        asked.append(cancel)
        if active_generations.count() and not cancel:
            raise LookupError("refused")
        return 0

    def _s(x):
        return struct.pack("<Q", len(x)) + x.encode()

    gguf = tmp_path / "tiny.gguf"
    gguf.write_bytes(
        b"GGUF"
        + struct.pack("<IQQ", 3, 0, 1)
        + _s("general.architecture")
        + struct.pack("<I", 8)
        + _s("llama")
    )

    async def _load():
        with active_generations.ActiveGeneration(
            threading.Event(), thread_id = "t1", run_id = "r1", model = "npu", kind = "chat"
        ):
            with pytest.raises(Exception) as exc:
                await inf._load_model_impl(
                    LoadRequest(model_path = str(gguf)), None, "s", on_reload_confirmed = _gate
                )
        return exc

    exc = asyncio.run(_load())
    assert asked == [False] and not torn_down, exc.value


def test_active_generations_for_a_model_lists_only_its_chats(backends):
    _, extra = backends
    app = FastAPI()
    app.include_router(inf.studio_router, prefix = "/api/inference")
    app.dependency_overrides[get_current_subject] = lambda: "test-subject"
    on_a, on_b = threading.Event(), threading.Event()
    extra.generations.add(on_b)
    with (
        active_generations.ActiveGeneration(on_a, thread_id = "chat-on-A"),
        active_generations.ActiveGeneration(on_b, thread_id = "chat-on-B"),
        TestClient(app) as client,
    ):
        get = lambda q: client.get("/api/inference/active-generations" + q).json()["thread_ids"]
        assert get("?model=org/B-GGUF") == ["chat-on-B"]
        assert get("?model=org/A-GGUF") == ["chat-on-A"]
        assert sorted(get("")) == ["chat-on-A", "chat-on-B"]
        assert get("?model=org/gone-GGUF") == []
        extra.llama.effective_parallel_slots = 4
        slots = client.get("/api/inference/active-generations?model=org/B-GGUF").json()
        assert slots["parallel_slots"] == 4


def test_active_generations_for_a_cached_primary_model_lists_the_primary_chats(
    backends, monkeypatch
):
    _, extra = backends
    standard = FakeOrchestrator("org/Active")
    standard.models["org/Cached"] = {}
    monkeypatch.setattr(orchestrator, "_inference_backend", standard)
    monkeypatch.setattr(inf, "_llama_cpp_backend", FakeLlama())
    app = FastAPI()
    app.include_router(inf.studio_router, prefix = "/api/inference")
    app.dependency_overrides[get_current_subject] = lambda: "test-subject"
    on_primary, on_slot = threading.Event(), threading.Event()
    extra.generations.add(on_slot)
    with (
        active_generations.ActiveGeneration(on_primary, thread_id = "chat-on-primary"),
        active_generations.ActiveGeneration(on_slot, thread_id = "chat-on-B"),
        TestClient(app) as client,
    ):
        get = lambda q: client.get("/api/inference/active-generations" + q).json()["thread_ids"]
        assert get("?model=org/Cached") == ["chat-on-primary"]
        assert get("?model=org/gone") == []


def test_an_integrated_gpu_keeps_its_free_memory_next_to_a_loaded_model():
    from core.inference.llama_cpp import _net_of_held_vram

    held = {0: 900}
    assert _net_of_held_vram([(0, 20_000, 24_000)], held) == [(0, 20_000, 24_000)]
    assert _net_of_held_vram([(0, 23_500, 24_000)], held) == [(0, 23_100, 24_000)]
    assert _net_of_held_vram([(0, 60_000, 0)], held) == [(0, 60_000, 0)]
    assert _net_of_held_vram([(0, 60_000, 0)], {}) == [(0, 60_000, 0)]


def test_a_model_that_fits_one_card_takes_one_no_other_model_runs_on():
    from core.inference.llama_cpp import LlamaCppBackend

    gpus = [(0, 170_000), (1, 150_000), (2, 20_000)]
    total = {0: 180_000, 1: 180_000, 2: 180_000}
    pick = lambda **kw: LlamaCppBackend._select_gpus(
        10 * 1024**3, gpus, usable_fraction = 0.9, total_by_idx = total, **kw
    )
    assert pick() == ([0], False)
    assert pick(shared = frozenset({0})) == ([1], False)
    assert pick(shared = frozenset({0, 1})) == ([0], False)
    assert LlamaCppBackend._select_gpus_split_aware(
        10 * 1024**3, gpus, usable_fraction = 0.9, total_by_idx = total, shared = frozenset({0})
    ) == ([1], False)


def test_a_public_preview_never_reaches_a_model_kept_alongside(backends):
    primary, extra = backends
    preview = SimpleNamespace(scope = {}, headers = {}, state = SimpleNamespace())
    inf.disable_openai_auto_switch_for_request(preview.scope)

    async def chat(model):
        await inf._maybe_auto_switch_model(model, preview, "s")
        return inf.get_llama_cpp_backend()

    assert asyncio.run(chat("org/B-GGUF")) is primary


def test_a_transformers_install_stops_the_workers_of_models_kept_alongside(backends, monkeypatch):
    import core.export as export
    import core.training as training
    import utils.transformers_latest as transformers_latest
    from models.inference import InstallLatestTransformersRequest

    kept = model_slots.ExtraSlot(FakeLlama(), FakeOrchestrator("org/C"), "owner")
    kept.orchestrator.is_worker_alive = lambda: kept.orchestrator.active_model_name is not None
    kept.orchestrator._cleanup = lambda: setattr(kept.orchestrator, "active_model_name", None)
    model_slots.slots.append(kept)
    idle = SimpleNamespace(
        is_training_active = lambda: False,
        is_export_active = lambda: False,
        current_checkpoint = None,
        cleanup_memory = lambda: None,
        is_worker_alive = lambda: False,
    )
    monkeypatch.setattr(training, "get_training_backend", lambda: idle)
    monkeypatch.setattr(export, "get_export_backend", lambda: idle)
    monkeypatch.setattr(keepwarm, "other_inference_request_count", lambda **kwargs: 0)
    alive_at_swap = []

    def install(version, before_swap, *args):
        before_swap()
        alive_at_swap.append(kept.orchestrator.is_worker_alive())
        return {"success": False, "message": "stop here"}

    monkeypatch.setattr(transformers_latest, "install_latest_transformers", install)
    with pytest.raises(HTTPException):
        asyncio.run(
            inf.install_latest_transformers_route(
                InstallLatestTransformersRequest(version = "9.9.9"), "s"
            )
        )
    assert alive_at_swap == [False] and kept not in model_slots.slots


def test_a_zero_vram_primary_keeps_the_chat_claim_a_kept_model_holds(backends, monkeypatch):
    _, extra = backends
    _hold_chat_claim(monkeypatch)
    source = inspect.getsource(inf._load_model_impl)
    assert "release, CHAT)" not in source and "release(CHAT)" not in source
    assert source.count("_release_chat_for_zero_vram_primary") == 3
    inf._release_chat_for_zero_vram_primary()
    assert gpu_arbiter.current_owner() == gpu_arbiter.CHAT
    model_slots.drop(extra)
    inf._release_chat_for_zero_vram_primary()
    assert gpu_arbiter.current_owner() is None


@pytest.mark.parametrize("primary_on_gpu", [False, True])
def test_ejecting_the_last_kept_model_beside_a_cpu_primary_drops_the_claim(
    backends, monkeypatch, primary_on_gpu
):
    primary, extra = backends
    _hold_chat_claim(monkeypatch)
    primary.holds_no_vram = not primary_on_gpu
    response = asyncio.run(inf._unload_model_impl(UnloadRequest(model_path = "org/B-GGUF"), "s"))
    assert response.status == "unloaded" and primary.is_loaded
    expected = gpu_arbiter.CHAT if primary_on_gpu else None
    assert gpu_arbiter.current_owner() == expected


def test_turning_the_setting_off_drops_the_claim_its_kept_models_held(backends, monkeypatch):
    from fastapi import BackgroundTasks
    import routes.settings as settings_routes

    primary, extra = backends
    _hold_chat_claim(monkeypatch)
    primary.unload_model()
    monkeypatch.setattr(settings_routes, "set_multi_model_mode", lambda mode: True)
    tasks = BackgroundTasks()
    settings_routes.update_multi_model(settings_routes.MultiModelPayload(enabled = False), tasks, "s")
    assert gpu_arbiter.current_owner() == gpu_arbiter.CHAT
    asyncio.run(tasks())
    assert model_slots.slots == [] and gpu_arbiter.current_owner() is None


def test_a_trained_model_still_filling_a_slot_cannot_be_deleted(backends, monkeypatch, tmp_path):
    import routes.models as models_routes

    run = tmp_path / "run1"
    run.mkdir()
    monkeypatch.setattr(models_routes, "outputs_root", lambda: tmp_path)
    starting = model_slots.ExtraSlot(FakeLlama(), FakeOrchestrator(), "owner")
    model_slots.slots.append(starting)
    monkeypatch.setattr(model_slots, "loading", (starting, str(run)))
    with pytest.raises(HTTPException) as excinfo:
        asyncio.run(
            models_routes.delete_finetuned_model(
                model_path = str(run), source = "training", current_subject = "s"
            )
        )
    assert excinfo.value.status_code == 409 and run.exists()


def test_reloading_a_kept_model_leaves_the_primarys_npu_model_loaded(
    backends, monkeypatch, tmp_path
):
    import struct

    primary, extra = backends
    primary.unload_model()
    unloaded = []
    npu = SimpleNamespace(
        is_loaded = True,
        loaded_model = SimpleNamespace(model_path = "lemonade:qwen3-0.6b-FLM", id = "qwen3-0.6b-FLM"),
        resident = lambda: None,
        unload = lambda: unloaded.append(True),
    )
    monkeypatch.setattr(npu_backend, "peek_npu_backend", lambda: npu)

    def _s(x):
        return struct.pack("<Q", len(x)) + x.encode()

    gguf = tmp_path / "tiny.gguf"
    gguf.write_bytes(
        b"GGUF"
        + struct.pack("<IQQ", 3, 0, 1)
        + _s("general.architecture")
        + struct.pack("<I", 8)
        + _s("llama")
    )

    from core.inference.llama_cpp import LlamaCppBackend

    extra.llama = LlamaCppBackend(manages_processes = False)
    monkeypatch.setattr(inf, "_llama_cpp_backend", LlamaCppBackend(manages_processes = False))
    monkeypatch.setattr(inf, "LlamaCppBackend", LlamaCppBackend)

    async def reload_kept():
        inf.routed_slot.set(extra)
        with pytest.raises(Exception):
            await inf._load_model_impl(
                LoadRequest(model_path = str(gguf), force_reload = True),
                None,
                "s",
                on_reload_confirmed = lambda *, cancel: 0,
            )

    asyncio.run(reload_kept())
    assert unloaded == []


def test_an_api_key_status_does_not_leak_a_local_path_through_serving():
    from hub.utils.host_paths import redact_host_paths

    local = "/home/alice/models/my-finetune"
    status = InferenceStatusResponse(
        active_model = local, loaded = [local], serving = [local, "org/B-GGUF"]
    )
    redacted = redact_host_paths(status, via_api_key = True)
    assert local not in redacted["active_model"] and local not in redacted["loaded"]
    assert local not in redacted["serving"] and "org/B-GGUF" in redacted["serving"]
    assert redact_host_paths(status, via_api_key = False).serving == [local, "org/B-GGUF"]


def test_a_managed_engine_load_never_lands_in_a_kept_slot(backends):
    _, extra = backends
    kept = LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q8_0", engine = "vllm")
    assert _selected(kept) is None
    assert _selected(kept.model_copy(update = {"alongside": True})) is None


def test_a_managed_engine_load_refuses_while_a_kept_model_generates(backends):
    _, extra = backends
    generating = threading.Event()
    extra.generations.add(generating)
    request = LoadRequest(model_path = "org/C", engine = "vllm")
    with pytest.raises(HTTPException) as excinfo:
        inf._gate_kept_models(request, cancel = False)
    assert excinfo.value.status_code == 409 and not generating.is_set()
    forced = request.model_copy(update = {"force_cancel_active": True})
    inf._gate_kept_models(forced, cancel = False)
    assert not generating.is_set() and model_slots.slots == [extra]
    asyncio.run(inf._retire_kept_models(forced))
    assert generating.is_set() and model_slots.slots == [] and not extra.llama.is_active


def test_kept_models_go_only_once_nothing_can_reject_a_managed_load():
    source = inspect.getsource(inf._load_model_impl)
    gate = source.index("_gate_kept_models(request, cancel = False)")
    primary_cancel = source.index("on_reload_confirmed(cancel = True)", gate)
    retire = source.index("await _retire_kept_models(request)", primary_cancel)
    assert source.index("_reject_unsupported_managed_kind(request, config)") < gate
    assert source.index("account_access.require_idle_other_accounts()") < gate < retire


def test_a_kept_model_that_would_not_unload_is_retried_and_still_priced(backends, monkeypatch):
    _, extra = backends
    registered = []
    monkeypatch.setattr(llama_cpp, "unregister_serving_backend", registered.append)
    real = extra.llama.unload_model
    monkeypatch.setattr(
        extra.llama, "unload_model", lambda: (_ for _ in ()).throw(RuntimeError("stuck"))
    )
    with pytest.raises(RuntimeError):
        model_slots.unload_extra_models(strict = True)
    assert model_slots.slots == [] and model_slots.stuck == [extra] and registered == []
    assert model_slots.holds_vram()
    monkeypatch.setattr(extra.llama, "unload_model", real)
    assert model_slots.unload_extra_models(strict = True) == 0
    assert model_slots.stuck == [] and registered == [extra.llama]


def test_a_gpu_handoff_stops_on_a_kept_model_that_would_not_unload(backends, monkeypatch):
    _, extra = backends

    def stuck():
        raise RuntimeError("llama-server ignored SIGKILL")

    monkeypatch.setattr(extra.llama, "unload_model", stuck)
    assert model_slots.unload_extra_models() == 1
    model_slots.slots.append(extra)
    with pytest.raises(RuntimeError, match = "kept alongside"):
        model_slots.unload_extra_models(strict = True)


def test_a_non_gguf_load_out_of_memory_makes_room_then_replaces(backends, monkeypatch):
    primary, extra = backends
    extra.request = LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q8_0")
    monkeypatch.setattr(inf, "release_chat_gpu_claim", lambda: None)
    failures = [HTTPException(500, "Failed to load model: CUDA out of memory")] * 2
    seen = []

    async def load(request, *args, **kwargs):
        seen.append((request.model_path, request.alongside))
        if request.alongside and failures:
            raise failures.pop()
        primary.model_identifier, primary.is_loaded = request.model_path, True
        return LoadResponse.model_construct(status = "loaded", model = request.model_path)

    monkeypatch.setattr(inf, "_run_tracked_load_model_impl", load)
    response = asyncio.run(
        inf.load_model_gated(LoadRequest(model_path = "org/C", alongside = True), None, "s")
    )
    assert response.evicted == ["org/B-GGUF:Q8_0", "org/A-GGUF"] and model_slots.slots == []
    assert seen == [("org/C", True), ("org/C", True), ("org/C", False)]
    other = HTTPException(500, "Failed to load model: no such repo")
    failures[:] = [other]
    with pytest.raises(HTTPException):
        asyncio.run(
            inf.load_model_gated(LoadRequest(model_path = "org/D", alongside = True), None, "s")
        )


def test_turning_the_setting_off_unloads_only_idle_kept_models(backends):
    _, extra = backends
    busy = _slot("org/D-GGUF")
    busy.generations.add(threading.Event())
    model_slots.slots.append(busy)
    assert model_slots.unload_idle() == 1
    assert model_slots.slots == [busy] and not extra.llama.is_active


def test_the_replace_fallback_refuses_while_the_active_model_generates(backends, monkeypatch):
    primary, extra = backends
    extra.request = LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q8_0")
    _gated_load_fakes(monkeypatch, short_fits = [1, 1])
    seen = []
    monkeypatch.setattr(
        inf,
        "_raise_or_cancel_active_generations",
        lambda **kw: seen.append(kw) or (_ for _ in ()).throw(HTTPException(409, "chats running")),
    )
    with pytest.raises(HTTPException) as excinfo:
        asyncio.run(
            inf.load_model_gated(LoadRequest(model_path = "org/C-GGUF", alongside = True), None, "s")
        )
    assert excinfo.value.status_code == 409 and seen[0]["cancel"] is False
    assert primary.model_identifier == "org/A-GGUF" and primary.is_loaded
    assert model_slots.slots == []


def test_an_automatic_replace_fallback_waits_for_the_active_chats(backends, monkeypatch):
    primary, extra = backends
    extra.request = LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q8_0")
    _gated_load_fakes(monkeypatch, short_fits = [1, 1])

    def refuse(**_kw):
        raise AssertionError("an automatic switch refused instead of waiting")

    waited = []

    async def wait(**kw):
        waited.append(primary.model_identifier)

    monkeypatch.setattr(inf, "_raise_or_cancel_active_generations", refuse)
    monkeypatch.setattr(inf, "_wait_for_model_switch_idle", wait)
    request = LoadRequest(model_path = "org/C-GGUF", alongside = True)
    assert asyncio.run(inf.load_model_gated(request, None, "s", background = True)) == "loaded"
    assert waited == ["org/A-GGUF"] and primary.model_identifier == "org/C-GGUF"


def test_an_automatic_replace_fallback_rechecks_the_sidecar_after_the_drain(backends, monkeypatch):
    primary, extra = backends
    extra.request = LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q8_0")
    _gated_load_fakes(monkeypatch, short_fits = [1, 1])
    installing = []

    def sidecar():
        if installing:
            raise HTTPException(status_code = 409, detail = "installing")

    async def wait(**kw):
        installing.append(True)  # a transformers install claims its window meanwhile

    monkeypatch.setattr(inf, "_raise_if_sidecar_swap_in_progress", sidecar)
    monkeypatch.setattr(inf, "_wait_for_model_switch_idle", wait)
    request = LoadRequest(model_path = "org/C-GGUF", alongside = True)
    with pytest.raises(HTTPException) as exc:
        asyncio.run(inf.load_model_gated(request, None, "s", background = True))
    assert exc.value.status_code == 409 and primary.model_identifier == "org/A-GGUF"


def test_an_automatic_replace_fallback_cancelled_during_the_drain_keeps_the_active_model(
    backends, monkeypatch
):
    primary, extra = backends
    extra.request = LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q8_0")
    _gated_load_fakes(monkeypatch, short_fits = [1, 1])
    cancel = threading.Event()

    async def wait(**kw):
        cancel.set()  # the API request is cancelled while running chats drain

    monkeypatch.setattr(inf, "_wait_for_model_switch_idle", wait)
    request = LoadRequest(model_path = "org/C-GGUF", alongside = True)
    with pytest.raises(HTTPException) as exc:
        asyncio.run(
            inf.load_model_gated(
                request, None, "s", background = True, impl_kwargs = {"load_cancel_event": cancel}
            )
        )
    assert exc.value.status_code == 409 and primary.model_identifier == "org/A-GGUF"


def test_training_stops_on_a_kept_model_that_would_not_unload(backends, monkeypatch):
    _, extra = backends

    def stuck():
        raise RuntimeError("llama-server ignored SIGKILL")

    monkeypatch.setattr(extra.llama, "unload_model", stuck)
    with pytest.raises(training_vram.ManagedEngineStillRunning):
        training_vram.free_kept_models_for_training("test")


def test_training_sees_and_retries_a_kept_model_that_failed_to_unload(backends, monkeypatch):
    primary, extra = backends
    primary.unload_model()

    def refuse():
        raise RuntimeError("llama-server ignored SIGKILL")

    monkeypatch.setattr(extra.llama, "unload_model", refuse)
    with pytest.raises(RuntimeError):
        model_slots.drop(extra)
    assert model_slots.slots == [] and model_slots.stuck == [extra]
    _hold_chat_claim(monkeypatch)
    inf._release_chat_for_zero_vram_primary()
    assert gpu_arbiter.current_owner() == gpu_arbiter.CHAT
    assert training_vram.summarize_resident_chat()["gguf"] == "org/B-GGUF"
    monkeypatch.setattr(training_vram, "summarize_resident_stt", lambda: {"any": False})
    with pytest.raises(training_vram.ManagedEngineStillRunning):
        training_vram.coordinate_models_for_training(lambda: (False, {}))
    monkeypatch.delattr(extra.llama, "unload_model")
    assert training_vram.free_kept_models_for_training("test") == ["kept:org/B-GGUF"]
    assert model_slots.stuck == [] and not extra.llama.is_active


def test_a_kept_worker_that_outlives_its_kill_stays_tracked(backends, monkeypatch):
    _, extra = backends
    extra.orchestrator.active_model_name = "org/S"
    alive = [True]
    extra.orchestrator.is_worker_alive = lambda: alive[0]
    extra.orchestrator._cleanup = lambda: setattr(extra.orchestrator, "active_model_name", None)
    with pytest.raises(RuntimeError):
        model_slots.unload_extra_models(strict = True)
    assert model_slots.slots == [] and model_slots.stuck == [extra] and model_slots.holds_vram()
    alive[0] = False
    model_slots.unload_extra_models(strict = True)
    assert model_slots.stuck == [] and not model_slots.holds_vram()


def test_a_stuck_server_still_blocks_deleting_its_files(backends, monkeypatch):
    _, extra = backends

    def refuse():
        raise RuntimeError("llama-server ignored SIGKILL")

    monkeypatch.setattr(extra.llama, "unload_model", refuse)
    with pytest.raises(RuntimeError):
        model_slots.drop(extra)
    assert model_slots.stuck == [extra]
    assert deletion._llama_cpp_blocks_delete("org/B-GGUF", None)


def test_a_partly_offloaded_model_plans_only_the_vram_its_cards_had():
    from core.inference.llama_cpp import _gpu_plan_mib

    gib = 1024**3
    assert _gpu_plan_mib(20 * gib, [0, 1], [(0, 30_000), (1, 30_000)]) == {0: 10240, 1: 10240}
    assert _gpu_plan_mib(20 * gib, [0], [(0, 6_000)]) == {0: 6_000}
    assert _gpu_plan_mib(4 * gib, [2], []) == {2: 4096}


def test_the_replace_fallback_frees_the_active_model_before_placing_the_new_one(
    backends, monkeypatch
):
    primary, extra = backends
    _gated_load_fakes(monkeypatch, short_fits = [1, 1])
    seen = []
    real = inf._run_tracked_load_model_impl

    async def load(request, *args, **kwargs):
        if not request.alongside:
            seen.append(primary.is_active)
        return await real(request, *args, **kwargs)

    monkeypatch.setattr(inf, "_run_tracked_load_model_impl", load)
    asyncio.run(
        inf.load_model_gated(LoadRequest(model_path = "org/C-GGUF", alongside = True), None, "s")
    )
    assert seen == [False] and primary.model_identifier == "org/C-GGUF"


def test_a_cpu_only_kept_model_is_never_evicted_for_gpu_room(backends):
    _, extra = backends
    extra.llama._gpu_offload_active = False
    on_gpu = _slot("org/D-GGUF", last_used = 5.0)
    model_slots.slots.append(on_gpu)
    assert model_slots.eviction_victims(None, 5000, (0,), discrete = True) == [on_gpu]
    model_slots.slots.remove(on_gpu)
    assert model_slots.eviction_victims(None, 5000, (0,), discrete = True) == []
    # A pool shared with host RAM (APU, Metal): the CPU-only server holds it too.
    assert model_slots.eviction_victims(None, 5000, (0,)) == [extra]
    # A GPU server that fell back to the CPU keeps the plan it is still charged for.
    extra.llama._planned_vram_mib = {0: 6000}
    assert model_slots.eviction_victims(None, 5000, (0,), discrete = True) == [extra]


def _short_on_gpu_one(monkeypatch, discrete = True):
    monkeypatch.setattr(inf, "release_chat_gpu_claim", lambda: None)
    calls = []

    async def load(request, *args, **kwargs):
        calls.append((request.alongside, request.force_alongside))
        if request.alongside and not request.force_alongside:
            raise GpuMemoryShortError(
                "needs 13 GB, 8 GB free", short_mib = 5000, gpu_indices = (1,), discrete = discrete
            )
        llama = inf.get_llama_cpp_backend()
        llama.model_identifier, llama.hf_variant = request.model_path, request.gguf_variant
        llama.is_loaded = llama.is_active = True
        return "loaded"

    monkeypatch.setattr(inf, "_run_tracked_load_model_impl", load)
    return calls


def test_a_model_short_on_its_own_gpu_keeps_the_models_on_others(backends, monkeypatch):
    primary, extra = backends
    primary._requested_gpu_ids = primary._gpu_ids = [0]
    extra.request = LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q8_0")
    extra.llama._gpu_offload_active = False
    calls = _short_on_gpu_one(monkeypatch)
    request = LoadRequest(model_path = "org/C-GGUF", alongside = True, gpu_ids = [1])
    assert asyncio.run(inf.load_model_gated(request, None, "s")) == "loaded"
    assert calls == [(True, False), (True, True)]
    assert primary.model_identifier == "org/A-GGUF" and primary.is_active
    # A pin to the short GPU with no GPU layers (manual gpu_layers=0) holds nothing there either.
    primary._requested_gpu_ids = primary._gpu_ids = [1]
    primary._gpu_offload_active = False
    short = GpuMemoryShortError("short", gpu_indices = (1,), discrete = True)
    assert model_slots.in_slot(None, lambda: inf._active_model_holds_none_of(short))
    assert [s.llama.model_identifier for s in model_slots.slots] == ["org/B-GGUF", "org/C-GGUF"]


def test_a_model_short_where_the_active_model_runs_still_replaces_it(backends, monkeypatch):
    primary, extra = backends
    primary._requested_gpu_ids = primary._gpu_ids = [1]
    extra.request = LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q8_0")
    extra.llama._gpu_offload_active = False
    calls = _short_on_gpu_one(monkeypatch)
    request = LoadRequest(model_path = "org/C-GGUF", alongside = True, gpu_ids = [1])
    assert asyncio.run(inf.load_model_gated(request, None, "s")) == "loaded"
    assert calls == [(True, False), (False, False)]
    assert primary.model_identifier == "org/C-GGUF"
    assert [s.llama.model_identifier for s in model_slots.slots] == ["org/B-GGUF"]


def test_metal_shortfall_evicts_then_rechecks_shared_memory(backends, monkeypatch, tmp_path):
    from .test_metal_explicit_context_guard import _launch

    primary, extra = backends
    extra.request = LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q8_0")
    attempts, launches = [], []
    monkeypatch.setattr(inf, "release_chat_gpu_claim", lambda: None)

    async def load(request, *args, **kwargs):
        budget_gib = 9 if extra.llama.is_active else 32
        attempts.append(budget_gib)
        _launch(
            tmp_path,
            monkeypatch,
            n_ctx = 0,
            real_fit = True,
            budget_bytes = budget_gib * 1024**3,
            weights_bytes = 16 * 1024**3,
            refuse_partial_gpu_fit = not request.force_alongside,
            launches = launches,
        )
        llama = inf.get_llama_cpp_backend()
        llama.model_identifier = request.model_path
        llama.is_loaded = llama.is_active = True
        return LoadResponse.model_construct(status = "loaded", model = request.model_path)

    monkeypatch.setattr(inf, "_run_tracked_load_model_impl", load)
    response = asyncio.run(
        inf.load_model_gated(LoadRequest(model_path = "org/C-GGUF", alongside = True), None, "s")
    )
    assert attempts == [9, 32]
    assert len(launches) == 1
    assert response.evicted == ["org/B-GGUF:Q8_0"]
    assert primary.is_loaded


@pytest.mark.parametrize("case", ["shared_pool", "unpinned", "companion_widened"])
def test_a_short_load_still_replaces_an_active_model_it_may_share_with(backends, monkeypatch, case):
    primary, extra = backends
    primary._requested_gpu_ids = primary._gpu_ids = [0]
    if case == "unpinned":
        # Auto placement: a user --device or an inherited LLAMA_ARG_DEVICE may outrank the plan.
        primary._requested_gpu_ids, primary._gpu_ids = None, None
        primary._planned_vram_mib = {0: 9000}
    elif case == "companion_widened":
        primary._gpu_ids = [0, 1]
    extra.request = LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q8_0")
    extra.llama._gpu_offload_active = False
    calls = _short_on_gpu_one(monkeypatch, discrete = case != "shared_pool")
    request = LoadRequest(model_path = "org/C-GGUF", alongside = True, gpu_ids = [1])
    assert asyncio.run(inf.load_model_gated(request, None, "s")) == "loaded"
    assert calls[-1] == (False, False)
    assert primary.model_identifier == "org/C-GGUF"


def _away_request(check):
    from starlette.requests import Request

    request = Request({"type": "http", "method": "POST", "path": "/v1/chat/completions"})
    inf.require_for_model_switch(request.scope, check)
    return request


def test_an_unattended_replace_fallback_rechecks_the_user_is_away_after_the_drain(
    backends, monkeypatch
):
    primary, extra = backends
    extra.request = LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q8_0")
    _gated_load_fakes(monkeypatch, short_fits = [1, 1])
    back = []

    def still_away():
        if back:
            raise HTTPException(status_code = 409, detail = "model_busy")

    async def wait(**kw):
        back.append(True)  # the user's chat on the active model ends during the drain

    monkeypatch.setattr(inf, "_wait_for_model_switch_idle", wait)
    request = LoadRequest(model_path = "org/C-GGUF", alongside = True)
    with pytest.raises(HTTPException) as exc:
        asyncio.run(inf.load_model_gated(request, _away_request(still_away), "s", background = True))
    assert exc.value.status_code == 409 and exc.value.detail == "model_busy"
    assert primary.model_identifier == "org/A-GGUF" and primary.is_loaded


def test_an_unattended_load_that_fits_beside_the_primary_is_not_rechecked(backends, monkeypatch):
    primary, _ = backends
    _gated_load_fakes(monkeypatch, short_fits = [])

    def refuse():
        raise AssertionError("an alongside load that touches no loaded model was rechecked")

    request = LoadRequest(model_path = "org/C-GGUF", alongside = True)
    assert (
        asyncio.run(inf.load_model_gated(request, _away_request(refuse), "s", background = True))
        == "loaded"
    )
    assert primary.model_identifier == "org/A-GGUF"
    assert model_slots.slots[-1].request.model_path == "org/C-GGUF"


def test_long_narration_is_budgeted_against_the_kept_speech_model_not_the_chat_primary(monkeypatch):
    from models.inference import ChatCompletionRequest

    kept = object()

    class Reached(Exception):
        pass

    async def route(workflow):
        inf.routed_slot.set(kept)

    async def no_switch(*a, **kw):
        return None

    def no_further(*a, **kw):
        raise Reached

    # The chat primary has a 64-token context; the kept speech model takes the narration.
    monkeypatch.setattr(inf, "_serves_audio_workflow", lambda workflow: False)
    monkeypatch.setattr(inf, "_maybe_auto_switch_model", no_switch)
    monkeypatch.setattr(inf, "_route_to_kept_audio_model", route)
    monkeypatch.setattr(
        inf, "_monitor_context_length", lambda: 8192 if inf.routed_slot.get() is kept else 64
    )
    monkeypatch.setattr(inf, "_audio_run_event", no_further)
    payload = ChatCompletionRequest(messages = [{"role": "user", "content": "x"}])

    async def run():
        token = inf.routed_slot.set(None)
        try:
            await inf._generate_tts_wav("word " * 400, payload, request = None, current_subject = "t")
        finally:
            inf.routed_slot.reset(token)

    with pytest.raises(Reached):
        asyncio.run(run())