Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

82.9 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""llama-server GGUF embedder tests, every boundary mocked."""

import os
import struct
import subprocess
import sys
import textwrap
import threading
import time
import types
from pathlib import Path

import numpy as np
import pytest

from core.rag import config, embeddings
from core.rag import embed_llama_server as mod
from core.rag.embed_llama_server import LlamaServerBackend


def _shared_setup_1(monkeypatch):
    import utils.embedding_model_settings as ems

    repo = "org/pending-GGUF"
    cleared = []
    monkeypatch.setattr(config, "effective_embedding_model", lambda: "org/pending")
    monkeypatch.setattr(config, "effective_gguf_repo", lambda: repo)
    monkeypatch.setattr(config, "EMBED_GGUF_VARIANT", "F16")
    monkeypatch.setattr(ems, "get_stored_download_pending", lambda model: True)
    return cleared, ems, repo


def _shared_setup_2(monkeypatch, tmp_path):
    import contextlib
    import huggingface_hub
    from core.inference import llama_cpp

    repo = config.effective_gguf_repo()
    _seed_cache(tmp_path / "hub", repo, ["bge-Q8_0.gguf"])
    _use_cache_root(monkeypatch, tmp_path / "hub")
    monkeypatch.setattr(config, "EMBED_GGUF_VARIANT", "F16")
    monkeypatch.setattr(llama_cpp, "_hf_offline_if_unreachable", contextlib.nullcontext)
    return huggingface_hub


def _shared_setup_3(monkeypatch):
    import utils.embedding_model_settings as ems

    repo = "org/pending-GGUF"
    monkeypatch.setattr(config, "effective_embedding_model", lambda: "org/pending")
    monkeypatch.setattr(config, "effective_gguf_repo", lambda: repo)
    monkeypatch.setattr(config, "EMBED_GGUF_VARIANT", "F16")
    return ems, repo


def _shared_setup_4(monkeypatch, tmp_path):
    _use_cache_root(monkeypatch, tmp_path / "hub")
    backend = LlamaServerBackend()

    assert backend._resolve_model_path().endswith("pending-Q8_0.gguf")


@pytest.fixture(autouse = True)
def _reset_backend_singleton():
    embeddings._reset_backend()
    yield
    embeddings._reset_backend()


class _FakeProc:
    """subprocess.Popen stand-in with controllable liveness."""

    def __init__(
        self,
        alive = True,
        returncode = 0,
    ):
        self._alive = alive
        self.returncode = returncode
        self.pid = 424242
        self.stdout = iter(())

    def poll(self):
        return None if self._alive else self.returncode

    def terminate(self):
        self._alive = False

    def kill(self):
        self._alive = False

    def wait(self, timeout = None):
        return self.returncode


def _mock_auto(monkeypatch, *, gpus, binary):
    from core.inference.llama_cpp import LlamaCppBackend

    monkeypatch.setattr(config, "EMBED_BACKEND", "auto")
    monkeypatch.setattr(LlamaCppBackend, "_get_gpu_free_memory", staticmethod(lambda **_kw: gpus))
    monkeypatch.setattr(LlamaCppBackend, "_find_llama_server_binary", staticmethod(lambda: binary))


def _stub_st_load(monkeypatch):
    # Fake the ST probe: sentence-transformers is absent in the torch-free backend CI.
    monkeypatch.setattr(embeddings, "_get", lambda *a, **k: object())


def test_auto_uses_st_with_cuda(monkeypatch):
    _stub_st_load(monkeypatch)
    _mock_auto(monkeypatch, gpus = [(0, 40000)], binary = "/bin/llama-server")
    assert type(embeddings._get_backend()).__name__ == "_SentenceTransformersBackend"


def test_auto_uses_llama_without_cuda(monkeypatch):
    _mock_auto(monkeypatch, gpus = [], binary = "/bin/llama-server")
    assert isinstance(embeddings._get_backend(), LlamaServerBackend)


def test_auto_falls_back_to_st_without_binary(monkeypatch):
    _stub_st_load(monkeypatch)
    _mock_auto(monkeypatch, gpus = [], binary = None)
    assert type(embeddings._get_backend()).__name__ == "_SentenceTransformersBackend"


def test_llama_backend_selected_by_config(monkeypatch):
    monkeypatch.setattr(config, "EMBED_BACKEND", "llama-server")
    assert isinstance(embeddings._get_backend(), LlamaServerBackend)


def test_unknown_backend_raises(monkeypatch):
    monkeypatch.setattr(config, "EMBED_BACKEND", "bogus")
    with pytest.raises(ValueError, match = "Unknown RAG_EMBED_BACKEND"):
        embeddings._get_backend()


def test_shutdown_waits_for_an_active_embedding_operation(monkeypatch):
    backend = LlamaServerBackend()
    entered = threading.Event()
    finish = threading.Event()
    shutdown_done = threading.Event()

    def _post(path, payload, **_kwargs):
        entered.set()
        assert finish.wait(timeout = 2)
        return {"data": [{"index": 0, "embedding": [1.0, 0.0]}]}

    monkeypatch.setattr(backend, "_post", _post)
    worker = threading.Thread(target = lambda: backend.encode(["x"]))
    worker.start()
    assert entered.wait(timeout = 2)

    closer = threading.Thread(target = lambda: (backend._shutdown(), shutdown_done.set()))
    closer.start()
    assert shutdown_done.wait(timeout = 0.05) is False
    finish.set()
    worker.join(timeout = 2)
    closer.join(timeout = 2)
    assert shutdown_done.is_set()
    with pytest.raises(RuntimeError, match = "unloaded"):
        backend.encode(["x"])


def test_explicit_backend_overrides_auto(monkeypatch):
    _stub_st_load(monkeypatch)
    monkeypatch.setattr(config, "EMBED_BACKEND", "sentence-transformers")
    assert type(embeddings._get_backend()).__name__ == "_SentenceTransformersBackend"
    monkeypatch.setattr(config, "EMBED_BACKEND", "llama-server")
    assert isinstance(embeddings._get_backend(), LlamaServerBackend)


def test_llama_backend_imports_no_torch():
    backend_dir = Path(__file__).resolve().parents[1]
    code = textwrap.dedent(
        """
        import sys
        from core.rag import embeddings
        b = embeddings._get_backend()
        assert type(b).__name__ == "LlamaServerBackend", type(b).__name__
        assert "torch" not in sys.modules, "torch was imported"
        assert "sentence_transformers" not in sys.modules, "ST was imported"
        print("OK")
        """
    )
    env = {
        **__import__("os").environ,
        "RAG_EMBED_BACKEND": "llama-server",
        "PYTHONPATH": str(backend_dir),
    }
    proc = subprocess.run([sys.executable, "-c", code], capture_output = True, text = True, env = env)
    assert proc.returncode == 0, proc.stderr
    assert "OK" in proc.stdout


def test_build_cmd_cpu_flags():
    b = LlamaServerBackend()
    cmd = b._build_cmd("/bin/llama-server", "/m/bge.gguf", 9999, use_gpu = False)
    assert "--embedding" in cmd
    assert cmd[cmd.index("--pooling") + 1] == "cls"
    assert cmd[cmd.index("--fit") + 1] == "off"
    assert cmd[cmd.index("-ngl") + 1] == "0"
    assert cmd[cmd.index("--port") + 1] == "9999"


def test_build_cmd_gpu_offloads():
    b = LlamaServerBackend()
    cmd = b._build_cmd("/bin/llama-server", "/m/bge.gguf", 1, use_gpu = True)
    assert cmd[cmd.index("-ngl") + 1] == "-1"


def test_build_env_cpu_hides_gpus():
    b = LlamaServerBackend()
    env = b._build_env("/bin/llama-server", use_gpu = False)
    assert env["CUDA_VISIBLE_DEVICES"] == ""
    assert env["LLAMA_SET_ROWS"] == "1"


def test_build_env_gpu_inherits_devices(monkeypatch):
    monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0,1")
    b = LlamaServerBackend()
    env = b._build_env("/bin/llama-server", use_gpu = True)
    assert env.get("CUDA_VISIBLE_DEVICES") == "0,1"


def test_build_env_gpu_on_macos_uses_the_dyld_search_path(monkeypatch, tmp_path):
    # dyld ignores LD_LIBRARY_PATH and Apple Silicon takes the use_gpu branch.
    import os
    import sys as _sys

    monkeypatch.setattr(_sys, "platform", "darwin")
    monkeypatch.setenv("DYLD_LIBRARY_PATH", "/opt/inherited")
    # The child env copies os.environ, so pin an inherited value rather than assert absence.
    monkeypatch.setenv("LD_LIBRARY_PATH", "/opt/sentinel")
    # Host-native dir: on a Windows runner '/opt/llama/bin' resolves drive-anchored.
    bin_dir = tmp_path / "llama" / "bin"
    bin_dir.mkdir(parents = True)
    (bin_dir / "llama-server").write_bytes(b"\xcf\xfa\xed\xfe")
    b = LlamaServerBackend()
    env = b._build_env(str(bin_dir / "llama-server"), use_gpu = True)
    entries = env["DYLD_LIBRARY_PATH"].split(os.pathsep)
    assert entries == [str(bin_dir), "/opt/inherited"]
    assert env["LD_LIBRARY_PATH"] == "/opt/sentinel"


def test_build_env_cpu_on_macos_still_gets_the_dyld_search_path(monkeypatch, tmp_path):
    # The CPU paths load the same sibling dylibs; the loader path cannot be gated on use_gpu.
    import os
    import sys as _sys

    monkeypatch.setattr(_sys, "platform", "darwin")
    monkeypatch.delenv("DYLD_LIBRARY_PATH", raising = False)
    bin_dir = tmp_path / "llama" / "bin"
    bin_dir.mkdir(parents = True)
    (bin_dir / "llama-server").write_bytes(b"\xcf\xfa\xed\xfe")
    b = LlamaServerBackend()
    env = b._build_env(str(bin_dir / "llama-server"), use_gpu = False)
    assert env["DYLD_LIBRARY_PATH"].split(os.pathsep)[0] == str(bin_dir)
    assert env["CUDA_VISIBLE_DEVICES"] == ""


def test_build_env_resolves_a_wrapper_entrypoint(monkeypatch, tmp_path):
    # The managed install wraps the server; dylibs sit next to the target, not the wrapper.
    import os
    import sys as _sys

    real_dir = tmp_path / "llama.cpp" / "build" / "bin"
    real_dir.mkdir(parents = True)
    (real_dir / "llama-server").write_text("")
    wrapper = tmp_path / "llama.cpp" / "llama-server"
    wrapper.write_text('#!/bin/sh\nexec "$(dirname "$0")/build/bin/llama-server" "$@"\n')
    monkeypatch.setattr(_sys, "platform", "darwin")
    monkeypatch.delenv("DYLD_LIBRARY_PATH", raising = False)
    b = LlamaServerBackend()
    env = b._build_env(str(wrapper), use_gpu = True)
    assert env["DYLD_LIBRARY_PATH"].split(os.pathsep)[0] == str(real_dir)


def test_use_gpu_explicit_modes(monkeypatch):
    b = LlamaServerBackend()
    monkeypatch.setattr(config, "EMBED_DEVICE", "gpu")
    assert b._use_gpu() is True
    monkeypatch.setattr(config, "EMBED_DEVICE", "cpu")
    assert b._use_gpu() is False


def test_use_gpu_auto_follows_probe(monkeypatch):
    b = LlamaServerBackend()
    monkeypatch.setattr(config, "EMBED_DEVICE", "auto")
    monkeypatch.setattr(LlamaServerBackend, "_gpu_available", staticmethod(lambda: True))
    assert b._use_gpu() is True
    monkeypatch.setattr(LlamaServerBackend, "_gpu_available", staticmethod(lambda: False))
    assert b._use_gpu() is False


def test_use_gpu_sticky_cpu_fallback(monkeypatch):
    b = LlamaServerBackend()
    monkeypatch.setattr(config, "EMBED_DEVICE", "auto")
    monkeypatch.setattr(LlamaServerBackend, "_gpu_available", staticmethod(lambda: True))
    b._force_cpu = True
    assert b._use_gpu() is False


def test_gpu_available_reuses_studio_probe(monkeypatch):
    import utils.hardware as uh
    from core.inference.llama_cpp import LlamaCppBackend

    monkeypatch.setattr(uh, "is_apple_silicon", lambda: False)
    monkeypatch.setattr(
        LlamaCppBackend, "_get_gpu_free_memory", staticmethod(lambda **_kw: [(0, 40000)])
    )
    assert LlamaServerBackend._gpu_available() is True
    monkeypatch.setattr(
        LlamaCppBackend, "_get_gpu_free_memory", staticmethod(lambda **_kw: [(0, 100)])
    )
    assert LlamaServerBackend._gpu_available() is False
    monkeypatch.setattr(LlamaCppBackend, "_get_gpu_free_memory", staticmethod(lambda **_kw: []))
    assert LlamaServerBackend._gpu_available() is False


def test_gpu_available_apple_metal(monkeypatch):
    import utils.hardware as uh
    monkeypatch.setattr(uh, "is_apple_silicon", lambda: True)
    assert LlamaServerBackend._gpu_available() is True


_REAL_POPEN = subprocess.Popen
_FAKE_BINARY = "/bin/llama-server"


def _intercept_server_popen(
    monkeypatch,
    proc,
    captured = None,
):
    """Answer only the embed server's own launch; a process-wide Popen patch replaced other threads'
    argv."""

    def popen(cmd, *args, **kwargs):
        if isinstance(cmd, (list, tuple)) and cmd and cmd[0] == _FAKE_BINARY:
            if captured is not None:
                captured["cmd"] = list(cmd)
            return proc
        return _REAL_POPEN(cmd, *args, **kwargs)

    monkeypatch.setattr(mod.subprocess, "Popen", popen)


def _patch_spawn_deps(
    monkeypatch,
    proc,
    *,
    free_port = 54321,
):
    monkeypatch.setattr(config, "EMBED_DEVICE", "cpu")
    monkeypatch.setattr(LlamaServerBackend, "_resolve_binary", lambda self: _FAKE_BINARY)
    monkeypatch.setattr(
        LlamaServerBackend, "_resolve_model_path", lambda self, model_name = None: "/m/bge.gguf"
    )
    monkeypatch.setattr(LlamaServerBackend, "_find_free_port", staticmethod(lambda: free_port))
    _intercept_server_popen(monkeypatch, proc)


def test_spawn_uses_explicit_port(monkeypatch):
    monkeypatch.setattr(config, "EMBED_PORT", 8123)
    b = LlamaServerBackend()
    _patch_spawn_deps(monkeypatch, _FakeProc(alive = True))
    monkeypatch.setattr(b, "_wait_for_health", lambda *a, **k: True)
    b._spawn()
    assert b._port == 8123


def test_spawn_uses_free_port_when_auto(monkeypatch):
    monkeypatch.setattr(config, "EMBED_PORT", 0)
    b = LlamaServerBackend()
    _patch_spawn_deps(monkeypatch, _FakeProc(alive = True), free_port = 47000)
    monkeypatch.setattr(b, "_wait_for_health", lambda *a, **k: True)
    b._spawn()
    assert b._port == 47000


def test_spawn_refreshes_pooling_for_a_replaced_model(monkeypatch, tmp_path):
    """A cached path survives the reaper, but its file can be replaced between server
    lifetimes. The relaunched command and the vectors' identity must capture the same pooling."""
    path = tmp_path / "embed.gguf"
    _write_gguf(path, "qwen3", 1)
    backend = LlamaServerBackend()
    backend._model_pooling = "last"
    proc = _FakeProc(alive = True)
    captured = {}

    monkeypatch.setattr(backend, "_resolve_binary", lambda: _FAKE_BINARY)
    monkeypatch.setattr(backend, "_resolve_model_path", lambda model_name = None: str(path))
    monkeypatch.setattr(backend, "_find_free_port", lambda: 47000)
    monkeypatch.setattr(backend, "_wait_for_health", lambda *a, **k: True)
    monkeypatch.setattr(backend, "_build_env", lambda *a, **k: {})

    _intercept_server_popen(monkeypatch, proc, captured)
    backend._spawn_once(False)

    assert captured["cmd"][captured["cmd"].index("--pooling") + 1] == "mean"
    assert backend._model_pooling == "mean"


def test_a_process_started_elsewhere_during_spawn_does_not_replace_the_server_command(
    monkeypatch, tmp_path
):
    """A concurrent spawn must not take the fake server or overwrite the command under test."""
    path = tmp_path / "embed.gguf"
    _write_gguf(path, "qwen3", 1)
    backend = LlamaServerBackend()
    proc = _FakeProc(alive = True)
    captured = {}
    foreign = {}

    def other_thread_starts_a_process():
        child = subprocess.Popen(
            [sys.executable, "-c", "pass"], stdout = subprocess.DEVNULL, stderr = subprocess.DEVNULL
        )
        foreign["real"] = child is not proc
        child.wait(timeout = 30)

    def health_while_another_thread_spawns(*_a, **_k):
        thread = threading.Thread(target = other_thread_starts_a_process)
        thread.start()
        thread.join(timeout = 30)
        return True

    monkeypatch.setattr(backend, "_resolve_binary", lambda: _FAKE_BINARY)
    monkeypatch.setattr(backend, "_resolve_model_path", lambda model_name = None: str(path))
    monkeypatch.setattr(backend, "_find_free_port", lambda: 47000)
    monkeypatch.setattr(backend, "_wait_for_health", health_while_another_thread_spawns)
    monkeypatch.setattr(backend, "_build_env", lambda *a, **k: {})
    _intercept_server_popen(monkeypatch, proc, captured)
    backend._spawn_once(False)

    assert foreign.get("real") is True, "the other thread was handed the fake server"
    assert captured["cmd"][0] == _FAKE_BINARY
    assert captured["cmd"][captured["cmd"].index("--pooling") + 1] == "mean"


def test_spawn_fails_loud_on_early_exit(monkeypatch):
    monkeypatch.setattr(config, "EMBED_PORT", 8124)
    b = LlamaServerBackend()
    _patch_spawn_deps(monkeypatch, _FakeProc(alive = False, returncode = 1))
    with pytest.raises(RuntimeError, match = "failed to become healthy"):
        b._spawn()


def test_spawn_auto_falls_back_to_cpu_on_gpu_failure(monkeypatch):
    monkeypatch.setattr(config, "EMBED_DEVICE", "auto")
    monkeypatch.setattr(LlamaServerBackend, "_gpu_available", staticmethod(lambda: True))
    b = LlamaServerBackend()
    calls = []

    def fake_spawn_once(use_gpu, model_name = None):
        calls.append(use_gpu)
        if use_gpu:
            raise RuntimeError("CUDA out of memory")

    monkeypatch.setattr(b, "_spawn_once", fake_spawn_once)
    b._spawn()
    assert calls == [True, False]
    assert b._force_cpu is True


def test_spawn_explicit_gpu_does_not_fall_back(monkeypatch):
    monkeypatch.setattr(config, "EMBED_DEVICE", "gpu")
    b = LlamaServerBackend()

    def fake_spawn_once(use_gpu, model_name = None):
        raise RuntimeError("CUDA out of memory")

    monkeypatch.setattr(b, "_spawn_once", fake_spawn_once)
    with pytest.raises(RuntimeError, match = "out of memory"):
        b._spawn()
    assert b._force_cpu is False


def _embed_response(vectors):
    items = [{"index": i, "embedding": v} for i, v in enumerate(vectors)]
    return {"data": list(reversed(items))}


def test_encode_orders_and_returns_float32(monkeypatch):
    b = LlamaServerBackend()
    monkeypatch.setattr(b, "_ensure_ready", lambda *_a, **_k: None)
    captured = {}

    def fake_post(path, payload, **_kwargs):
        captured["path"] = path
        captured["input"] = payload["input"]
        return _embed_response([[3.0, 4.0], [0.0, 5.0]])

    monkeypatch.setattr(b, "_post", fake_post)
    out = b.encode(["a", "b"], normalize = False)
    assert captured["path"] == "/v1/embeddings"
    assert out.dtype == np.float32
    assert out.shape == (2, 2)
    assert out[0].tolist() == [3.0, 4.0]


def test_encode_normalizes(monkeypatch):
    b = LlamaServerBackend()
    monkeypatch.setattr(b, "_ensure_ready", lambda *_a, **_k: None)
    monkeypatch.setattr(b, "_post", lambda p, pl, **_k: _embed_response([[3.0, 4.0]]))
    out = b.encode(["a"], normalize = True)
    np.testing.assert_allclose(np.linalg.norm(out, axis = 1), [1.0], rtol = 1e-6)


def test_encode_empty_returns_zero_rows(monkeypatch):
    b = LlamaServerBackend()
    b._dim = 384
    monkeypatch.setattr(b, "_ensure_ready", lambda *_a, **_k: None)
    out = b.encode([])
    assert out.shape == (0, 384)
    assert out.dtype == np.float32


def test_encode_rejects_count_mismatch(monkeypatch):
    b = LlamaServerBackend()
    monkeypatch.setattr(b, "_ensure_ready", lambda *_a, **_k: None)
    monkeypatch.setattr(
        b, "_post", lambda p, pl, **_k: {"data": [{"index": 0, "embedding": [1.0]}]}
    )
    with pytest.raises(RuntimeError, match = "vectors for"):
        b.encode(["a", "b"], normalize = False)


def test_encode_batches(monkeypatch):
    monkeypatch.setattr(config, "EMBED_BATCH", 2)
    b = LlamaServerBackend()
    monkeypatch.setattr(b, "_ensure_ready", lambda *_a, **_k: None)
    calls = []

    def fake_post(path, payload, **_kwargs):
        chunk = payload["input"]
        calls.append(len(chunk))
        return _embed_response([[1.0, 0.0]] * len(chunk))

    monkeypatch.setattr(b, "_post", fake_post)
    out = b.encode(["a", "b", "c"], normalize = False)
    assert out.shape == (3, 2)
    assert calls == [2, 1]


def test_dim_probes_once_and_caches(monkeypatch):
    b = LlamaServerBackend()
    monkeypatch.setattr(b, "_ensure_ready", lambda *_a, **_k: None)
    n_calls = {"n": 0}

    def fake_post(path, payload, **_kwargs):
        n_calls["n"] += 1
        return _embed_response([[0.1] * 384])

    monkeypatch.setattr(b, "_post", fake_post)
    assert b.dim() == 384
    assert b.dim() == 384
    assert n_calls["n"] == 1


def test_token_counter_hits_tokenize(monkeypatch):
    b = LlamaServerBackend()
    monkeypatch.setattr(b, "_ensure_ready", lambda *_a, **_k: None)
    seen = {}

    def fake_post(path, payload, **_kwargs):
        seen["path"] = path
        seen["content"] = payload["content"]
        return {"tokens": [1, 2, 3, 4]}

    monkeypatch.setattr(b, "_post", fake_post)
    count = b.token_counter()
    assert count("hello world") == 4
    assert seen["path"] == "/tokenize"
    assert seen["content"] == "hello world"


def test_ensure_ready_respawns_dead_process(monkeypatch):
    b = LlamaServerBackend()
    b._process = _FakeProc(alive = False, returncode = 0)
    spawned = {"n": 0}

    def fake_spawn(model_name = None):
        spawned["n"] += 1
        b._process = _FakeProc(alive = True)
        b._model_repo = config.effective_gguf_repo()
        from utils.llama_cpp_path_settings import custom_llama_cpp_path_revision

        b._binary_path_revision = custom_llama_cpp_path_revision()

    monkeypatch.setattr(b, "_spawn", fake_spawn)
    b._ensure_ready()
    assert spawned["n"] == 1
    assert b._process_alive()
    b._ensure_ready()
    assert spawned["n"] == 1


def test_path_save_restarts_embedder_with_new_binary(monkeypatch):
    from utils import llama_cpp_path_settings

    backend = LlamaServerBackend()
    backend._process = _FakeProc(alive = True)
    backend._model_repo = config.effective_gguf_repo()
    backend._binary = "/old/llama-server"
    backend._binary_path_revision = llama_cpp_path_settings.custom_llama_cpp_path_revision()
    monkeypatch.setattr(
        llama_cpp_path_settings, "_path_revision", backend._binary_path_revision + 1
    )
    spawned = []

    def fake_spawn(model_name = None):
        spawned.append(backend._binary)
        backend._process = _FakeProc(alive = True)
        backend._model_repo = config.effective_gguf_repo()
        backend._binary = "/new/llama-server"
        backend._binary_path_revision = llama_cpp_path_settings.custom_llama_cpp_path_revision()

    monkeypatch.setattr(backend, "_spawn", fake_spawn)
    backend._ensure_ready()

    assert spawned == [None]
    assert backend._binary == "/new/llama-server"


def test_post_restarts_once_on_connect_error(monkeypatch):
    import httpx

    b = LlamaServerBackend()
    b._port = 9000
    monkeypatch.setattr(b, "_ensure_ready", lambda *_a, **_k: None)
    restarts = {"n": 0}
    monkeypatch.setattr(
        b, "_restart", lambda *_a, **_k: restarts.__setitem__("n", restarts["n"] + 1)
    )

    attempts = {"n": 0}

    class _Client:
        def post(
            self,
            url,
            json,
            headers = None,
        ):
            attempts["n"] += 1
            if attempts["n"] == 1:
                raise httpx.ConnectError("boom")

            class _R:
                def raise_for_status(self_inner):
                    return None

                def json(self_inner):
                    return {"tokens": [1]}

            return _R()

    b._client = _Client()
    out = b._post("/tokenize", {"content": "x"})
    assert out == {"tokens": [1]}
    assert restarts["n"] == 1


def test_post_restarts_once_on_read_timeout(monkeypatch):
    import httpx

    b = LlamaServerBackend()
    b._port = 9000
    monkeypatch.setattr(b, "_ensure_ready", lambda *_a, **_k: None)
    restarts = {"n": 0}
    monkeypatch.setattr(
        b, "_restart", lambda *_a, **_k: restarts.__setitem__("n", restarts["n"] + 1)
    )

    attempts = {"n": 0}

    class _Client:
        def post(
            self,
            url,
            json,
            headers = None,
        ):
            attempts["n"] += 1
            if attempts["n"] == 1:
                raise httpx.ReadTimeout("timed out")

            class _R:
                def raise_for_status(self_inner):
                    return None

                def json(self_inner):
                    return {"data": [{"index": 0, "embedding": [1.0, 0.0]}]}

            return _R()

    b._client = _Client()
    out = b._post("/v1/embeddings", {"input": ["x"]})
    assert out["data"][0]["embedding"] == [1.0, 0.0]
    assert restarts["n"] == 1


class _CrashingGpuServer:
    """Spawn and client stand-ins: a GPU server dies under every request (#13663), a CPU one answers."""

    def __init__(
        self,
        monkeypatch,
        backend,
        *,
        timeout = False,
        cpu_crashes = False,
    ):
        import httpx

        self.spawns = []
        monkeypatch.setattr(backend, "_ensure_ready", lambda *_a, **_k: None)
        monkeypatch.setattr(LlamaServerBackend, "_gpu_available", staticmethod(lambda: True))

        def fake_spawn_once(use_gpu, model_name = None):
            self.spawns.append("gpu" if use_gpu else "cpu")
            backend._process = _FakeProc(alive = True, returncode = 0)
            backend._on_gpu = use_gpu
            backend._port = 9000

        monkeypatch.setattr(backend, "_spawn_once", fake_spawn_once)
        fake_spawn_once(backend._use_gpu())

        class _Client:
            def post(
                self,
                url,
                json,
                headers = None,
            ):
                if backend._on_gpu or cpu_crashes:
                    if timeout:
                        raise httpx.ReadTimeout("timed out")
                    backend._process._alive = False
                    backend._process.returncode = 3221226505
                    raise httpx.ReadError("connection reset by peer")

                class _R:
                    def raise_for_status(self_inner):
                        return None

                    def json(self_inner):
                        return {"data": [{"index": 0, "embedding": [1.0, 0.0]}]}

                return _R()

        backend._client = _Client()


def test_a_gpu_server_crashing_twice_on_one_request_finishes_on_cpu(monkeypatch):
    monkeypatch.setattr(config, "EMBED_DEVICE", "auto")
    b = LlamaServerBackend()
    fake = _CrashingGpuServer(monkeypatch, b)
    out = b._post("/v1/embeddings", {"input": ["x"]})
    assert out["data"][0]["embedding"] == [1.0, 0.0]
    assert fake.spawns == ["gpu", "gpu", "cpu"]
    b._post("/v1/embeddings", {"input": ["y"]})
    assert fake.spawns == ["gpu", "gpu", "cpu"]


def test_a_settings_gpu_choice_still_falls_back_after_repeated_crashes(monkeypatch):
    import utils.embedding_model_settings as ems

    monkeypatch.setattr(config, "EMBED_DEVICE", "auto")
    monkeypatch.setattr(ems, "get_stored_embedding_device", lambda: "gpu")
    b = LlamaServerBackend()
    fake = _CrashingGpuServer(monkeypatch, b)
    b._post("/v1/embeddings", {"input": ["x"]})
    assert fake.spawns == ["gpu", "gpu", "cpu"]


def test_literal_gpu_env_raises_instead_of_moving_a_crashing_server_to_cpu(monkeypatch):
    monkeypatch.setattr(config, "EMBED_DEVICE", "gpu")
    b = LlamaServerBackend()
    fake = _CrashingGpuServer(monkeypatch, b)
    with pytest.raises(RuntimeError, match = "failed after retry"):
        b._post("/v1/embeddings", {"input": ["x"]})
    assert fake.spawns == ["gpu", "gpu"]
    assert b._force_cpu is False


def test_repeated_timeouts_on_a_live_gpu_server_do_not_move_it_to_cpu(monkeypatch):
    monkeypatch.setattr(config, "EMBED_DEVICE", "auto")
    b = LlamaServerBackend()
    fake = _CrashingGpuServer(monkeypatch, b, timeout = True)
    with pytest.raises(RuntimeError, match = "failed after retry"):
        b._post("/v1/embeddings", {"input": ["x"]})
    assert fake.spawns == ["gpu", "gpu"]
    assert b._force_cpu is False


def test_a_cpu_server_crashing_twice_raises_without_a_third_spawn(monkeypatch):
    monkeypatch.setattr(config, "EMBED_DEVICE", "cpu")
    b = LlamaServerBackend()
    fake = _CrashingGpuServer(monkeypatch, b, cpu_crashes = True)
    with pytest.raises(RuntimeError, match = "failed after retry"):
        b._post("/v1/embeddings", {"input": ["x"]})
    assert fake.spawns == ["cpu", "cpu"]


def test_a_crash_logs_the_server_exit_code_and_last_output(monkeypatch, caplog):
    monkeypatch.setattr(config, "EMBED_DEVICE", "auto")
    b = LlamaServerBackend()
    _CrashingGpuServer(monkeypatch, b)
    b._stdout_lines = ["rocBLAS error from hip error code: 'hipErrorInvalidKernelFile':218"]
    with caplog.at_level("WARNING", logger = mod.logger.name):
        b._post("/v1/embeddings", {"input": ["x"]})
    assert "exited during a request (code 3221226505)" in caplog.text
    assert "hipErrorInvalidKernelFile" in caplog.text
    assert "continuing on CPU" in caplog.text


class TestTheEmbeddingLoaderChangesAreMacOsOnly:
    """The Linux and Windows embed server launch must stay as it was; only macOS may change."""

    def test_the_help_probe_still_inherits_the_environment_off_mac(self, monkeypatch):
        seen = {}

        def fake_run(cmd, **kwargs):
            seen.update(kwargs)
            return types.SimpleNamespace(stdout = "--embedding", stderr = "")

        monkeypatch.setattr(mod.sys, "platform", "linux")
        monkeypatch.setattr(mod.subprocess, "run", fake_run)
        mod.LlamaServerBackend._help_text.cache_clear()
        mod.LlamaServerBackend._help_text("/opt/mine/llama-server")
        assert seen.get("env") is None

    def test_the_help_probe_gets_the_loader_environment_on_mac(self, monkeypatch, tmp_path):
        binary = tmp_path / "llama-server"
        binary.write_bytes(b"\xcf\xfa\xed\xfe")
        seen = {}

        def fake_run(cmd, **kwargs):
            seen.update(kwargs)
            return types.SimpleNamespace(stdout = "--embedding", stderr = "")

        monkeypatch.setattr(mod.sys, "platform", "darwin")
        monkeypatch.setattr(mod.subprocess, "run", fake_run)
        mod.LlamaServerBackend._help_text.cache_clear()
        mod.LlamaServerBackend._help_text(str(binary))
        assert seen.get("env") is not None

    def test_linux_gpu_libs_still_hang_off_the_given_path(self, monkeypatch, tmp_path):
        real = tmp_path / "build" / "bin" / "llama-server"
        real.parent.mkdir(parents = True)
        real.write_bytes(b"\xcf\xfa\xed\xfe")
        wrapper = tmp_path / "llama-server"
        wrapper.write_text('#!/bin/sh\nexec "$(dirname "$0")/build/bin/llama-server" "$@"\n')
        wrapper.chmod(0o755)
        seen = {}
        monkeypatch.setattr(mod.sys, "platform", "linux")
        monkeypatch.setattr(
            mod.LlamaServerBackend,
            "_add_linux_cuda_libs",
            staticmethod(lambda env, d: seen.setdefault("dir", d)),
        )
        mod.LlamaServerBackend()._build_env(str(wrapper), use_gpu = True)
        assert seen["dir"] == str(wrapper.parent)


def test_device_preference_is_shared_between_both_backends(monkeypatch):
    """One reader, so the two backends cannot disagree about the same string.

    They did: this one compared a bare .lower() with no strip, so " gpu " forced the
    GPU for sentence-transformers and fell through to auto here, and an Intel user
    writing their own accelerator's name (xpu) got neither.
    """
    for requested, expected in (
        ("gpu", "gpu"),
        ("GPU", "gpu"),
        ("  gpu  ", "gpu"),
        ("cuda", "gpu"),
        ("rocm", "gpu"),
        ("hip", "gpu"),
        ("xpu", "gpu"),
        ("mps", "gpu"),
        ("metal", "gpu"),
        ("cpu", "cpu"),
        ("CPU", "cpu"),
        (" cpu ", "cpu"),
        ("auto", "auto"),
        ("", "auto"),
        ("   ", "auto"),
        ("banana", "auto"),
    ):
        monkeypatch.setattr(config, "EMBED_DEVICE", requested)
        assert config.embed_device_preference() == expected, requested


def test_use_gpu_honours_the_normalized_preference(monkeypatch):
    """A whitespace-padded or device-named setting reaches this backend too."""
    backend = LlamaServerBackend()
    for requested in ("  gpu  ", "GPU", "xpu", "cuda"):
        monkeypatch.setattr(config, "EMBED_DEVICE", requested)
        monkeypatch.setattr(
            LlamaServerBackend,
            "_gpu_available",
            staticmethod(lambda: False),
        )
        assert backend._use_gpu() is True, requested
    for requested in (" cpu ", "CPU"):
        monkeypatch.setattr(config, "EMBED_DEVICE", requested)
        monkeypatch.setattr(
            LlamaServerBackend,
            "_gpu_available",
            staticmethod(lambda: True),
        )
        assert backend._use_gpu() is False, requested


def test_gpu_opt_in_still_falls_back_to_cpu(monkeypatch):
    """A GPU opt-in spelling still falls back to CPU on a failed start; it must not become fatal."""
    for requested in ("cuda", "rocm", "hip", "xpu", "mps", "metal", "  gpu  ", "  CUDA  "):
        monkeypatch.setattr(config, "EMBED_DEVICE", requested)
        backend = LlamaServerBackend()
        calls: list[bool] = []

        def fake_spawn_once(
            use_gpu,
            model_name = None,
            _calls = calls,
        ):
            _calls.append(use_gpu)
            if use_gpu:
                raise RuntimeError("GPU start failed")

        monkeypatch.setattr(backend, "_spawn_once", fake_spawn_once)
        backend._spawn()
        assert calls == [True, False], requested
        assert backend._force_cpu is True, requested


def test_literal_gpu_remains_a_hard_request(monkeypatch):
    """The documented value keeps forcing the GPU, as it always has."""
    for requested in ("gpu", "GPU"):
        monkeypatch.setattr(config, "EMBED_DEVICE", requested)
        backend = LlamaServerBackend()
        monkeypatch.setattr(
            backend,
            "_spawn_once",
            lambda use_gpu, *_a: (_ for _ in ()).throw(RuntimeError("no cuda")),
        )
        with pytest.raises(RuntimeError, match = "no cuda"):
            backend._spawn()
        assert backend._force_cpu is False, requested


def test_a_soft_gpu_opt_in_keeps_its_own_cpu_fallback(monkeypatch):
    """A soft GPU opt-in keeps its CPU fallback, so respawns do not retry a failed GPU start."""
    for requested in ("cuda", "xpu", "  gpu  "):
        monkeypatch.setattr(config, "EMBED_DEVICE", requested)
        backend = LlamaServerBackend()
        calls: list[bool] = []

        def fake_spawn_once(
            use_gpu,
            model_name = None,
            _calls = calls,
        ):
            _calls.append(use_gpu)
            if use_gpu:
                raise RuntimeError("GPU start failed")

        monkeypatch.setattr(backend, "_spawn_once", fake_spawn_once)
        backend._spawn()
        assert calls == [True, False], requested
        backend._spawn()
        assert calls == [True, False, False], requested
        assert backend._use_gpu() is False, requested


def _seed_cache(
    root,
    repo_id,
    filenames,
    *,
    revision = "abc123",
    ref = True,
):
    """A hub cache tree built the way hf_hub_download builds one: the snapshot holds
    symlinks into blobs/, not regular files."""
    repo_dir = root / f"models--{repo_id.replace('/', '--')}"
    snapshot = repo_dir / "snapshots" / revision
    blobs = repo_dir / "blobs"
    blobs.mkdir(parents = True, exist_ok = True)
    for name in filenames:
        blob = blobs / name.replace("/", "_")
        blob.write_text("gguf")
        link = snapshot / name
        link.parent.mkdir(parents = True, exist_ok = True)
        link.symlink_to(os.path.relpath(blob, link.parent))
    if ref:
        (repo_dir / "refs").mkdir(parents = True, exist_ok = True)
        (repo_dir / "refs" / "main").write_text(revision)
    return snapshot


def _use_cache_root(monkeypatch, root):
    import utils.hf_cache_settings as hcs
    monkeypatch.setattr(hcs, "active_hf_hub_cache", lambda: str(root))


def _no_hub(monkeypatch):
    monkeypatch.setattr(
        LlamaServerBackend,
        "_resolve_uncached_model_path",
        lambda self, *a, **k: pytest.fail("resolved over the hub despite a usable cache"),
    )


def test_cached_gguf_resolves_without_touching_the_hub(monkeypatch, tmp_path):
    """A repo already on disk must never be re-listed: _model_path is per-process, so
    every restart used to pay a hub call to name a file it already held (#8778)."""
    repo = config.effective_gguf_repo()
    _seed_cache(tmp_path / "hub", repo, ["bge-F16.gguf"])
    _use_cache_root(monkeypatch, tmp_path / "hub")
    _no_hub(monkeypatch)
    backend = LlamaServerBackend()
    backend._dim = 384
    assert backend._resolve_model_path().endswith("bge-F16.gguf")
    assert backend._model_repo == repo
    assert backend._dim is None


def test_the_pick_takes_the_shortest_variant_match_and_never_an_mmproj(monkeypatch):
    """Wanted file is not first or shortest, and the shortest F16 match is an mmproj to skip."""
    monkeypatch.setattr(config, "EMBED_GGUF_VARIANT", "F16")
    names = [
        "bge-small-en-v1.5-F16-alt.gguf",
        "mmproj-F16.gguf",
        "bge-small-en-v1.5-F16.gguf",
        "bge-Q8_0.gguf",
        "README.md",
    ]
    assert LlamaServerBackend._pick_gguf(names) == "bge-small-en-v1.5-F16.gguf"


def test_cached_gguf_skips_mmproj_and_honours_the_variant(monkeypatch, tmp_path):
    """The same rule reached through the cache, over a real snapshot layout."""
    repo = config.effective_gguf_repo()
    _seed_cache(
        tmp_path / "hub",
        repo,
        ["mmproj-F16.gguf", "bge-small-en-v1.5-F16.gguf", "bge-Q8_0.gguf", "README.md"],
    )
    _use_cache_root(monkeypatch, tmp_path / "hub")
    monkeypatch.setattr(config, "EMBED_GGUF_VARIANT", "F16")
    assert LlamaServerBackend._resolve_cached_gguf(repo).endswith("bge-small-en-v1.5-F16.gguf")


def test_the_pick_is_stable_whatever_order_the_names_arrive_in(monkeypatch):
    """Directory scans are unordered, and among equal-length shards only the first has the metadata."""
    monkeypatch.setattr(config, "EMBED_GGUF_VARIANT", "F16")
    shards = ["model-F16-00001-of-00002.gguf", "model-F16-00002-of-00002.gguf"]
    assert LlamaServerBackend._pick_gguf(shards) == shards[0]
    assert LlamaServerBackend._pick_gguf(list(reversed(shards))) == shards[0]


def test_an_mtp_drafter_is_never_taken_for_the_embedder(monkeypatch):
    """An MTP drafter is a companion, not a model: a cache holding only one must read as no embedder."""
    monkeypatch.setattr(config, "EMBED_GGUF_VARIANT", "BF16")
    assert LlamaServerBackend._pick_gguf(["MTP/mtp-model-BF16.gguf"]) is None
    assert (
        LlamaServerBackend._pick_gguf(["MTP/mtp-model-BF16.gguf", "model-BF16.gguf"])
        == "model-BF16.gguf"
    )


def test_cached_gguf_is_found_in_a_subdirectory(monkeypatch, tmp_path):
    """Quant files sit in subdirectories, so a root-only scan would re-download a cached model."""
    repo = config.effective_gguf_repo()
    _seed_cache(tmp_path / "hub", repo, ["F16/bge-small.gguf"])
    _use_cache_root(monkeypatch, tmp_path / "hub")
    resolved = LlamaServerBackend._resolve_cached_gguf(repo)
    assert resolved is not None and Path(resolved).parts[-2:] == ("F16", "bge-small.gguf")


def test_an_incomplete_cached_shard_set_is_not_a_hit(monkeypatch, tmp_path):
    """A lone trailing shard cannot serve the model; llama-server needs the first shard's metadata."""
    repo = config.effective_gguf_repo()
    _seed_cache(tmp_path / "hub", repo, ["model-F16-00002-of-00002.gguf"])
    _use_cache_root(monkeypatch, tmp_path / "hub")
    assert LlamaServerBackend._resolve_cached_gguf(repo) is None


def test_a_complete_cached_shard_set_resolves_to_its_first_shard(monkeypatch, tmp_path):
    """A complete shard set is servable and resolves to shard 1 whatever order the scan returns."""
    repo = config.effective_gguf_repo()
    _seed_cache(
        tmp_path / "hub",
        repo,
        ["model-F16-00002-of-00002.gguf", "model-F16-00001-of-00002.gguf"],
    )
    _use_cache_root(monkeypatch, tmp_path / "hub")
    resolved = LlamaServerBackend._resolve_cached_gguf(repo)
    assert resolved is not None and resolved.endswith("model-F16-00001-of-00002.gguf")


def test_an_incomplete_set_does_not_shadow_a_complete_one(monkeypatch, tmp_path):
    """Completeness must narrow the candidates, not veto the winner, or a servable family goes unseen."""
    repo = config.effective_gguf_repo()
    _seed_cache(
        tmp_path / "hub",
        repo,
        [
            "a-F16-00001-of-00002.gguf",
            "embedding-model-F16-00001-of-00002.gguf",
            "embedding-model-F16-00002-of-00002.gguf",
        ],
    )
    _use_cache_root(monkeypatch, tmp_path / "hub")
    resolved = LlamaServerBackend._resolve_cached_gguf(repo)
    assert resolved is not None
    assert resolved.endswith("embedding-model-F16-00001-of-00002.gguf")


def test_an_incomplete_variant_match_does_not_hide_a_whole_other_variant(monkeypatch, tmp_path):
    """Offline, the relaxed pass takes any variant. An unservable F16 must not stand in the
    way of a Q8 that is whole -- the difference between a working embedder and none."""
    repo = config.effective_gguf_repo()
    _seed_cache(tmp_path / "hub", repo, ["model-F16-00002-of-00002.gguf", "model-Q8_0.gguf"])
    _use_cache_root(monkeypatch, tmp_path / "hub")
    monkeypatch.setattr(config, "EMBED_GGUF_VARIANT", "F16")
    assert LlamaServerBackend._resolve_cached_gguf(repo) is None
    relaxed = LlamaServerBackend._resolve_cached_gguf(repo, require_variant = False)
    assert relaxed is not None and relaxed.endswith("model-Q8_0.gguf")


def test_cached_gguf_uses_only_the_revision_refs_main_names(monkeypatch, tmp_path):
    """hf_hub_download serves refs/main. The current revision here sorts neither first nor
    last, so no directory ordering can stand in for reading the ref."""
    repo = config.effective_gguf_repo()
    root = tmp_path / "hub"
    _seed_cache(root, repo, ["aaa-F16.gguf"], revision = "aaa111", ref = False)
    _seed_cache(root, repo, ["live-F16.gguf"], revision = "mmm555", ref = True)
    _seed_cache(root, repo, ["zzz-F16.gguf"], revision = "zzz999", ref = False)
    _use_cache_root(monkeypatch, root)
    assert LlamaServerBackend._resolve_cached_gguf(repo).endswith("live-F16.gguf")


def test_a_commit_pinned_cache_with_no_ref_defers_to_the_hub(monkeypatch, tmp_path):
    """Snapshot directories are commit hashes, which order by nothing; guessing among them
    could serve a superseded model, so an unnamed revision is no hit at all."""
    repo = config.effective_gguf_repo()
    _seed_cache(tmp_path / "hub", repo, ["bge-F16.gguf"], revision = "deadbeef", ref = False)
    _use_cache_root(monkeypatch, tmp_path / "hub")
    assert LlamaServerBackend._resolve_cached_gguf(repo) is None


def test_a_cached_other_variant_does_not_satisfy_the_configured_one(monkeypatch, tmp_path):
    """Falling back on a listing means the variant is not published; falling back on a
    cache would serve a Q8 left from an earlier setting instead of the configured F16."""
    repo = config.effective_gguf_repo()
    _seed_cache(tmp_path / "hub", repo, ["bge-Q8_0.gguf"])
    _use_cache_root(monkeypatch, tmp_path / "hub")
    monkeypatch.setattr(config, "EMBED_GGUF_VARIANT", "F16")
    assert LlamaServerBackend._resolve_cached_gguf(repo) is None


def test_an_unreachable_hub_falls_back_to_whatever_variant_is_cached(monkeypatch, tmp_path):
    """A cached other variant beats no embedder at all, the degrade issue 8778 asks for."""
    huggingface_hub = _shared_setup_2(monkeypatch, tmp_path)
    monkeypatch.setattr(
        huggingface_hub,
        "list_repo_files",
        lambda *a, **k: (_ for _ in ()).throw(OSError("offline mode is enabled")),
    )
    backend = LlamaServerBackend()
    assert backend._resolve_model_path().endswith("bge-Q8_0.gguf")


def test_a_pending_picker_download_never_falls_through_to_the_hub(monkeypatch, tmp_path):
    import utils.embedding_model_settings as ems

    monkeypatch.setattr(config, "effective_embedding_model", lambda: "org/pending")
    monkeypatch.setattr(config, "effective_gguf_repo", lambda: "org/pending-GGUF")
    monkeypatch.setattr(ems, "get_stored_download_pending", lambda model: True)
    _use_cache_root(monkeypatch, tmp_path / "empty")
    backend = LlamaServerBackend()
    monkeypatch.setattr(
        backend,
        "_resolve_uncached_model_path",
        lambda *a: (_ for _ in ()).throw(AssertionError("pending model reached the Hub")),
    )

    with pytest.raises(RuntimeError, match = "not downloaded yet"):
        backend._resolve_model_path()


def test_a_completed_pending_download_uses_its_cached_fallback_quant(monkeypatch, tmp_path):
    ems, repo = _shared_setup_3(monkeypatch)
    monkeypatch.setattr(ems, "get_stored_download_pending", lambda model: True)
    _seed_cache(tmp_path / "hub", repo, ["pending-Q8_0.gguf"])
    _use_cache_root(monkeypatch, tmp_path / "hub")
    backend = LlamaServerBackend()
    monkeypatch.setattr(
        backend,
        "_resolve_uncached_model_path",
        lambda *a: (_ for _ in ()).throw(AssertionError("cached pending model reached Hub")),
    )

    assert backend._resolve_model_path().endswith("pending-Q8_0.gguf")


def test_an_unreachable_hub_still_reaches_the_fallback_repo_s_cache(monkeypatch, tmp_path):
    """Cache-first consults only the preferred repo to keep a reachable companion ahead of the
    fallback; offline the relaxed pass must still check the fallback repo cache."""
    import contextlib
    import huggingface_hub
    from core.inference import llama_cpp

    monkeypatch.setattr(config, "effective_embedding_model", lambda: "org/foo")
    monkeypatch.setattr(config, "effective_gguf_repo", lambda: "org/foo-GGUF")
    _seed_cache(tmp_path / "hub", "org/foo", ["fallback-F16.gguf"])
    _use_cache_root(monkeypatch, tmp_path / "hub")
    monkeypatch.setattr(llama_cpp, "_hf_offline_if_unreachable", contextlib.nullcontext)
    monkeypatch.setattr(
        huggingface_hub,
        "list_repo_files",
        lambda *a, **k: (_ for _ in ()).throw(OSError("offline mode is enabled")),
    )
    backend = LlamaServerBackend()
    assert backend._resolve_model_path().endswith("fallback-F16.gguf")


def test_a_reachable_hub_is_preferred_over_a_cached_other_variant(monkeypatch, tmp_path):
    """A cached Q8 is a last resort and must not pre-empt a reachable hub naming the variant."""
    huggingface_hub = _shared_setup_2(monkeypatch, tmp_path)
    monkeypatch.setattr(huggingface_hub, "list_repo_files", lambda *a, **k: ["bge-F16.gguf"])
    monkeypatch.setattr(huggingface_hub, "hf_hub_download", lambda **kw: "/cache/bge-F16.gguf")
    backend = LlamaServerBackend()
    assert backend._resolve_model_path() == "/cache/bge-F16.gguf"


def test_a_failed_transfer_surfaces_instead_of_serving_another_variant(monkeypatch, tmp_path):
    """A download failing for its own reasons (no disk, bad checksum) is no evidence the
    variant is unobtainable, so it must not be answered with a different model."""
    huggingface_hub = _shared_setup_2(monkeypatch, tmp_path)
    monkeypatch.setattr(huggingface_hub, "list_repo_files", lambda *a, **k: ["bge-F16.gguf"])
    monkeypatch.setattr(
        huggingface_hub,
        "hf_hub_download",
        lambda **kw: (_ for _ in ()).throw(OSError("No space left on device")),
    )
    backend = LlamaServerBackend()
    with pytest.raises(OSError, match = "No space left on device"):
        backend._resolve_model_path()


def test_a_cached_gguf_is_found_whatever_the_extension_case(monkeypatch, tmp_path):
    """The hub path always matched the suffix case-insensitively; a stricter cache scan
    would re-download a model sitting on disk, or fail outright when offline."""
    repo = config.effective_gguf_repo()
    _seed_cache(tmp_path / "hub", repo, ["bge-F16.GGUF"])
    _use_cache_root(monkeypatch, tmp_path / "hub")
    assert LlamaServerBackend._resolve_cached_gguf(repo).endswith("bge-F16.GGUF")


def test_the_adopted_path_is_tagged_with_the_repo_it_was_resolved_for(monkeypatch, tmp_path):
    """Tag the path with the repo it was resolved for, so a Settings change reads as stale."""
    repo = config.effective_gguf_repo()
    _seed_cache(tmp_path / "hub", repo, ["bge-F16.gguf"])
    _use_cache_root(monkeypatch, tmp_path / "hub")
    _no_hub(monkeypatch)
    backend = LlamaServerBackend()

    def _resolve_then_change_the_setting(repo_id, **_kw):
        monkeypatch.setattr(config, "effective_gguf_repo", lambda: "someone/else-GGUF")
        return str(tmp_path / "hub" / "bge-F16.gguf")

    monkeypatch.setattr(
        LlamaServerBackend,
        "_resolve_cached_gguf",
        staticmethod(_resolve_then_change_the_setting),
    )
    backend._resolve_model_path()
    assert backend._model_repo == repo
    assert not backend._current()


def test_a_cached_fallback_repo_does_not_pre_empt_the_preferred_one(monkeypatch, tmp_path):
    """A custom model resolves through its "-GGUF" companion first and only falls back to
    the model repo when the companion has none. A file cached under the fallback must not
    reorder that, or the backend serves the wrong weights and tags them as current."""
    import contextlib
    import huggingface_hub
    from core.inference import llama_cpp

    monkeypatch.setattr(config, "effective_embedding_model", lambda: "org/foo")
    monkeypatch.setattr(config, "effective_gguf_repo", lambda: "org/foo-GGUF")
    _seed_cache(tmp_path / "hub", "org/foo", ["fallback-F16.gguf"])
    _use_cache_root(monkeypatch, tmp_path / "hub")
    monkeypatch.setattr(llama_cpp, "_hf_offline_if_unreachable", contextlib.nullcontext)
    monkeypatch.setattr(
        huggingface_hub,
        "list_repo_files",
        lambda repo_id, **k: ["preferred-F16.gguf"] if repo_id == "org/foo-GGUF" else [],
    )
    monkeypatch.setattr(
        huggingface_hub, "hf_hub_download", lambda **kw: "/cache/preferred-F16.gguf"
    )
    backend = LlamaServerBackend()
    assert backend._resolve_model_path() == "/cache/preferred-F16.gguf"


def test_the_cache_folder_is_named_from_the_resolved_repo_casing(monkeypatch, tmp_path):
    """Cache folders take the resolver's casing; a mismatch re-downloads and fails offline."""
    import utils.paths as paths_pkg

    _seed_cache(tmp_path / "hub", "Cached/Spelling-GGUF", ["bge-F16.gguf"])
    _use_cache_root(monkeypatch, tmp_path / "hub")
    monkeypatch.setattr(
        paths_pkg, "resolve_cached_repo_id_case", lambda name: "Cached/Spelling-GGUF"
    )
    resolved = LlamaServerBackend._resolve_cached_gguf("as/typed-GGUF")
    assert resolved is not None and resolved.endswith("bge-F16.gguf")


def test_the_listing_cannot_outlive_its_deadline(monkeypatch, tmp_path):
    """list_repo_files takes no timeout and pagination overrides any client-level default,
    so a blackholed address holds this call, and _lifecycle_lock with it, until the kernel
    exhausts its SYN retries."""
    import contextlib
    import huggingface_hub
    from core.inference import llama_cpp

    @contextlib.contextmanager
    def reachable():
        yield False

    def _stalls(*_a, **_k):
        time.sleep(30)
        raise AssertionError("deadline did not cut the listing short")

    _use_cache_root(monkeypatch, tmp_path / "empty")
    monkeypatch.setattr(llama_cpp, "_hf_offline_if_unreachable", reachable)
    monkeypatch.setattr(huggingface_hub, "list_repo_files", _stalls)
    monkeypatch.setattr(LlamaServerBackend, "_LIST_DEADLINE_S", 0.3)
    backend = LlamaServerBackend()
    started = time.monotonic()
    with pytest.raises(RuntimeError, match = "no .gguf embedder found"):
        backend._resolve_model_path()
    assert time.monotonic() - started < 3.0


def test_the_hub_resolve_runs_inside_the_unreachable_guard(monkeypatch, tmp_path):
    """Deadline covers only the listing; the guard blocks transfers to an unreachable endpoint."""
    import contextlib
    import huggingface_hub
    from core.inference import llama_cpp

    active = []

    @contextlib.contextmanager
    def recording_guard():
        active.append(True)
        try:
            yield False
        finally:
            active.pop()

    def _require_guard(*_a, **_k):
        assert active, "hub call made outside the guard"
        return ["bge-F16.gguf"]

    def _download(**_kw):
        assert active, "download made outside the guard"
        return "/cache/bge-F16.gguf"

    _use_cache_root(monkeypatch, tmp_path / "empty")
    monkeypatch.setattr(llama_cpp, "_hf_offline_if_unreachable", recording_guard)
    monkeypatch.setattr(huggingface_hub, "list_repo_files", _require_guard)
    monkeypatch.setattr(huggingface_hub, "hf_hub_download", _download)
    backend = LlamaServerBackend()
    assert backend._resolve_model_path() == "/cache/bge-F16.gguf"
    assert active == []


def test_a_split_family_is_downloaded_whole_not_just_its_first_shard(monkeypatch, tmp_path):
    """Download the whole split GGUF family: llama-server opens the sibling shards itself with -m shard1."""
    import contextlib
    import huggingface_hub
    from core.inference import llama_cpp

    names = [
        "bge-F16-00001-of-00003.gguf",
        "bge-F16-00002-of-00003.gguf",
        "bge-F16-00003-of-00003.gguf",
    ]
    fetched = []
    _use_cache_root(monkeypatch, tmp_path / "hub")
    monkeypatch.setattr(config, "EMBED_GGUF_VARIANT", "F16")
    monkeypatch.setattr(llama_cpp, "_hf_offline_if_unreachable", contextlib.nullcontext)
    monkeypatch.setattr(huggingface_hub, "list_repo_files", lambda *a, **k: names)
    monkeypatch.setattr(
        huggingface_hub,
        "hf_hub_download",
        lambda **kw: (fetched.append(kw["filename"]), f"/cache/{kw['filename']}")[1],
    )
    backend = LlamaServerBackend()

    assert backend._resolve_model_path() == "/cache/bge-F16-00001-of-00003.gguf"
    assert fetched == names


def test_a_torn_published_family_is_skipped_rather_than_half_fetched(monkeypatch, tmp_path):
    """Shard 2 missing on the hub is not a repo to adopt; the candidate loop moves on."""
    import contextlib
    import huggingface_hub
    from core.inference import llama_cpp

    _use_cache_root(monkeypatch, tmp_path / "hub")
    monkeypatch.setattr(config, "EMBED_GGUF_VARIANT", "F16")
    monkeypatch.setattr(llama_cpp, "_hf_offline_if_unreachable", contextlib.nullcontext)
    monkeypatch.setattr(
        huggingface_hub,
        "list_repo_files",
        lambda *a, **k: ["bge-F16-00001-of-00003.gguf", "bge-F16-00003-of-00003.gguf"],
    )
    monkeypatch.setattr(
        huggingface_hub,
        "hf_hub_download",
        lambda **kw: pytest.fail("a torn family must not be fetched"),
    )
    backend = LlamaServerBackend()
    with pytest.raises(RuntimeError, match = "no .gguf embedder found"):
        backend._resolve_model_path()


def test_a_local_dir_of_shards_resolves_to_shard_one_in_any_scan_order(monkeypatch, tmp_path):
    """`iterdir` guarantees no order, and every shard name here is the same length,
    so a length-only sort let the directory decide which shard llama-server opened."""
    names = [
        "bge-F16-00001-of-00003.gguf",
        "bge-F16-00002-of-00003.gguf",
        "bge-F16-00003-of-00003.gguf",
    ]
    monkeypatch.setattr(config, "EMBED_GGUF_VARIANT", "F16")
    orders = [names, list(reversed(names)), [names[1], names[2], names[0]]]
    real_iterdir = Path.iterdir
    for index, order in enumerate(orders):
        d = tmp_path / f"dir{index}"
        d.mkdir()
        for name in order:
            (d / name).write_bytes(b"")
        monkeypatch.setattr(
            Path,
            "iterdir",
            lambda self, o = order, r = real_iterdir: (
                iter([self / n for n in o]) if self == d else r(self)
            ),
        )
        picked = LlamaServerBackend._resolve_local_gguf(str(d))
        assert Path(picked).name == "bge-F16-00001-of-00003.gguf"


def test_a_local_dir_with_a_torn_family_is_refused(monkeypatch, tmp_path):
    monkeypatch.setattr(config, "EMBED_GGUF_VARIANT", "F16")
    d = tmp_path / "torn"
    d.mkdir()
    for n in ("bge-F16-00001-of-00003.gguf", "bge-F16-00003-of-00003.gguf"):
        (d / n).write_bytes(b"")
    with pytest.raises(RuntimeError, match = "no .gguf file found"):
        LlamaServerBackend._resolve_local_gguf(str(d))


def test_a_local_dir_still_skips_mmproj_and_appledouble_sidecars(tmp_path):
    """macOS copies leave `._name` sidecars beside the real file."""
    d = tmp_path / "local"
    d.mkdir()
    (d / "mmproj-bge.gguf").write_bytes(b"")
    (d / "._bge.gguf").write_bytes(b"\x00\x05\x16\x07")
    (d / "bge.gguf").write_bytes(b"")
    assert Path(LlamaServerBackend._resolve_local_gguf(str(d))).name == "bge.gguf"


def test_a_wsl_drive_letter_dir_resolves_like_the_other_local_probes(monkeypatch, tmp_path):
    """Drive-letter folders must resolve under WSL via normalize_path, as the other local probes do."""
    from utils.paths import path_utils

    drive = tmp_path / "mnt" / "c" / "models"
    drive.mkdir(parents = True)
    (drive / "bge-F16.gguf").write_bytes(b"")
    monkeypatch.setattr(config, "EMBED_GGUF_VARIANT", "F16")
    monkeypatch.setattr(path_utils, "_IS_WSL", True)
    monkeypatch.setattr(
        path_utils,
        "normalize_path",
        lambda p: (
            str(tmp_path / "mnt" / p[0].lower() / p[3:].replace("\\", "/"))
            if len(p) >= 3 and p[1] == ":"
            else p
        ),
    )
    import utils.paths as paths_pkg

    monkeypatch.setattr(paths_pkg, "normalize_path", path_utils.normalize_path)

    picked = LlamaServerBackend._resolve_local_gguf(r"C:\models")
    assert picked is not None and Path(picked).name == "bge-F16.gguf"


def test_a_posix_path_is_untouched_by_the_normalization(tmp_path):
    """The rewrite must not start mangling ordinary POSIX paths."""
    d = tmp_path / "plain"
    d.mkdir()
    (d / "bge.gguf").write_bytes(b"")
    assert Path(LlamaServerBackend._resolve_local_gguf(str(d))).name == "bge.gguf"


def test_a_stand_in_quant_does_not_retire_the_pending_marker(monkeypatch, tmp_path):
    """A stand-in quant is a leftover from an earlier setting; clearing the marker would pin it for good."""
    cleared, ems, repo = _shared_setup_1(monkeypatch)
    monkeypatch.setattr(ems, "clear_stored_download_pending", lambda model: cleared.append(model))
    _seed_cache(tmp_path / "hub", repo, ["pending-Q8_0.gguf"])
    _shared_setup_4(monkeypatch, tmp_path)
    assert cleared == [], "a stand-in quant is not the transfer that was advertised"


def test_the_planned_variant_landing_retires_the_pending_marker(monkeypatch, tmp_path):
    """The planned variant landing in the preferred repo answers the marker, so it is cleared."""
    cleared, ems, repo = _shared_setup_1(monkeypatch)
    monkeypatch.setattr(ems, "clear_stored_download_pending", lambda model: cleared.append(model))
    _seed_cache(tmp_path / "hub", repo, ["pending-F16.gguf"])
    _use_cache_root(monkeypatch, tmp_path / "hub")
    backend = LlamaServerBackend()

    assert backend._resolve_model_path().endswith("pending-F16.gguf")
    assert cleared == ["org/pending"]


def test_the_planned_fallback_quant_landing_retires_the_pending_marker(monkeypatch, tmp_path):
    """Fallback quant landings must clear the pending marker, or eviction later reports not downloaded."""
    cleared, ems, repo = _shared_setup_1(monkeypatch)
    monkeypatch.setattr(ems, "get_stored_gguf_files", lambda model: ["pending-Q8_0.gguf"])
    monkeypatch.setattr(ems, "clear_stored_download_pending", lambda model: cleared.append(model))
    _seed_cache(tmp_path / "hub", repo, ["pending-Q8_0.gguf"])
    _shared_setup_4(monkeypatch, tmp_path)
    assert cleared == ["org/pending"]


def test_a_pinned_request_serves_the_pinned_models_gguf_not_the_live_setting(monkeypatch):
    """One subprocess serves one GGUF, so a pinned request must not read the live setting."""
    from core.rag import config

    b = LlamaServerBackend()
    monkeypatch.setattr(config, "effective_embedding_model", lambda: "org/model-b")
    monkeypatch.setattr(
        config,
        "effective_gguf_repo_for_embedding_model",
        lambda model: f"{model}-GGUF",
    )
    monkeypatch.setattr(b, "_process_alive", lambda: True)
    monkeypatch.setattr(b, "_binary_path_revision", None, raising = False)
    import utils.llama_cpp_path_settings as paths

    monkeypatch.setattr(paths, "custom_llama_cpp_path_revision", lambda: None)

    b._model_repo = "org/model-a-GGUF"

    assert b._current() is False
    assert b._current("org/model-a") is True
    assert b._current("org/model-b") is False


def test_resolution_follows_the_pinned_model(monkeypatch, tmp_path):
    """_resolve_model_path derived the repo and the local-path probe from the live
    setting, so the pinned model reached neither."""
    from core.rag import config

    gguf = tmp_path / "pinned.gguf"
    gguf.write_bytes(b"GGUF")
    b = LlamaServerBackend()
    monkeypatch.setattr(config, "effective_embedding_model", lambda: "org/model-b")
    monkeypatch.setattr(
        config, "effective_gguf_repo_for_embedding_model", lambda model: f"{model}-GGUF"
    )
    seen = []
    monkeypatch.setattr(
        LlamaServerBackend,
        "_resolve_local_gguf",
        staticmethod(
            lambda model: seen.append(model) or (str(gguf) if "model-a" in model else None)
        ),
    )

    assert b._resolve_model_path("org/model-a") == str(gguf)
    assert seen == ["org/model-a"]
    # Tagged with the repo it was resolved for, so an unpinned request sees it as stale.
    assert b._model_repo == "org/model-a-GGUF"


def test_a_swap_cannot_land_between_readiness_and_the_request(monkeypatch):
    """Readiness and the POST were two steps, so a swap in between landed A's
    request on B's server, storing B's vectors under A's identity."""
    import threading

    b = LlamaServerBackend()
    order: list[str] = []
    swapped = threading.Event()

    def fake_ensure_ready(model_name = None):
        order.append(f"ready:{model_name}")

    def fake_post(
        url,
        json = None,
        headers = None,
    ):  # noqa: A002 - httpx's parameter name
        order.append("post")
        swapper = threading.Thread(target = _swap)
        swapper.start()
        swapper.join(timeout = 0.5)
        order.append("post-done")

        class _R:
            @staticmethod
            def raise_for_status():
                return None

            @staticmethod
            def json():
                return {"tokens": [1]}

        return _R()

    def _swap():
        with b._serve_lock:
            order.append("swap")
            swapped.set()

    monkeypatch.setattr(b, "_ensure_ready", fake_ensure_ready)
    monkeypatch.setattr(b._client, "post", fake_post)
    monkeypatch.setattr(b, "_port", 1)

    b._post("/tokenize", {"content": "x"}, model_name = "org/model-a")
    swapped.wait(timeout = 2)

    assert order[:3] == ["ready:org/model-a", "post", "post-done"]
    assert order[-1] == "swap"


def test_a_dim_probe_is_serialized_with_a_model_switch(monkeypatch):
    """The dim probe shares the model-switch lock; _dim belongs to whichever GGUF is loaded."""
    b = LlamaServerBackend()
    monkeypatch.setattr(b, "_ensure_ready", lambda model_name = None: None)
    observed = {}

    def _probe_encode(
        texts,
        normalize = False,
        model_name = None,
    ):
        acquired = []
        switcher = threading.Thread(
            target = lambda: acquired.append(b._serve_lock.acquire(timeout = 0.5))
        )
        switcher.start()
        switcher.join()
        if acquired[0]:
            b._serve_lock.release()
        observed["excluded"] = not acquired[0]
        return np.zeros((1, 384), dtype = np.float32)

    monkeypatch.setattr(b, "encode", _probe_encode)

    assert b.dim() == 384
    assert observed["excluded"] is True, "a model switch could land inside the probe"


def test_the_runtime_probe_does_not_wait_on_a_model_load(monkeypatch):
    """The runtime probe must not wait on the load lock, or Settings stalls for the whole first load."""
    monkeypatch.setattr(embeddings, "_load_device", lambda: "cpu")
    finished = threading.Event()

    def _probe():
        embeddings.sentence_transformers_runtime_available()
        finished.set()

    with embeddings._lock:
        worker = threading.Thread(target = _probe, daemon = True)
        worker.start()
        assert finished.wait(timeout = 60), "the resolve preflight waited on the model-load lock"


def test_a_stale_quant_in_another_directory_is_not_the_planned_family(monkeypatch, tmp_path):
    """A repo that files each quant in its own directory publishes Q8_0/model.gguf
    beside Q4_K_M/model.gguf. Matching the plan on base names alone let the stale
    one pass for the planned one, retiring the marker and pinning the wrong quant."""
    cleared, ems, repo = _shared_setup_1(monkeypatch)
    monkeypatch.setattr(ems, "get_stored_gguf_files", lambda model: ["Q8_0/model.gguf"])
    monkeypatch.setattr(ems, "clear_stored_download_pending", lambda model: cleared.append(model))
    _seed_cache(tmp_path / "hub", repo, ["Q4_K_M/model.gguf"])
    _use_cache_root(monkeypatch, tmp_path / "hub")
    backend = LlamaServerBackend()

    assert backend._resolve_model_path().endswith("model.gguf")
    assert cleared == [], "a quant from another directory is not the planned family"


def test_the_planned_family_matches_by_its_directory_too(monkeypatch, tmp_path):
    """The other half: the planned copy in its own directory still retires it."""
    cleared, ems, repo = _shared_setup_1(monkeypatch)
    monkeypatch.setattr(ems, "get_stored_gguf_files", lambda model: ["Q8_0/model.gguf"])
    monkeypatch.setattr(ems, "clear_stored_download_pending", lambda model: cleared.append(model))
    _seed_cache(tmp_path / "hub", repo, ["Q8_0/model.gguf"])
    _use_cache_root(monkeypatch, tmp_path / "hub")
    backend = LlamaServerBackend()

    assert backend._resolve_model_path().endswith("model.gguf")
    assert cleared == ["org/pending"]


def test_warming_one_model_cannot_respawn_under_another_models_request(monkeypatch):
    """Readying the server outside the lock could respawn it under another model's request."""
    b = LlamaServerBackend()
    unlocked = []

    def _lock_is_free() -> bool:
        # Ownership is per thread for an RLock, so ask from another one.
        seen = []
        probe = threading.Thread(target = lambda: seen.append(b._serve_lock.acquire(timeout = 0.5)))
        probe.start()
        probe.join()
        if seen[0]:
            b._serve_lock.release()
        return seen[0]

    def _recording_ensure_ready(model_name = None):
        if _lock_is_free():
            unlocked.append(model_name)

    monkeypatch.setattr(b, "_ensure_ready", _recording_ensure_ready)
    monkeypatch.setattr(
        b,
        "encode",
        lambda texts, normalize = False, model_name = None: np.zeros((1, 384), dtype = np.float32),
    )
    b.warm(model_name = "org/model-b")

    assert unlocked == [], "a respawn during warm-up could land outside the serving lock"


def test_the_planned_family_outranks_a_variant_that_arrives_later(monkeypatch, tmp_path):
    """Identity records model and repo, not the file; a later variant must not change weights unnoticed."""
    ems, repo = _shared_setup_3(monkeypatch)
    monkeypatch.setattr(ems, "get_stored_download_pending", lambda model: False)
    monkeypatch.setattr(ems, "get_stored_gguf_files", lambda model: ["pending-Q8_0.gguf"])
    monkeypatch.setattr(ems, "clear_stored_download_pending", lambda model: False)
    _seed_cache(tmp_path / "hub", repo, ["pending-Q8_0.gguf", "pending-F16.gguf"])
    _shared_setup_4(monkeypatch, tmp_path)


def test_a_partly_present_planned_family_is_not_served(monkeypatch, tmp_path):
    """Whole family or nothing: a torn set entered at shard 1 cannot be opened by llama-server."""
    ems, repo = _shared_setup_3(monkeypatch)
    monkeypatch.setattr(ems, "get_stored_download_pending", lambda model: False)
    monkeypatch.setattr(
        ems,
        "get_stored_gguf_files",
        lambda model: ["p-00001-of-00002.gguf", "p-00002-of-00002.gguf"],
    )
    _seed_cache(tmp_path / "hub", repo, ["p-00001-of-00002.gguf", "pending-F16.gguf"])
    _use_cache_root(monkeypatch, tmp_path / "hub")
    backend = LlamaServerBackend()

    assert backend._planned_family_path("org/pending", repo) is None
    assert backend._resolve_model_path().endswith("pending-F16.gguf")


def _write_gguf(
    path,
    arch,
    pooling = None,
):
    kvs = [
        struct.pack("<Q", 20)
        + b"general.architecture"
        + struct.pack("<IQ", 8, len(arch))
        + arch.encode()
    ]
    if pooling is not None:
        key = f"{arch}.pooling_type".encode()
        kvs.append(struct.pack("<Q", len(key)) + key + struct.pack("<II", 4, pooling))
    Path(path).write_bytes(b"GGUF" + struct.pack("<IQQ", 3, 0, len(kvs)) + b"".join(kvs))


@pytest.mark.parametrize(
    "pooling, expected",
    [(3, "last"), (1, "mean"), (2, "cls"), (None, "cls"), (0, "cls")],
)
def test_build_cmd_serves_the_pooling_the_gguf_declares(tmp_path, pooling, expected):
    """Forcing CLS on a last-token (Qwen3-Embedding) or mean (nomic) GGUF pooled one
    token, so unrelated sentences embedded nearly alike. A GGUF declaring none keeps
    CLS, since llama-server would fall back to NONE and /v1/embeddings refuses that."""
    path = tmp_path / "embed.gguf"
    _write_gguf(path, "qwen3", pooling)
    cmd = LlamaServerBackend()._build_cmd("/bin/llama-server", str(path), 1, use_gpu = False)
    assert cmd[cmd.index("--pooling") + 1] == expected


@pytest.mark.parametrize("arch", ["nomic-bert", "nomic-bert-moe"])
def test_build_cmd_uses_mean_for_older_nomic_ggufs_without_pooling_metadata(tmp_path, arch):
    """Dedicated Nomic embedding architectures predate reliable pooling metadata, but
    their model family is mean-pooled. An explicit unsupported value still falls back safely."""
    path = tmp_path / "nomic.gguf"
    _write_gguf(path, arch)
    backend = LlamaServerBackend()
    cmd = backend._build_cmd("/bin/llama-server", str(path), 1, use_gpu = False)
    assert cmd[cmd.index("--pooling") + 1] == "mean"
    assert backend._adopt_model_path(str(path), "org/nomic-GGUF") == str(path)
    assert backend._model_pooling == "mean"

    _write_gguf(path, arch, 0)
    cmd = backend._build_cmd("/bin/llama-server", str(path), 1, use_gpu = False)
    assert cmd[cmd.index("--pooling") + 1] == "cls"


@pytest.mark.parametrize("pooling, suffix", [(3, ":last"), (1, ":mean"), (2, "")])
def test_llama_identity_changes_only_for_a_non_cls_gguf(monkeypatch, tmp_path, pooling, suffix):
    """Vectors indexed under the forced CLS must read as stale once a mean or last
    GGUF is served natively, while a CLS GGUF such as bge-small keeps its identity."""
    model, repo = "org/embed", "org/embed-GGUF"
    monkeypatch.setattr(config, "effective_gguf_repo_for_embedding_model", lambda m: repo)
    snapshot = _seed_cache(tmp_path / "hub", repo, ["embed-F16.gguf"])
    _write_gguf((snapshot / "embed-F16.gguf").resolve(), "qwen3", pooling)
    _use_cache_root(monkeypatch, tmp_path / "hub")
    legacy = config.embedding_identity("llama-server", model, gguf_repo = repo)
    assert embeddings._identity(True, model) == legacy + suffix


def test_llama_identity_follows_the_served_gguf_after_the_cache_moves(monkeypatch, tmp_path):
    """Moving the HF cache in Settings leaves the server on the old file, so a fresh
    cache search dropped the ``:last`` suffix and tagged last-pooled vectors as CLS."""
    model, repo = "org/embed", "org/embed-GGUF"
    monkeypatch.setattr(config, "effective_gguf_repo_for_embedding_model", lambda m: repo)
    snapshot = _seed_cache(tmp_path / "hub-a", repo, ["embed-F16.gguf"])
    served = (snapshot / "embed-F16.gguf").resolve()
    _write_gguf(served, "qwen3", 3)
    backend = LlamaServerBackend()
    backend._model_path, backend._model_repo = str(served), repo
    monkeypatch.setattr(embeddings, "_backend", backend)
    (tmp_path / "hub-b").mkdir()
    _use_cache_root(monkeypatch, tmp_path / "hub-b")
    legacy = config.embedding_identity("llama-server", model, gguf_repo = repo)
    assert embeddings._identity(True, model) == legacy + ":last"
    assert embeddings._identity(True, model, backend) == legacy + ":last"


def test_llama_identity_with_no_gguf_on_disk_matches_no_forced_cls_row(monkeypatch, tmp_path):
    """With the GGUF evicted and no server up, the prediction fell back to the forced-CLS
    identity, so a re-upload deduplicated onto those stale vectors instead of re-indexing."""
    model, repo = "org/embed", "org/embed-GGUF"
    monkeypatch.setattr(config, "effective_gguf_repo_for_embedding_model", lambda m: repo)
    (tmp_path / "hub").mkdir()
    _use_cache_root(monkeypatch, tmp_path / "hub")
    legacy = config.embedding_identity("llama-server", model, gguf_repo = repo)
    predicted = embeddings._identity(True, model)
    assert not config.embedding_identity_matches(legacy, predicted)
    assert config.embedding_identity_model(predicted) == model


def test_llama_identity_keeps_the_launched_pooling_when_the_file_vanishes(monkeypatch, tmp_path):
    """A running server keeps serving its open GGUF after the cache entry is evicted, so
    re-reading the gone path answered CLS and tagged last-pooled vectors as CLS."""
    model, repo = "org/embed", "org/embed-GGUF"
    monkeypatch.setattr(config, "effective_gguf_repo_for_embedding_model", lambda m: repo)
    snapshot = _seed_cache(tmp_path / "hub", repo, ["embed-F16.gguf"])
    served = (snapshot / "embed-F16.gguf").resolve()
    _write_gguf(served, "qwen3", 3)
    _use_cache_root(monkeypatch, tmp_path / "hub")
    backend = LlamaServerBackend()
    backend._adopt_model_path(str(served), repo)
    monkeypatch.setattr(embeddings, "_backend", backend)
    served.unlink()
    legacy = config.embedding_identity("llama-server", model, gguf_repo = repo)
    assert embeddings._identity(True, model, backend) == legacy + ":last"


def test_llama_identity_rechecks_a_stopped_backends_replaced_gguf(monkeypatch, tmp_path):
    """Before an encode, a stopped server will respawn from the current file and must not
    deduplicate with its old pooling. A completed encode still keeps the value it actually used."""
    model, repo = "org/embed", "org/embed-GGUF"
    monkeypatch.setattr(config, "effective_gguf_repo_for_embedding_model", lambda m: repo)
    snapshot = _seed_cache(tmp_path / "hub", repo, ["embed-F16.gguf"])
    served = (snapshot / "embed-F16.gguf").resolve()
    _write_gguf(served, "qwen3", 3)
    _use_cache_root(monkeypatch, tmp_path / "hub")
    backend = LlamaServerBackend()
    backend._adopt_model_path(str(served), repo)
    monkeypatch.setattr(embeddings, "_backend", backend)
    _write_gguf(served, "qwen3", 1)
    legacy = config.embedding_identity("llama-server", model, gguf_repo = repo)

    assert embeddings._identity(True, model) == legacy + ":mean"
    assert embeddings._identity(True, model, backend) == legacy + ":last"


def test_llama_identity_treats_an_evicted_stopped_model_as_unresolved(monkeypatch, tmp_path):
    """A stopped backend cannot keep serving a deleted path. Pre-encode deduplication must
    miss old CLS rows, and the next spawn must run the normal resolver instead of that path."""
    model, repo = "org/embed", "org/embed-GGUF"
    monkeypatch.setattr(config, "effective_gguf_repo_for_embedding_model", lambda m: repo)
    snapshot = _seed_cache(tmp_path / "hub", repo, ["embed-F16.gguf"])
    served = (snapshot / "embed-F16.gguf").resolve()
    _write_gguf(served, "qwen3", 2)
    _use_cache_root(monkeypatch, tmp_path / "hub")
    backend = LlamaServerBackend()
    backend._adopt_model_path(str(served), repo)
    monkeypatch.setattr(embeddings, "_backend", backend)
    served.unlink()
    legacy = config.embedding_identity("llama-server", model, gguf_repo = repo)

    assert embeddings._identity(True, model) == legacy + ":unresolved"
    monkeypatch.setattr(backend, "_resolve_uncached_model_path", lambda *args: "/fresh.gguf")
    assert backend._resolve_model_path(model) == "/fresh.gguf"


def test_preencode_identity_takes_one_locked_backend_snapshot(monkeypatch):
    """A concurrent model switch cannot combine model A's repo check with model B's pooling."""

    class SwitchingBackend(LlamaServerBackend):
        def __init__(self):
            self.gate = False
            self.repo_read = threading.Event()
            self.allow_repo_read = threading.Event()
            super().__init__()

        @property
        def _model_repo(self):
            value = self._repo_value
            if self.gate:
                self.repo_read.set()
                assert self.allow_repo_read.wait(timeout = 2)
            return value

        @_model_repo.setter
        def _model_repo(self, value):
            self._repo_value = value

        @property
        def _model_pooling(self):
            return self._pooling_value

        @_model_pooling.setter
        def _model_pooling(self, value):
            self._pooling_value = value

    model, repo_a, repo_b = "org/a", "org/a-GGUF", "org/b-GGUF"
    backend = SwitchingBackend()
    backend._model_path = "/served/a.gguf"
    backend._model_repo = repo_a
    backend._model_pooling = "last"
    backend._process = _FakeProc(alive = True)
    backend.gate = True
    monkeypatch.setattr(config, "effective_gguf_repo_for_embedding_model", lambda _model: repo_a)
    monkeypatch.setattr(embeddings, "_backend", backend)
    result = {}
    switched_before_snapshot = []

    def predict():
        result["identity"] = embeddings._identity(True, model)

    def switch_model():
        assert backend.repo_read.wait(timeout = 2)
        acquired = backend._serve_lock.acquire(timeout = 0.05)
        switched_before_snapshot.append(acquired)
        if acquired:
            backend._repo_value = repo_b
            backend._pooling_value = "mean"
            backend._serve_lock.release()
        backend.allow_repo_read.set()
        if not acquired:
            with backend._serve_lock:
                backend._repo_value = repo_b
                backend._pooling_value = "mean"

    reader = threading.Thread(target = predict)
    writer = threading.Thread(target = switch_model)
    reader.start()
    writer.start()
    reader.join(timeout = 2)
    writer.join(timeout = 2)
    backend._process = None

    assert not reader.is_alive() and not writer.is_alive()
    assert switched_before_snapshot == [False]
    assert result["identity"] == config.embedding_identity(
        "llama-server", model, gguf_repo = repo_a, pooling = "last"
    )


def test_encode_identity_keeps_the_callers_snapshot_when_another_model_takes_over(monkeypatch):
    """The shared backend can switch models after an encode returns. Each thread must retain
    the immutable repo and pooling that served its own vectors, not the backend's later state."""
    backend = LlamaServerBackend()
    model_a, model_b = "org/a", "org/b"
    repo_a, repo_b = "org/a-GGUF", "org/b-GGUF"
    identities = {}
    first_waiting = threading.Event()
    second_done = threading.Event()

    monkeypatch.setattr(embeddings, "_get_backend", lambda _model = None: backend)
    monkeypatch.setattr(
        config,
        "effective_gguf_repo_for_embedding_model",
        lambda model: repo_a if model == model_a else repo_b,
    )

    def fake_ready(model):
        backend._model_repo = repo_a if model == model_a else repo_b
        backend._model_pooling = "last" if model == model_a else "mean"

    monkeypatch.setattr(backend, "_ensure_ready", fake_ready)

    def fake_encode_active(
        texts,
        *,
        model_name = None,
        **_kwargs,
    ):
        backend._ensure_ready(model_name)
        return np.zeros((len(texts), 2), dtype = np.float32)

    monkeypatch.setattr(backend, "_encode_active", fake_encode_active)
    real_identity = embeddings._identity

    def delayed_identity(
        is_llama,
        name,
        served = None,
        served_identity = None,
    ):
        if name == model_a:
            first_waiting.set()
            assert second_done.wait(timeout = 2)
        else:
            second_done.set()
        if served_identity is None:
            return real_identity(is_llama, name, served)
        return real_identity(is_llama, name, served, served_identity)

    monkeypatch.setattr(embeddings, "_identity", delayed_identity)

    def run(model):
        identities[model] = embeddings.encode_with_identity(["x"], model_name = model)[1]

    first = threading.Thread(target = run, args = (model_a,))
    second = threading.Thread(target = run, args = (model_b,))
    first.start()
    assert first_waiting.wait(timeout = 2)
    second.start()
    first.join(timeout = 2)
    second.join(timeout = 2)

    assert not first.is_alive() and not second.is_alive()
    assert identities[model_a] == config.embedding_identity(
        "llama-server", model_a, gguf_repo = repo_a, pooling = "last"
    )
    assert identities[model_b] == config.embedding_identity(
        "llama-server", model_b, gguf_repo = repo_b, pooling = "mean"
    )


def test_llama_identity_is_not_predicted_from_a_fallback_repo(monkeypatch, tmp_path):
    """Only the desired repo's cache is read before going online; a fallback's GGUF may differ."""
    model, repo = "org/embed", "org/embed-GGUF"
    monkeypatch.setattr(config, "effective_gguf_repo_for_embedding_model", lambda m: repo)
    snapshot = _seed_cache(tmp_path / "hub", model, ["embed-F16.gguf"])
    _write_gguf((snapshot / "embed-F16.gguf").resolve(), "qwen3", 3)
    _use_cache_root(monkeypatch, tmp_path / "hub")
    legacy = config.embedding_identity("llama-server", model, gguf_repo = repo)
    assert embeddings._identity(True, model) == legacy + ":unresolved"


def test_llama_identity_follows_the_planned_family_before_a_later_variant(monkeypatch, tmp_path):
    """The loader pins a completed download family before considering the configured
    variant, so prediction must not take pooling from a different file that arrived later."""
    import utils.embedding_model_settings as ems

    model, repo = "org/embed", "org/embed-GGUF"
    monkeypatch.setattr(config, "effective_gguf_repo_for_embedding_model", lambda m: repo)
    monkeypatch.setattr(config, "EMBED_GGUF_VARIANT", "F16")
    monkeypatch.setattr(ems, "get_stored_gguf_files", lambda m: ["embed-Q8_0.gguf"])
    snapshot = _seed_cache(tmp_path / "hub", repo, ["embed-Q8_0.gguf", "embed-F16.gguf"])
    _write_gguf((snapshot / "embed-Q8_0.gguf").resolve(), "qwen3", 3)
    _write_gguf((snapshot / "embed-F16.gguf").resolve(), "bert", 2)
    _use_cache_root(monkeypatch, tmp_path / "hub")
    legacy = config.embedding_identity("llama-server", model, gguf_repo = repo)

    assert embeddings._identity(True, model) == legacy + ":last"


@pytest.mark.parametrize("download_pending, suffix", [(False, ":unresolved"), (True, ":last")])
def test_llama_identity_only_predicts_a_stand_in_quant_while_its_download_is_pending(
    monkeypatch, tmp_path, download_pending, suffix
):
    """Without the configured variant, the loader goes online unless an active download
    makes the cached quant a deliberate stand-in. Only that stand-in is predictable."""
    import utils.embedding_model_settings as ems

    model, repo = "org/embed", "org/embed-GGUF"
    monkeypatch.setattr(config, "effective_gguf_repo_for_embedding_model", lambda m: repo)
    monkeypatch.setattr(config, "EMBED_GGUF_VARIANT", "F16")
    monkeypatch.setattr(ems, "get_stored_gguf_files", lambda m: None)
    monkeypatch.setattr(ems, "get_stored_download_pending", lambda m: download_pending)
    snapshot = _seed_cache(tmp_path / "hub", repo, ["embed-Q8_0.gguf"])
    _write_gguf((snapshot / "embed-Q8_0.gguf").resolve(), "qwen3", 3)
    _use_cache_root(monkeypatch, tmp_path / "hub")
    legacy = config.embedding_identity("llama-server", model, gguf_repo = repo)

    assert embeddings._identity(True, model) == legacy + suffix


def test_spawn_uses_a_per_spawn_key_that_requests_send(monkeypatch, tmp_path):
    import utils.paths.storage_roots as roots

    monkeypatch.setattr(roots, "auth_root", lambda: tmp_path)
    monkeypatch.delenv("UNSLOTH_LLAMA_SERVER_API_KEY", raising = False)
    monkeypatch.setattr(config, "EMBED_PORT", 0)
    b = LlamaServerBackend()
    proc = _FakeProc(alive = True)
    captured, keys = {}, []
    _patch_spawn_deps(monkeypatch, proc)
    _intercept_server_popen(monkeypatch, proc, captured)

    def healthy(*_a, **_k):
        cmd = captured["cmd"]
        keys.append(Path(cmd[cmd.index("--api-key-file") + 1]).read_text())
        return True

    monkeypatch.setattr(b, "_wait_for_health", healthy)
    b._spawn()
    assert keys == [b._api_key] and b._auth_headers() == {"Authorization": f"Bearer {b._api_key}"}
    # llama-server read it at startup; nothing is left on disk.
    assert not list(tmp_path.glob("llama_api_key_*"))