Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""llama-server GGUF embedder tests, every boundary mocked."""
import os
import struct
import subprocess
import sys
import textwrap
import threading
import time
import types
from pathlib import Path
import numpy as np
import pytest
from core.rag import config, embeddings
from core.rag import embed_llama_server as mod
from core.rag.embed_llama_server import LlamaServerBackend
def _shared_setup_1(monkeypatch):
import utils.embedding_model_settings as ems
repo = "org/pending-GGUF"
cleared = []
monkeypatch.setattr(config, "effective_embedding_model", lambda: "org/pending")
monkeypatch.setattr(config, "effective_gguf_repo", lambda: repo)
monkeypatch.setattr(config, "EMBED_GGUF_VARIANT", "F16")
monkeypatch.setattr(ems, "get_stored_download_pending", lambda model: True)
return cleared, ems, repo
def _shared_setup_2(monkeypatch, tmp_path):
import contextlib
import huggingface_hub
from core.inference import llama_cpp
repo = config.effective_gguf_repo()
_seed_cache(tmp_path / "hub", repo, ["bge-Q8_0.gguf"])
_use_cache_root(monkeypatch, tmp_path / "hub")
monkeypatch.setattr(config, "EMBED_GGUF_VARIANT", "F16")
monkeypatch.setattr(llama_cpp, "_hf_offline_if_unreachable", contextlib.nullcontext)
return huggingface_hub
def _shared_setup_3(monkeypatch):
import utils.embedding_model_settings as ems
repo = "org/pending-GGUF"
monkeypatch.setattr(config, "effective_embedding_model", lambda: "org/pending")
monkeypatch.setattr(config, "effective_gguf_repo", lambda: repo)
monkeypatch.setattr(config, "EMBED_GGUF_VARIANT", "F16")
return ems, repo
def _shared_setup_4(monkeypatch, tmp_path):
_use_cache_root(monkeypatch, tmp_path / "hub")
backend = LlamaServerBackend()
assert backend._resolve_model_path().endswith("pending-Q8_0.gguf")
@pytest.fixture(autouse = True)
def _reset_backend_singleton():
embeddings._reset_backend()
yield
embeddings._reset_backend()
class _FakeProc:
"""subprocess.Popen stand-in with controllable liveness."""
def __init__(
self,
alive = True,
returncode = 0,
):
self._alive = alive
self.returncode = returncode
self.pid = 424242
self.stdout = iter(())
def poll(self):
return None if self._alive else self.returncode
def terminate(self):
self._alive = False
def kill(self):
self._alive = False
def wait(self, timeout = None):
return self.returncode
def _mock_auto(monkeypatch, *, gpus, binary):
from core.inference.llama_cpp import LlamaCppBackend
monkeypatch.setattr(config, "EMBED_BACKEND", "auto")
monkeypatch.setattr(LlamaCppBackend, "_get_gpu_free_memory", staticmethod(lambda **_kw: gpus))
monkeypatch.setattr(LlamaCppBackend, "_find_llama_server_binary", staticmethod(lambda: binary))
def _stub_st_load(monkeypatch):
# Fake the ST probe: sentence-transformers is absent in the torch-free backend CI.
monkeypatch.setattr(embeddings, "_get", lambda *a, **k: object())
def test_auto_uses_st_with_cuda(monkeypatch):
_stub_st_load(monkeypatch)
_mock_auto(monkeypatch, gpus = [(0, 40000)], binary = "/bin/llama-server")
assert type(embeddings._get_backend()).__name__ == "_SentenceTransformersBackend"
def test_auto_uses_llama_without_cuda(monkeypatch):
_mock_auto(monkeypatch, gpus = [], binary = "/bin/llama-server")
assert isinstance(embeddings._get_backend(), LlamaServerBackend)
def test_auto_falls_back_to_st_without_binary(monkeypatch):
_stub_st_load(monkeypatch)
_mock_auto(monkeypatch, gpus = [], binary = None)
assert type(embeddings._get_backend()).__name__ == "_SentenceTransformersBackend"
def test_llama_backend_selected_by_config(monkeypatch):
monkeypatch.setattr(config, "EMBED_BACKEND", "llama-server")
assert isinstance(embeddings._get_backend(), LlamaServerBackend)
def test_unknown_backend_raises(monkeypatch):
monkeypatch.setattr(config, "EMBED_BACKEND", "bogus")
with pytest.raises(ValueError, match = "Unknown RAG_EMBED_BACKEND"):
embeddings._get_backend()
def test_shutdown_waits_for_an_active_embedding_operation(monkeypatch):
backend = LlamaServerBackend()
entered = threading.Event()
finish = threading.Event()
shutdown_done = threading.Event()
def _post(path, payload, **_kwargs):
entered.set()
assert finish.wait(timeout = 2)
return {"data": [{"index": 0, "embedding": [1.0, 0.0]}]}
monkeypatch.setattr(backend, "_post", _post)
worker = threading.Thread(target = lambda: backend.encode(["x"]))
worker.start()
assert entered.wait(timeout = 2)
closer = threading.Thread(target = lambda: (backend._shutdown(), shutdown_done.set()))
closer.start()
assert shutdown_done.wait(timeout = 0.05) is False
finish.set()
worker.join(timeout = 2)
closer.join(timeout = 2)
assert shutdown_done.is_set()
with pytest.raises(RuntimeError, match = "unloaded"):
backend.encode(["x"])
def test_explicit_backend_overrides_auto(monkeypatch):
_stub_st_load(monkeypatch)
monkeypatch.setattr(config, "EMBED_BACKEND", "sentence-transformers")
assert type(embeddings._get_backend()).__name__ == "_SentenceTransformersBackend"
monkeypatch.setattr(config, "EMBED_BACKEND", "llama-server")
assert isinstance(embeddings._get_backend(), LlamaServerBackend)
def test_llama_backend_imports_no_torch():
backend_dir = Path(__file__).resolve().parents[1]
code = textwrap.dedent(
"""
import sys
from core.rag import embeddings
b = embeddings._get_backend()
assert type(b).__name__ == "LlamaServerBackend", type(b).__name__
assert "torch" not in sys.modules, "torch was imported"
assert "sentence_transformers" not in sys.modules, "ST was imported"
print("OK")
"""
)
env = {
**__import__("os").environ,
"RAG_EMBED_BACKEND": "llama-server",
"PYTHONPATH": str(backend_dir),
}
proc = subprocess.run([sys.executable, "-c", code], capture_output = True, text = True, env = env)
assert proc.returncode == 0, proc.stderr
assert "OK" in proc.stdout
def test_build_cmd_cpu_flags():
b = LlamaServerBackend()
cmd = b._build_cmd("/bin/llama-server", "/m/bge.gguf", 9999, use_gpu = False)
assert "--embedding" in cmd
assert cmd[cmd.index("--pooling") + 1] == "cls"
assert cmd[cmd.index("--fit") + 1] == "off"
assert cmd[cmd.index("-ngl") + 1] == "0"
assert cmd[cmd.index("--port") + 1] == "9999"
def test_build_cmd_gpu_offloads():
b = LlamaServerBackend()
cmd = b._build_cmd("/bin/llama-server", "/m/bge.gguf", 1, use_gpu = True)
assert cmd[cmd.index("-ngl") + 1] == "-1"
def test_build_env_cpu_hides_gpus():
b = LlamaServerBackend()
env = b._build_env("/bin/llama-server", use_gpu = False)
assert env["CUDA_VISIBLE_DEVICES"] == ""
assert env["LLAMA_SET_ROWS"] == "1"
def test_build_env_gpu_inherits_devices(monkeypatch):
monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0,1")
b = LlamaServerBackend()
env = b._build_env("/bin/llama-server", use_gpu = True)
assert env.get("CUDA_VISIBLE_DEVICES") == "0,1"
def test_build_env_gpu_on_macos_uses_the_dyld_search_path(monkeypatch, tmp_path):
# dyld ignores LD_LIBRARY_PATH and Apple Silicon takes the use_gpu branch.
import os
import sys as _sys
monkeypatch.setattr(_sys, "platform", "darwin")
monkeypatch.setenv("DYLD_LIBRARY_PATH", "/opt/inherited")
# The child env copies os.environ, so pin an inherited value rather than assert absence.
monkeypatch.setenv("LD_LIBRARY_PATH", "/opt/sentinel")
# Host-native dir: on a Windows runner '/opt/llama/bin' resolves drive-anchored.
bin_dir = tmp_path / "llama" / "bin"
bin_dir.mkdir(parents = True)
(bin_dir / "llama-server").write_bytes(b"\xcf\xfa\xed\xfe")
b = LlamaServerBackend()
env = b._build_env(str(bin_dir / "llama-server"), use_gpu = True)
entries = env["DYLD_LIBRARY_PATH"].split(os.pathsep)
assert entries == [str(bin_dir), "/opt/inherited"]
assert env["LD_LIBRARY_PATH"] == "/opt/sentinel"
def test_build_env_cpu_on_macos_still_gets_the_dyld_search_path(monkeypatch, tmp_path):
# The CPU paths load the same sibling dylibs; the loader path cannot be gated on use_gpu.
import os
import sys as _sys
monkeypatch.setattr(_sys, "platform", "darwin")
monkeypatch.delenv("DYLD_LIBRARY_PATH", raising = False)
bin_dir = tmp_path / "llama" / "bin"
bin_dir.mkdir(parents = True)
(bin_dir / "llama-server").write_bytes(b"\xcf\xfa\xed\xfe")
b = LlamaServerBackend()
env = b._build_env(str(bin_dir / "llama-server"), use_gpu = False)
assert env["DYLD_LIBRARY_PATH"].split(os.pathsep)[0] == str(bin_dir)
assert env["CUDA_VISIBLE_DEVICES"] == ""
def test_build_env_resolves_a_wrapper_entrypoint(monkeypatch, tmp_path):
# The managed install wraps the server; dylibs sit next to the target, not the wrapper.
import os
import sys as _sys
real_dir = tmp_path / "llama.cpp" / "build" / "bin"
real_dir.mkdir(parents = True)
(real_dir / "llama-server").write_text("")
wrapper = tmp_path / "llama.cpp" / "llama-server"
wrapper.write_text('#!/bin/sh\nexec "$(dirname "$0")/build/bin/llama-server" "$@"\n')
monkeypatch.setattr(_sys, "platform", "darwin")
monkeypatch.delenv("DYLD_LIBRARY_PATH", raising = False)
b = LlamaServerBackend()
env = b._build_env(str(wrapper), use_gpu = True)
assert env["DYLD_LIBRARY_PATH"].split(os.pathsep)[0] == str(real_dir)
def test_use_gpu_explicit_modes(monkeypatch):
b = LlamaServerBackend()
monkeypatch.setattr(config, "EMBED_DEVICE", "gpu")
assert b._use_gpu() is True
monkeypatch.setattr(config, "EMBED_DEVICE", "cpu")
assert b._use_gpu() is False
def test_use_gpu_auto_follows_probe(monkeypatch):
b = LlamaServerBackend()
monkeypatch.setattr(config, "EMBED_DEVICE", "auto")
monkeypatch.setattr(LlamaServerBackend, "_gpu_available", staticmethod(lambda: True))
assert b._use_gpu() is True
monkeypatch.setattr(LlamaServerBackend, "_gpu_available", staticmethod(lambda: False))
assert b._use_gpu() is False
def test_use_gpu_sticky_cpu_fallback(monkeypatch):
b = LlamaServerBackend()
monkeypatch.setattr(config, "EMBED_DEVICE", "auto")
monkeypatch.setattr(LlamaServerBackend, "_gpu_available", staticmethod(lambda: True))
b._force_cpu = True
assert b._use_gpu() is False
def test_gpu_available_reuses_studio_probe(monkeypatch):
import utils.hardware as uh
from core.inference.llama_cpp import LlamaCppBackend
monkeypatch.setattr(uh, "is_apple_silicon", lambda: False)
monkeypatch.setattr(
LlamaCppBackend, "_get_gpu_free_memory", staticmethod(lambda **_kw: [(0, 40000)])
)
assert LlamaServerBackend._gpu_available() is True
monkeypatch.setattr(
LlamaCppBackend, "_get_gpu_free_memory", staticmethod(lambda **_kw: [(0, 100)])
)
assert LlamaServerBackend._gpu_available() is False
monkeypatch.setattr(LlamaCppBackend, "_get_gpu_free_memory", staticmethod(lambda **_kw: []))
assert LlamaServerBackend._gpu_available() is False
def test_gpu_available_apple_metal(monkeypatch):
import utils.hardware as uh
monkeypatch.setattr(uh, "is_apple_silicon", lambda: True)
assert LlamaServerBackend._gpu_available() is True
_REAL_POPEN = subprocess.Popen
_FAKE_BINARY = "/bin/llama-server"
def _intercept_server_popen(
monkeypatch,
proc,
captured = None,
):
"""Answer only the embed server's own launch; a process-wide Popen patch replaced other threads'
argv."""
def popen(cmd, *args, **kwargs):
if isinstance(cmd, (list, tuple)) and cmd and cmd[0] == _FAKE_BINARY:
if captured is not None:
captured["cmd"] = list(cmd)
return proc
return _REAL_POPEN(cmd, *args, **kwargs)
monkeypatch.setattr(mod.subprocess, "Popen", popen)
def _patch_spawn_deps(
monkeypatch,
proc,
*,
free_port = 54321,
):
monkeypatch.setattr(config, "EMBED_DEVICE", "cpu")
monkeypatch.setattr(LlamaServerBackend, "_resolve_binary", lambda self: _FAKE_BINARY)
monkeypatch.setattr(
LlamaServerBackend, "_resolve_model_path", lambda self, model_name = None: "/m/bge.gguf"
)
monkeypatch.setattr(LlamaServerBackend, "_find_free_port", staticmethod(lambda: free_port))
_intercept_server_popen(monkeypatch, proc)
def test_spawn_uses_explicit_port(monkeypatch):
monkeypatch.setattr(config, "EMBED_PORT", 8123)
b = LlamaServerBackend()
_patch_spawn_deps(monkeypatch, _FakeProc(alive = True))
monkeypatch.setattr(b, "_wait_for_health", lambda *a, **k: True)
b._spawn()
assert b._port == 8123
def test_spawn_uses_free_port_when_auto(monkeypatch):
monkeypatch.setattr(config, "EMBED_PORT", 0)
b = LlamaServerBackend()
_patch_spawn_deps(monkeypatch, _FakeProc(alive = True), free_port = 47000)
monkeypatch.setattr(b, "_wait_for_health", lambda *a, **k: True)
b._spawn()
assert b._port == 47000
def test_spawn_refreshes_pooling_for_a_replaced_model(monkeypatch, tmp_path):
"""A cached path survives the reaper, but its file can be replaced between server
lifetimes. The relaunched command and the vectors' identity must capture the same pooling."""
path = tmp_path / "embed.gguf"
_write_gguf(path, "qwen3", 1)
backend = LlamaServerBackend()
backend._model_pooling = "last"
proc = _FakeProc(alive = True)
captured = {}
monkeypatch.setattr(backend, "_resolve_binary", lambda: _FAKE_BINARY)
monkeypatch.setattr(backend, "_resolve_model_path", lambda model_name = None: str(path))
monkeypatch.setattr(backend, "_find_free_port", lambda: 47000)
monkeypatch.setattr(backend, "_wait_for_health", lambda *a, **k: True)
monkeypatch.setattr(backend, "_build_env", lambda *a, **k: {})
_intercept_server_popen(monkeypatch, proc, captured)
backend._spawn_once(False)
assert captured["cmd"][captured["cmd"].index("--pooling") + 1] == "mean"
assert backend._model_pooling == "mean"
def test_a_process_started_elsewhere_during_spawn_does_not_replace_the_server_command(
monkeypatch, tmp_path
):
"""A concurrent spawn must not take the fake server or overwrite the command under test."""
path = tmp_path / "embed.gguf"
_write_gguf(path, "qwen3", 1)
backend = LlamaServerBackend()
proc = _FakeProc(alive = True)
captured = {}
foreign = {}
def other_thread_starts_a_process():
child = subprocess.Popen(
[sys.executable, "-c", "pass"], stdout = subprocess.DEVNULL, stderr = subprocess.DEVNULL
)
foreign["real"] = child is not proc
child.wait(timeout = 30)
def health_while_another_thread_spawns(*_a, **_k):
thread = threading.Thread(target = other_thread_starts_a_process)
thread.start()
thread.join(timeout = 30)
return True
monkeypatch.setattr(backend, "_resolve_binary", lambda: _FAKE_BINARY)
monkeypatch.setattr(backend, "_resolve_model_path", lambda model_name = None: str(path))
monkeypatch.setattr(backend, "_find_free_port", lambda: 47000)
monkeypatch.setattr(backend, "_wait_for_health", health_while_another_thread_spawns)
monkeypatch.setattr(backend, "_build_env", lambda *a, **k: {})
_intercept_server_popen(monkeypatch, proc, captured)
backend._spawn_once(False)
assert foreign.get("real") is True, "the other thread was handed the fake server"
assert captured["cmd"][0] == _FAKE_BINARY
assert captured["cmd"][captured["cmd"].index("--pooling") + 1] == "mean"
def test_spawn_fails_loud_on_early_exit(monkeypatch):
monkeypatch.setattr(config, "EMBED_PORT", 8124)
b = LlamaServerBackend()
_patch_spawn_deps(monkeypatch, _FakeProc(alive = False, returncode = 1))
with pytest.raises(RuntimeError, match = "failed to become healthy"):
b._spawn()
def test_spawn_auto_falls_back_to_cpu_on_gpu_failure(monkeypatch):
monkeypatch.setattr(config, "EMBED_DEVICE", "auto")
monkeypatch.setattr(LlamaServerBackend, "_gpu_available", staticmethod(lambda: True))
b = LlamaServerBackend()
calls = []
def fake_spawn_once(use_gpu, model_name = None):
calls.append(use_gpu)
if use_gpu:
raise RuntimeError("CUDA out of memory")
monkeypatch.setattr(b, "_spawn_once", fake_spawn_once)
b._spawn()
assert calls == [True, False]
assert b._force_cpu is True
def test_spawn_explicit_gpu_does_not_fall_back(monkeypatch):
monkeypatch.setattr(config, "EMBED_DEVICE", "gpu")
b = LlamaServerBackend()
def fake_spawn_once(use_gpu, model_name = None):
raise RuntimeError("CUDA out of memory")
monkeypatch.setattr(b, "_spawn_once", fake_spawn_once)
with pytest.raises(RuntimeError, match = "out of memory"):
b._spawn()
assert b._force_cpu is False
def _embed_response(vectors):
items = [{"index": i, "embedding": v} for i, v in enumerate(vectors)]
return {"data": list(reversed(items))}
def test_encode_orders_and_returns_float32(monkeypatch):
b = LlamaServerBackend()
monkeypatch.setattr(b, "_ensure_ready", lambda *_a, **_k: None)
captured = {}
def fake_post(path, payload, **_kwargs):
captured["path"] = path
captured["input"] = payload["input"]
return _embed_response([[3.0, 4.0], [0.0, 5.0]])
monkeypatch.setattr(b, "_post", fake_post)
out = b.encode(["a", "b"], normalize = False)
assert captured["path"] == "/v1/embeddings"
assert out.dtype == np.float32
assert out.shape == (2, 2)
assert out[0].tolist() == [3.0, 4.0]
def test_encode_normalizes(monkeypatch):
b = LlamaServerBackend()
monkeypatch.setattr(b, "_ensure_ready", lambda *_a, **_k: None)
monkeypatch.setattr(b, "_post", lambda p, pl, **_k: _embed_response([[3.0, 4.0]]))
out = b.encode(["a"], normalize = True)
np.testing.assert_allclose(np.linalg.norm(out, axis = 1), [1.0], rtol = 1e-6)
def test_encode_empty_returns_zero_rows(monkeypatch):
b = LlamaServerBackend()
b._dim = 384
monkeypatch.setattr(b, "_ensure_ready", lambda *_a, **_k: None)
out = b.encode([])
assert out.shape == (0, 384)
assert out.dtype == np.float32
def test_encode_rejects_count_mismatch(monkeypatch):
b = LlamaServerBackend()
monkeypatch.setattr(b, "_ensure_ready", lambda *_a, **_k: None)
monkeypatch.setattr(
b, "_post", lambda p, pl, **_k: {"data": [{"index": 0, "embedding": [1.0]}]}
)
with pytest.raises(RuntimeError, match = "vectors for"):
b.encode(["a", "b"], normalize = False)
def test_encode_batches(monkeypatch):
monkeypatch.setattr(config, "EMBED_BATCH", 2)
b = LlamaServerBackend()
monkeypatch.setattr(b, "_ensure_ready", lambda *_a, **_k: None)
calls = []
def fake_post(path, payload, **_kwargs):
chunk = payload["input"]
calls.append(len(chunk))
return _embed_response([[1.0, 0.0]] * len(chunk))
monkeypatch.setattr(b, "_post", fake_post)
out = b.encode(["a", "b", "c"], normalize = False)
assert out.shape == (3, 2)
assert calls == [2, 1]
def test_dim_probes_once_and_caches(monkeypatch):
b = LlamaServerBackend()
monkeypatch.setattr(b, "_ensure_ready", lambda *_a, **_k: None)
n_calls = {"n": 0}
def fake_post(path, payload, **_kwargs):
n_calls["n"] += 1
return _embed_response([[0.1] * 384])
monkeypatch.setattr(b, "_post", fake_post)
assert b.dim() == 384
assert b.dim() == 384
assert n_calls["n"] == 1
def test_token_counter_hits_tokenize(monkeypatch):
b = LlamaServerBackend()
monkeypatch.setattr(b, "_ensure_ready", lambda *_a, **_k: None)
seen = {}
def fake_post(path, payload, **_kwargs):
seen["path"] = path
seen["content"] = payload["content"]
return {"tokens": [1, 2, 3, 4]}
monkeypatch.setattr(b, "_post", fake_post)
count = b.token_counter()
assert count("hello world") == 4
assert seen["path"] == "/tokenize"
assert seen["content"] == "hello world"
def test_ensure_ready_respawns_dead_process(monkeypatch):
b = LlamaServerBackend()
b._process = _FakeProc(alive = False, returncode = 0)
spawned = {"n": 0}
def fake_spawn(model_name = None):
spawned["n"] += 1
b._process = _FakeProc(alive = True)
b._model_repo = config.effective_gguf_repo()
from utils.llama_cpp_path_settings import custom_llama_cpp_path_revision
b._binary_path_revision = custom_llama_cpp_path_revision()
monkeypatch.setattr(b, "_spawn", fake_spawn)
b._ensure_ready()
assert spawned["n"] == 1
assert b._process_alive()
b._ensure_ready()
assert spawned["n"] == 1
def test_path_save_restarts_embedder_with_new_binary(monkeypatch):
from utils import llama_cpp_path_settings
backend = LlamaServerBackend()
backend._process = _FakeProc(alive = True)
backend._model_repo = config.effective_gguf_repo()
backend._binary = "/old/llama-server"
backend._binary_path_revision = llama_cpp_path_settings.custom_llama_cpp_path_revision()
monkeypatch.setattr(
llama_cpp_path_settings, "_path_revision", backend._binary_path_revision + 1
)
spawned = []
def fake_spawn(model_name = None):
spawned.append(backend._binary)
backend._process = _FakeProc(alive = True)
backend._model_repo = config.effective_gguf_repo()
backend._binary = "/new/llama-server"
backend._binary_path_revision = llama_cpp_path_settings.custom_llama_cpp_path_revision()
monkeypatch.setattr(backend, "_spawn", fake_spawn)
backend._ensure_ready()
assert spawned == [None]
assert backend._binary == "/new/llama-server"
def test_post_restarts_once_on_connect_error(monkeypatch):
import httpx
b = LlamaServerBackend()
b._port = 9000
monkeypatch.setattr(b, "_ensure_ready", lambda *_a, **_k: None)
restarts = {"n": 0}
monkeypatch.setattr(
b, "_restart", lambda *_a, **_k: restarts.__setitem__("n", restarts["n"] + 1)
)
attempts = {"n": 0}
class _Client:
def post(
self,
url,
json,
headers = None,
):
attempts["n"] += 1
if attempts["n"] == 1:
raise httpx.ConnectError("boom")
class _R:
def raise_for_status(self_inner):
return None
def json(self_inner):
return {"tokens": [1]}
return _R()
b._client = _Client()
out = b._post("/tokenize", {"content": "x"})
assert out == {"tokens": [1]}
assert restarts["n"] == 1
def test_post_restarts_once_on_read_timeout(monkeypatch):
import httpx
b = LlamaServerBackend()
b._port = 9000
monkeypatch.setattr(b, "_ensure_ready", lambda *_a, **_k: None)
restarts = {"n": 0}
monkeypatch.setattr(
b, "_restart", lambda *_a, **_k: restarts.__setitem__("n", restarts["n"] + 1)
)
attempts = {"n": 0}
class _Client:
def post(
self,
url,
json,
headers = None,
):
attempts["n"] += 1
if attempts["n"] == 1:
raise httpx.ReadTimeout("timed out")
class _R:
def raise_for_status(self_inner):
return None
def json(self_inner):
return {"data": [{"index": 0, "embedding": [1.0, 0.0]}]}
return _R()
b._client = _Client()
out = b._post("/v1/embeddings", {"input": ["x"]})
assert out["data"][0]["embedding"] == [1.0, 0.0]
assert restarts["n"] == 1
class _CrashingGpuServer:
"""Spawn and client stand-ins: a GPU server dies under every request (#13663), a CPU one answers."""
def __init__(
self,
monkeypatch,
backend,
*,
timeout = False,
cpu_crashes = False,
):
import httpx
self.spawns = []
monkeypatch.setattr(backend, "_ensure_ready", lambda *_a, **_k: None)
monkeypatch.setattr(LlamaServerBackend, "_gpu_available", staticmethod(lambda: True))
def fake_spawn_once(use_gpu, model_name = None):
self.spawns.append("gpu" if use_gpu else "cpu")
backend._process = _FakeProc(alive = True, returncode = 0)
backend._on_gpu = use_gpu
backend._port = 9000
monkeypatch.setattr(backend, "_spawn_once", fake_spawn_once)
fake_spawn_once(backend._use_gpu())
class _Client:
def post(
self,
url,
json,
headers = None,
):
if backend._on_gpu or cpu_crashes:
if timeout:
raise httpx.ReadTimeout("timed out")
backend._process._alive = False
backend._process.returncode = 3221226505
raise httpx.ReadError("connection reset by peer")
class _R:
def raise_for_status(self_inner):
return None
def json(self_inner):
return {"data": [{"index": 0, "embedding": [1.0, 0.0]}]}
return _R()
backend._client = _Client()
def test_a_gpu_server_crashing_twice_on_one_request_finishes_on_cpu(monkeypatch):
monkeypatch.setattr(config, "EMBED_DEVICE", "auto")
b = LlamaServerBackend()
fake = _CrashingGpuServer(monkeypatch, b)
out = b._post("/v1/embeddings", {"input": ["x"]})
assert out["data"][0]["embedding"] == [1.0, 0.0]
assert fake.spawns == ["gpu", "gpu", "cpu"]
b._post("/v1/embeddings", {"input": ["y"]})
assert fake.spawns == ["gpu", "gpu", "cpu"]
def test_a_settings_gpu_choice_still_falls_back_after_repeated_crashes(monkeypatch):
import utils.embedding_model_settings as ems
monkeypatch.setattr(config, "EMBED_DEVICE", "auto")
monkeypatch.setattr(ems, "get_stored_embedding_device", lambda: "gpu")
b = LlamaServerBackend()
fake = _CrashingGpuServer(monkeypatch, b)
b._post("/v1/embeddings", {"input": ["x"]})
assert fake.spawns == ["gpu", "gpu", "cpu"]
def test_literal_gpu_env_raises_instead_of_moving_a_crashing_server_to_cpu(monkeypatch):
monkeypatch.setattr(config, "EMBED_DEVICE", "gpu")
b = LlamaServerBackend()
fake = _CrashingGpuServer(monkeypatch, b)
with pytest.raises(RuntimeError, match = "failed after retry"):
b._post("/v1/embeddings", {"input": ["x"]})
assert fake.spawns == ["gpu", "gpu"]
assert b._force_cpu is False
def test_repeated_timeouts_on_a_live_gpu_server_do_not_move_it_to_cpu(monkeypatch):
monkeypatch.setattr(config, "EMBED_DEVICE", "auto")
b = LlamaServerBackend()
fake = _CrashingGpuServer(monkeypatch, b, timeout = True)
with pytest.raises(RuntimeError, match = "failed after retry"):
b._post("/v1/embeddings", {"input": ["x"]})
assert fake.spawns == ["gpu", "gpu"]
assert b._force_cpu is False
def test_a_cpu_server_crashing_twice_raises_without_a_third_spawn(monkeypatch):
monkeypatch.setattr(config, "EMBED_DEVICE", "cpu")
b = LlamaServerBackend()
fake = _CrashingGpuServer(monkeypatch, b, cpu_crashes = True)
with pytest.raises(RuntimeError, match = "failed after retry"):
b._post("/v1/embeddings", {"input": ["x"]})
assert fake.spawns == ["cpu", "cpu"]
def test_a_crash_logs_the_server_exit_code_and_last_output(monkeypatch, caplog):
monkeypatch.setattr(config, "EMBED_DEVICE", "auto")
b = LlamaServerBackend()
_CrashingGpuServer(monkeypatch, b)
b._stdout_lines = ["rocBLAS error from hip error code: 'hipErrorInvalidKernelFile':218"]
with caplog.at_level("WARNING", logger = mod.logger.name):
b._post("/v1/embeddings", {"input": ["x"]})
assert "exited during a request (code 3221226505)" in caplog.text
assert "hipErrorInvalidKernelFile" in caplog.text
assert "continuing on CPU" in caplog.text
class TestTheEmbeddingLoaderChangesAreMacOsOnly:
"""The Linux and Windows embed server launch must stay as it was; only macOS may change."""
def test_the_help_probe_still_inherits_the_environment_off_mac(self, monkeypatch):
seen = {}
def fake_run(cmd, **kwargs):
seen.update(kwargs)
return types.SimpleNamespace(stdout = "--embedding", stderr = "")
monkeypatch.setattr(mod.sys, "platform", "linux")
monkeypatch.setattr(mod.subprocess, "run", fake_run)
mod.LlamaServerBackend._help_text.cache_clear()
mod.LlamaServerBackend._help_text("/opt/mine/llama-server")
assert seen.get("env") is None
def test_the_help_probe_gets_the_loader_environment_on_mac(self, monkeypatch, tmp_path):
binary = tmp_path / "llama-server"
binary.write_bytes(b"\xcf\xfa\xed\xfe")
seen = {}
def fake_run(cmd, **kwargs):
seen.update(kwargs)
return types.SimpleNamespace(stdout = "--embedding", stderr = "")
monkeypatch.setattr(mod.sys, "platform", "darwin")
monkeypatch.setattr(mod.subprocess, "run", fake_run)
mod.LlamaServerBackend._help_text.cache_clear()
mod.LlamaServerBackend._help_text(str(binary))
assert seen.get("env") is not None
def test_linux_gpu_libs_still_hang_off_the_given_path(self, monkeypatch, tmp_path):
real = tmp_path / "build" / "bin" / "llama-server"
real.parent.mkdir(parents = True)
real.write_bytes(b"\xcf\xfa\xed\xfe")
wrapper = tmp_path / "llama-server"
wrapper.write_text('#!/bin/sh\nexec "$(dirname "$0")/build/bin/llama-server" "$@"\n')
wrapper.chmod(0o755)
seen = {}
monkeypatch.setattr(mod.sys, "platform", "linux")
monkeypatch.setattr(
mod.LlamaServerBackend,
"_add_linux_cuda_libs",
staticmethod(lambda env, d: seen.setdefault("dir", d)),
)
mod.LlamaServerBackend()._build_env(str(wrapper), use_gpu = True)
assert seen["dir"] == str(wrapper.parent)
def test_device_preference_is_shared_between_both_backends(monkeypatch):
"""One reader, so the two backends cannot disagree about the same string.
They did: this one compared a bare .lower() with no strip, so " gpu " forced the
GPU for sentence-transformers and fell through to auto here, and an Intel user
writing their own accelerator's name (xpu) got neither.
"""
for requested, expected in (
("gpu", "gpu"),
("GPU", "gpu"),
(" gpu ", "gpu"),
("cuda", "gpu"),
("rocm", "gpu"),
("hip", "gpu"),
("xpu", "gpu"),
("mps", "gpu"),
("metal", "gpu"),
("cpu", "cpu"),
("CPU", "cpu"),
(" cpu ", "cpu"),
("auto", "auto"),
("", "auto"),
(" ", "auto"),
("banana", "auto"),
):
monkeypatch.setattr(config, "EMBED_DEVICE", requested)
assert config.embed_device_preference() == expected, requested
def test_use_gpu_honours_the_normalized_preference(monkeypatch):
"""A whitespace-padded or device-named setting reaches this backend too."""
backend = LlamaServerBackend()
for requested in (" gpu ", "GPU", "xpu", "cuda"):
monkeypatch.setattr(config, "EMBED_DEVICE", requested)
monkeypatch.setattr(
LlamaServerBackend,
"_gpu_available",
staticmethod(lambda: False),
)
assert backend._use_gpu() is True, requested
for requested in (" cpu ", "CPU"):
monkeypatch.setattr(config, "EMBED_DEVICE", requested)
monkeypatch.setattr(
LlamaServerBackend,
"_gpu_available",
staticmethod(lambda: True),
)
assert backend._use_gpu() is False, requested
def test_gpu_opt_in_still_falls_back_to_cpu(monkeypatch):
"""A GPU opt-in spelling still falls back to CPU on a failed start; it must not become fatal."""
for requested in ("cuda", "rocm", "hip", "xpu", "mps", "metal", " gpu ", " CUDA "):
monkeypatch.setattr(config, "EMBED_DEVICE", requested)
backend = LlamaServerBackend()
calls: list[bool] = []
def fake_spawn_once(
use_gpu,
model_name = None,
_calls = calls,
):
_calls.append(use_gpu)
if use_gpu:
raise RuntimeError("GPU start failed")
monkeypatch.setattr(backend, "_spawn_once", fake_spawn_once)
backend._spawn()
assert calls == [True, False], requested
assert backend._force_cpu is True, requested
def test_literal_gpu_remains_a_hard_request(monkeypatch):
"""The documented value keeps forcing the GPU, as it always has."""
for requested in ("gpu", "GPU"):
monkeypatch.setattr(config, "EMBED_DEVICE", requested)
backend = LlamaServerBackend()
monkeypatch.setattr(
backend,
"_spawn_once",
lambda use_gpu, *_a: (_ for _ in ()).throw(RuntimeError("no cuda")),
)
with pytest.raises(RuntimeError, match = "no cuda"):
backend._spawn()
assert backend._force_cpu is False, requested
def test_a_soft_gpu_opt_in_keeps_its_own_cpu_fallback(monkeypatch):
"""A soft GPU opt-in keeps its CPU fallback, so respawns do not retry a failed GPU start."""
for requested in ("cuda", "xpu", " gpu "):
monkeypatch.setattr(config, "EMBED_DEVICE", requested)
backend = LlamaServerBackend()
calls: list[bool] = []
def fake_spawn_once(
use_gpu,
model_name = None,
_calls = calls,
):
_calls.append(use_gpu)
if use_gpu:
raise RuntimeError("GPU start failed")
monkeypatch.setattr(backend, "_spawn_once", fake_spawn_once)
backend._spawn()
assert calls == [True, False], requested
backend._spawn()
assert calls == [True, False, False], requested
assert backend._use_gpu() is False, requested
def _seed_cache(
root,
repo_id,
filenames,
*,
revision = "abc123",
ref = True,
):
"""A hub cache tree built the way hf_hub_download builds one: the snapshot holds
symlinks into blobs/, not regular files."""
repo_dir = root / f"models--{repo_id.replace('/', '--')}"
snapshot = repo_dir / "snapshots" / revision
blobs = repo_dir / "blobs"
blobs.mkdir(parents = True, exist_ok = True)
for name in filenames:
blob = blobs / name.replace("/", "_")
blob.write_text("gguf")
link = snapshot / name
link.parent.mkdir(parents = True, exist_ok = True)
link.symlink_to(os.path.relpath(blob, link.parent))
if ref:
(repo_dir / "refs").mkdir(parents = True, exist_ok = True)
(repo_dir / "refs" / "main").write_text(revision)
return snapshot
def _use_cache_root(monkeypatch, root):
import utils.hf_cache_settings as hcs
monkeypatch.setattr(hcs, "active_hf_hub_cache", lambda: str(root))
def _no_hub(monkeypatch):
monkeypatch.setattr(
LlamaServerBackend,
"_resolve_uncached_model_path",
lambda self, *a, **k: pytest.fail("resolved over the hub despite a usable cache"),
)
def test_cached_gguf_resolves_without_touching_the_hub(monkeypatch, tmp_path):
"""A repo already on disk must never be re-listed: _model_path is per-process, so
every restart used to pay a hub call to name a file it already held (#8778)."""
repo = config.effective_gguf_repo()
_seed_cache(tmp_path / "hub", repo, ["bge-F16.gguf"])
_use_cache_root(monkeypatch, tmp_path / "hub")
_no_hub(monkeypatch)
backend = LlamaServerBackend()
backend._dim = 384
assert backend._resolve_model_path().endswith("bge-F16.gguf")
assert backend._model_repo == repo
assert backend._dim is None
def test_the_pick_takes_the_shortest_variant_match_and_never_an_mmproj(monkeypatch):
"""Wanted file is not first or shortest, and the shortest F16 match is an mmproj to skip."""
monkeypatch.setattr(config, "EMBED_GGUF_VARIANT", "F16")
names = [
"bge-small-en-v1.5-F16-alt.gguf",
"mmproj-F16.gguf",
"bge-small-en-v1.5-F16.gguf",
"bge-Q8_0.gguf",
"README.md",
]
assert LlamaServerBackend._pick_gguf(names) == "bge-small-en-v1.5-F16.gguf"
def test_cached_gguf_skips_mmproj_and_honours_the_variant(monkeypatch, tmp_path):
"""The same rule reached through the cache, over a real snapshot layout."""
repo = config.effective_gguf_repo()
_seed_cache(
tmp_path / "hub",
repo,
["mmproj-F16.gguf", "bge-small-en-v1.5-F16.gguf", "bge-Q8_0.gguf", "README.md"],
)
_use_cache_root(monkeypatch, tmp_path / "hub")
monkeypatch.setattr(config, "EMBED_GGUF_VARIANT", "F16")
assert LlamaServerBackend._resolve_cached_gguf(repo).endswith("bge-small-en-v1.5-F16.gguf")
def test_the_pick_is_stable_whatever_order_the_names_arrive_in(monkeypatch):
"""Directory scans are unordered, and among equal-length shards only the first has the metadata."""
monkeypatch.setattr(config, "EMBED_GGUF_VARIANT", "F16")
shards = ["model-F16-00001-of-00002.gguf", "model-F16-00002-of-00002.gguf"]
assert LlamaServerBackend._pick_gguf(shards) == shards[0]
assert LlamaServerBackend._pick_gguf(list(reversed(shards))) == shards[0]
def test_an_mtp_drafter_is_never_taken_for_the_embedder(monkeypatch):
"""An MTP drafter is a companion, not a model: a cache holding only one must read as no embedder."""
monkeypatch.setattr(config, "EMBED_GGUF_VARIANT", "BF16")
assert LlamaServerBackend._pick_gguf(["MTP/mtp-model-BF16.gguf"]) is None
assert (
LlamaServerBackend._pick_gguf(["MTP/mtp-model-BF16.gguf", "model-BF16.gguf"])
== "model-BF16.gguf"
)
def test_cached_gguf_is_found_in_a_subdirectory(monkeypatch, tmp_path):
"""Quant files sit in subdirectories, so a root-only scan would re-download a cached model."""
repo = config.effective_gguf_repo()
_seed_cache(tmp_path / "hub", repo, ["F16/bge-small.gguf"])
_use_cache_root(monkeypatch, tmp_path / "hub")
resolved = LlamaServerBackend._resolve_cached_gguf(repo)
assert resolved is not None and Path(resolved).parts[-2:] == ("F16", "bge-small.gguf")
def test_an_incomplete_cached_shard_set_is_not_a_hit(monkeypatch, tmp_path):
"""A lone trailing shard cannot serve the model; llama-server needs the first shard's metadata."""
repo = config.effective_gguf_repo()
_seed_cache(tmp_path / "hub", repo, ["model-F16-00002-of-00002.gguf"])
_use_cache_root(monkeypatch, tmp_path / "hub")
assert LlamaServerBackend._resolve_cached_gguf(repo) is None
def test_a_complete_cached_shard_set_resolves_to_its_first_shard(monkeypatch, tmp_path):
"""A complete shard set is servable and resolves to shard 1 whatever order the scan returns."""
repo = config.effective_gguf_repo()
_seed_cache(
tmp_path / "hub",
repo,
["model-F16-00002-of-00002.gguf", "model-F16-00001-of-00002.gguf"],
)
_use_cache_root(monkeypatch, tmp_path / "hub")
resolved = LlamaServerBackend._resolve_cached_gguf(repo)
assert resolved is not None and resolved.endswith("model-F16-00001-of-00002.gguf")
def test_an_incomplete_set_does_not_shadow_a_complete_one(monkeypatch, tmp_path):
"""Completeness must narrow the candidates, not veto the winner, or a servable family goes unseen."""
repo = config.effective_gguf_repo()
_seed_cache(
tmp_path / "hub",
repo,
[
"a-F16-00001-of-00002.gguf",
"embedding-model-F16-00001-of-00002.gguf",
"embedding-model-F16-00002-of-00002.gguf",
],
)
_use_cache_root(monkeypatch, tmp_path / "hub")
resolved = LlamaServerBackend._resolve_cached_gguf(repo)
assert resolved is not None
assert resolved.endswith("embedding-model-F16-00001-of-00002.gguf")
def test_an_incomplete_variant_match_does_not_hide_a_whole_other_variant(monkeypatch, tmp_path):
"""Offline, the relaxed pass takes any variant. An unservable F16 must not stand in the
way of a Q8 that is whole -- the difference between a working embedder and none."""
repo = config.effective_gguf_repo()
_seed_cache(tmp_path / "hub", repo, ["model-F16-00002-of-00002.gguf", "model-Q8_0.gguf"])
_use_cache_root(monkeypatch, tmp_path / "hub")
monkeypatch.setattr(config, "EMBED_GGUF_VARIANT", "F16")
assert LlamaServerBackend._resolve_cached_gguf(repo) is None
relaxed = LlamaServerBackend._resolve_cached_gguf(repo, require_variant = False)
assert relaxed is not None and relaxed.endswith("model-Q8_0.gguf")
def test_cached_gguf_uses_only_the_revision_refs_main_names(monkeypatch, tmp_path):
"""hf_hub_download serves refs/main. The current revision here sorts neither first nor
last, so no directory ordering can stand in for reading the ref."""
repo = config.effective_gguf_repo()
root = tmp_path / "hub"
_seed_cache(root, repo, ["aaa-F16.gguf"], revision = "aaa111", ref = False)
_seed_cache(root, repo, ["live-F16.gguf"], revision = "mmm555", ref = True)
_seed_cache(root, repo, ["zzz-F16.gguf"], revision = "zzz999", ref = False)
_use_cache_root(monkeypatch, root)
assert LlamaServerBackend._resolve_cached_gguf(repo).endswith("live-F16.gguf")
def test_a_commit_pinned_cache_with_no_ref_defers_to_the_hub(monkeypatch, tmp_path):
"""Snapshot directories are commit hashes, which order by nothing; guessing among them
could serve a superseded model, so an unnamed revision is no hit at all."""
repo = config.effective_gguf_repo()
_seed_cache(tmp_path / "hub", repo, ["bge-F16.gguf"], revision = "deadbeef", ref = False)
_use_cache_root(monkeypatch, tmp_path / "hub")
assert LlamaServerBackend._resolve_cached_gguf(repo) is None
def test_a_cached_other_variant_does_not_satisfy_the_configured_one(monkeypatch, tmp_path):
"""Falling back on a listing means the variant is not published; falling back on a
cache would serve a Q8 left from an earlier setting instead of the configured F16."""
repo = config.effective_gguf_repo()
_seed_cache(tmp_path / "hub", repo, ["bge-Q8_0.gguf"])
_use_cache_root(monkeypatch, tmp_path / "hub")
monkeypatch.setattr(config, "EMBED_GGUF_VARIANT", "F16")
assert LlamaServerBackend._resolve_cached_gguf(repo) is None
def test_an_unreachable_hub_falls_back_to_whatever_variant_is_cached(monkeypatch, tmp_path):
"""A cached other variant beats no embedder at all, the degrade issue 8778 asks for."""
huggingface_hub = _shared_setup_2(monkeypatch, tmp_path)
monkeypatch.setattr(
huggingface_hub,
"list_repo_files",
lambda *a, **k: (_ for _ in ()).throw(OSError("offline mode is enabled")),
)
backend = LlamaServerBackend()
assert backend._resolve_model_path().endswith("bge-Q8_0.gguf")
def test_a_pending_picker_download_never_falls_through_to_the_hub(monkeypatch, tmp_path):
import utils.embedding_model_settings as ems
monkeypatch.setattr(config, "effective_embedding_model", lambda: "org/pending")
monkeypatch.setattr(config, "effective_gguf_repo", lambda: "org/pending-GGUF")
monkeypatch.setattr(ems, "get_stored_download_pending", lambda model: True)
_use_cache_root(monkeypatch, tmp_path / "empty")
backend = LlamaServerBackend()
monkeypatch.setattr(
backend,
"_resolve_uncached_model_path",
lambda *a: (_ for _ in ()).throw(AssertionError("pending model reached the Hub")),
)
with pytest.raises(RuntimeError, match = "not downloaded yet"):
backend._resolve_model_path()
def test_a_completed_pending_download_uses_its_cached_fallback_quant(monkeypatch, tmp_path):
ems, repo = _shared_setup_3(monkeypatch)
monkeypatch.setattr(ems, "get_stored_download_pending", lambda model: True)
_seed_cache(tmp_path / "hub", repo, ["pending-Q8_0.gguf"])
_use_cache_root(monkeypatch, tmp_path / "hub")
backend = LlamaServerBackend()
monkeypatch.setattr(
backend,
"_resolve_uncached_model_path",
lambda *a: (_ for _ in ()).throw(AssertionError("cached pending model reached Hub")),
)
assert backend._resolve_model_path().endswith("pending-Q8_0.gguf")
def test_an_unreachable_hub_still_reaches_the_fallback_repo_s_cache(monkeypatch, tmp_path):
"""Cache-first consults only the preferred repo to keep a reachable companion ahead of the
fallback; offline the relaxed pass must still check the fallback repo cache."""
import contextlib
import huggingface_hub
from core.inference import llama_cpp
monkeypatch.setattr(config, "effective_embedding_model", lambda: "org/foo")
monkeypatch.setattr(config, "effective_gguf_repo", lambda: "org/foo-GGUF")
_seed_cache(tmp_path / "hub", "org/foo", ["fallback-F16.gguf"])
_use_cache_root(monkeypatch, tmp_path / "hub")
monkeypatch.setattr(llama_cpp, "_hf_offline_if_unreachable", contextlib.nullcontext)
monkeypatch.setattr(
huggingface_hub,
"list_repo_files",
lambda *a, **k: (_ for _ in ()).throw(OSError("offline mode is enabled")),
)
backend = LlamaServerBackend()
assert backend._resolve_model_path().endswith("fallback-F16.gguf")
def test_a_reachable_hub_is_preferred_over_a_cached_other_variant(monkeypatch, tmp_path):
"""A cached Q8 is a last resort and must not pre-empt a reachable hub naming the variant."""
huggingface_hub = _shared_setup_2(monkeypatch, tmp_path)
monkeypatch.setattr(huggingface_hub, "list_repo_files", lambda *a, **k: ["bge-F16.gguf"])
monkeypatch.setattr(huggingface_hub, "hf_hub_download", lambda **kw: "/cache/bge-F16.gguf")
backend = LlamaServerBackend()
assert backend._resolve_model_path() == "/cache/bge-F16.gguf"
def test_a_failed_transfer_surfaces_instead_of_serving_another_variant(monkeypatch, tmp_path):
"""A download failing for its own reasons (no disk, bad checksum) is no evidence the
variant is unobtainable, so it must not be answered with a different model."""
huggingface_hub = _shared_setup_2(monkeypatch, tmp_path)
monkeypatch.setattr(huggingface_hub, "list_repo_files", lambda *a, **k: ["bge-F16.gguf"])
monkeypatch.setattr(
huggingface_hub,
"hf_hub_download",
lambda **kw: (_ for _ in ()).throw(OSError("No space left on device")),
)
backend = LlamaServerBackend()
with pytest.raises(OSError, match = "No space left on device"):
backend._resolve_model_path()
def test_a_cached_gguf_is_found_whatever_the_extension_case(monkeypatch, tmp_path):
"""The hub path always matched the suffix case-insensitively; a stricter cache scan
would re-download a model sitting on disk, or fail outright when offline."""
repo = config.effective_gguf_repo()
_seed_cache(tmp_path / "hub", repo, ["bge-F16.GGUF"])
_use_cache_root(monkeypatch, tmp_path / "hub")
assert LlamaServerBackend._resolve_cached_gguf(repo).endswith("bge-F16.GGUF")
def test_the_adopted_path_is_tagged_with_the_repo_it_was_resolved_for(monkeypatch, tmp_path):
"""Tag the path with the repo it was resolved for, so a Settings change reads as stale."""
repo = config.effective_gguf_repo()
_seed_cache(tmp_path / "hub", repo, ["bge-F16.gguf"])
_use_cache_root(monkeypatch, tmp_path / "hub")
_no_hub(monkeypatch)
backend = LlamaServerBackend()
def _resolve_then_change_the_setting(repo_id, **_kw):
monkeypatch.setattr(config, "effective_gguf_repo", lambda: "someone/else-GGUF")
return str(tmp_path / "hub" / "bge-F16.gguf")
monkeypatch.setattr(
LlamaServerBackend,
"_resolve_cached_gguf",
staticmethod(_resolve_then_change_the_setting),
)
backend._resolve_model_path()
assert backend._model_repo == repo
assert not backend._current()
def test_a_cached_fallback_repo_does_not_pre_empt_the_preferred_one(monkeypatch, tmp_path):
"""A custom model resolves through its "-GGUF" companion first and only falls back to
the model repo when the companion has none. A file cached under the fallback must not
reorder that, or the backend serves the wrong weights and tags them as current."""
import contextlib
import huggingface_hub
from core.inference import llama_cpp
monkeypatch.setattr(config, "effective_embedding_model", lambda: "org/foo")
monkeypatch.setattr(config, "effective_gguf_repo", lambda: "org/foo-GGUF")
_seed_cache(tmp_path / "hub", "org/foo", ["fallback-F16.gguf"])
_use_cache_root(monkeypatch, tmp_path / "hub")
monkeypatch.setattr(llama_cpp, "_hf_offline_if_unreachable", contextlib.nullcontext)
monkeypatch.setattr(
huggingface_hub,
"list_repo_files",
lambda repo_id, **k: ["preferred-F16.gguf"] if repo_id == "org/foo-GGUF" else [],
)
monkeypatch.setattr(
huggingface_hub, "hf_hub_download", lambda **kw: "/cache/preferred-F16.gguf"
)
backend = LlamaServerBackend()
assert backend._resolve_model_path() == "/cache/preferred-F16.gguf"
def test_the_cache_folder_is_named_from_the_resolved_repo_casing(monkeypatch, tmp_path):
"""Cache folders take the resolver's casing; a mismatch re-downloads and fails offline."""
import utils.paths as paths_pkg
_seed_cache(tmp_path / "hub", "Cached/Spelling-GGUF", ["bge-F16.gguf"])
_use_cache_root(monkeypatch, tmp_path / "hub")
monkeypatch.setattr(
paths_pkg, "resolve_cached_repo_id_case", lambda name: "Cached/Spelling-GGUF"
)
resolved = LlamaServerBackend._resolve_cached_gguf("as/typed-GGUF")
assert resolved is not None and resolved.endswith("bge-F16.gguf")
def test_the_listing_cannot_outlive_its_deadline(monkeypatch, tmp_path):
"""list_repo_files takes no timeout and pagination overrides any client-level default,
so a blackholed address holds this call, and _lifecycle_lock with it, until the kernel
exhausts its SYN retries."""
import contextlib
import huggingface_hub
from core.inference import llama_cpp
@contextlib.contextmanager
def reachable():
yield False
def _stalls(*_a, **_k):
time.sleep(30)
raise AssertionError("deadline did not cut the listing short")
_use_cache_root(monkeypatch, tmp_path / "empty")
monkeypatch.setattr(llama_cpp, "_hf_offline_if_unreachable", reachable)
monkeypatch.setattr(huggingface_hub, "list_repo_files", _stalls)
monkeypatch.setattr(LlamaServerBackend, "_LIST_DEADLINE_S", 0.3)
backend = LlamaServerBackend()
started = time.monotonic()
with pytest.raises(RuntimeError, match = "no .gguf embedder found"):
backend._resolve_model_path()
assert time.monotonic() - started < 3.0
def test_the_hub_resolve_runs_inside_the_unreachable_guard(monkeypatch, tmp_path):
"""Deadline covers only the listing; the guard blocks transfers to an unreachable endpoint."""
import contextlib
import huggingface_hub
from core.inference import llama_cpp
active = []
@contextlib.contextmanager
def recording_guard():
active.append(True)
try:
yield False
finally:
active.pop()
def _require_guard(*_a, **_k):
assert active, "hub call made outside the guard"
return ["bge-F16.gguf"]
def _download(**_kw):
assert active, "download made outside the guard"
return "/cache/bge-F16.gguf"
_use_cache_root(monkeypatch, tmp_path / "empty")
monkeypatch.setattr(llama_cpp, "_hf_offline_if_unreachable", recording_guard)
monkeypatch.setattr(huggingface_hub, "list_repo_files", _require_guard)
monkeypatch.setattr(huggingface_hub, "hf_hub_download", _download)
backend = LlamaServerBackend()
assert backend._resolve_model_path() == "/cache/bge-F16.gguf"
assert active == []
def test_a_split_family_is_downloaded_whole_not_just_its_first_shard(monkeypatch, tmp_path):
"""Download the whole split GGUF family: llama-server opens the sibling shards itself with -m shard1."""
import contextlib
import huggingface_hub
from core.inference import llama_cpp
names = [
"bge-F16-00001-of-00003.gguf",
"bge-F16-00002-of-00003.gguf",
"bge-F16-00003-of-00003.gguf",
]
fetched = []
_use_cache_root(monkeypatch, tmp_path / "hub")
monkeypatch.setattr(config, "EMBED_GGUF_VARIANT", "F16")
monkeypatch.setattr(llama_cpp, "_hf_offline_if_unreachable", contextlib.nullcontext)
monkeypatch.setattr(huggingface_hub, "list_repo_files", lambda *a, **k: names)
monkeypatch.setattr(
huggingface_hub,
"hf_hub_download",
lambda **kw: (fetched.append(kw["filename"]), f"/cache/{kw['filename']}")[1],
)
backend = LlamaServerBackend()
assert backend._resolve_model_path() == "/cache/bge-F16-00001-of-00003.gguf"
assert fetched == names
def test_a_torn_published_family_is_skipped_rather_than_half_fetched(monkeypatch, tmp_path):
"""Shard 2 missing on the hub is not a repo to adopt; the candidate loop moves on."""
import contextlib
import huggingface_hub
from core.inference import llama_cpp
_use_cache_root(monkeypatch, tmp_path / "hub")
monkeypatch.setattr(config, "EMBED_GGUF_VARIANT", "F16")
monkeypatch.setattr(llama_cpp, "_hf_offline_if_unreachable", contextlib.nullcontext)
monkeypatch.setattr(
huggingface_hub,
"list_repo_files",
lambda *a, **k: ["bge-F16-00001-of-00003.gguf", "bge-F16-00003-of-00003.gguf"],
)
monkeypatch.setattr(
huggingface_hub,
"hf_hub_download",
lambda **kw: pytest.fail("a torn family must not be fetched"),
)
backend = LlamaServerBackend()
with pytest.raises(RuntimeError, match = "no .gguf embedder found"):
backend._resolve_model_path()
def test_a_local_dir_of_shards_resolves_to_shard_one_in_any_scan_order(monkeypatch, tmp_path):
"""`iterdir` guarantees no order, and every shard name here is the same length,
so a length-only sort let the directory decide which shard llama-server opened."""
names = [
"bge-F16-00001-of-00003.gguf",
"bge-F16-00002-of-00003.gguf",
"bge-F16-00003-of-00003.gguf",
]
monkeypatch.setattr(config, "EMBED_GGUF_VARIANT", "F16")
orders = [names, list(reversed(names)), [names[1], names[2], names[0]]]
real_iterdir = Path.iterdir
for index, order in enumerate(orders):
d = tmp_path / f"dir{index}"
d.mkdir()
for name in order:
(d / name).write_bytes(b"")
monkeypatch.setattr(
Path,
"iterdir",
lambda self, o = order, r = real_iterdir: (
iter([self / n for n in o]) if self == d else r(self)
),
)
picked = LlamaServerBackend._resolve_local_gguf(str(d))
assert Path(picked).name == "bge-F16-00001-of-00003.gguf"
def test_a_local_dir_with_a_torn_family_is_refused(monkeypatch, tmp_path):
monkeypatch.setattr(config, "EMBED_GGUF_VARIANT", "F16")
d = tmp_path / "torn"
d.mkdir()
for n in ("bge-F16-00001-of-00003.gguf", "bge-F16-00003-of-00003.gguf"):
(d / n).write_bytes(b"")
with pytest.raises(RuntimeError, match = "no .gguf file found"):
LlamaServerBackend._resolve_local_gguf(str(d))
def test_a_local_dir_still_skips_mmproj_and_appledouble_sidecars(tmp_path):
"""macOS copies leave `._name` sidecars beside the real file."""
d = tmp_path / "local"
d.mkdir()
(d / "mmproj-bge.gguf").write_bytes(b"")
(d / "._bge.gguf").write_bytes(b"\x00\x05\x16\x07")
(d / "bge.gguf").write_bytes(b"")
assert Path(LlamaServerBackend._resolve_local_gguf(str(d))).name == "bge.gguf"
def test_a_wsl_drive_letter_dir_resolves_like_the_other_local_probes(monkeypatch, tmp_path):
"""Drive-letter folders must resolve under WSL via normalize_path, as the other local probes do."""
from utils.paths import path_utils
drive = tmp_path / "mnt" / "c" / "models"
drive.mkdir(parents = True)
(drive / "bge-F16.gguf").write_bytes(b"")
monkeypatch.setattr(config, "EMBED_GGUF_VARIANT", "F16")
monkeypatch.setattr(path_utils, "_IS_WSL", True)
monkeypatch.setattr(
path_utils,
"normalize_path",
lambda p: (
str(tmp_path / "mnt" / p[0].lower() / p[3:].replace("\\", "/"))
if len(p) >= 3 and p[1] == ":"
else p
),
)
import utils.paths as paths_pkg
monkeypatch.setattr(paths_pkg, "normalize_path", path_utils.normalize_path)
picked = LlamaServerBackend._resolve_local_gguf(r"C:\models")
assert picked is not None and Path(picked).name == "bge-F16.gguf"
def test_a_posix_path_is_untouched_by_the_normalization(tmp_path):
"""The rewrite must not start mangling ordinary POSIX paths."""
d = tmp_path / "plain"
d.mkdir()
(d / "bge.gguf").write_bytes(b"")
assert Path(LlamaServerBackend._resolve_local_gguf(str(d))).name == "bge.gguf"
def test_a_stand_in_quant_does_not_retire_the_pending_marker(monkeypatch, tmp_path):
"""A stand-in quant is a leftover from an earlier setting; clearing the marker would pin it for good."""
cleared, ems, repo = _shared_setup_1(monkeypatch)
monkeypatch.setattr(ems, "clear_stored_download_pending", lambda model: cleared.append(model))
_seed_cache(tmp_path / "hub", repo, ["pending-Q8_0.gguf"])
_shared_setup_4(monkeypatch, tmp_path)
assert cleared == [], "a stand-in quant is not the transfer that was advertised"
def test_the_planned_variant_landing_retires_the_pending_marker(monkeypatch, tmp_path):
"""The planned variant landing in the preferred repo answers the marker, so it is cleared."""
cleared, ems, repo = _shared_setup_1(monkeypatch)
monkeypatch.setattr(ems, "clear_stored_download_pending", lambda model: cleared.append(model))
_seed_cache(tmp_path / "hub", repo, ["pending-F16.gguf"])
_use_cache_root(monkeypatch, tmp_path / "hub")
backend = LlamaServerBackend()
assert backend._resolve_model_path().endswith("pending-F16.gguf")
assert cleared == ["org/pending"]
def test_the_planned_fallback_quant_landing_retires_the_pending_marker(monkeypatch, tmp_path):
"""Fallback quant landings must clear the pending marker, or eviction later reports not downloaded."""
cleared, ems, repo = _shared_setup_1(monkeypatch)
monkeypatch.setattr(ems, "get_stored_gguf_files", lambda model: ["pending-Q8_0.gguf"])
monkeypatch.setattr(ems, "clear_stored_download_pending", lambda model: cleared.append(model))
_seed_cache(tmp_path / "hub", repo, ["pending-Q8_0.gguf"])
_shared_setup_4(monkeypatch, tmp_path)
assert cleared == ["org/pending"]
def test_a_pinned_request_serves_the_pinned_models_gguf_not_the_live_setting(monkeypatch):
"""One subprocess serves one GGUF, so a pinned request must not read the live setting."""
from core.rag import config
b = LlamaServerBackend()
monkeypatch.setattr(config, "effective_embedding_model", lambda: "org/model-b")
monkeypatch.setattr(
config,
"effective_gguf_repo_for_embedding_model",
lambda model: f"{model}-GGUF",
)
monkeypatch.setattr(b, "_process_alive", lambda: True)
monkeypatch.setattr(b, "_binary_path_revision", None, raising = False)
import utils.llama_cpp_path_settings as paths
monkeypatch.setattr(paths, "custom_llama_cpp_path_revision", lambda: None)
b._model_repo = "org/model-a-GGUF"
assert b._current() is False
assert b._current("org/model-a") is True
assert b._current("org/model-b") is False
def test_resolution_follows_the_pinned_model(monkeypatch, tmp_path):
"""_resolve_model_path derived the repo and the local-path probe from the live
setting, so the pinned model reached neither."""
from core.rag import config
gguf = tmp_path / "pinned.gguf"
gguf.write_bytes(b"GGUF")
b = LlamaServerBackend()
monkeypatch.setattr(config, "effective_embedding_model", lambda: "org/model-b")
monkeypatch.setattr(
config, "effective_gguf_repo_for_embedding_model", lambda model: f"{model}-GGUF"
)
seen = []
monkeypatch.setattr(
LlamaServerBackend,
"_resolve_local_gguf",
staticmethod(
lambda model: seen.append(model) or (str(gguf) if "model-a" in model else None)
),
)
assert b._resolve_model_path("org/model-a") == str(gguf)
assert seen == ["org/model-a"]
# Tagged with the repo it was resolved for, so an unpinned request sees it as stale.
assert b._model_repo == "org/model-a-GGUF"
def test_a_swap_cannot_land_between_readiness_and_the_request(monkeypatch):
"""Readiness and the POST were two steps, so a swap in between landed A's
request on B's server, storing B's vectors under A's identity."""
import threading
b = LlamaServerBackend()
order: list[str] = []
swapped = threading.Event()
def fake_ensure_ready(model_name = None):
order.append(f"ready:{model_name}")
def fake_post(
url,
json = None,
headers = None,
): # noqa: A002 - httpx's parameter name
order.append("post")
swapper = threading.Thread(target = _swap)
swapper.start()
swapper.join(timeout = 0.5)
order.append("post-done")
class _R:
@staticmethod
def raise_for_status():
return None
@staticmethod
def json():
return {"tokens": [1]}
return _R()
def _swap():
with b._serve_lock:
order.append("swap")
swapped.set()
monkeypatch.setattr(b, "_ensure_ready", fake_ensure_ready)
monkeypatch.setattr(b._client, "post", fake_post)
monkeypatch.setattr(b, "_port", 1)
b._post("/tokenize", {"content": "x"}, model_name = "org/model-a")
swapped.wait(timeout = 2)
assert order[:3] == ["ready:org/model-a", "post", "post-done"]
assert order[-1] == "swap"
def test_a_dim_probe_is_serialized_with_a_model_switch(monkeypatch):
"""The dim probe shares the model-switch lock; _dim belongs to whichever GGUF is loaded."""
b = LlamaServerBackend()
monkeypatch.setattr(b, "_ensure_ready", lambda model_name = None: None)
observed = {}
def _probe_encode(
texts,
normalize = False,
model_name = None,
):
acquired = []
switcher = threading.Thread(
target = lambda: acquired.append(b._serve_lock.acquire(timeout = 0.5))
)
switcher.start()
switcher.join()
if acquired[0]:
b._serve_lock.release()
observed["excluded"] = not acquired[0]
return np.zeros((1, 384), dtype = np.float32)
monkeypatch.setattr(b, "encode", _probe_encode)
assert b.dim() == 384
assert observed["excluded"] is True, "a model switch could land inside the probe"
def test_the_runtime_probe_does_not_wait_on_a_model_load(monkeypatch):
"""The runtime probe must not wait on the load lock, or Settings stalls for the whole first load."""
monkeypatch.setattr(embeddings, "_load_device", lambda: "cpu")
finished = threading.Event()
def _probe():
embeddings.sentence_transformers_runtime_available()
finished.set()
with embeddings._lock:
worker = threading.Thread(target = _probe, daemon = True)
worker.start()
assert finished.wait(timeout = 60), "the resolve preflight waited on the model-load lock"
def test_a_stale_quant_in_another_directory_is_not_the_planned_family(monkeypatch, tmp_path):
"""A repo that files each quant in its own directory publishes Q8_0/model.gguf
beside Q4_K_M/model.gguf. Matching the plan on base names alone let the stale
one pass for the planned one, retiring the marker and pinning the wrong quant."""
cleared, ems, repo = _shared_setup_1(monkeypatch)
monkeypatch.setattr(ems, "get_stored_gguf_files", lambda model: ["Q8_0/model.gguf"])
monkeypatch.setattr(ems, "clear_stored_download_pending", lambda model: cleared.append(model))
_seed_cache(tmp_path / "hub", repo, ["Q4_K_M/model.gguf"])
_use_cache_root(monkeypatch, tmp_path / "hub")
backend = LlamaServerBackend()
assert backend._resolve_model_path().endswith("model.gguf")
assert cleared == [], "a quant from another directory is not the planned family"
def test_the_planned_family_matches_by_its_directory_too(monkeypatch, tmp_path):
"""The other half: the planned copy in its own directory still retires it."""
cleared, ems, repo = _shared_setup_1(monkeypatch)
monkeypatch.setattr(ems, "get_stored_gguf_files", lambda model: ["Q8_0/model.gguf"])
monkeypatch.setattr(ems, "clear_stored_download_pending", lambda model: cleared.append(model))
_seed_cache(tmp_path / "hub", repo, ["Q8_0/model.gguf"])
_use_cache_root(monkeypatch, tmp_path / "hub")
backend = LlamaServerBackend()
assert backend._resolve_model_path().endswith("model.gguf")
assert cleared == ["org/pending"]
def test_warming_one_model_cannot_respawn_under_another_models_request(monkeypatch):
"""Readying the server outside the lock could respawn it under another model's request."""
b = LlamaServerBackend()
unlocked = []
def _lock_is_free() -> bool:
# Ownership is per thread for an RLock, so ask from another one.
seen = []
probe = threading.Thread(target = lambda: seen.append(b._serve_lock.acquire(timeout = 0.5)))
probe.start()
probe.join()
if seen[0]:
b._serve_lock.release()
return seen[0]
def _recording_ensure_ready(model_name = None):
if _lock_is_free():
unlocked.append(model_name)
monkeypatch.setattr(b, "_ensure_ready", _recording_ensure_ready)
monkeypatch.setattr(
b,
"encode",
lambda texts, normalize = False, model_name = None: np.zeros((1, 384), dtype = np.float32),
)
b.warm(model_name = "org/model-b")
assert unlocked == [], "a respawn during warm-up could land outside the serving lock"
def test_the_planned_family_outranks_a_variant_that_arrives_later(monkeypatch, tmp_path):
"""Identity records model and repo, not the file; a later variant must not change weights unnoticed."""
ems, repo = _shared_setup_3(monkeypatch)
monkeypatch.setattr(ems, "get_stored_download_pending", lambda model: False)
monkeypatch.setattr(ems, "get_stored_gguf_files", lambda model: ["pending-Q8_0.gguf"])
monkeypatch.setattr(ems, "clear_stored_download_pending", lambda model: False)
_seed_cache(tmp_path / "hub", repo, ["pending-Q8_0.gguf", "pending-F16.gguf"])
_shared_setup_4(monkeypatch, tmp_path)
def test_a_partly_present_planned_family_is_not_served(monkeypatch, tmp_path):
"""Whole family or nothing: a torn set entered at shard 1 cannot be opened by llama-server."""
ems, repo = _shared_setup_3(monkeypatch)
monkeypatch.setattr(ems, "get_stored_download_pending", lambda model: False)
monkeypatch.setattr(
ems,
"get_stored_gguf_files",
lambda model: ["p-00001-of-00002.gguf", "p-00002-of-00002.gguf"],
)
_seed_cache(tmp_path / "hub", repo, ["p-00001-of-00002.gguf", "pending-F16.gguf"])
_use_cache_root(monkeypatch, tmp_path / "hub")
backend = LlamaServerBackend()
assert backend._planned_family_path("org/pending", repo) is None
assert backend._resolve_model_path().endswith("pending-F16.gguf")
def _write_gguf(
path,
arch,
pooling = None,
):
kvs = [
struct.pack("<Q", 20)
+ b"general.architecture"
+ struct.pack("<IQ", 8, len(arch))
+ arch.encode()
]
if pooling is not None:
key = f"{arch}.pooling_type".encode()
kvs.append(struct.pack("<Q", len(key)) + key + struct.pack("<II", 4, pooling))
Path(path).write_bytes(b"GGUF" + struct.pack("<IQQ", 3, 0, len(kvs)) + b"".join(kvs))
@pytest.mark.parametrize(
"pooling, expected",
[(3, "last"), (1, "mean"), (2, "cls"), (None, "cls"), (0, "cls")],
)
def test_build_cmd_serves_the_pooling_the_gguf_declares(tmp_path, pooling, expected):
"""Forcing CLS on a last-token (Qwen3-Embedding) or mean (nomic) GGUF pooled one
token, so unrelated sentences embedded nearly alike. A GGUF declaring none keeps
CLS, since llama-server would fall back to NONE and /v1/embeddings refuses that."""
path = tmp_path / "embed.gguf"
_write_gguf(path, "qwen3", pooling)
cmd = LlamaServerBackend()._build_cmd("/bin/llama-server", str(path), 1, use_gpu = False)
assert cmd[cmd.index("--pooling") + 1] == expected
@pytest.mark.parametrize("arch", ["nomic-bert", "nomic-bert-moe"])
def test_build_cmd_uses_mean_for_older_nomic_ggufs_without_pooling_metadata(tmp_path, arch):
"""Dedicated Nomic embedding architectures predate reliable pooling metadata, but
their model family is mean-pooled. An explicit unsupported value still falls back safely."""
path = tmp_path / "nomic.gguf"
_write_gguf(path, arch)
backend = LlamaServerBackend()
cmd = backend._build_cmd("/bin/llama-server", str(path), 1, use_gpu = False)
assert cmd[cmd.index("--pooling") + 1] == "mean"
assert backend._adopt_model_path(str(path), "org/nomic-GGUF") == str(path)
assert backend._model_pooling == "mean"
_write_gguf(path, arch, 0)
cmd = backend._build_cmd("/bin/llama-server", str(path), 1, use_gpu = False)
assert cmd[cmd.index("--pooling") + 1] == "cls"
@pytest.mark.parametrize("pooling, suffix", [(3, ":last"), (1, ":mean"), (2, "")])
def test_llama_identity_changes_only_for_a_non_cls_gguf(monkeypatch, tmp_path, pooling, suffix):
"""Vectors indexed under the forced CLS must read as stale once a mean or last
GGUF is served natively, while a CLS GGUF such as bge-small keeps its identity."""
model, repo = "org/embed", "org/embed-GGUF"
monkeypatch.setattr(config, "effective_gguf_repo_for_embedding_model", lambda m: repo)
snapshot = _seed_cache(tmp_path / "hub", repo, ["embed-F16.gguf"])
_write_gguf((snapshot / "embed-F16.gguf").resolve(), "qwen3", pooling)
_use_cache_root(monkeypatch, tmp_path / "hub")
legacy = config.embedding_identity("llama-server", model, gguf_repo = repo)
assert embeddings._identity(True, model) == legacy + suffix
def test_llama_identity_follows_the_served_gguf_after_the_cache_moves(monkeypatch, tmp_path):
"""Moving the HF cache in Settings leaves the server on the old file, so a fresh
cache search dropped the ``:last`` suffix and tagged last-pooled vectors as CLS."""
model, repo = "org/embed", "org/embed-GGUF"
monkeypatch.setattr(config, "effective_gguf_repo_for_embedding_model", lambda m: repo)
snapshot = _seed_cache(tmp_path / "hub-a", repo, ["embed-F16.gguf"])
served = (snapshot / "embed-F16.gguf").resolve()
_write_gguf(served, "qwen3", 3)
backend = LlamaServerBackend()
backend._model_path, backend._model_repo = str(served), repo
monkeypatch.setattr(embeddings, "_backend", backend)
(tmp_path / "hub-b").mkdir()
_use_cache_root(monkeypatch, tmp_path / "hub-b")
legacy = config.embedding_identity("llama-server", model, gguf_repo = repo)
assert embeddings._identity(True, model) == legacy + ":last"
assert embeddings._identity(True, model, backend) == legacy + ":last"
def test_llama_identity_with_no_gguf_on_disk_matches_no_forced_cls_row(monkeypatch, tmp_path):
"""With the GGUF evicted and no server up, the prediction fell back to the forced-CLS
identity, so a re-upload deduplicated onto those stale vectors instead of re-indexing."""
model, repo = "org/embed", "org/embed-GGUF"
monkeypatch.setattr(config, "effective_gguf_repo_for_embedding_model", lambda m: repo)
(tmp_path / "hub").mkdir()
_use_cache_root(monkeypatch, tmp_path / "hub")
legacy = config.embedding_identity("llama-server", model, gguf_repo = repo)
predicted = embeddings._identity(True, model)
assert not config.embedding_identity_matches(legacy, predicted)
assert config.embedding_identity_model(predicted) == model
def test_llama_identity_keeps_the_launched_pooling_when_the_file_vanishes(monkeypatch, tmp_path):
"""A running server keeps serving its open GGUF after the cache entry is evicted, so
re-reading the gone path answered CLS and tagged last-pooled vectors as CLS."""
model, repo = "org/embed", "org/embed-GGUF"
monkeypatch.setattr(config, "effective_gguf_repo_for_embedding_model", lambda m: repo)
snapshot = _seed_cache(tmp_path / "hub", repo, ["embed-F16.gguf"])
served = (snapshot / "embed-F16.gguf").resolve()
_write_gguf(served, "qwen3", 3)
_use_cache_root(monkeypatch, tmp_path / "hub")
backend = LlamaServerBackend()
backend._adopt_model_path(str(served), repo)
monkeypatch.setattr(embeddings, "_backend", backend)
served.unlink()
legacy = config.embedding_identity("llama-server", model, gguf_repo = repo)
assert embeddings._identity(True, model, backend) == legacy + ":last"
def test_llama_identity_rechecks_a_stopped_backends_replaced_gguf(monkeypatch, tmp_path):
"""Before an encode, a stopped server will respawn from the current file and must not
deduplicate with its old pooling. A completed encode still keeps the value it actually used."""
model, repo = "org/embed", "org/embed-GGUF"
monkeypatch.setattr(config, "effective_gguf_repo_for_embedding_model", lambda m: repo)
snapshot = _seed_cache(tmp_path / "hub", repo, ["embed-F16.gguf"])
served = (snapshot / "embed-F16.gguf").resolve()
_write_gguf(served, "qwen3", 3)
_use_cache_root(monkeypatch, tmp_path / "hub")
backend = LlamaServerBackend()
backend._adopt_model_path(str(served), repo)
monkeypatch.setattr(embeddings, "_backend", backend)
_write_gguf(served, "qwen3", 1)
legacy = config.embedding_identity("llama-server", model, gguf_repo = repo)
assert embeddings._identity(True, model) == legacy + ":mean"
assert embeddings._identity(True, model, backend) == legacy + ":last"
def test_llama_identity_treats_an_evicted_stopped_model_as_unresolved(monkeypatch, tmp_path):
"""A stopped backend cannot keep serving a deleted path. Pre-encode deduplication must
miss old CLS rows, and the next spawn must run the normal resolver instead of that path."""
model, repo = "org/embed", "org/embed-GGUF"
monkeypatch.setattr(config, "effective_gguf_repo_for_embedding_model", lambda m: repo)
snapshot = _seed_cache(tmp_path / "hub", repo, ["embed-F16.gguf"])
served = (snapshot / "embed-F16.gguf").resolve()
_write_gguf(served, "qwen3", 2)
_use_cache_root(monkeypatch, tmp_path / "hub")
backend = LlamaServerBackend()
backend._adopt_model_path(str(served), repo)
monkeypatch.setattr(embeddings, "_backend", backend)
served.unlink()
legacy = config.embedding_identity("llama-server", model, gguf_repo = repo)
assert embeddings._identity(True, model) == legacy + ":unresolved"
monkeypatch.setattr(backend, "_resolve_uncached_model_path", lambda *args: "/fresh.gguf")
assert backend._resolve_model_path(model) == "/fresh.gguf"
def test_preencode_identity_takes_one_locked_backend_snapshot(monkeypatch):
"""A concurrent model switch cannot combine model A's repo check with model B's pooling."""
class SwitchingBackend(LlamaServerBackend):
def __init__(self):
self.gate = False
self.repo_read = threading.Event()
self.allow_repo_read = threading.Event()
super().__init__()
@property
def _model_repo(self):
value = self._repo_value
if self.gate:
self.repo_read.set()
assert self.allow_repo_read.wait(timeout = 2)
return value
@_model_repo.setter
def _model_repo(self, value):
self._repo_value = value
@property
def _model_pooling(self):
return self._pooling_value
@_model_pooling.setter
def _model_pooling(self, value):
self._pooling_value = value
model, repo_a, repo_b = "org/a", "org/a-GGUF", "org/b-GGUF"
backend = SwitchingBackend()
backend._model_path = "/served/a.gguf"
backend._model_repo = repo_a
backend._model_pooling = "last"
backend._process = _FakeProc(alive = True)
backend.gate = True
monkeypatch.setattr(config, "effective_gguf_repo_for_embedding_model", lambda _model: repo_a)
monkeypatch.setattr(embeddings, "_backend", backend)
result = {}
switched_before_snapshot = []
def predict():
result["identity"] = embeddings._identity(True, model)
def switch_model():
assert backend.repo_read.wait(timeout = 2)
acquired = backend._serve_lock.acquire(timeout = 0.05)
switched_before_snapshot.append(acquired)
if acquired:
backend._repo_value = repo_b
backend._pooling_value = "mean"
backend._serve_lock.release()
backend.allow_repo_read.set()
if not acquired:
with backend._serve_lock:
backend._repo_value = repo_b
backend._pooling_value = "mean"
reader = threading.Thread(target = predict)
writer = threading.Thread(target = switch_model)
reader.start()
writer.start()
reader.join(timeout = 2)
writer.join(timeout = 2)
backend._process = None
assert not reader.is_alive() and not writer.is_alive()
assert switched_before_snapshot == [False]
assert result["identity"] == config.embedding_identity(
"llama-server", model, gguf_repo = repo_a, pooling = "last"
)
def test_encode_identity_keeps_the_callers_snapshot_when_another_model_takes_over(monkeypatch):
"""The shared backend can switch models after an encode returns. Each thread must retain
the immutable repo and pooling that served its own vectors, not the backend's later state."""
backend = LlamaServerBackend()
model_a, model_b = "org/a", "org/b"
repo_a, repo_b = "org/a-GGUF", "org/b-GGUF"
identities = {}
first_waiting = threading.Event()
second_done = threading.Event()
monkeypatch.setattr(embeddings, "_get_backend", lambda _model = None: backend)
monkeypatch.setattr(
config,
"effective_gguf_repo_for_embedding_model",
lambda model: repo_a if model == model_a else repo_b,
)
def fake_ready(model):
backend._model_repo = repo_a if model == model_a else repo_b
backend._model_pooling = "last" if model == model_a else "mean"
monkeypatch.setattr(backend, "_ensure_ready", fake_ready)
def fake_encode_active(
texts,
*,
model_name = None,
**_kwargs,
):
backend._ensure_ready(model_name)
return np.zeros((len(texts), 2), dtype = np.float32)
monkeypatch.setattr(backend, "_encode_active", fake_encode_active)
real_identity = embeddings._identity
def delayed_identity(
is_llama,
name,
served = None,
served_identity = None,
):
if name == model_a:
first_waiting.set()
assert second_done.wait(timeout = 2)
else:
second_done.set()
if served_identity is None:
return real_identity(is_llama, name, served)
return real_identity(is_llama, name, served, served_identity)
monkeypatch.setattr(embeddings, "_identity", delayed_identity)
def run(model):
identities[model] = embeddings.encode_with_identity(["x"], model_name = model)[1]
first = threading.Thread(target = run, args = (model_a,))
second = threading.Thread(target = run, args = (model_b,))
first.start()
assert first_waiting.wait(timeout = 2)
second.start()
first.join(timeout = 2)
second.join(timeout = 2)
assert not first.is_alive() and not second.is_alive()
assert identities[model_a] == config.embedding_identity(
"llama-server", model_a, gguf_repo = repo_a, pooling = "last"
)
assert identities[model_b] == config.embedding_identity(
"llama-server", model_b, gguf_repo = repo_b, pooling = "mean"
)
def test_llama_identity_is_not_predicted_from_a_fallback_repo(monkeypatch, tmp_path):
"""Only the desired repo's cache is read before going online; a fallback's GGUF may differ."""
model, repo = "org/embed", "org/embed-GGUF"
monkeypatch.setattr(config, "effective_gguf_repo_for_embedding_model", lambda m: repo)
snapshot = _seed_cache(tmp_path / "hub", model, ["embed-F16.gguf"])
_write_gguf((snapshot / "embed-F16.gguf").resolve(), "qwen3", 3)
_use_cache_root(monkeypatch, tmp_path / "hub")
legacy = config.embedding_identity("llama-server", model, gguf_repo = repo)
assert embeddings._identity(True, model) == legacy + ":unresolved"
def test_llama_identity_follows_the_planned_family_before_a_later_variant(monkeypatch, tmp_path):
"""The loader pins a completed download family before considering the configured
variant, so prediction must not take pooling from a different file that arrived later."""
import utils.embedding_model_settings as ems
model, repo = "org/embed", "org/embed-GGUF"
monkeypatch.setattr(config, "effective_gguf_repo_for_embedding_model", lambda m: repo)
monkeypatch.setattr(config, "EMBED_GGUF_VARIANT", "F16")
monkeypatch.setattr(ems, "get_stored_gguf_files", lambda m: ["embed-Q8_0.gguf"])
snapshot = _seed_cache(tmp_path / "hub", repo, ["embed-Q8_0.gguf", "embed-F16.gguf"])
_write_gguf((snapshot / "embed-Q8_0.gguf").resolve(), "qwen3", 3)
_write_gguf((snapshot / "embed-F16.gguf").resolve(), "bert", 2)
_use_cache_root(monkeypatch, tmp_path / "hub")
legacy = config.embedding_identity("llama-server", model, gguf_repo = repo)
assert embeddings._identity(True, model) == legacy + ":last"
@pytest.mark.parametrize("download_pending, suffix", [(False, ":unresolved"), (True, ":last")])
def test_llama_identity_only_predicts_a_stand_in_quant_while_its_download_is_pending(
monkeypatch, tmp_path, download_pending, suffix
):
"""Without the configured variant, the loader goes online unless an active download
makes the cached quant a deliberate stand-in. Only that stand-in is predictable."""
import utils.embedding_model_settings as ems
model, repo = "org/embed", "org/embed-GGUF"
monkeypatch.setattr(config, "effective_gguf_repo_for_embedding_model", lambda m: repo)
monkeypatch.setattr(config, "EMBED_GGUF_VARIANT", "F16")
monkeypatch.setattr(ems, "get_stored_gguf_files", lambda m: None)
monkeypatch.setattr(ems, "get_stored_download_pending", lambda m: download_pending)
snapshot = _seed_cache(tmp_path / "hub", repo, ["embed-Q8_0.gguf"])
_write_gguf((snapshot / "embed-Q8_0.gguf").resolve(), "qwen3", 3)
_use_cache_root(monkeypatch, tmp_path / "hub")
legacy = config.embedding_identity("llama-server", model, gguf_repo = repo)
assert embeddings._identity(True, model) == legacy + suffix
def test_spawn_uses_a_per_spawn_key_that_requests_send(monkeypatch, tmp_path):
import utils.paths.storage_roots as roots
monkeypatch.setattr(roots, "auth_root", lambda: tmp_path)
monkeypatch.delenv("UNSLOTH_LLAMA_SERVER_API_KEY", raising = False)
monkeypatch.setattr(config, "EMBED_PORT", 0)
b = LlamaServerBackend()
proc = _FakeProc(alive = True)
captured, keys = {}, []
_patch_spawn_deps(monkeypatch, proc)
_intercept_server_popen(monkeypatch, proc, captured)
def healthy(*_a, **_k):
cmd = captured["cmd"]
keys.append(Path(cmd[cmd.index("--api-key-file") + 1]).read_text())
return True
monkeypatch.setattr(b, "_wait_for_health", healthy)
b._spawn()
assert keys == [b._api_key] and b._auth_headers() == {"Authorization": f"Bearer {b._api_key}"}
# llama-server read it at startup; nothing is left on disk.
assert not list(tmp_path.glob("llama_api_key_*"))