Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Embedder concurrency tests: the fast tokenizer isn't thread-safe, so encode
and token counting must be serialized (else threads panic "Already borrowed")."""
import json
import os
import sys
import threading
import time
from types import SimpleNamespace
import numpy as np
import pytest
from core.rag import config, embeddings
def _shared_setup_1(monkeypatch):
observed = {}
class FakeSentenceTransformer:
def __init__(self, name, **kwargs):
observed.update(kwargs)
monkeypatch.setitem(
sys.modules,
"sentence_transformers",
SimpleNamespace(SentenceTransformer = FakeSentenceTransformer),
)
monkeypatch.setattr(embeddings, "_install_torchao_stub_once", lambda: None)
monkeypatch.setattr(embeddings, "_guard_model_security", lambda *_a, **_k: None)
return observed
def _shared_setup_2(
monkeypatch,
tmp_path,
name = "Org/Embedder",
):
monkeypatch.setattr(
"utils.hf_cache_settings.active_hf_hub_cache",
lambda: str(tmp_path / "selected-hub"),
)
embeddings._model = None
embeddings._name = None
# _get publishes into globals nothing restores; snapshot after the reset so teardown restores None.
monkeypatch.setattr(embeddings, "_model", None)
monkeypatch.setattr(embeddings, "_name", None)
embeddings._get(name)
def _shared_setup_3(monkeypatch):
monkeypatch.setattr(
embeddings,
"get_device",
lambda: embeddings.DeviceType.CUDA,
)
def _shared_setup_4(monkeypatch):
monkeypatch.setattr(
embeddings,
"_resolve_auto_for_model",
lambda model = None: "sentence-transformers",
)
# Mark the child non-dumpable: piped core_pattern (apport) ignores RLIMIT_CORE. Linux-only.
_CRASHING_UNLESS_CPU_SCRIPT = (
"import ctypes, os, sys\n"
"if sys.argv[1] != 'cpu':\n"
# First: every crash arm, abort() included, must follow the PR_SET_DUMPABLE clear.
" try:\n"
" ctypes.CDLL(None).prctl(4, 0, 0, 0, 0) # PR_SET_DUMPABLE = 0\n"
" except Exception:\n"
" pass\n"
# On Windows ctypes turns the null read into OSError; abort() (CRT exit 3) is the real shape.
" if sys.platform == 'win32':\n"
" os.abort()\n"
" ctypes.string_at(0)\n"
)
@pytest.fixture(autouse = True)
def _pin_st_backend(monkeypatch):
monkeypatch.setattr(config, "EMBED_BACKEND", "sentence-transformers")
embeddings._reset_backend()
yield
embeddings._reset_backend()
class _ConcurrencyProbe:
"""Records whether two callers were in the guarded body at once."""
def __init__(self):
self.inside = 0
self.saw_overlap = False
self._g = threading.Lock()
def enter(self):
with self._g:
self.inside += 1
if self.inside > 1:
self.saw_overlap = True
time.sleep(0.005)
with self._g:
self.inside -= 1
class _FakeModel:
def __init__(self, probe):
self._probe = probe
self.tokenizer = _FakeTokenizer(probe)
def encode(self, texts, **_kw):
self._probe.enter()
return np.zeros((len(texts), 4), dtype = np.float32)
class _FakeTokenizer:
def __init__(self, probe):
self._probe = probe
def encode(self, text, **_kw):
self._probe.enter()
return list(range(len(text.split())))
def _hammer(fn, n = 8):
errors: list[Exception] = []
def worker():
try:
fn()
except Exception as exc: # noqa: BLE001
errors.append(exc)
threads = [threading.Thread(target = worker) for _ in range(n)]
for t in threads:
t.start()
for t in threads:
t.join()
return errors
def test_first_encode_builds_the_selected_backend_once(monkeypatch):
"""Importing the facade is inert; the first real vector operation owns construction."""
builds: list[str] = []
class _Backend:
def encode(self, texts, **_kwargs):
return np.zeros((len(texts), 4), dtype = np.float32)
def _build(model_name = None):
builds.append("backend")
return _Backend()
monkeypatch.setattr(embeddings, "_build_st_backend_or_fallback", _build)
assert embeddings._backend is None
embeddings.encode(["first"])
embeddings.encode(["second"])
assert builds == ["backend"]
class _AutoBackend:
def encode(self, texts, **_kwargs):
return np.zeros((len(texts), 4), dtype = np.float32)
def token_counter(self, **_kwargs):
return lambda text: len(text.split())
def dim(self, **_kwargs):
return 4
def _resolve_auto_for(monkeypatch, *, stored = None):
import utils.embedding_model_settings as ems
asked: list[str] = []
monkeypatch.setattr(config, "EMBED_BACKEND", "auto")
monkeypatch.setattr(ems, "get_stored_backend", lambda _model: stored and stored["backend"])
monkeypatch.setattr(
embeddings, "_resolve_auto", lambda: asked.append("probe") or "sentence-transformers"
)
monkeypatch.setattr(
embeddings, "_build_st_backend_or_fallback", lambda model_name = None: _AutoBackend()
)
return asked
def test_auto_asks_the_hardware_once_for_the_backend_it_built(monkeypatch):
"""Every encode, token count and identity check resolves ``auto``, and the GPU probe
behind it starts a subprocess, so asking per call started ~3 per indexed document (#10390)."""
asked = _resolve_auto_for(monkeypatch)
embeddings.encode(["first"], model_name = "org/embedder")
assert asked == ["probe"]
for _ in range(3):
embeddings.encode(["again"], model_name = "org/embedder")
embeddings.token_counter("org/embedder")("some words")
embeddings.dim("org/embedder")
embeddings.embedding_identity("org/embedder")
assert asked == ["probe"]
embeddings.release_backend()
embeddings.encode(["after unload"], model_name = "org/embedder")
assert asked == ["probe", "probe"]
def test_a_saved_backend_still_replaces_a_resident_auto_backend(monkeypatch):
"""Saving a backend rebuilds even a resident auto pick (issue 9739) instead of keeping it."""
stored = {"backend": None}
_resolve_auto_for(monkeypatch, stored = stored)
_patch_llama_backend(monkeypatch, binary = "/fake/llama-server")
assert isinstance(embeddings._get_backend("org/embedder"), _AutoBackend)
stored["backend"] = "llama-server"
assert isinstance(embeddings._get_backend("org/embedder"), _SentinelLlamaBackend)
def test_a_replaced_backend_asks_the_hardware_again(monkeypatch):
"""The kept answer is tied to the object it was kept for, so a rebuild re-asks the hardware."""
asked = _resolve_auto_for(monkeypatch)
_patch_llama_backend(monkeypatch, binary = "/fake/llama-server")
first = embeddings._get_backend("org/embedder")
assert asked == ["probe"]
monkeypatch.setattr(embeddings, "_model_names_gguf_repo", lambda _model: True)
second = embeddings._get_backend("org/embedder")
assert second is not first
assert asked == ["probe"]
monkeypatch.setattr(embeddings, "_model_names_gguf_repo", lambda _model: False)
assert embeddings._resolve_auto_for_model("org/embedder") == "sentence-transformers"
assert asked == ["probe", "probe"]
def test_picker_plans_embeddinggemma_on_llama_server(monkeypatch):
stored = {"backend": None}
_resolve_auto_for(monkeypatch, stored = stored)
monkeypatch.setattr(embeddings, "_forced_backends", {})
monkeypatch.setattr(config, "EMBED_DEVICE", "auto")
monkeypatch.delenv("RAG_EMBED_GGUF_REPO", raising = False)
monkeypatch.setattr(embeddings, "sentence_transformers_runtime_available", lambda: True)
monkeypatch.setattr(embeddings, "_llama_server_runtime_available", lambda: True)
monkeypatch.setattr(embeddings, "sentence_transformers_can_load", lambda *a, **k: True)
effective = {"model": "org/previous-embedder"}
monkeypatch.setattr(config, "effective_embedding_model", lambda: effective["model"])
plan = embeddings.resolved_backend_for_model
assert plan("unsloth/embeddinggemma-300m") == "llama-server"
assert plan("unsloth/embeddinggemma-2") == "llama-server"
# Exact ids: a fine-tune need not publish the GGUF the preference relies on.
assert plan("org/embeddinggemma-300m-finetune") == "sentence-transformers"
# Only the plan: the runtime follows the saved record, so an env-configured model keeps its space.
assert embeddings._resolve_auto_for_model("unsloth/embeddinggemma-300m") == (
"sentence-transformers"
)
# The model in effect without a record runs on that default, and its Settings status must say so.
effective["model"] = "unsloth/embeddinggemma-300m"
assert plan("unsloth/embeddinggemma-300m") == "sentence-transformers"
effective["model"] = "org/previous-embedder"
# A model saved on sentence-transformers keeps the space its documents were indexed in.
stored["backend"] = "sentence-transformers"
assert plan("unsloth/embeddinggemma-300m") == "sentence-transformers"
stored["backend"] = None
monkeypatch.setattr(config, "EMBED_DEVICE", "gpu")
assert plan("unsloth/embeddinggemma-300m") == "sentence-transformers"
monkeypatch.setattr(config, "EMBED_DEVICE", "auto")
monkeypatch.setenv("RAG_EMBED_GGUF_REPO", "org/other-GGUF")
assert plan("unsloth/embeddinggemma-300m") == "sentence-transformers"
monkeypatch.delenv("RAG_EMBED_GGUF_REPO")
monkeypatch.setattr(config, "EMBED_BACKEND", "sentence-transformers")
assert plan("unsloth/embeddinggemma-300m") == "sentence-transformers"
monkeypatch.setattr(config, "EMBED_BACKEND", "auto")
monkeypatch.setattr(embeddings, "_llama_server_runtime_available", lambda: False)
assert plan("unsloth/embeddinggemma-300m") == "sentence-transformers"
def test_a_resolution_that_never_asked_the_hardware_keeps_nothing(monkeypatch):
"""_get_backend clears the per-thread answer first; otherwise a stale earlier answer gets published."""
asked = _resolve_auto_for(monkeypatch)
_patch_llama_backend(monkeypatch, binary = "/fake/llama-server")
embeddings.active_backend_is_llama("org/embedder")
assert asked == ["probe"]
monkeypatch.setattr(embeddings, "_model_names_gguf_repo", lambda _model: True)
built = embeddings._get_backend("org/embedder")
assert isinstance(built, _SentinelLlamaBackend)
assert embeddings._resident_hardware is None
def test_encode_is_serialized(monkeypatch):
probe = _ConcurrencyProbe()
monkeypatch.setattr(embeddings, "_get", lambda model_name = None: _FakeModel(probe))
errors = _hammer(lambda: embeddings.encode(["alpha beta", "gamma"]))
assert errors == []
assert probe.saw_overlap is False
def test_token_counter_is_serialized(monkeypatch):
probe = _ConcurrencyProbe()
monkeypatch.setattr(embeddings, "_get", lambda model_name = None: _FakeModel(probe))
count = embeddings.token_counter()
errors = _hammer(lambda: count("one two three four"))
assert errors == []
assert probe.saw_overlap is False
def test_encode_enables_parallelism_only_during_call(monkeypatch):
seen = {}
class _M:
tokenizer = None
def encode(self, texts, **_kw):
seen["during"] = os.environ.get("TOKENIZERS_PARALLELISM")
return np.zeros((len(texts), 4), dtype = np.float32)
monkeypatch.setattr(embeddings, "_get", lambda model_name = None: _M())
os.environ["TOKENIZERS_PARALLELISM"] = "false"
embeddings.encode(["alpha", "beta"])
assert seen["during"] == "true"
assert os.environ.get("TOKENIZERS_PARALLELISM") == "false"
def test_token_counter_enables_parallelism_only_during_call(monkeypatch):
seen = {}
class _Tok:
def encode(self, text, **_kw):
seen["during"] = os.environ.get("TOKENIZERS_PARALLELISM")
return list(range(len(text.split())))
class _M:
tokenizer = _Tok()
monkeypatch.setattr(embeddings, "_get", lambda model_name = None: _M())
os.environ["TOKENIZERS_PARALLELISM"] = "false"
count = embeddings.token_counter()
count("alpha beta gamma")
assert seen["during"] == "true"
assert os.environ.get("TOKENIZERS_PARALLELISM") == "false"
def test_token_counter_reacquires_backend_retired_between_chunk_calls(monkeypatch):
from core.rag.embed_llama_server import LlamaServerBackend
retired = LlamaServerBackend()
replacement = LlamaServerBackend()
monkeypatch.setattr(embeddings, "_get_backend", lambda *_a, **_k: retired)
count = embeddings.token_counter("org/embedder")
retired._closed = True
monkeypatch.setattr(embeddings, "_get_backend", lambda *_a, **_k: replacement)
monkeypatch.setattr(
replacement,
"_post",
lambda path, payload, **_k: {"tokens": [1, 2, 3]},
)
assert count("the next chunk") == 3
def test_token_counter_does_not_hide_non_lifecycle_errors(monkeypatch):
class _BrokenCounterBackend:
_closed = False
def token_counter(self, *, model_name = None):
def _raise(_text):
raise RuntimeError("invalid tokenizer response")
return _raise
monkeypatch.setattr(embeddings, "_get_backend", lambda *_a, **_k: _BrokenCounterBackend())
with pytest.raises(RuntimeError, match = "invalid tokenizer response"):
embeddings.token_counter()("chunk")
def test_st_unload_waits_for_encode_admitted_before_model_lookup(monkeypatch):
entered_lookup = threading.Event()
finish_lookup = threading.Event()
unload_done = threading.Event()
order = []
errors = []
class _Model:
def encode(self, texts, **kwargs):
order.append("encode")
return np.zeros((len(texts), 2), dtype = np.float32)
def _get(model_name = None):
entered_lookup.set()
assert finish_lookup.wait(timeout = 2)
return _Model()
def _encode():
try:
embeddings._st_encode(["chunk"])
except Exception as exc: # pragma: no cover - surfaced below
errors.append(exc)
monkeypatch.setattr(embeddings, "_get", _get)
monkeypatch.setattr(embeddings, "_model", object())
monkeypatch.setattr(embeddings, "_name", "org/embedder")
worker = threading.Thread(target = _encode)
worker.start()
assert entered_lookup.wait(timeout = 2)
def _unload():
embeddings._release_st_model()
order.append("unload")
unload_done.set()
closer = threading.Thread(target = _unload)
closer.start()
assert unload_done.wait(timeout = 0.05) is False
finish_lookup.set()
# The release runs gc.collect() and, once a device is detected, a GPU cache clear: late in
# a busy xdist worker that took over 2s and the old join returned before "unload" landed.
worker.join(timeout = 30)
closer.join(timeout = 30)
assert not worker.is_alive(), "the encode never finished"
assert not closer.is_alive(), "the unload never finished"
assert errors == []
assert order == ["encode", "unload"]
assert unload_done.is_set()
def test_sentence_transformer_load_uses_live_cache(monkeypatch, tmp_path):
observed = {}
class FakeSentenceTransformer:
def __init__(self, name, **kwargs):
observed["name"] = name
observed.update(kwargs)
monkeypatch.setitem(
sys.modules,
"sentence_transformers",
SimpleNamespace(SentenceTransformer = FakeSentenceTransformer),
)
monkeypatch.setattr(embeddings, "_install_torchao_stub_once", lambda: None)
monkeypatch.setattr(embeddings, "_guard_model_security", lambda *_a, **_k: None)
monkeypatch.setattr(embeddings, "_device", lambda: "cpu")
_shared_setup_2(monkeypatch, tmp_path)
assert observed["name"] == "Org/Embedder"
assert observed["cache_folder"] == str(tmp_path / "selected-hub")
# fp32: dtype follows the device actually loaded on.
assert list(observed["model_kwargs"].values()) == ["float32"]
def test_device_defaults_to_cpu_on_an_accelerator_host(monkeypatch):
"""Auto device stays on CPU: a CUDA allocation pins a primary context nothing releases."""
monkeypatch.setattr(embeddings.config, "EMBED_DEVICE", "auto")
_shared_setup_3(monkeypatch)
assert embeddings._device() == "cpu"
def test_device_opts_in_to_the_accelerator(monkeypatch):
"""Every accelerator spelling, including xpu and rocm, must opt in and resolve to cuda."""
_shared_setup_3(monkeypatch)
for requested in ("gpu", "GPU", " cuda ", "rocm", "hip", "xpu", "mps", "metal"):
monkeypatch.setattr(embeddings.config, "EMBED_DEVICE", requested)
assert embeddings._device() == "cuda", requested
def test_device_opt_in_still_yields_cpu_without_an_accelerator(monkeypatch):
monkeypatch.setattr(embeddings.config, "EMBED_DEVICE", "gpu")
monkeypatch.setattr(embeddings, "get_device", lambda: embeddings.DeviceType.CPU)
assert embeddings._device() == "cpu"
def test_device_opt_in_on_apple_stays_on_cpu(monkeypatch):
"""MLX is not a torch device. Asking for a GPU must not produce a device string
torch cannot open, which is why this stays a lookup in _TORCH_DEVICE."""
monkeypatch.setattr(embeddings.config, "EMBED_DEVICE", "gpu")
monkeypatch.setattr(embeddings, "get_device", lambda: embeddings.DeviceType.MLX)
assert embeddings._device() == "cpu"
def test_unrecognized_device_setting_falls_back_without_raising(monkeypatch):
_shared_setup_3(monkeypatch)
for requested in ("", " ", "banana", "auto", None):
monkeypatch.setattr(embeddings.config, "EMBED_DEVICE", requested)
assert embeddings._device() == "cpu", requested
def test_cpu_never_loads_float16(monkeypatch, tmp_path):
"""CPU must never load float16; torch 2.2 has no CPU Half LayerNorm kernel for BERT models."""
observed = _shared_setup_1(monkeypatch)
monkeypatch.setattr(
"utils.hf_cache_settings.active_hf_hub_cache",
lambda: str(tmp_path / "hub"),
)
for embed_device, hardware, load_device in (
("auto", embeddings.DeviceType.CUDA, None),
("cpu", embeddings.DeviceType.CUDA, None),
("gpu", embeddings.DeviceType.CPU, None),
("gpu", embeddings.DeviceType.CUDA, "cpu"),
):
monkeypatch.setattr(embeddings.config, "EMBED_DEVICE", embed_device)
monkeypatch.setattr(embeddings, "get_device", lambda hw = hardware: hw)
if load_device is not None:
monkeypatch.setattr(embeddings, "_load_device", lambda d = load_device: d)
embeddings._model = None
embeddings._name = None
observed.clear()
embeddings._get("Org/Embedder")
assert observed["device"] == "cpu", (embed_device, hardware)
assert list(observed["model_kwargs"].values()) == ["float32"], (embed_device, hardware)
embeddings._model = None
embeddings._name = None
def test_opted_in_accelerator_loads_float16(monkeypatch, tmp_path):
observed = _shared_setup_1(monkeypatch)
monkeypatch.setattr(embeddings, "_load_device", lambda: "cuda")
_shared_setup_2(monkeypatch, tmp_path)
assert observed["device"] == "cuda"
assert list(observed["model_kwargs"].values()) == ["float16"]
@pytest.mark.parametrize(
"device, native, expected",
[
("cuda", True, "bfloat16"),
("cuda", False, "float32"),
("xpu", True, "bfloat16"),
("xpu", False, "float32"),
],
)
def test_float16_unsafe_model_avoids_float16(monkeypatch, tmp_path, device, native, expected):
import core.training.diffusion_train_common as dtc
observed = _shared_setup_1(monkeypatch)
monkeypatch.setattr(embeddings, "_load_device", lambda: device)
monkeypatch.setattr(dtc, "native_bf16_supported", lambda: native)
monkeypatch.setattr(dtc, "native_bf16_supported_xpu", lambda: native)
_shared_setup_2(monkeypatch, tmp_path, name = "unsloth/EmbeddingGemma-300m")
assert observed["device"] == device
assert list(observed["model_kwargs"].values()) == [expected]
def test_accelerator_fallback_loads_float32_on_cpu(monkeypatch, tmp_path):
observed = _shared_setup_1(monkeypatch)
monkeypatch.setattr(embeddings, "_load_device", lambda: "cpu")
monkeypatch.setattr(embeddings, "_device", lambda: "cuda")
_shared_setup_2(monkeypatch, tmp_path)
assert observed["device"] == "cpu"
assert list(observed["model_kwargs"].values()) == ["float32"]
class _SentinelLlamaBackend:
"""Stand-in for LlamaServerBackend; never spawns a real server."""
def _force_st_load_failure(monkeypatch):
"""Make the ST warm-probe raise."""
def _boom(model_name = None):
raise RuntimeError("torch is broken on this machine")
monkeypatch.setattr(embeddings, "_get", _boom)
def _patch_llama_backend(monkeypatch, *, binary):
from core.inference.llama_cpp import LlamaCppBackend
from core.rag import embed_llama_server
monkeypatch.setattr(LlamaCppBackend, "_find_llama_server_binary", staticmethod(lambda: binary))
monkeypatch.setattr(embed_llama_server, "LlamaServerBackend", _SentinelLlamaBackend)
def test_st_failure_falls_back_to_llama_server(monkeypatch):
_force_st_load_failure(monkeypatch)
_patch_llama_backend(monkeypatch, binary = "/fake/llama-server")
embeddings._reset_backend()
backend = embeddings._get_backend()
assert isinstance(backend, _SentinelLlamaBackend)
def test_st_failure_without_llama_binary_reraises(monkeypatch):
_force_st_load_failure(monkeypatch)
_patch_llama_backend(monkeypatch, binary = None)
embeddings._reset_backend()
with pytest.raises(RuntimeError, match = "torch is broken"):
embeddings._get_backend()
def test_st_success_keeps_sentence_transformers(monkeypatch):
monkeypatch.setattr(embeddings, "_get", lambda model_name = None: object())
_patch_llama_backend(monkeypatch, binary = "/fake/llama-server")
embeddings._reset_backend()
backend = embeddings._get_backend()
assert isinstance(backend, embeddings._SentenceTransformersBackend)
def test_loaded_state_belongs_to_the_resident_sentence_transformer(monkeypatch):
monkeypatch.setattr(embeddings, "_backend", embeddings._SentenceTransformersBackend())
monkeypatch.setattr(embeddings, "_model", object())
monkeypatch.setattr(embeddings, "_name", "org/resident")
assert embeddings.backend_is_loaded() is True
assert embeddings.backend_is_loaded("org/resident") is True
assert embeddings.backend_is_loaded("org/new-selection") is False
def test_loaded_state_belongs_to_the_resident_gguf_repo(monkeypatch):
# A live process: residency means the subprocess is really there.
backend = SimpleNamespace(_model_repo = "org/resident-GGUF", _process_alive = lambda: True)
monkeypatch.setattr(embeddings, "_backend", backend)
monkeypatch.setattr(embeddings, "_is_llama_backend", lambda value: value is backend)
monkeypatch.setattr(
embeddings.config,
"effective_gguf_repo_for_embedding_model",
lambda model: f"{model}-GGUF",
)
assert embeddings.backend_is_loaded("org/resident") is True
assert embeddings.backend_is_loaded("org/new-selection") is False
def test_unload_clears_the_sentence_transformer_weights(monkeypatch):
embeddings._backend = embeddings._SentenceTransformersBackend()
embeddings._backend_key = embeddings._current_backend_key()
embeddings._model = object()
embeddings._name = "org/embedder"
assert embeddings.release_backend() is True
assert embeddings._model is None
assert embeddings._name is None
def test_pending_sentence_transformer_refuses_implicit_download_and_llama_fallback(monkeypatch):
import utils.embedding_model_settings as ems
import utils.utils as utils
class _MustNotLoad:
def __init__(self, *args, **kwargs): # pragma: no cover - failure is the assertion
raise AssertionError("pending model reached SentenceTransformer")
monkeypatch.setitem(
sys.modules,
"sentence_transformers",
SimpleNamespace(SentenceTransformer = _MustNotLoad),
)
monkeypatch.setattr(ems, "get_stored_download_pending", lambda model: True)
monkeypatch.setattr(utils, "hf_cache_snapshot_is_loadable", lambda model: False)
monkeypatch.setattr(embeddings, "_install_torchao_stub_once", lambda: None)
monkeypatch.setattr(embeddings, "_load_device", lambda: "cpu")
monkeypatch.setattr(
embeddings,
"_try_make_llama_backend",
lambda: (_ for _ in ()).throw(AssertionError("pending ST fell back to llama")),
)
embeddings._model = None
embeddings._name = None
with pytest.raises(embeddings.EmbeddingModelDownloadRequiredError, match = "not downloaded"):
embeddings._build_st_backend_or_fallback()
def test_replacing_a_llama_backend_shuts_it_down(monkeypatch):
calls = []
class _OldLlama:
def _shutdown(self):
calls.append("shutdown")
monkeypatch.setattr(embeddings.config, "EMBED_BACKEND", "sentence-transformers")
monkeypatch.setattr(
embeddings,
"_build_st_backend_or_fallback",
lambda *_a, **_k: embeddings._SentenceTransformersBackend(),
)
embeddings._backend = _OldLlama()
embeddings._backend_key = "stale"
assert isinstance(embeddings._get_backend(), embeddings._SentenceTransformersBackend)
assert calls == ["shutdown"]
def test_explicit_llama_backend_disallows_st_resolution(monkeypatch):
monkeypatch.setattr(embeddings.config, "EMBED_BACKEND", "llama-server")
monkeypatch.setattr(embeddings, "_forced_backends", {})
assert embeddings.sentence_transformers_fallback_allowed() is False
monkeypatch.setattr(embeddings.config, "EMBED_BACKEND", "auto")
assert embeddings.sentence_transformers_fallback_allowed() is True
monkeypatch.setattr(
embeddings,
"_forced_backends",
{embeddings.config.effective_embedding_model(): "llama-server"},
)
assert embeddings.sentence_transformers_fallback_allowed() is False
def test_forced_llama_fallback_is_scoped_to_the_model_that_failed(monkeypatch):
monkeypatch.setattr(embeddings.config, "EMBED_BACKEND", "auto")
monkeypatch.setattr(embeddings, "_forced_backends", {"org/failed": "llama-server"})
_shared_setup_4(monkeypatch)
assert embeddings.sentence_transformers_fallback_allowed("org/failed") is False
assert embeddings.sentence_transformers_fallback_allowed("org/new-model") is True
assert embeddings.resolved_backend_for_model("org/failed") == "llama-server"
assert embeddings.resolved_backend_for_model("org/new-model") == "sentence-transformers"
def test_runtime_preflight_predicts_the_supported_llama_fallback(monkeypatch):
monkeypatch.setattr(embeddings.config, "EMBED_BACKEND", "auto")
monkeypatch.setattr(embeddings, "_forced_backends", {})
_shared_setup_4(monkeypatch)
monkeypatch.setattr(embeddings, "sentence_transformers_runtime_available", lambda: False)
monkeypatch.setattr(embeddings, "_llama_server_runtime_available", lambda: True)
assert embeddings.resolved_backend_for_model("org/embedder") == "llama-server"
def test_runtime_preflight_keeps_st_when_no_fallback_exists(monkeypatch):
monkeypatch.setattr(embeddings.config, "EMBED_BACKEND", "auto")
monkeypatch.setattr(embeddings, "_forced_backends", {})
_shared_setup_4(monkeypatch)
monkeypatch.setattr(embeddings, "sentence_transformers_runtime_available", lambda: False)
monkeypatch.setattr(embeddings, "_llama_server_runtime_available", lambda: False)
assert embeddings.resolved_backend_for_model("org/embedder") == "sentence-transformers"
def _st_folder(tmp_path, module_type, model_config):
import json
folder = tmp_path / "embedder"
folder.mkdir()
(folder / "modules.json").write_text(
json.dumps([{"idx": 0, "name": "0", "path": "", "type": module_type}])
)
(folder / "config.json").write_text(json.dumps(model_config))
return str(folder)
@pytest.mark.parametrize(
"module_type, model_config, loadable",
[
("sentence_transformers.models.Transformer", {"model_type": "bert"}, True),
# How a repo saved by sentence-transformers 6 names its modules (embeddinggemma-2).
(
"sentence_transformers.no_such_module.transformer.Transformer",
{"model_type": "bert"},
False,
),
("sentence_transformers.models.Transformer", {"model_type": "no_such_model_type"}, False),
# The loader never trusts remote code, so an auto_map does not make an unknown type loadable.
(
"sentence_transformers.models.Transformer",
{"model_type": "no_such_model_type", "auto_map": {}},
False,
),
],
)
def test_st_can_load_reads_modules_and_model_type(tmp_path, module_type, model_config, loadable):
pytest.importorskip("sentence_transformers")
pytest.importorskip("transformers")
assert (
embeddings.sentence_transformers_can_load(_st_folder(tmp_path, module_type, model_config))
is loadable
)
@pytest.mark.parametrize(
"root_model_type,module_model_type,loadable",
[
("bert", "no_such_model_type", False),
("no_such_model_type", "bert", True),
],
)
def test_st_can_load_reads_the_transformer_modules_config(
tmp_path, root_model_type, module_model_type, loadable
):
pytest.importorskip("sentence_transformers")
pytest.importorskip("transformers")
folder = tmp_path / "embedder"
module = folder / "0_Transformer"
module.mkdir(parents = True)
(folder / "modules.json").write_text(
json.dumps(
[
{
"idx": 0,
"name": "0",
"path": "0_Transformer",
"type": "sentence_transformers.models.Transformer",
}
]
)
)
(folder / "config.json").write_text(json.dumps({"model_type": root_model_type}))
(module / "config.json").write_text(json.dumps({"model_type": module_model_type}))
assert embeddings.sentence_transformers_can_load(str(folder)) is loadable
def test_st_can_load_preserves_anonymous_hub_access(monkeypatch):
pytest.importorskip("sentence_transformers")
seen = []
def _read(name, filename, token):
seen.append((filename, token))
return None if filename == "modules.json" else {}
monkeypatch.setattr(embeddings, "_repo_json", _read)
assert embeddings.sentence_transformers_can_load("org/public", False) is True
assert seen == [("modules.json", False), ("config.json", False)]
def test_st_can_load_without_the_files_keeps_the_plan(tmp_path):
assert embeddings.sentence_transformers_can_load(str(tmp_path)) is True
def test_st_can_load_gives_up_on_a_stalled_hub(monkeypatch):
pytest.importorskip("sentence_transformers")
monkeypatch.setattr(embeddings, "_ST_LOAD_PREFLIGHT_TIMEOUT_S", 0.05)
monkeypatch.setattr(embeddings, "_repo_json", lambda *_: time.sleep(1))
started = time.monotonic()
assert embeddings.sentence_transformers_can_load("org/stalled") is True
assert time.monotonic() - started < 0.5
def test_model_st_cannot_open_is_planned_on_llama(monkeypatch):
monkeypatch.setattr(embeddings.config, "EMBED_BACKEND", "auto")
monkeypatch.setattr(embeddings, "_forced_backends", {})
_shared_setup_4(monkeypatch)
monkeypatch.setattr(embeddings, "sentence_transformers_runtime_available", lambda: True)
monkeypatch.setattr(
embeddings,
"sentence_transformers_can_load",
lambda model, token = None: model != "org/st6-model",
)
monkeypatch.setattr(embeddings, "_llama_server_runtime_available", lambda: True)
assert embeddings.resolved_backend_for_model("org/st6-model") == "llama-server"
assert embeddings.resolved_backend_for_model("org/embedder") == "sentence-transformers"
# An explicit ST policy makes the runtime ignore a stored llama backend, so the plan must not promise one.
monkeypatch.setattr(embeddings.config, "EMBED_BACKEND", "sentence-transformers")
assert embeddings.resolved_backend_for_model("org/st6-model") == "sentence-transformers"
monkeypatch.setattr(embeddings.config, "EMBED_BACKEND", "auto")
monkeypatch.setattr(embeddings, "_llama_server_runtime_available", lambda: False)
assert embeddings.resolved_backend_for_model("org/st6-model") == "sentence-transformers"
def test_runtime_preflight_catches_a_fatal_torch_device_mismatch(monkeypatch):
monkeypatch.setattr(
embeddings,
"_load_device",
lambda: (_ for _ in ()).throw(embeddings.TorchDeviceUnusableError("broken torch")),
)
assert embeddings.sentence_transformers_runtime_available() is False
class _BoomOnEncodeModel:
"""Loads fine (init probe passes) but raises when encoding."""
tokenizer = None
def encode(self, texts, **_kw):
raise RuntimeError("CUDA error during encode")
def test_st_encode_runtime_failure_switches_to_llama(monkeypatch):
monkeypatch.setattr(embeddings, "_get", lambda model_name = None: _BoomOnEncodeModel())
_patch_llama_backend(monkeypatch, binary = "/fake/llama-server")
calls = {}
embeddings._model = object()
embeddings._name = "org/embedder"
def _sentinel_encode(
self,
texts,
*,
model_name = None,
normalize = True,
):
calls["used"] = True
return np.zeros((len(texts), 4), dtype = np.float32)
monkeypatch.setattr(_SentinelLlamaBackend, "encode", _sentinel_encode, raising = False)
embeddings._reset_backend()
out = embeddings.encode(["alpha", "beta"])
assert calls.get("used") is True
assert out.shape == (2, 4)
assert embeddings._model is None
assert embeddings._name is None
assert embeddings.config.effective_embedding_model() in embeddings._forced_backends
assert isinstance(embeddings._get_backend(), _SentinelLlamaBackend)
# It outranks the saved model's resolution so ST cannot be re-chosen after failing.
monkeypatch.setattr(embeddings, "_resolve_auto_for_model", lambda: "sentence-transformers")
assert isinstance(embeddings._get_backend(), _SentinelLlamaBackend)
embeddings._reset_backend()
assert embeddings._forced_backends == {}
def test_st_encode_failure_without_llama_binary_reraises(monkeypatch):
monkeypatch.setattr(embeddings, "_get", lambda model_name = None: _BoomOnEncodeModel())
_patch_llama_backend(monkeypatch, binary = None)
embeddings._reset_backend()
with pytest.raises(RuntimeError, match = "CUDA error during encode"):
embeddings.encode(["alpha", "beta"])
def _patch_probe(monkeypatch, usable):
"""Return configured probe results and record the devices checked."""
from utils import torch_device_probe
asked = []
def _can_allocate(device):
asked.append(device)
return usable[device]
monkeypatch.setattr(torch_device_probe, "device_can_allocate", _can_allocate)
return asked
class _ImportIsACrash:
"""Fail if sentence-transformers is reached after both device probes crash."""
def __getattr__(self, name):
raise AssertionError(f"sentence-transformers was reached ({name}) on a crashing host")
def test_load_device_keeps_the_accelerator_when_it_is_usable(monkeypatch):
monkeypatch.setattr(embeddings, "_device", lambda: "cuda")
asked = _patch_probe(monkeypatch, {"cuda": True})
assert embeddings._load_device() == "cuda"
assert asked == ["cuda"]
def test_load_device_degrades_to_cpu_when_the_accelerator_crashes(monkeypatch):
monkeypatch.setattr(embeddings, "_device", lambda: "cuda")
_patch_probe(monkeypatch, {"cuda": False, "cpu": True})
assert embeddings._load_device() == "cpu"
def test_load_device_raises_when_torch_crashes_on_cpu_too(monkeypatch):
monkeypatch.setattr(embeddings, "_device", lambda: "cuda")
_patch_probe(monkeypatch, {"cuda": False, "cpu": False})
with pytest.raises(embeddings.TorchDeviceUnusableError):
embeddings._load_device()
def test_load_device_does_not_probe_a_cpu_only_host(monkeypatch):
monkeypatch.setattr(embeddings, "_device", lambda: "cpu")
asked = _patch_probe(monkeypatch, {})
assert embeddings._load_device() == "cpu"
assert asked == []
def test_a_real_crashing_child_moves_the_load_to_cpu(monkeypatch):
from utils import torch_device_probe
monkeypatch.setenv(torch_device_probe.DISABLE_ENV_VAR, "0")
monkeypatch.setattr(
torch_device_probe,
"_PROBE_SCRIPT",
_CRASHING_UNLESS_CPU_SCRIPT,
)
torch_device_probe.device_can_allocate.cache_clear()
monkeypatch.setattr(embeddings, "_device", lambda: "cuda")
try:
assert embeddings._load_device() == "cpu"
finally:
torch_device_probe.device_can_allocate.cache_clear()
def test_crashing_torch_falls_back_to_llama_server(monkeypatch):
monkeypatch.setattr(embeddings, "_device", lambda: "cuda")
_patch_probe(monkeypatch, {"cuda": False, "cpu": False})
_patch_llama_backend(monkeypatch, binary = "/fake/llama-server")
monkeypatch.setitem(sys.modules, "sentence_transformers", _ImportIsACrash())
embeddings._model = None
embeddings._name = None
embeddings._reset_backend()
assert isinstance(embeddings._get_backend(), _SentinelLlamaBackend)
def test_encode_reacquires_a_backend_retired_between_batches(monkeypatch):
"""`release_backend` promises the next embed rebuilds. Without a reacquire here
that promise held only for `token_counter`, and pressing Unload mid-ingestion
failed the document being indexed instead of continuing it."""
from core.rag.embed_llama_server import LlamaServerBackend
retired = LlamaServerBackend()
replacement = LlamaServerBackend()
retired._closed = True
served = np.zeros((1, 4), dtype = np.float32)
monkeypatch.setattr(
replacement,
"encode",
lambda texts, **kwargs: served,
)
backends = iter([retired, replacement])
monkeypatch.setattr(embeddings, "_get_backend", lambda *_a, **_k: next(backends))
assert embeddings.encode(["chunk"]) is served
assert embeddings._served_by.backend is replacement
def test_encode_does_not_hide_a_non_lifecycle_runtime_error(monkeypatch):
class _BrokenBackend:
_closed = False
def encode(self, texts, **kwargs):
raise RuntimeError("llama-server returned no embedding")
monkeypatch.setattr(embeddings, "_get_backend", lambda *_a, **_k: _BrokenBackend())
with pytest.raises(RuntimeError, match = "returned no embedding"):
embeddings.encode(["chunk"])
def test_encode_surfaces_the_unload_when_no_replacement_is_published(monkeypatch):
from core.rag.embed_llama_server import LlamaServerBackend
retired = LlamaServerBackend()
retired._closed = True
monkeypatch.setattr(embeddings, "_get_backend", lambda *_a, **_k: retired)
with pytest.raises(RuntimeError, match = "was unloaded"):
embeddings.encode(["chunk"])
def test_the_token_counter_follows_an_unloaded_sentence_transformer(monkeypatch):
"""Read the tokenizer per call under the compute lock, so an unloaded model stops counting."""
looked_up = []
class _Tok:
def __init__(self, n):
self.n = n
def encode(
self,
text,
add_special_tokens = False,
):
return list(range(self.n))
class _Model:
def __init__(self, n):
self.tokenizer = _Tok(n)
models = [_Model(3), _Model(7)]
def _get(model_name = None):
looked_up.append(model_name)
return models[min(len(looked_up) - 1, 1)]
monkeypatch.setattr(embeddings, "_get", _get)
count = embeddings._st_token_counter("org/embedder")
assert looked_up == [], "the lookup must not happen before the first call"
assert count("first") == 3
assert count("second") == 7
assert looked_up == ["org/embedder", "org/embedder"]
def test_a_local_gguf_selects_llama_even_on_a_gpu_box(monkeypatch, tmp_path):
"""_resolve_auto answers sentence-transformers whenever a GPU is present, so a
GPU box pointed at a local .gguf planned an ST load of a GGUF file."""
gguf = tmp_path / "embed.gguf"
gguf.write_bytes(b"GGUF")
monkeypatch.setattr(embeddings.config, "EMBED_BACKEND", "auto")
monkeypatch.setattr(embeddings, "_forced_backends", {})
monkeypatch.setattr(embeddings, "_resolve_auto", lambda: "sentence-transformers")
assert embeddings._resolve_auto_for_model(str(gguf)) == "llama-server"
assert embeddings.resolved_backend_for_model(str(gguf)) == "llama-server"
assert embeddings._resolve_auto_for_model(str(tmp_path)) == "llama-server"
assert embeddings._resolve_auto_for_model("unsloth/bge-small-en-v1.5") == (
"sentence-transformers"
)
def test_a_local_gguf_beats_a_stored_sentence_transformers_record(monkeypatch, tmp_path):
"""A stored ST record for a .gguf only comes from a failed force-save, so it must not win."""
gguf = tmp_path / "embed.gguf"
gguf.write_bytes(b"GGUF")
monkeypatch.setattr(embeddings.config, "EMBED_BACKEND", "auto")
monkeypatch.setattr(embeddings, "_resolve_auto", lambda: "sentence-transformers")
import utils.embedding_model_settings as ems
monkeypatch.setattr(ems, "get_stored_backend", lambda _m: "sentence-transformers")
assert embeddings._resolve_auto_for_model(str(gguf)) == "llama-server"
def test_an_explicit_backend_is_not_overridden_by_a_local_gguf(monkeypatch, tmp_path):
"""Only ``auto`` consults the probe; an explicit setting stays verbatim."""
gguf = tmp_path / "embed.gguf"
gguf.write_bytes(b"GGUF")
monkeypatch.setattr(embeddings.config, "EMBED_BACKEND", "sentence-transformers")
monkeypatch.setattr(embeddings, "_forced_backends", {})
monkeypatch.setattr(embeddings, "sentence_transformers_runtime_available", lambda: True)
assert embeddings.resolved_backend_for_model(str(gguf)) == "sentence-transformers"
def test_the_st_probe_warms_the_pinned_model_not_the_live_setting(monkeypatch):
"""The probe must warm the pinned model, not the live setting, or a job pinned to A probes B."""
warmed = []
class _Probe:
def warm(self, model_name = None):
warmed.append(model_name)
monkeypatch.setattr(embeddings, "_SentenceTransformersBackend", _Probe)
monkeypatch.setattr(embeddings.config, "EMBED_BACKEND", "sentence-transformers")
monkeypatch.setattr(embeddings, "_forced_backends", {})
embeddings._reset_backend()
try:
embeddings._get_backend("org/pinned")
finally:
embeddings._reset_backend()
assert warmed == ["org/pinned"]
def test_the_security_gate_scans_the_snapshot_that_is_actually_loaded(monkeypatch, tmp_path):
"""The security gate must scan the snapshot that is actually loaded, not the Hub's current commit."""
snapshot = tmp_path / "snap"
snapshot.mkdir()
(snapshot / "model.safetensors").write_bytes(b"ST")
scanned = []
monkeypatch.setattr(
embeddings,
"_guard_model_security",
lambda target, local_only = False, display = None: scanned.append(target),
)
class _ST:
def __init__(self, target, **_kwargs):
self.loaded = target
self.tokenizer = None
def get_sentence_embedding_dimension(self):
return 8
monkeypatch.setattr(embeddings, "_load_device", lambda: "cpu")
monkeypatch.setattr(embeddings, "_install_torchao_stub_once", lambda: None)
monkeypatch.setattr(embeddings, "_st_accepts_local_files_only", lambda _c: False)
import utils.utils as utils
monkeypatch.setattr(utils, "cached_st_source", lambda m: ("org/embedder", snapshot))
monkeypatch.setattr(utils, "hf_cache_snapshot_dir", lambda m: snapshot)
import sys as _sys
import types as _t
st_mod = _t.ModuleType("sentence_transformers")
st_mod.SentenceTransformer = _ST
monkeypatch.setitem(_sys.modules, "sentence_transformers", st_mod)
embeddings._model = None
embeddings._name = None
try:
model = embeddings._get("org/embedder")
finally:
embeddings._model = None
embeddings._name = None
assert model.loaded == str(snapshot)
assert scanned == [str(snapshot)]
def test_the_shared_load_setup_does_not_strand_module_weights(tmp_path):
"""The shared load setup must not strand module weights for later tests in the same xdist worker."""
with pytest.MonkeyPatch.context() as mp:
_shared_setup_1(mp)
mp.setattr(embeddings, "_device", lambda: "cpu")
_shared_setup_2(mp, tmp_path)
assert embeddings._model is not None, "the setup loaded nothing, so this proves nothing"
assert embeddings._name == "Org/Embedder"
assert embeddings._model is None
assert embeddings._name is None
@pytest.mark.parametrize("resident", [False, True])
def test_the_residency_probe_does_not_wait_on_a_model_load(monkeypatch, resident):
"""The residency probe must answer under the load locks, not wait on a model load."""
import threading
embeddings._reset_backend()
# backend_is_loaded falls through to module-level weights; set them rather than inherit xdist state.
monkeypatch.setattr(embeddings, "_model", object() if resident else None)
monkeypatch.setattr(embeddings, "_name", "org/embedder" if resident else None)
answered = threading.Event()
result = {}
with embeddings._backend_lock, embeddings._lock:
def _probe():
result["any"] = embeddings.backend_is_loaded()
result["named"] = embeddings.backend_is_loaded("org/embedder")
answered.set()
threading.Thread(target = _probe, daemon = True).start()
assert answered.wait(timeout = 5), "the status probe blocked on the load locks"
assert result == {"any": resident, "named": resident}
def test_a_dead_llama_process_is_not_reported_as_loaded(monkeypatch):
"""Dead llama processes keep _model_repo, so liveness must gate the loaded state."""
alive = {"value": True}
backend = SimpleNamespace(
_model_repo = "org/resident-GGUF", _process_alive = lambda: alive["value"]
)
monkeypatch.setattr(embeddings, "_backend", backend)
monkeypatch.setattr(embeddings, "_is_llama_backend", lambda value: value is backend)
monkeypatch.setattr(
embeddings.config,
"effective_gguf_repo_for_embedding_model",
lambda model: f"{model}-GGUF",
)
assert embeddings.backend_is_loaded("org/resident") is True
assert embeddings.backend_is_loaded() is True
alive["value"] = False
assert embeddings.backend_is_loaded("org/resident") is False
assert embeddings.backend_is_loaded() is False
def test_two_models_can_each_hold_their_own_llama_fallback_pin(monkeypatch):
"""Each (key, model) keeps its own llama fallback pin; one model's failure must not erase another's."""
monkeypatch.setattr(embeddings.config, "EMBED_BACKEND", "auto")
monkeypatch.setattr(embeddings, "_forced_backends", {})
monkeypatch.setattr(
embeddings, "_resolve_auto_for_model", lambda model = None: "sentence-transformers"
)
embeddings._forced_backends["org/a"] = "llama-server"
embeddings._forced_backends["org/b"] = "llama-server"
assert embeddings.resolved_backend_for_model("org/a") == "llama-server"
assert embeddings.resolved_backend_for_model("org/b") == "llama-server"
assert embeddings.sentence_transformers_fallback_allowed("org/a") is False
assert embeddings.sentence_transformers_fallback_allowed("org/b") is False
assert embeddings.resolved_backend_for_model("org/c") == "sentence-transformers"
assert embeddings.sentence_transformers_fallback_allowed("org/c") is True
def test_the_forced_backend_probe_does_not_wait_on_a_model_load(monkeypatch):
"""_get_backend holds _backend_lock across a model load, so the forced probe must not wait on it."""
import threading
monkeypatch.setattr(embeddings.config, "EMBED_BACKEND", "auto")
monkeypatch.setattr(embeddings, "_forced_backends", {"org/pinned": "llama-server"})
answered = threading.Event()
result = {}
with embeddings._backend_lock:
def _probe():
result["resolved"] = embeddings.resolved_backend_for_model("org/pinned")
result["fallback"] = embeddings.sentence_transformers_fallback_allowed("org/pinned")
answered.set()
threading.Thread(target = _probe, daemon = True).start()
assert answered.wait(timeout = 5), "the resolver probe blocked on the construction lock"
assert result == {"resolved": "llama-server", "fallback": False}
def test_a_model_reloaded_behind_an_unload_is_still_reported_and_freed(monkeypatch):
"""A model reloaded behind an unload must still be reported and freed, not treated as nothing loaded."""
monkeypatch.setattr(embeddings, "_backend", None, raising = False)
monkeypatch.setattr(embeddings, "_backend_key", None, raising = False)
monkeypatch.setattr(embeddings, "_model", object(), raising = False)
monkeypatch.setattr(embeddings, "_name", "org/embedder", raising = False)
assert embeddings.backend_is_loaded() is True
assert embeddings.backend_is_loaded("org/embedder") is True
assert embeddings.backend_is_loaded("org/other") is False
assert embeddings.release_backend() is True
assert embeddings._model is None
assert embeddings.backend_is_loaded() is False
assert embeddings.release_backend() is False