Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

48.9 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Embedder concurrency tests: the fast tokenizer isn't thread-safe, so encode
and token counting must be serialized (else threads panic "Already borrowed")."""

import json
import os
import sys
import threading
import time
from types import SimpleNamespace

import numpy as np
import pytest

from core.rag import config, embeddings


def _shared_setup_1(monkeypatch):
    observed = {}

    class FakeSentenceTransformer:
        def __init__(self, name, **kwargs):
            observed.update(kwargs)

    monkeypatch.setitem(
        sys.modules,
        "sentence_transformers",
        SimpleNamespace(SentenceTransformer = FakeSentenceTransformer),
    )
    monkeypatch.setattr(embeddings, "_install_torchao_stub_once", lambda: None)
    monkeypatch.setattr(embeddings, "_guard_model_security", lambda *_a, **_k: None)
    return observed


def _shared_setup_2(
    monkeypatch,
    tmp_path,
    name = "Org/Embedder",
):
    monkeypatch.setattr(
        "utils.hf_cache_settings.active_hf_hub_cache",
        lambda: str(tmp_path / "selected-hub"),
    )
    embeddings._model = None
    embeddings._name = None
    # _get publishes into globals nothing restores; snapshot after the reset so teardown restores None.
    monkeypatch.setattr(embeddings, "_model", None)
    monkeypatch.setattr(embeddings, "_name", None)

    embeddings._get(name)


def _shared_setup_3(monkeypatch):
    monkeypatch.setattr(
        embeddings,
        "get_device",
        lambda: embeddings.DeviceType.CUDA,
    )


def _shared_setup_4(monkeypatch):
    monkeypatch.setattr(
        embeddings,
        "_resolve_auto_for_model",
        lambda model = None: "sentence-transformers",
    )


# Mark the child non-dumpable: piped core_pattern (apport) ignores RLIMIT_CORE. Linux-only.
_CRASHING_UNLESS_CPU_SCRIPT = (
    "import ctypes, os, sys\n"
    "if sys.argv[1] != 'cpu':\n"
    # First: every crash arm, abort() included, must follow the PR_SET_DUMPABLE clear.
    "    try:\n"
    "        ctypes.CDLL(None).prctl(4, 0, 0, 0, 0)  # PR_SET_DUMPABLE = 0\n"
    "    except Exception:\n"
    "        pass\n"
    # On Windows ctypes turns the null read into OSError; abort() (CRT exit 3) is the real shape.
    "    if sys.platform == 'win32':\n"
    "        os.abort()\n"
    "    ctypes.string_at(0)\n"
)


@pytest.fixture(autouse = True)
def _pin_st_backend(monkeypatch):
    monkeypatch.setattr(config, "EMBED_BACKEND", "sentence-transformers")
    embeddings._reset_backend()
    yield
    embeddings._reset_backend()


class _ConcurrencyProbe:
    """Records whether two callers were in the guarded body at once."""

    def __init__(self):
        self.inside = 0
        self.saw_overlap = False
        self._g = threading.Lock()

    def enter(self):
        with self._g:
            self.inside += 1
            if self.inside > 1:
                self.saw_overlap = True
        time.sleep(0.005)
        with self._g:
            self.inside -= 1


class _FakeModel:
    def __init__(self, probe):
        self._probe = probe
        self.tokenizer = _FakeTokenizer(probe)

    def encode(self, texts, **_kw):
        self._probe.enter()
        return np.zeros((len(texts), 4), dtype = np.float32)


class _FakeTokenizer:
    def __init__(self, probe):
        self._probe = probe

    def encode(self, text, **_kw):
        self._probe.enter()
        return list(range(len(text.split())))


def _hammer(fn, n = 8):
    errors: list[Exception] = []

    def worker():
        try:
            fn()
        except Exception as exc:  # noqa: BLE001
            errors.append(exc)

    threads = [threading.Thread(target = worker) for _ in range(n)]
    for t in threads:
        t.start()
    for t in threads:
        t.join()
    return errors


def test_first_encode_builds_the_selected_backend_once(monkeypatch):
    """Importing the facade is inert; the first real vector operation owns construction."""
    builds: list[str] = []

    class _Backend:
        def encode(self, texts, **_kwargs):
            return np.zeros((len(texts), 4), dtype = np.float32)

    def _build(model_name = None):
        builds.append("backend")
        return _Backend()

    monkeypatch.setattr(embeddings, "_build_st_backend_or_fallback", _build)

    assert embeddings._backend is None
    embeddings.encode(["first"])
    embeddings.encode(["second"])

    assert builds == ["backend"]


class _AutoBackend:
    def encode(self, texts, **_kwargs):
        return np.zeros((len(texts), 4), dtype = np.float32)

    def token_counter(self, **_kwargs):
        return lambda text: len(text.split())

    def dim(self, **_kwargs):
        return 4


def _resolve_auto_for(monkeypatch, *, stored = None):
    import utils.embedding_model_settings as ems

    asked: list[str] = []
    monkeypatch.setattr(config, "EMBED_BACKEND", "auto")
    monkeypatch.setattr(ems, "get_stored_backend", lambda _model: stored and stored["backend"])
    monkeypatch.setattr(
        embeddings, "_resolve_auto", lambda: asked.append("probe") or "sentence-transformers"
    )
    monkeypatch.setattr(
        embeddings, "_build_st_backend_or_fallback", lambda model_name = None: _AutoBackend()
    )
    return asked


def test_auto_asks_the_hardware_once_for_the_backend_it_built(monkeypatch):
    """Every encode, token count and identity check resolves ``auto``, and the GPU probe
    behind it starts a subprocess, so asking per call started ~3 per indexed document (#10390)."""
    asked = _resolve_auto_for(monkeypatch)

    embeddings.encode(["first"], model_name = "org/embedder")
    assert asked == ["probe"]
    for _ in range(3):
        embeddings.encode(["again"], model_name = "org/embedder")
        embeddings.token_counter("org/embedder")("some words")
        embeddings.dim("org/embedder")
        embeddings.embedding_identity("org/embedder")
    assert asked == ["probe"]

    embeddings.release_backend()
    embeddings.encode(["after unload"], model_name = "org/embedder")
    assert asked == ["probe", "probe"]


def test_a_saved_backend_still_replaces_a_resident_auto_backend(monkeypatch):
    """Saving a backend rebuilds even a resident auto pick (issue 9739) instead of keeping it."""
    stored = {"backend": None}
    _resolve_auto_for(monkeypatch, stored = stored)
    _patch_llama_backend(monkeypatch, binary = "/fake/llama-server")

    assert isinstance(embeddings._get_backend("org/embedder"), _AutoBackend)
    stored["backend"] = "llama-server"
    assert isinstance(embeddings._get_backend("org/embedder"), _SentinelLlamaBackend)


def test_a_replaced_backend_asks_the_hardware_again(monkeypatch):
    """The kept answer is tied to the object it was kept for, so a rebuild re-asks the hardware."""
    asked = _resolve_auto_for(monkeypatch)
    _patch_llama_backend(monkeypatch, binary = "/fake/llama-server")

    first = embeddings._get_backend("org/embedder")
    assert asked == ["probe"]

    monkeypatch.setattr(embeddings, "_model_names_gguf_repo", lambda _model: True)
    second = embeddings._get_backend("org/embedder")
    assert second is not first
    assert asked == ["probe"]

    monkeypatch.setattr(embeddings, "_model_names_gguf_repo", lambda _model: False)
    assert embeddings._resolve_auto_for_model("org/embedder") == "sentence-transformers"
    assert asked == ["probe", "probe"]


def test_picker_plans_embeddinggemma_on_llama_server(monkeypatch):
    stored = {"backend": None}
    _resolve_auto_for(monkeypatch, stored = stored)
    monkeypatch.setattr(embeddings, "_forced_backends", {})
    monkeypatch.setattr(config, "EMBED_DEVICE", "auto")
    monkeypatch.delenv("RAG_EMBED_GGUF_REPO", raising = False)
    monkeypatch.setattr(embeddings, "sentence_transformers_runtime_available", lambda: True)
    monkeypatch.setattr(embeddings, "_llama_server_runtime_available", lambda: True)
    monkeypatch.setattr(embeddings, "sentence_transformers_can_load", lambda *a, **k: True)
    effective = {"model": "org/previous-embedder"}
    monkeypatch.setattr(config, "effective_embedding_model", lambda: effective["model"])
    plan = embeddings.resolved_backend_for_model

    assert plan("unsloth/embeddinggemma-300m") == "llama-server"
    assert plan("unsloth/embeddinggemma-2") == "llama-server"
    # Exact ids: a fine-tune need not publish the GGUF the preference relies on.
    assert plan("org/embeddinggemma-300m-finetune") == "sentence-transformers"
    # Only the plan: the runtime follows the saved record, so an env-configured model keeps its space.
    assert embeddings._resolve_auto_for_model("unsloth/embeddinggemma-300m") == (
        "sentence-transformers"
    )
    # The model in effect without a record runs on that default, and its Settings status must say so.
    effective["model"] = "unsloth/embeddinggemma-300m"
    assert plan("unsloth/embeddinggemma-300m") == "sentence-transformers"
    effective["model"] = "org/previous-embedder"

    # A model saved on sentence-transformers keeps the space its documents were indexed in.
    stored["backend"] = "sentence-transformers"
    assert plan("unsloth/embeddinggemma-300m") == "sentence-transformers"
    stored["backend"] = None

    monkeypatch.setattr(config, "EMBED_DEVICE", "gpu")
    assert plan("unsloth/embeddinggemma-300m") == "sentence-transformers"
    monkeypatch.setattr(config, "EMBED_DEVICE", "auto")

    monkeypatch.setenv("RAG_EMBED_GGUF_REPO", "org/other-GGUF")
    assert plan("unsloth/embeddinggemma-300m") == "sentence-transformers"
    monkeypatch.delenv("RAG_EMBED_GGUF_REPO")

    monkeypatch.setattr(config, "EMBED_BACKEND", "sentence-transformers")
    assert plan("unsloth/embeddinggemma-300m") == "sentence-transformers"
    monkeypatch.setattr(config, "EMBED_BACKEND", "auto")

    monkeypatch.setattr(embeddings, "_llama_server_runtime_available", lambda: False)
    assert plan("unsloth/embeddinggemma-300m") == "sentence-transformers"


def test_a_resolution_that_never_asked_the_hardware_keeps_nothing(monkeypatch):
    """_get_backend clears the per-thread answer first; otherwise a stale earlier answer gets published."""
    asked = _resolve_auto_for(monkeypatch)
    _patch_llama_backend(monkeypatch, binary = "/fake/llama-server")

    embeddings.active_backend_is_llama("org/embedder")
    assert asked == ["probe"]

    monkeypatch.setattr(embeddings, "_model_names_gguf_repo", lambda _model: True)
    built = embeddings._get_backend("org/embedder")
    assert isinstance(built, _SentinelLlamaBackend)

    assert embeddings._resident_hardware is None


def test_encode_is_serialized(monkeypatch):
    probe = _ConcurrencyProbe()
    monkeypatch.setattr(embeddings, "_get", lambda model_name = None: _FakeModel(probe))
    errors = _hammer(lambda: embeddings.encode(["alpha beta", "gamma"]))
    assert errors == []
    assert probe.saw_overlap is False


def test_token_counter_is_serialized(monkeypatch):
    probe = _ConcurrencyProbe()
    monkeypatch.setattr(embeddings, "_get", lambda model_name = None: _FakeModel(probe))
    count = embeddings.token_counter()
    errors = _hammer(lambda: count("one two three four"))
    assert errors == []
    assert probe.saw_overlap is False


def test_encode_enables_parallelism_only_during_call(monkeypatch):
    seen = {}

    class _M:
        tokenizer = None

        def encode(self, texts, **_kw):
            seen["during"] = os.environ.get("TOKENIZERS_PARALLELISM")
            return np.zeros((len(texts), 4), dtype = np.float32)

    monkeypatch.setattr(embeddings, "_get", lambda model_name = None: _M())
    os.environ["TOKENIZERS_PARALLELISM"] = "false"
    embeddings.encode(["alpha", "beta"])
    assert seen["during"] == "true"
    assert os.environ.get("TOKENIZERS_PARALLELISM") == "false"


def test_token_counter_enables_parallelism_only_during_call(monkeypatch):
    seen = {}

    class _Tok:
        def encode(self, text, **_kw):
            seen["during"] = os.environ.get("TOKENIZERS_PARALLELISM")
            return list(range(len(text.split())))

    class _M:
        tokenizer = _Tok()

    monkeypatch.setattr(embeddings, "_get", lambda model_name = None: _M())
    os.environ["TOKENIZERS_PARALLELISM"] = "false"
    count = embeddings.token_counter()
    count("alpha beta gamma")
    assert seen["during"] == "true"
    assert os.environ.get("TOKENIZERS_PARALLELISM") == "false"


def test_token_counter_reacquires_backend_retired_between_chunk_calls(monkeypatch):
    from core.rag.embed_llama_server import LlamaServerBackend

    retired = LlamaServerBackend()
    replacement = LlamaServerBackend()
    monkeypatch.setattr(embeddings, "_get_backend", lambda *_a, **_k: retired)
    count = embeddings.token_counter("org/embedder")

    retired._closed = True
    monkeypatch.setattr(embeddings, "_get_backend", lambda *_a, **_k: replacement)
    monkeypatch.setattr(
        replacement,
        "_post",
        lambda path, payload, **_k: {"tokens": [1, 2, 3]},
    )

    assert count("the next chunk") == 3


def test_token_counter_does_not_hide_non_lifecycle_errors(monkeypatch):
    class _BrokenCounterBackend:
        _closed = False

        def token_counter(self, *, model_name = None):
            def _raise(_text):
                raise RuntimeError("invalid tokenizer response")

            return _raise

    monkeypatch.setattr(embeddings, "_get_backend", lambda *_a, **_k: _BrokenCounterBackend())
    with pytest.raises(RuntimeError, match = "invalid tokenizer response"):
        embeddings.token_counter()("chunk")


def test_st_unload_waits_for_encode_admitted_before_model_lookup(monkeypatch):
    entered_lookup = threading.Event()
    finish_lookup = threading.Event()
    unload_done = threading.Event()
    order = []
    errors = []

    class _Model:
        def encode(self, texts, **kwargs):
            order.append("encode")
            return np.zeros((len(texts), 2), dtype = np.float32)

    def _get(model_name = None):
        entered_lookup.set()
        assert finish_lookup.wait(timeout = 2)
        return _Model()

    def _encode():
        try:
            embeddings._st_encode(["chunk"])
        except Exception as exc:  # pragma: no cover - surfaced below
            errors.append(exc)

    monkeypatch.setattr(embeddings, "_get", _get)
    monkeypatch.setattr(embeddings, "_model", object())
    monkeypatch.setattr(embeddings, "_name", "org/embedder")
    worker = threading.Thread(target = _encode)
    worker.start()
    assert entered_lookup.wait(timeout = 2)

    def _unload():
        embeddings._release_st_model()
        order.append("unload")
        unload_done.set()

    closer = threading.Thread(target = _unload)
    closer.start()
    assert unload_done.wait(timeout = 0.05) is False
    finish_lookup.set()
    # The release runs gc.collect() and, once a device is detected, a GPU cache clear: late in
    # a busy xdist worker that took over 2s and the old join returned before "unload" landed.
    worker.join(timeout = 30)
    closer.join(timeout = 30)
    assert not worker.is_alive(), "the encode never finished"
    assert not closer.is_alive(), "the unload never finished"

    assert errors == []
    assert order == ["encode", "unload"]
    assert unload_done.is_set()


def test_sentence_transformer_load_uses_live_cache(monkeypatch, tmp_path):
    observed = {}

    class FakeSentenceTransformer:
        def __init__(self, name, **kwargs):
            observed["name"] = name
            observed.update(kwargs)

    monkeypatch.setitem(
        sys.modules,
        "sentence_transformers",
        SimpleNamespace(SentenceTransformer = FakeSentenceTransformer),
    )
    monkeypatch.setattr(embeddings, "_install_torchao_stub_once", lambda: None)
    monkeypatch.setattr(embeddings, "_guard_model_security", lambda *_a, **_k: None)
    monkeypatch.setattr(embeddings, "_device", lambda: "cpu")
    _shared_setup_2(monkeypatch, tmp_path)

    assert observed["name"] == "Org/Embedder"
    assert observed["cache_folder"] == str(tmp_path / "selected-hub")
    # fp32: dtype follows the device actually loaded on.
    assert list(observed["model_kwargs"].values()) == ["float32"]


def test_device_defaults_to_cpu_on_an_accelerator_host(monkeypatch):
    """Auto device stays on CPU: a CUDA allocation pins a primary context nothing releases."""
    monkeypatch.setattr(embeddings.config, "EMBED_DEVICE", "auto")
    _shared_setup_3(monkeypatch)
    assert embeddings._device() == "cpu"


def test_device_opts_in_to_the_accelerator(monkeypatch):
    """Every accelerator spelling, including xpu and rocm, must opt in and resolve to cuda."""
    _shared_setup_3(monkeypatch)
    for requested in ("gpu", "GPU", " cuda ", "rocm", "hip", "xpu", "mps", "metal"):
        monkeypatch.setattr(embeddings.config, "EMBED_DEVICE", requested)
        assert embeddings._device() == "cuda", requested


def test_device_opt_in_still_yields_cpu_without_an_accelerator(monkeypatch):
    monkeypatch.setattr(embeddings.config, "EMBED_DEVICE", "gpu")
    monkeypatch.setattr(embeddings, "get_device", lambda: embeddings.DeviceType.CPU)
    assert embeddings._device() == "cpu"


def test_device_opt_in_on_apple_stays_on_cpu(monkeypatch):
    """MLX is not a torch device. Asking for a GPU must not produce a device string
    torch cannot open, which is why this stays a lookup in _TORCH_DEVICE."""
    monkeypatch.setattr(embeddings.config, "EMBED_DEVICE", "gpu")
    monkeypatch.setattr(embeddings, "get_device", lambda: embeddings.DeviceType.MLX)
    assert embeddings._device() == "cpu"


def test_unrecognized_device_setting_falls_back_without_raising(monkeypatch):
    _shared_setup_3(monkeypatch)
    for requested in ("", "   ", "banana", "auto", None):
        monkeypatch.setattr(embeddings.config, "EMBED_DEVICE", requested)
        assert embeddings._device() == "cpu", requested


def test_cpu_never_loads_float16(monkeypatch, tmp_path):
    """CPU must never load float16; torch 2.2 has no CPU Half LayerNorm kernel for BERT models."""
    observed = _shared_setup_1(monkeypatch)
    monkeypatch.setattr(
        "utils.hf_cache_settings.active_hf_hub_cache",
        lambda: str(tmp_path / "hub"),
    )
    for embed_device, hardware, load_device in (
        ("auto", embeddings.DeviceType.CUDA, None),
        ("cpu", embeddings.DeviceType.CUDA, None),
        ("gpu", embeddings.DeviceType.CPU, None),
        ("gpu", embeddings.DeviceType.CUDA, "cpu"),
    ):
        monkeypatch.setattr(embeddings.config, "EMBED_DEVICE", embed_device)
        monkeypatch.setattr(embeddings, "get_device", lambda hw = hardware: hw)
        if load_device is not None:
            monkeypatch.setattr(embeddings, "_load_device", lambda d = load_device: d)
        embeddings._model = None
        embeddings._name = None
        observed.clear()

        embeddings._get("Org/Embedder")

        assert observed["device"] == "cpu", (embed_device, hardware)
        assert list(observed["model_kwargs"].values()) == ["float32"], (embed_device, hardware)
    embeddings._model = None
    embeddings._name = None


def test_opted_in_accelerator_loads_float16(monkeypatch, tmp_path):
    observed = _shared_setup_1(monkeypatch)
    monkeypatch.setattr(embeddings, "_load_device", lambda: "cuda")
    _shared_setup_2(monkeypatch, tmp_path)

    assert observed["device"] == "cuda"
    assert list(observed["model_kwargs"].values()) == ["float16"]


@pytest.mark.parametrize(
    "device, native, expected",
    [
        ("cuda", True, "bfloat16"),
        ("cuda", False, "float32"),
        ("xpu", True, "bfloat16"),
        ("xpu", False, "float32"),
    ],
)
def test_float16_unsafe_model_avoids_float16(monkeypatch, tmp_path, device, native, expected):
    import core.training.diffusion_train_common as dtc

    observed = _shared_setup_1(monkeypatch)
    monkeypatch.setattr(embeddings, "_load_device", lambda: device)
    monkeypatch.setattr(dtc, "native_bf16_supported", lambda: native)
    monkeypatch.setattr(dtc, "native_bf16_supported_xpu", lambda: native)
    _shared_setup_2(monkeypatch, tmp_path, name = "unsloth/EmbeddingGemma-300m")

    assert observed["device"] == device
    assert list(observed["model_kwargs"].values()) == [expected]


def test_accelerator_fallback_loads_float32_on_cpu(monkeypatch, tmp_path):
    observed = _shared_setup_1(monkeypatch)
    monkeypatch.setattr(embeddings, "_load_device", lambda: "cpu")
    monkeypatch.setattr(embeddings, "_device", lambda: "cuda")
    _shared_setup_2(monkeypatch, tmp_path)

    assert observed["device"] == "cpu"
    assert list(observed["model_kwargs"].values()) == ["float32"]


class _SentinelLlamaBackend:
    """Stand-in for LlamaServerBackend; never spawns a real server."""


def _force_st_load_failure(monkeypatch):
    """Make the ST warm-probe raise."""

    def _boom(model_name = None):
        raise RuntimeError("torch is broken on this machine")

    monkeypatch.setattr(embeddings, "_get", _boom)


def _patch_llama_backend(monkeypatch, *, binary):
    from core.inference.llama_cpp import LlamaCppBackend
    from core.rag import embed_llama_server

    monkeypatch.setattr(LlamaCppBackend, "_find_llama_server_binary", staticmethod(lambda: binary))
    monkeypatch.setattr(embed_llama_server, "LlamaServerBackend", _SentinelLlamaBackend)


def test_st_failure_falls_back_to_llama_server(monkeypatch):
    _force_st_load_failure(monkeypatch)
    _patch_llama_backend(monkeypatch, binary = "/fake/llama-server")
    embeddings._reset_backend()
    backend = embeddings._get_backend()
    assert isinstance(backend, _SentinelLlamaBackend)


def test_st_failure_without_llama_binary_reraises(monkeypatch):
    _force_st_load_failure(monkeypatch)
    _patch_llama_backend(monkeypatch, binary = None)
    embeddings._reset_backend()
    with pytest.raises(RuntimeError, match = "torch is broken"):
        embeddings._get_backend()


def test_st_success_keeps_sentence_transformers(monkeypatch):
    monkeypatch.setattr(embeddings, "_get", lambda model_name = None: object())
    _patch_llama_backend(monkeypatch, binary = "/fake/llama-server")
    embeddings._reset_backend()
    backend = embeddings._get_backend()
    assert isinstance(backend, embeddings._SentenceTransformersBackend)


def test_loaded_state_belongs_to_the_resident_sentence_transformer(monkeypatch):
    monkeypatch.setattr(embeddings, "_backend", embeddings._SentenceTransformersBackend())
    monkeypatch.setattr(embeddings, "_model", object())
    monkeypatch.setattr(embeddings, "_name", "org/resident")

    assert embeddings.backend_is_loaded() is True
    assert embeddings.backend_is_loaded("org/resident") is True
    assert embeddings.backend_is_loaded("org/new-selection") is False


def test_loaded_state_belongs_to_the_resident_gguf_repo(monkeypatch):
    # A live process: residency means the subprocess is really there.
    backend = SimpleNamespace(_model_repo = "org/resident-GGUF", _process_alive = lambda: True)
    monkeypatch.setattr(embeddings, "_backend", backend)
    monkeypatch.setattr(embeddings, "_is_llama_backend", lambda value: value is backend)
    monkeypatch.setattr(
        embeddings.config,
        "effective_gguf_repo_for_embedding_model",
        lambda model: f"{model}-GGUF",
    )

    assert embeddings.backend_is_loaded("org/resident") is True
    assert embeddings.backend_is_loaded("org/new-selection") is False


def test_unload_clears_the_sentence_transformer_weights(monkeypatch):
    embeddings._backend = embeddings._SentenceTransformersBackend()
    embeddings._backend_key = embeddings._current_backend_key()
    embeddings._model = object()
    embeddings._name = "org/embedder"

    assert embeddings.release_backend() is True
    assert embeddings._model is None
    assert embeddings._name is None


def test_pending_sentence_transformer_refuses_implicit_download_and_llama_fallback(monkeypatch):
    import utils.embedding_model_settings as ems
    import utils.utils as utils

    class _MustNotLoad:
        def __init__(self, *args, **kwargs):  # pragma: no cover - failure is the assertion
            raise AssertionError("pending model reached SentenceTransformer")

    monkeypatch.setitem(
        sys.modules,
        "sentence_transformers",
        SimpleNamespace(SentenceTransformer = _MustNotLoad),
    )
    monkeypatch.setattr(ems, "get_stored_download_pending", lambda model: True)
    monkeypatch.setattr(utils, "hf_cache_snapshot_is_loadable", lambda model: False)
    monkeypatch.setattr(embeddings, "_install_torchao_stub_once", lambda: None)
    monkeypatch.setattr(embeddings, "_load_device", lambda: "cpu")
    monkeypatch.setattr(
        embeddings,
        "_try_make_llama_backend",
        lambda: (_ for _ in ()).throw(AssertionError("pending ST fell back to llama")),
    )
    embeddings._model = None
    embeddings._name = None

    with pytest.raises(embeddings.EmbeddingModelDownloadRequiredError, match = "not downloaded"):
        embeddings._build_st_backend_or_fallback()


def test_replacing_a_llama_backend_shuts_it_down(monkeypatch):
    calls = []

    class _OldLlama:
        def _shutdown(self):
            calls.append("shutdown")

    monkeypatch.setattr(embeddings.config, "EMBED_BACKEND", "sentence-transformers")
    monkeypatch.setattr(
        embeddings,
        "_build_st_backend_or_fallback",
        lambda *_a, **_k: embeddings._SentenceTransformersBackend(),
    )
    embeddings._backend = _OldLlama()
    embeddings._backend_key = "stale"

    assert isinstance(embeddings._get_backend(), embeddings._SentenceTransformersBackend)
    assert calls == ["shutdown"]


def test_explicit_llama_backend_disallows_st_resolution(monkeypatch):
    monkeypatch.setattr(embeddings.config, "EMBED_BACKEND", "llama-server")
    monkeypatch.setattr(embeddings, "_forced_backends", {})
    assert embeddings.sentence_transformers_fallback_allowed() is False

    monkeypatch.setattr(embeddings.config, "EMBED_BACKEND", "auto")
    assert embeddings.sentence_transformers_fallback_allowed() is True
    monkeypatch.setattr(
        embeddings,
        "_forced_backends",
        {embeddings.config.effective_embedding_model(): "llama-server"},
    )
    assert embeddings.sentence_transformers_fallback_allowed() is False


def test_forced_llama_fallback_is_scoped_to_the_model_that_failed(monkeypatch):
    monkeypatch.setattr(embeddings.config, "EMBED_BACKEND", "auto")
    monkeypatch.setattr(embeddings, "_forced_backends", {"org/failed": "llama-server"})
    _shared_setup_4(monkeypatch)

    assert embeddings.sentence_transformers_fallback_allowed("org/failed") is False
    assert embeddings.sentence_transformers_fallback_allowed("org/new-model") is True
    assert embeddings.resolved_backend_for_model("org/failed") == "llama-server"
    assert embeddings.resolved_backend_for_model("org/new-model") == "sentence-transformers"


def test_runtime_preflight_predicts_the_supported_llama_fallback(monkeypatch):
    monkeypatch.setattr(embeddings.config, "EMBED_BACKEND", "auto")
    monkeypatch.setattr(embeddings, "_forced_backends", {})
    _shared_setup_4(monkeypatch)
    monkeypatch.setattr(embeddings, "sentence_transformers_runtime_available", lambda: False)
    monkeypatch.setattr(embeddings, "_llama_server_runtime_available", lambda: True)

    assert embeddings.resolved_backend_for_model("org/embedder") == "llama-server"


def test_runtime_preflight_keeps_st_when_no_fallback_exists(monkeypatch):
    monkeypatch.setattr(embeddings.config, "EMBED_BACKEND", "auto")
    monkeypatch.setattr(embeddings, "_forced_backends", {})
    _shared_setup_4(monkeypatch)
    monkeypatch.setattr(embeddings, "sentence_transformers_runtime_available", lambda: False)
    monkeypatch.setattr(embeddings, "_llama_server_runtime_available", lambda: False)

    assert embeddings.resolved_backend_for_model("org/embedder") == "sentence-transformers"


def _st_folder(tmp_path, module_type, model_config):
    import json

    folder = tmp_path / "embedder"
    folder.mkdir()
    (folder / "modules.json").write_text(
        json.dumps([{"idx": 0, "name": "0", "path": "", "type": module_type}])
    )
    (folder / "config.json").write_text(json.dumps(model_config))
    return str(folder)


@pytest.mark.parametrize(
    "module_type, model_config, loadable",
    [
        ("sentence_transformers.models.Transformer", {"model_type": "bert"}, True),
        # How a repo saved by sentence-transformers 6 names its modules (embeddinggemma-2).
        (
            "sentence_transformers.no_such_module.transformer.Transformer",
            {"model_type": "bert"},
            False,
        ),
        ("sentence_transformers.models.Transformer", {"model_type": "no_such_model_type"}, False),
        # The loader never trusts remote code, so an auto_map does not make an unknown type loadable.
        (
            "sentence_transformers.models.Transformer",
            {"model_type": "no_such_model_type", "auto_map": {}},
            False,
        ),
    ],
)
def test_st_can_load_reads_modules_and_model_type(tmp_path, module_type, model_config, loadable):
    pytest.importorskip("sentence_transformers")
    pytest.importorskip("transformers")
    assert (
        embeddings.sentence_transformers_can_load(_st_folder(tmp_path, module_type, model_config))
        is loadable
    )


@pytest.mark.parametrize(
    "root_model_type,module_model_type,loadable",
    [
        ("bert", "no_such_model_type", False),
        ("no_such_model_type", "bert", True),
    ],
)
def test_st_can_load_reads_the_transformer_modules_config(
    tmp_path, root_model_type, module_model_type, loadable
):
    pytest.importorskip("sentence_transformers")
    pytest.importorskip("transformers")
    folder = tmp_path / "embedder"
    module = folder / "0_Transformer"
    module.mkdir(parents = True)
    (folder / "modules.json").write_text(
        json.dumps(
            [
                {
                    "idx": 0,
                    "name": "0",
                    "path": "0_Transformer",
                    "type": "sentence_transformers.models.Transformer",
                }
            ]
        )
    )
    (folder / "config.json").write_text(json.dumps({"model_type": root_model_type}))
    (module / "config.json").write_text(json.dumps({"model_type": module_model_type}))

    assert embeddings.sentence_transformers_can_load(str(folder)) is loadable


def test_st_can_load_preserves_anonymous_hub_access(monkeypatch):
    pytest.importorskip("sentence_transformers")
    seen = []

    def _read(name, filename, token):
        seen.append((filename, token))
        return None if filename == "modules.json" else {}

    monkeypatch.setattr(embeddings, "_repo_json", _read)

    assert embeddings.sentence_transformers_can_load("org/public", False) is True
    assert seen == [("modules.json", False), ("config.json", False)]


def test_st_can_load_without_the_files_keeps_the_plan(tmp_path):
    assert embeddings.sentence_transformers_can_load(str(tmp_path)) is True


def test_st_can_load_gives_up_on_a_stalled_hub(monkeypatch):
    pytest.importorskip("sentence_transformers")
    monkeypatch.setattr(embeddings, "_ST_LOAD_PREFLIGHT_TIMEOUT_S", 0.05)
    monkeypatch.setattr(embeddings, "_repo_json", lambda *_: time.sleep(1))

    started = time.monotonic()
    assert embeddings.sentence_transformers_can_load("org/stalled") is True
    assert time.monotonic() - started < 0.5


def test_model_st_cannot_open_is_planned_on_llama(monkeypatch):
    monkeypatch.setattr(embeddings.config, "EMBED_BACKEND", "auto")
    monkeypatch.setattr(embeddings, "_forced_backends", {})
    _shared_setup_4(monkeypatch)
    monkeypatch.setattr(embeddings, "sentence_transformers_runtime_available", lambda: True)
    monkeypatch.setattr(
        embeddings,
        "sentence_transformers_can_load",
        lambda model, token = None: model != "org/st6-model",
    )
    monkeypatch.setattr(embeddings, "_llama_server_runtime_available", lambda: True)

    assert embeddings.resolved_backend_for_model("org/st6-model") == "llama-server"
    assert embeddings.resolved_backend_for_model("org/embedder") == "sentence-transformers"

    # An explicit ST policy makes the runtime ignore a stored llama backend, so the plan must not promise one.
    monkeypatch.setattr(embeddings.config, "EMBED_BACKEND", "sentence-transformers")
    assert embeddings.resolved_backend_for_model("org/st6-model") == "sentence-transformers"

    monkeypatch.setattr(embeddings.config, "EMBED_BACKEND", "auto")
    monkeypatch.setattr(embeddings, "_llama_server_runtime_available", lambda: False)
    assert embeddings.resolved_backend_for_model("org/st6-model") == "sentence-transformers"


def test_runtime_preflight_catches_a_fatal_torch_device_mismatch(monkeypatch):
    monkeypatch.setattr(
        embeddings,
        "_load_device",
        lambda: (_ for _ in ()).throw(embeddings.TorchDeviceUnusableError("broken torch")),
    )

    assert embeddings.sentence_transformers_runtime_available() is False


class _BoomOnEncodeModel:
    """Loads fine (init probe passes) but raises when encoding."""

    tokenizer = None

    def encode(self, texts, **_kw):
        raise RuntimeError("CUDA error during encode")


def test_st_encode_runtime_failure_switches_to_llama(monkeypatch):
    monkeypatch.setattr(embeddings, "_get", lambda model_name = None: _BoomOnEncodeModel())
    _patch_llama_backend(monkeypatch, binary = "/fake/llama-server")
    calls = {}
    embeddings._model = object()
    embeddings._name = "org/embedder"

    def _sentinel_encode(
        self,
        texts,
        *,
        model_name = None,
        normalize = True,
    ):
        calls["used"] = True
        return np.zeros((len(texts), 4), dtype = np.float32)

    monkeypatch.setattr(_SentinelLlamaBackend, "encode", _sentinel_encode, raising = False)
    embeddings._reset_backend()

    out = embeddings.encode(["alpha", "beta"])
    assert calls.get("used") is True
    assert out.shape == (2, 4)
    assert embeddings._model is None
    assert embeddings._name is None
    assert embeddings.config.effective_embedding_model() in embeddings._forced_backends
    assert isinstance(embeddings._get_backend(), _SentinelLlamaBackend)
    # It outranks the saved model's resolution so ST cannot be re-chosen after failing.
    monkeypatch.setattr(embeddings, "_resolve_auto_for_model", lambda: "sentence-transformers")
    assert isinstance(embeddings._get_backend(), _SentinelLlamaBackend)
    embeddings._reset_backend()
    assert embeddings._forced_backends == {}


def test_st_encode_failure_without_llama_binary_reraises(monkeypatch):
    monkeypatch.setattr(embeddings, "_get", lambda model_name = None: _BoomOnEncodeModel())
    _patch_llama_backend(monkeypatch, binary = None)
    embeddings._reset_backend()
    with pytest.raises(RuntimeError, match = "CUDA error during encode"):
        embeddings.encode(["alpha", "beta"])


def _patch_probe(monkeypatch, usable):
    """Return configured probe results and record the devices checked."""
    from utils import torch_device_probe

    asked = []

    def _can_allocate(device):
        asked.append(device)
        return usable[device]

    monkeypatch.setattr(torch_device_probe, "device_can_allocate", _can_allocate)
    return asked


class _ImportIsACrash:
    """Fail if sentence-transformers is reached after both device probes crash."""

    def __getattr__(self, name):
        raise AssertionError(f"sentence-transformers was reached ({name}) on a crashing host")


def test_load_device_keeps_the_accelerator_when_it_is_usable(monkeypatch):
    monkeypatch.setattr(embeddings, "_device", lambda: "cuda")
    asked = _patch_probe(monkeypatch, {"cuda": True})
    assert embeddings._load_device() == "cuda"
    assert asked == ["cuda"]


def test_load_device_degrades_to_cpu_when_the_accelerator_crashes(monkeypatch):
    monkeypatch.setattr(embeddings, "_device", lambda: "cuda")
    _patch_probe(monkeypatch, {"cuda": False, "cpu": True})
    assert embeddings._load_device() == "cpu"


def test_load_device_raises_when_torch_crashes_on_cpu_too(monkeypatch):
    monkeypatch.setattr(embeddings, "_device", lambda: "cuda")
    _patch_probe(monkeypatch, {"cuda": False, "cpu": False})
    with pytest.raises(embeddings.TorchDeviceUnusableError):
        embeddings._load_device()


def test_load_device_does_not_probe_a_cpu_only_host(monkeypatch):
    monkeypatch.setattr(embeddings, "_device", lambda: "cpu")
    asked = _patch_probe(monkeypatch, {})
    assert embeddings._load_device() == "cpu"
    assert asked == []


def test_a_real_crashing_child_moves_the_load_to_cpu(monkeypatch):
    from utils import torch_device_probe

    monkeypatch.setenv(torch_device_probe.DISABLE_ENV_VAR, "0")
    monkeypatch.setattr(
        torch_device_probe,
        "_PROBE_SCRIPT",
        _CRASHING_UNLESS_CPU_SCRIPT,
    )
    torch_device_probe.device_can_allocate.cache_clear()
    monkeypatch.setattr(embeddings, "_device", lambda: "cuda")
    try:
        assert embeddings._load_device() == "cpu"
    finally:
        torch_device_probe.device_can_allocate.cache_clear()


def test_crashing_torch_falls_back_to_llama_server(monkeypatch):
    monkeypatch.setattr(embeddings, "_device", lambda: "cuda")
    _patch_probe(monkeypatch, {"cuda": False, "cpu": False})
    _patch_llama_backend(monkeypatch, binary = "/fake/llama-server")
    monkeypatch.setitem(sys.modules, "sentence_transformers", _ImportIsACrash())
    embeddings._model = None
    embeddings._name = None
    embeddings._reset_backend()

    assert isinstance(embeddings._get_backend(), _SentinelLlamaBackend)


def test_encode_reacquires_a_backend_retired_between_batches(monkeypatch):
    """`release_backend` promises the next embed rebuilds. Without a reacquire here
    that promise held only for `token_counter`, and pressing Unload mid-ingestion
    failed the document being indexed instead of continuing it."""
    from core.rag.embed_llama_server import LlamaServerBackend

    retired = LlamaServerBackend()
    replacement = LlamaServerBackend()
    retired._closed = True
    served = np.zeros((1, 4), dtype = np.float32)
    monkeypatch.setattr(
        replacement,
        "encode",
        lambda texts, **kwargs: served,
    )
    backends = iter([retired, replacement])
    monkeypatch.setattr(embeddings, "_get_backend", lambda *_a, **_k: next(backends))

    assert embeddings.encode(["chunk"]) is served
    assert embeddings._served_by.backend is replacement


def test_encode_does_not_hide_a_non_lifecycle_runtime_error(monkeypatch):
    class _BrokenBackend:
        _closed = False

        def encode(self, texts, **kwargs):
            raise RuntimeError("llama-server returned no embedding")

    monkeypatch.setattr(embeddings, "_get_backend", lambda *_a, **_k: _BrokenBackend())
    with pytest.raises(RuntimeError, match = "returned no embedding"):
        embeddings.encode(["chunk"])


def test_encode_surfaces_the_unload_when_no_replacement_is_published(monkeypatch):
    from core.rag.embed_llama_server import LlamaServerBackend

    retired = LlamaServerBackend()
    retired._closed = True
    monkeypatch.setattr(embeddings, "_get_backend", lambda *_a, **_k: retired)
    with pytest.raises(RuntimeError, match = "was unloaded"):
        embeddings.encode(["chunk"])


def test_the_token_counter_follows_an_unloaded_sentence_transformer(monkeypatch):
    """Read the tokenizer per call under the compute lock, so an unloaded model stops counting."""
    looked_up = []

    class _Tok:
        def __init__(self, n):
            self.n = n

        def encode(
            self,
            text,
            add_special_tokens = False,
        ):
            return list(range(self.n))

    class _Model:
        def __init__(self, n):
            self.tokenizer = _Tok(n)

    models = [_Model(3), _Model(7)]

    def _get(model_name = None):
        looked_up.append(model_name)
        return models[min(len(looked_up) - 1, 1)]

    monkeypatch.setattr(embeddings, "_get", _get)
    count = embeddings._st_token_counter("org/embedder")

    assert looked_up == [], "the lookup must not happen before the first call"
    assert count("first") == 3
    assert count("second") == 7
    assert looked_up == ["org/embedder", "org/embedder"]


def test_a_local_gguf_selects_llama_even_on_a_gpu_box(monkeypatch, tmp_path):
    """_resolve_auto answers sentence-transformers whenever a GPU is present, so a
    GPU box pointed at a local .gguf planned an ST load of a GGUF file."""
    gguf = tmp_path / "embed.gguf"
    gguf.write_bytes(b"GGUF")
    monkeypatch.setattr(embeddings.config, "EMBED_BACKEND", "auto")
    monkeypatch.setattr(embeddings, "_forced_backends", {})
    monkeypatch.setattr(embeddings, "_resolve_auto", lambda: "sentence-transformers")

    assert embeddings._resolve_auto_for_model(str(gguf)) == "llama-server"
    assert embeddings.resolved_backend_for_model(str(gguf)) == "llama-server"
    assert embeddings._resolve_auto_for_model(str(tmp_path)) == "llama-server"
    assert embeddings._resolve_auto_for_model("unsloth/bge-small-en-v1.5") == (
        "sentence-transformers"
    )


def test_a_local_gguf_beats_a_stored_sentence_transformers_record(monkeypatch, tmp_path):
    """A stored ST record for a .gguf only comes from a failed force-save, so it must not win."""
    gguf = tmp_path / "embed.gguf"
    gguf.write_bytes(b"GGUF")
    monkeypatch.setattr(embeddings.config, "EMBED_BACKEND", "auto")
    monkeypatch.setattr(embeddings, "_resolve_auto", lambda: "sentence-transformers")
    import utils.embedding_model_settings as ems

    monkeypatch.setattr(ems, "get_stored_backend", lambda _m: "sentence-transformers")

    assert embeddings._resolve_auto_for_model(str(gguf)) == "llama-server"


def test_an_explicit_backend_is_not_overridden_by_a_local_gguf(monkeypatch, tmp_path):
    """Only ``auto`` consults the probe; an explicit setting stays verbatim."""
    gguf = tmp_path / "embed.gguf"
    gguf.write_bytes(b"GGUF")
    monkeypatch.setattr(embeddings.config, "EMBED_BACKEND", "sentence-transformers")
    monkeypatch.setattr(embeddings, "_forced_backends", {})
    monkeypatch.setattr(embeddings, "sentence_transformers_runtime_available", lambda: True)

    assert embeddings.resolved_backend_for_model(str(gguf)) == "sentence-transformers"


def test_the_st_probe_warms_the_pinned_model_not_the_live_setting(monkeypatch):
    """The probe must warm the pinned model, not the live setting, or a job pinned to A probes B."""
    warmed = []

    class _Probe:
        def warm(self, model_name = None):
            warmed.append(model_name)

    monkeypatch.setattr(embeddings, "_SentenceTransformersBackend", _Probe)
    monkeypatch.setattr(embeddings.config, "EMBED_BACKEND", "sentence-transformers")
    monkeypatch.setattr(embeddings, "_forced_backends", {})
    embeddings._reset_backend()
    try:
        embeddings._get_backend("org/pinned")
    finally:
        embeddings._reset_backend()

    assert warmed == ["org/pinned"]


def test_the_security_gate_scans_the_snapshot_that_is_actually_loaded(monkeypatch, tmp_path):
    """The security gate must scan the snapshot that is actually loaded, not the Hub's current commit."""
    snapshot = tmp_path / "snap"
    snapshot.mkdir()
    (snapshot / "model.safetensors").write_bytes(b"ST")
    scanned = []
    monkeypatch.setattr(
        embeddings,
        "_guard_model_security",
        lambda target, local_only = False, display = None: scanned.append(target),
    )

    class _ST:
        def __init__(self, target, **_kwargs):
            self.loaded = target
            self.tokenizer = None

        def get_sentence_embedding_dimension(self):
            return 8

    monkeypatch.setattr(embeddings, "_load_device", lambda: "cpu")
    monkeypatch.setattr(embeddings, "_install_torchao_stub_once", lambda: None)
    monkeypatch.setattr(embeddings, "_st_accepts_local_files_only", lambda _c: False)
    import utils.utils as utils

    monkeypatch.setattr(utils, "cached_st_source", lambda m: ("org/embedder", snapshot))
    monkeypatch.setattr(utils, "hf_cache_snapshot_dir", lambda m: snapshot)
    import sys as _sys
    import types as _t

    st_mod = _t.ModuleType("sentence_transformers")
    st_mod.SentenceTransformer = _ST
    monkeypatch.setitem(_sys.modules, "sentence_transformers", st_mod)
    embeddings._model = None
    embeddings._name = None
    try:
        model = embeddings._get("org/embedder")
    finally:
        embeddings._model = None
        embeddings._name = None

    assert model.loaded == str(snapshot)
    assert scanned == [str(snapshot)]


def test_the_shared_load_setup_does_not_strand_module_weights(tmp_path):
    """The shared load setup must not strand module weights for later tests in the same xdist worker."""
    with pytest.MonkeyPatch.context() as mp:
        _shared_setup_1(mp)
        mp.setattr(embeddings, "_device", lambda: "cpu")
        _shared_setup_2(mp, tmp_path)
        assert embeddings._model is not None, "the setup loaded nothing, so this proves nothing"
        assert embeddings._name == "Org/Embedder"

    assert embeddings._model is None
    assert embeddings._name is None


@pytest.mark.parametrize("resident", [False, True])
def test_the_residency_probe_does_not_wait_on_a_model_load(monkeypatch, resident):
    """The residency probe must answer under the load locks, not wait on a model load."""
    import threading

    embeddings._reset_backend()
    # backend_is_loaded falls through to module-level weights; set them rather than inherit xdist state.
    monkeypatch.setattr(embeddings, "_model", object() if resident else None)
    monkeypatch.setattr(embeddings, "_name", "org/embedder" if resident else None)
    answered = threading.Event()
    result = {}

    with embeddings._backend_lock, embeddings._lock:

        def _probe():
            result["any"] = embeddings.backend_is_loaded()
            result["named"] = embeddings.backend_is_loaded("org/embedder")
            answered.set()

        threading.Thread(target = _probe, daemon = True).start()
        assert answered.wait(timeout = 5), "the status probe blocked on the load locks"

    assert result == {"any": resident, "named": resident}


def test_a_dead_llama_process_is_not_reported_as_loaded(monkeypatch):
    """Dead llama processes keep _model_repo, so liveness must gate the loaded state."""
    alive = {"value": True}
    backend = SimpleNamespace(
        _model_repo = "org/resident-GGUF", _process_alive = lambda: alive["value"]
    )
    monkeypatch.setattr(embeddings, "_backend", backend)
    monkeypatch.setattr(embeddings, "_is_llama_backend", lambda value: value is backend)
    monkeypatch.setattr(
        embeddings.config,
        "effective_gguf_repo_for_embedding_model",
        lambda model: f"{model}-GGUF",
    )

    assert embeddings.backend_is_loaded("org/resident") is True
    assert embeddings.backend_is_loaded() is True

    alive["value"] = False
    assert embeddings.backend_is_loaded("org/resident") is False
    assert embeddings.backend_is_loaded() is False


def test_two_models_can_each_hold_their_own_llama_fallback_pin(monkeypatch):
    """Each (key, model) keeps its own llama fallback pin; one model's failure must not erase another's."""
    monkeypatch.setattr(embeddings.config, "EMBED_BACKEND", "auto")
    monkeypatch.setattr(embeddings, "_forced_backends", {})
    monkeypatch.setattr(
        embeddings, "_resolve_auto_for_model", lambda model = None: "sentence-transformers"
    )

    embeddings._forced_backends["org/a"] = "llama-server"
    embeddings._forced_backends["org/b"] = "llama-server"

    assert embeddings.resolved_backend_for_model("org/a") == "llama-server"
    assert embeddings.resolved_backend_for_model("org/b") == "llama-server"
    assert embeddings.sentence_transformers_fallback_allowed("org/a") is False
    assert embeddings.sentence_transformers_fallback_allowed("org/b") is False
    assert embeddings.resolved_backend_for_model("org/c") == "sentence-transformers"
    assert embeddings.sentence_transformers_fallback_allowed("org/c") is True


def test_the_forced_backend_probe_does_not_wait_on_a_model_load(monkeypatch):
    """_get_backend holds _backend_lock across a model load, so the forced probe must not wait on it."""
    import threading

    monkeypatch.setattr(embeddings.config, "EMBED_BACKEND", "auto")
    monkeypatch.setattr(embeddings, "_forced_backends", {"org/pinned": "llama-server"})
    answered = threading.Event()
    result = {}

    with embeddings._backend_lock:

        def _probe():
            result["resolved"] = embeddings.resolved_backend_for_model("org/pinned")
            result["fallback"] = embeddings.sentence_transformers_fallback_allowed("org/pinned")
            answered.set()

        threading.Thread(target = _probe, daemon = True).start()
        assert answered.wait(timeout = 5), "the resolver probe blocked on the construction lock"

    assert result == {"resolved": "llama-server", "fallback": False}


def test_a_model_reloaded_behind_an_unload_is_still_reported_and_freed(monkeypatch):
    """A model reloaded behind an unload must still be reported and freed, not treated as nothing loaded."""
    monkeypatch.setattr(embeddings, "_backend", None, raising = False)
    monkeypatch.setattr(embeddings, "_backend_key", None, raising = False)
    monkeypatch.setattr(embeddings, "_model", object(), raising = False)
    monkeypatch.setattr(embeddings, "_name", "org/embedder", raising = False)

    assert embeddings.backend_is_loaded() is True
    assert embeddings.backend_is_loaded("org/embedder") is True
    assert embeddings.backend_is_loaded("org/other") is False

    assert embeddings.release_backend() is True
    assert embeddings._model is None
    assert embeddings.backend_is_loaded() is False
    assert embeddings.release_backend() is False