Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: fix document search for embedding models other than the default (#11853)

Nilayff46f0f26586
diff --git a/studio/backend/core/rag/config.py b/studio/backend/core/rag/config.py
index e023242a492..ee14188003d 100644
--- a/studio/backend/core/rag/config.py
+++ b/studio/backend/core/rag/config.py
@@ -98,16 +98,19 @@ def embedding_identity(
model: str,
*,
gguf_repo: str | None = None,
+ pooling: str | None = None,
) -> str:
"""Tagged identity for ``documents.embedding_model``.
The configured model comes first so a row written before identities carried a tag
still compares equal on it. llama-server appends the GGUF repo it actually embeds
- through, which is the part that can differ from the model's ST form."""
+ through, which is the part that can differ from the model's ST form, and any pooling
+ other than the CLS it once forced on every GGUF."""
model = _escape_identity_segment(model)
if gguf_repo is None:
return f"{backend}:{model}"
- return f"{backend}:{model}:{_escape_identity_segment(gguf_repo)}"
+ identity = f"{backend}:{model}:{_escape_identity_segment(gguf_repo)}"
+ return identity if pooling is None else f"{identity}:{pooling}"
def embedding_identity_model(identity: str | None) -> str | None:
diff --git a/studio/backend/core/rag/embed_llama_server.py b/studio/backend/core/rag/embed_llama_server.py
index d0d560b7baf..b8e7a506b62 100644
--- a/studio/backend/core/rag/embed_llama_server.py
+++ b/studio/backend/core/rag/embed_llama_server.py
@@ -76,29 +76,58 @@ def _skip_gguf_value(f, vtype: int) -> None:
f.seek(_GGUF_SCALAR_WIDTHS[vtype], 1)
-def _gguf_context_length(path: str) -> int | None:
+def _gguf_arch_uint(path: str, field: str) -> tuple[str | None, int | None]:
try:
with open(path, "rb") as f:
if f.read(4) != b"GGUF":
- return None
+ return None, None
f.read(4)
_, kv_count = struct.unpack("<QQ", f.read(16))
arch = None
- lengths: dict[str, int] = {}
+ values: dict[str, int] = {}
for _ in range(kv_count):
key = f.read(struct.unpack("<Q", f.read(8))[0]).decode("utf-8")
vtype = struct.unpack("<I", f.read(4))[0]
if key == "general.architecture" and vtype == 8:
arch = f.read(struct.unpack("<Q", f.read(8))[0]).decode("utf-8")
- elif key.endswith(".context_length") and vtype in (4, 10):
- lengths[key] = int.from_bytes(f.read(_GGUF_SCALAR_WIDTHS[vtype]), "little")
+ elif key.endswith(f".{field}") and vtype in (4, 10):
+ values[key] = int.from_bytes(f.read(_GGUF_SCALAR_WIDTHS[vtype]), "little")
else:
_skip_gguf_value(f, vtype)
- if arch is not None and f"{arch}.context_length" in lengths:
- return lengths[f"{arch}.context_length"] or None
+ if arch is not None and f"{arch}.{field}" in values:
+ return arch, values[f"{arch}.{field}"]
except (OSError, struct.error, UnicodeDecodeError, KeyError, ValueError):
- return None
- return None
+ return None, None
+ return arch, None
+
+
+def _gguf_context_length(path: str) -> int | None:
+ return _gguf_arch_uint(path, "context_length")[1] or None
+
+
+_GGUF_POOLING = {1: "mean", 2: "cls", 3: "last"}
+_GGUF_POOLING_DEFAULT = {"nomic-bert": "mean", "nomic-bert-moe": "mean"}
+
+
+@lru_cache(maxsize = 32)
+def _gguf_pooling_at(path: str, mtime_ns: int, size: int) -> str:
+ arch, value = _gguf_arch_uint(path, "pooling_type")
+ if value in _GGUF_POOLING:
+ return _GGUF_POOLING[value]
+ if value is None:
+ # Older dedicated Nomic embedders commonly omit the SentenceTransformers pooling
+ # metadata. Their model family is mean-pooled; unknown architectures keep the legacy CLS
+ # fallback rather than guessing across encoder, decoder and reranker families.
+ return _GGUF_POOLING_DEFAULT.get((arch or "").lower(), "cls")
+ return "cls"
+
+
+def _gguf_pooling(path: str) -> str:
+ try:
+ st = os.stat(path)
+ except OSError:
+ return "cls"
+ return _gguf_pooling_at(path, st.st_mtime_ns, st.st_size)
def _resolve_entrypoint(binary: str) -> str:
@@ -154,6 +183,9 @@ class LlamaServerBackend:
self._operation_condition = threading.Condition()
self._active_operations = 0
self._operation_local = threading.local()
+ # Immutable repo/pooling pair from this thread's last completed encode. The backend object
+ # itself is shared and can switch models before its caller constructs the vector identity.
+ self._served_identity_local = threading.local()
self._closed = False
self._process: subprocess.Popen | None = None
self._port: int | None = None
@@ -167,6 +199,8 @@ class LlamaServerBackend:
# them lands A's POST on B's server.
self._serve_lock = threading.RLock()
self._model_path: str | None = None
+ # Read when the path is adopted: the running server keeps this pooling even if the file later vanishes.
+ self._model_pooling: str | None = None
# A Settings change makes the cached path/dim stale, forcing a re-resolve and respawn (see _ensure_ready).
self._model_repo: str | None = None
self._binary: str | None = None
@@ -453,6 +487,28 @@ class LlamaServerBackend:
files = [p for p in files if p != pick]
return None
+ @staticmethod
+ def cached_pooling(model: str) -> str | None:
+ """Pooling of ``model``'s GGUF already on disk, found without the network; None when
+ the loader cannot yet know which file it will serve. Match its local, planned and configured-
+ variant order so identity prediction never reads metadata from a different cached family."""
+ try:
+ desired = config.effective_gguf_repo_for_embedding_model(model)
+ path = LlamaServerBackend._resolve_local_gguf(model)
+ path = path or LlamaServerBackend._planned_family_path(model, desired)
+ path = path or LlamaServerBackend._resolve_cached_gguf(desired)
+ if path is None:
+ try:
+ from utils.embedding_model_settings import get_stored_download_pending
+ download_pending = get_stored_download_pending(model)
+ except Exception: # noqa: BLE001 - old/unavailable settings store
+ download_pending = False
+ if download_pending:
+ path = LlamaServerBackend._resolve_cached_gguf(desired, require_variant = False)
+ except Exception: # noqa: BLE001 - identity prediction must not block ingestion
+ return None
+ return None if path is None else _gguf_pooling(path)
+
def _resolve_model_path(self, model_name: str | None = None) -> str:
"""Download (or cache-hit) the variant-matching, non-mmproj GGUF embedder,
returning its local path. Re-resolves when the effective repo changed (a
@@ -466,7 +522,15 @@ class LlamaServerBackend:
# setting change reads as stale and respawns.
desired = config.effective_gguf_repo_for_embedding_model(model)
if self._model_path is not None and self._model_repo == desired:
- return self._model_path
+ if Path(self._model_path).is_file():
+ return self._model_path
+ # The process is stopped when this resolver runs. A remembered cache path can have
+ # been evicted since its last server lifetime, so let the normal cache/Hub order find
+ # what the next process can actually open.
+ self._model_path = None
+ self._model_pooling = None
+ self._dim = None
+ self._max_tokens = None
local = self._resolve_local_gguf(model)
if local is not None:
return self._adopt_model_path(local, desired)
@@ -592,6 +656,7 @@ class LlamaServerBackend:
"""Serve `path` for `desired`; the width is re-probed against whatever is adopted."""
self._model_path = path
self._model_repo = desired
+ self._model_pooling = _gguf_pooling(path) if path else None
self._dim = None
self._max_tokens = None
return self._model_path
@@ -711,7 +776,15 @@ class LlamaServerBackend:
from core.inference.llama_cpp import LlamaCppBackend
return LlamaCppBackend._arch_gate_survivors(binary)
- def _build_cmd(self, binary: str, model_path: str, port: int, *, use_gpu: bool) -> list[str]:
+ def _build_cmd(
+ self,
+ binary: str,
+ model_path: str,
+ port: int,
+ *,
+ use_gpu: bool,
+ pooling: str | None = None,
+ ) -> list[str]:
# No --embd-normalize (absent in some builds; we normalize in Python to match the ST path), and
# --fit off so ctx/offload are not auto-resized.
cmd = [
@@ -724,7 +797,7 @@ class LlamaServerBackend:
str(port),
"--embedding",
"--pooling",
- "cls",
+ pooling or _gguf_pooling(model_path),
"--fit",
"off",
]
@@ -855,9 +928,13 @@ class LlamaServerBackend:
) -> None:
binary = self._resolve_binary()
model_path = self._resolve_model_path(model_name)
+ # The cached path can be replaced in place between server lifetimes. Capture pooling once
+ # per spawn so the explicit llama-server flag and the vectors' recorded identity agree.
+ pooling = _gguf_pooling(model_path)
+ self._model_pooling = pooling
port = config.EMBED_PORT or self._find_free_port()
env = self._build_env(binary, use_gpu = use_gpu)
- cmd = self._build_cmd(binary, model_path, port, use_gpu = use_gpu)
+ cmd = self._build_cmd(binary, model_path, port, use_gpu = use_gpu, pooling = pooling)
logger.info(
"starting llama-server embedder (%s): %s",
"gpu" if use_gpu else "cpu",
@@ -952,6 +1029,11 @@ class LlamaServerBackend:
and self._binary_path_revision == custom_llama_cpp_path_revision()
)
+ def pooling_identity_snapshot(self) -> tuple[str | None, str | None, str | None, bool]:
+ """One coherent path/repo/pooling/process snapshot for pre-encode identity prediction."""
+ with self._serve_lock:
+ return self._model_path, self._model_repo, self._model_pooling, self._process_alive()
+
def _ensure_ready(self, model_name: str | None = None) -> None:
"""Guarantee a live server on ``model_name``, (re)spawning if needed. Double-checked so the
current path takes no lock; self-heals after the chat reaper kills us and re-resolves
@@ -1073,8 +1155,15 @@ class LlamaServerBackend:
"""Embed texts -> (N, dim) float32. ``model_name`` pins which GGUF serves
the request, so a Settings change cannot answer it from another model.
Normalizes in Python to match the ST backend."""
- with self._operation():
- return self._encode_active(texts, normalize = normalize, model_name = model_name)
+ self._served_identity_local.value = None
+ with self._operation(), self._serve_lock:
+ vectors = self._encode_active(texts, normalize = normalize, model_name = model_name)
+ self._served_identity_local.value = (self._model_repo, self._model_pooling)
+ return vectors
+
+ def served_embedding_identity(self) -> tuple[str | None, str | None] | None:
+ """Repo and pooling captured by this thread's last completed encode."""
+ return getattr(self._served_identity_local, "value", None)
def _encode_active(
self,
diff --git a/studio/backend/core/rag/embeddings.py b/studio/backend/core/rag/embeddings.py
index cd3c86b81cc..303e5869ef8 100644
--- a/studio/backend/core/rag/embeddings.py
+++ b/studio/backend/core/rag/embeddings.py
@@ -1074,12 +1074,55 @@ def active_backend_is_llama(model_name: str | None = None) -> bool:
return False
-def _identity(is_llama: bool, name: str) -> str:
+def _llama_pooling(name: str, served = None) -> str | None:
+ try:
+ from .embed_llama_server import LlamaServerBackend, _gguf_pooling
+ except Exception: # noqa: BLE001 - llama plumbing import must never block
+ return None
+ # The file the running server loaded wins over a cache search: moving the HF cache leaves it serving the old path.
+ backend = served if served is not None else _backend
+ desired = config.effective_gguf_repo_for_embedding_model(name)
+ if isinstance(backend, LlamaServerBackend):
+ path, repo, captured, alive = backend.pooling_identity_snapshot()
+ else:
+ path, repo, captured, alive = None, None, None, False
+ if path and repo == desired:
+ # A live server still serves the pooling captured at its spawn even if the file is
+ # replaced. ``served`` means an encode just completed on that same captured value. With a
+ # stopped ambient backend, however, the next spawn will read the file again, so pre-encode
+ # deduplication must do the same.
+ if served is not None or alive:
+ pooling = captured or _gguf_pooling(path)
+ elif os.path.isfile(path):
+ pooling = _gguf_pooling(path)
+ else:
+ pooling = LlamaServerBackend.cached_pooling(name)
+ else:
+ pooling = LlamaServerBackend.cached_pooling(name)
+ if pooling is None:
+ # Nothing on disk to read, so match no stored row: pre-encode dedupe must not take forced-CLS vectors as current.
+ return "unresolved"
+ return None if pooling == "cls" else pooling
+
+
+def _identity(
+ is_llama: bool,
+ name: str,
+ served = None,
+ served_identity: tuple[str | None, str | None] | None = None,
+) -> str:
if is_llama:
+ if served_identity is None:
+ repo = config.effective_gguf_repo_for_embedding_model(name)
+ pooling = _llama_pooling(name, served)
+ else:
+ repo, captured_pooling = served_identity
+ pooling = None if captured_pooling == "cls" else captured_pooling
return config.embedding_identity(
"llama-server",
name,
- gguf_repo = config.effective_gguf_repo_for_embedding_model(name),
+ gguf_repo = repo,
+ pooling = pooling,
)
return config.embedding_identity("sentence-transformers", name)
@@ -1144,12 +1187,14 @@ def encode_with_identity(
were never in, and a query then searches (or a document is stored against) the
wrong half of the index."""
_served_by.backend = None
+ _served_by.llama_identity = None
vectors = encode(texts, model_name = model_name, normalize = normalize)
served = getattr(_served_by, "backend", None)
+ served_identity = getattr(_served_by, "llama_identity", None)
name = model_name or config.effective_embedding_model()
if served is None:
return vectors, embedding_identity(name)
- return vectors, _identity(_is_llama_backend(served), name)
+ return vectors, _identity(_is_llama_backend(served), name, served, served_identity)
def warm(model_name: str | None = None) -> None:
@@ -1172,8 +1217,12 @@ def encode(
"""
backend = _get_backend(model_name)
_served_by.backend = backend
+ _served_by.llama_identity = None
try:
- return backend.encode(texts, model_name = model_name, normalize = normalize)
+ vectors = backend.encode(texts, model_name = model_name, normalize = normalize)
+ served = getattr(_served_by, "backend", None) or backend
+ _served_by.llama_identity = getattr(served, "served_embedding_identity", lambda: None)()
+ return vectors
except RuntimeError:
if not (_is_llama_backend(backend) and getattr(backend, "_closed", False)):
raise
@@ -1181,7 +1230,9 @@ def encode(
if replacement is backend:
raise RuntimeError("llama-server embedding backend was unloaded")
_served_by.backend = replacement
- return replacement.encode(texts, model_name = model_name, normalize = normalize)
+ vectors = replacement.encode(texts, model_name = model_name, normalize = normalize)
+ _served_by.llama_identity = getattr(replacement, "served_embedding_identity", lambda: None)()
+ return vectors
def dim(model_name: str | None = None) -> int:
diff --git a/studio/backend/tests/test_rag_embed_llama_server.py b/studio/backend/tests/test_rag_embed_llama_server.py
index c1e6f8af96e..f3b3fa73b0d 100644
--- a/studio/backend/tests/test_rag_embed_llama_server.py
+++ b/studio/backend/tests/test_rag_embed_llama_server.py
@@ -4,6 +4,7 @@
"""llama-server GGUF embedder tests, every boundary mocked."""
import os
+import struct
import subprocess
import sys
import textwrap
@@ -371,6 +372,33 @@ def test_spawn_uses_free_port_when_auto(monkeypatch):
assert b._port == 47000
+def test_spawn_refreshes_pooling_for_a_replaced_model(monkeypatch, tmp_path):
+ """A cached path survives the reaper, but its file can be replaced between server
+ lifetimes. The relaunched command and the vectors' identity must capture the same pooling."""
+ path = tmp_path / "embed.gguf"
+ _write_gguf(path, "qwen3", 1)
+ backend = LlamaServerBackend()
+ backend._model_pooling = "last"
+ proc = _FakeProc(alive = True)
+ captured = {}
+
+ monkeypatch.setattr(backend, "_resolve_binary", lambda: "/bin/llama-server")
+ monkeypatch.setattr(backend, "_resolve_model_path", lambda model_name = None: str(path))
+ monkeypatch.setattr(backend, "_find_free_port", lambda: 47000)
+ monkeypatch.setattr(backend, "_wait_for_health", lambda *a, **k: True)
+ monkeypatch.setattr(backend, "_build_env", lambda *a, **k: {})
+
+ def fake_popen(cmd, **_kwargs):
+ captured["cmd"] = cmd
+ return proc
+
+ monkeypatch.setattr(mod.subprocess, "Popen", fake_popen)
+ backend._spawn_once(False)
+
+ assert captured["cmd"][captured["cmd"].index("--pooling") + 1] == "mean"
+ assert backend._model_pooling == "mean"
+
+
def test_spawn_fails_loud_on_early_exit(monkeypatch):
monkeypatch.setattr(config, "EMBED_PORT", 8124)
b = LlamaServerBackend()
@@ -1656,3 +1684,348 @@ def test_a_partly_present_planned_family_is_not_served(monkeypatch, tmp_path):
assert backend._planned_family_path("org/pending", repo) is None
assert backend._resolve_model_path().endswith("pending-F16.gguf")
+
+
+def _write_gguf(
+ path,
+ arch,
+ pooling = None,
+):
+ kvs = [
+ struct.pack("<Q", 20)
+ + b"general.architecture"
+ + struct.pack("<IQ", 8, len(arch))
+ + arch.encode()
+ ]
+ if pooling is not None:
+ key = f"{arch}.pooling_type".encode()
+ kvs.append(struct.pack("<Q", len(key)) + key + struct.pack("<II", 4, pooling))
+ Path(path).write_bytes(b"GGUF" + struct.pack("<IQQ", 3, 0, len(kvs)) + b"".join(kvs))
+
+
+ "pooling, expected",
+ [(3, "last"), (1, "mean"), (2, "cls"), (None, "cls"), (0, "cls")],
+)
+def test_build_cmd_serves_the_pooling_the_gguf_declares(tmp_path, pooling, expected):
+ """Forcing CLS on a last-token (Qwen3-Embedding) or mean (nomic) GGUF pooled one
+ token, so unrelated sentences embedded nearly alike. A GGUF declaring none keeps
+ CLS, since llama-server would fall back to NONE and /v1/embeddings refuses that."""
+ path = tmp_path / "embed.gguf"
+ _write_gguf(path, "qwen3", pooling)
+ cmd = LlamaServerBackend()._build_cmd("/bin/llama-server", str(path), 1, use_gpu = False)
+ assert cmd[cmd.index("--pooling") + 1] == expected
+
+
[email protected]("arch", ["nomic-bert", "nomic-bert-moe"])
+def test_build_cmd_uses_mean_for_older_nomic_ggufs_without_pooling_metadata(tmp_path, arch):
+ """Dedicated Nomic embedding architectures predate reliable pooling metadata, but
+ their model family is mean-pooled. An explicit unsupported value still falls back safely."""
+ path = tmp_path / "nomic.gguf"
+ _write_gguf(path, arch)
+ backend = LlamaServerBackend()
+ cmd = backend._build_cmd("/bin/llama-server", str(path), 1, use_gpu = False)
+ assert cmd[cmd.index("--pooling") + 1] == "mean"
+ assert backend._adopt_model_path(str(path), "org/nomic-GGUF") == str(path)
+ assert backend._model_pooling == "mean"
+
+ _write_gguf(path, arch, 0)
+ cmd = backend._build_cmd("/bin/llama-server", str(path), 1, use_gpu = False)
+ assert cmd[cmd.index("--pooling") + 1] == "cls"
+
+
[email protected]("pooling, suffix", [(3, ":last"), (1, ":mean"), (2, "")])
+def test_llama_identity_changes_only_for_a_non_cls_gguf(monkeypatch, tmp_path, pooling, suffix):
+ """Vectors indexed under the forced CLS must read as stale once a mean or last
+ GGUF is served natively, while a CLS GGUF such as bge-small keeps its identity."""
+ model, repo = "org/embed", "org/embed-GGUF"
+ monkeypatch.setattr(config, "effective_gguf_repo_for_embedding_model", lambda m: repo)
+ snapshot = _seed_cache(tmp_path / "hub", repo, ["embed-F16.gguf"])
+ _write_gguf((snapshot / "embed-F16.gguf").resolve(), "qwen3", pooling)
+ _use_cache_root(monkeypatch, tmp_path / "hub")
+ legacy = config.embedding_identity("llama-server", model, gguf_repo = repo)
+ assert embeddings._identity(True, model) == legacy + suffix
+
+
+def test_llama_identity_follows_the_served_gguf_after_the_cache_moves(monkeypatch, tmp_path):
+ """Moving the HF cache in Settings leaves the server on the old file, so a fresh
+ cache search dropped the ``:last`` suffix and tagged last-pooled vectors as CLS."""
+ model, repo = "org/embed", "org/embed-GGUF"
+ monkeypatch.setattr(config, "effective_gguf_repo_for_embedding_model", lambda m: repo)
+ snapshot = _seed_cache(tmp_path / "hub-a", repo, ["embed-F16.gguf"])
+ served = (snapshot / "embed-F16.gguf").resolve()
+ _write_gguf(served, "qwen3", 3)
+ backend = LlamaServerBackend()
+ backend._model_path, backend._model_repo = str(served), repo
+ monkeypatch.setattr(embeddings, "_backend", backend)
+ (tmp_path / "hub-b").mkdir()
+ _use_cache_root(monkeypatch, tmp_path / "hub-b")
+ legacy = config.embedding_identity("llama-server", model, gguf_repo = repo)
+ assert embeddings._identity(True, model) == legacy + ":last"
+ assert embeddings._identity(True, model, backend) == legacy + ":last"
+
+
+def test_llama_identity_with_no_gguf_on_disk_matches_no_forced_cls_row(monkeypatch, tmp_path):
+ """With the GGUF evicted and no server up, the prediction fell back to the forced-CLS
+ identity, so a re-upload deduplicated onto those stale vectors instead of re-indexing."""
+ model, repo = "org/embed", "org/embed-GGUF"
+ monkeypatch.setattr(config, "effective_gguf_repo_for_embedding_model", lambda m: repo)
+ (tmp_path / "hub").mkdir()
+ _use_cache_root(monkeypatch, tmp_path / "hub")
+ legacy = config.embedding_identity("llama-server", model, gguf_repo = repo)
+ predicted = embeddings._identity(True, model)
+ assert not config.embedding_identity_matches(legacy, predicted)
+ assert config.embedding_identity_model(predicted) == model
+
+
+def test_llama_identity_keeps_the_launched_pooling_when_the_file_vanishes(monkeypatch, tmp_path):
+ """A running server keeps serving its open GGUF after the cache entry is evicted, so
+ re-reading the gone path answered CLS and tagged last-pooled vectors as CLS."""
+ model, repo = "org/embed", "org/embed-GGUF"
+ monkeypatch.setattr(config, "effective_gguf_repo_for_embedding_model", lambda m: repo)
+ snapshot = _seed_cache(tmp_path / "hub", repo, ["embed-F16.gguf"])
+ served = (snapshot / "embed-F16.gguf").resolve()
+ _write_gguf(served, "qwen3", 3)
+ _use_cache_root(monkeypatch, tmp_path / "hub")
+ backend = LlamaServerBackend()
+ backend._adopt_model_path(str(served), repo)
+ monkeypatch.setattr(embeddings, "_backend", backend)
+ served.unlink()
+ legacy = config.embedding_identity("llama-server", model, gguf_repo = repo)
+ assert embeddings._identity(True, model, backend) == legacy + ":last"
+
+
+def test_llama_identity_rechecks_a_stopped_backends_replaced_gguf(monkeypatch, tmp_path):
+ """Before an encode, a stopped server will respawn from the current file and must not
+ deduplicate with its old pooling. A completed encode still keeps the value it actually used."""
+ model, repo = "org/embed", "org/embed-GGUF"
+ monkeypatch.setattr(config, "effective_gguf_repo_for_embedding_model", lambda m: repo)
+ snapshot = _seed_cache(tmp_path / "hub", repo, ["embed-F16.gguf"])
+ served = (snapshot / "embed-F16.gguf").resolve()
+ _write_gguf(served, "qwen3", 3)
+ _use_cache_root(monkeypatch, tmp_path / "hub")
+ backend = LlamaServerBackend()
+ backend._adopt_model_path(str(served), repo)
+ monkeypatch.setattr(embeddings, "_backend", backend)
+ _write_gguf(served, "qwen3", 1)
+ legacy = config.embedding_identity("llama-server", model, gguf_repo = repo)
+
+ assert embeddings._identity(True, model) == legacy + ":mean"
+ assert embeddings._identity(True, model, backend) == legacy + ":last"
+
+
+def test_llama_identity_treats_an_evicted_stopped_model_as_unresolved(monkeypatch, tmp_path):
+ """A stopped backend cannot keep serving a deleted path. Pre-encode deduplication must
+ miss old CLS rows, and the next spawn must run the normal resolver instead of that path."""
+ model, repo = "org/embed", "org/embed-GGUF"
+ monkeypatch.setattr(config, "effective_gguf_repo_for_embedding_model", lambda m: repo)
+ snapshot = _seed_cache(tmp_path / "hub", repo, ["embed-F16.gguf"])
+ served = (snapshot / "embed-F16.gguf").resolve()
+ _write_gguf(served, "qwen3", 2)
+ _use_cache_root(monkeypatch, tmp_path / "hub")
+ backend = LlamaServerBackend()
+ backend._adopt_model_path(str(served), repo)
+ monkeypatch.setattr(embeddings, "_backend", backend)
+ served.unlink()
+ legacy = config.embedding_identity("llama-server", model, gguf_repo = repo)
+
+ assert embeddings._identity(True, model) == legacy + ":unresolved"
+ monkeypatch.setattr(backend, "_resolve_uncached_model_path", lambda *args: "/fresh.gguf")
+ assert backend._resolve_model_path(model) == "/fresh.gguf"
+
+
+def test_preencode_identity_takes_one_locked_backend_snapshot(monkeypatch):
+ """A concurrent model switch cannot combine model A's repo check with model B's pooling."""
+
+ class SwitchingBackend(LlamaServerBackend):
+ def __init__(self):
+ self.gate = False
+ self.repo_read = threading.Event()
+ self.allow_repo_read = threading.Event()
+ super().__init__()
+
+ @property
+ def _model_repo(self):
+ value = self._repo_value
+ if self.gate:
+ self.repo_read.set()
+ assert self.allow_repo_read.wait(timeout = 2)
+ return value
+
+ @_model_repo.setter
+ def _model_repo(self, value):
+ self._repo_value = value
+
+ @property
+ def _model_pooling(self):
+ return self._pooling_value
+
+ @_model_pooling.setter
+ def _model_pooling(self, value):
+ self._pooling_value = value
+
+ model, repo_a, repo_b = "org/a", "org/a-GGUF", "org/b-GGUF"
+ backend = SwitchingBackend()
+ backend._model_path = "/served/a.gguf"
+ backend._model_repo = repo_a
+ backend._model_pooling = "last"
+ backend._process = _FakeProc(alive = True)
+ backend.gate = True
+ monkeypatch.setattr(config, "effective_gguf_repo_for_embedding_model", lambda _model: repo_a)
+ monkeypatch.setattr(embeddings, "_backend", backend)
+ result = {}
+ switched_before_snapshot = []
+
+ def predict():
+ result["identity"] = embeddings._identity(True, model)
+
+ def switch_model():
+ assert backend.repo_read.wait(timeout = 2)
+ acquired = backend._serve_lock.acquire(timeout = 0.05)
+ switched_before_snapshot.append(acquired)
+ if acquired:
+ backend._repo_value = repo_b
+ backend._pooling_value = "mean"
+ backend._serve_lock.release()
+ backend.allow_repo_read.set()
+ if not acquired:
+ with backend._serve_lock:
+ backend._repo_value = repo_b
+ backend._pooling_value = "mean"
+
+ reader = threading.Thread(target = predict)
+ writer = threading.Thread(target = switch_model)
+ reader.start()
+ writer.start()
+ reader.join(timeout = 2)
+ writer.join(timeout = 2)
+ backend._process = None
+
+ assert not reader.is_alive() and not writer.is_alive()
+ assert switched_before_snapshot == [False]
+ assert result["identity"] == config.embedding_identity(
+ "llama-server", model, gguf_repo = repo_a, pooling = "last"
+ )
+
+
+def test_encode_identity_keeps_the_callers_snapshot_when_another_model_takes_over(monkeypatch):
+ """The shared backend can switch models after an encode returns. Each thread must retain
+ the immutable repo and pooling that served its own vectors, not the backend's later state."""
+ backend = LlamaServerBackend()
+ model_a, model_b = "org/a", "org/b"
+ repo_a, repo_b = "org/a-GGUF", "org/b-GGUF"
+ identities = {}
+ first_waiting = threading.Event()
+ second_done = threading.Event()
+
+ monkeypatch.setattr(embeddings, "_get_backend", lambda _model = None: backend)
+ monkeypatch.setattr(
+ config,
+ "effective_gguf_repo_for_embedding_model",
+ lambda model: repo_a if model == model_a else repo_b,
+ )
+
+ def fake_ready(model):
+ backend._model_repo = repo_a if model == model_a else repo_b
+ backend._model_pooling = "last" if model == model_a else "mean"
+
+ monkeypatch.setattr(backend, "_ensure_ready", fake_ready)
+
+ def fake_encode_active(
+ texts,
+ *,
+ model_name = None,
+ **_kwargs,
+ ):
+ backend._ensure_ready(model_name)
+ return np.zeros((len(texts), 2), dtype = np.float32)
+
+ monkeypatch.setattr(backend, "_encode_active", fake_encode_active)
+ real_identity = embeddings._identity
+
+ def delayed_identity(
+ is_llama,
+ name,
+ served = None,
+ served_identity = None,
+ ):
+ if name == model_a:
+ first_waiting.set()
+ assert second_done.wait(timeout = 2)
+ else:
+ second_done.set()
+ if served_identity is None:
+ return real_identity(is_llama, name, served)
+ return real_identity(is_llama, name, served, served_identity)
+
+ monkeypatch.setattr(embeddings, "_identity", delayed_identity)
+
+ def run(model):
+ identities[model] = embeddings.encode_with_identity(["x"], model_name = model)[1]
+
+ first = threading.Thread(target = run, args = (model_a,))
+ second = threading.Thread(target = run, args = (model_b,))
+ first.start()
+ assert first_waiting.wait(timeout = 2)
+ second.start()
+ first.join(timeout = 2)
+ second.join(timeout = 2)
+
+ assert not first.is_alive() and not second.is_alive()
+ assert identities[model_a] == config.embedding_identity(
+ "llama-server", model_a, gguf_repo = repo_a, pooling = "last"
+ )
+ assert identities[model_b] == config.embedding_identity(
+ "llama-server", model_b, gguf_repo = repo_b, pooling = "mean"
+ )
+
+
+def test_llama_identity_is_not_predicted_from_a_fallback_repo(monkeypatch, tmp_path):
+ """The loader serves only the desired repo from cache before going online, so a
+ fallback repo's cached GGUF may not be what the next encode loads."""
+ model, repo = "org/embed", "org/embed-GGUF"
+ monkeypatch.setattr(config, "effective_gguf_repo_for_embedding_model", lambda m: repo)
+ snapshot = _seed_cache(tmp_path / "hub", model, ["embed-F16.gguf"])
+ _write_gguf((snapshot / "embed-F16.gguf").resolve(), "qwen3", 3)
+ _use_cache_root(monkeypatch, tmp_path / "hub")
+ legacy = config.embedding_identity("llama-server", model, gguf_repo = repo)
+ assert embeddings._identity(True, model) == legacy + ":unresolved"
+
+
+def test_llama_identity_follows_the_planned_family_before_a_later_variant(monkeypatch, tmp_path):
+ """The loader pins a completed download family before considering the configured
+ variant, so prediction must not take pooling from a different file that arrived later."""
+ import utils.embedding_model_settings as ems
+
+ model, repo = "org/embed", "org/embed-GGUF"
+ monkeypatch.setattr(config, "effective_gguf_repo_for_embedding_model", lambda m: repo)
+ monkeypatch.setattr(config, "EMBED_GGUF_VARIANT", "F16")
+ monkeypatch.setattr(ems, "get_stored_gguf_files", lambda m: ["embed-Q8_0.gguf"])
+ snapshot = _seed_cache(tmp_path / "hub", repo, ["embed-Q8_0.gguf", "embed-F16.gguf"])
+ _write_gguf((snapshot / "embed-Q8_0.gguf").resolve(), "qwen3", 3)
+ _write_gguf((snapshot / "embed-F16.gguf").resolve(), "bert", 2)
+ _use_cache_root(monkeypatch, tmp_path / "hub")
+ legacy = config.embedding_identity("llama-server", model, gguf_repo = repo)
+
+ assert embeddings._identity(True, model) == legacy + ":last"
+
+
[email protected]("download_pending, suffix", [(False, ":unresolved"), (True, ":last")])
+def test_llama_identity_only_predicts_a_stand_in_quant_while_its_download_is_pending(
+ monkeypatch, tmp_path, download_pending, suffix
+):
+ """Without the configured variant, the loader goes online unless an active download
+ makes the cached quant a deliberate stand-in. Only that stand-in is predictable."""
+ import utils.embedding_model_settings as ems
+
+ model, repo = "org/embed", "org/embed-GGUF"
+ monkeypatch.setattr(config, "effective_gguf_repo_for_embedding_model", lambda m: repo)
+ monkeypatch.setattr(config, "EMBED_GGUF_VARIANT", "F16")
+ monkeypatch.setattr(ems, "get_stored_gguf_files", lambda m: None)
+ monkeypatch.setattr(ems, "get_stored_download_pending", lambda m: download_pending)
+ snapshot = _seed_cache(tmp_path / "hub", repo, ["embed-Q8_0.gguf"])
+ _write_gguf((snapshot / "embed-Q8_0.gguf").resolve(), "qwen3", 3)
+ _use_cache_root(monkeypatch, tmp_path / "hub")
+ legacy = config.embedding_identity("llama-server", model, gguf_repo = repo)
+
+ assert embeddings._identity(True, model) == legacy + suffix
diff --git a/studio/backend/tests/test_rag_embedding_identity.py b/studio/backend/tests/test_rag_embedding_identity.py
index bec168057e5..fb2626ca19e 100644
--- a/studio/backend/tests/test_rag_embedding_identity.py
+++ b/studio/backend/tests/test_rag_embedding_identity.py
@@ -220,6 +220,7 @@ def test_identity_distinguishes_the_backends_and_the_gguf_repo(monkeypatch):
def test_llama_identity_uses_the_resolved_stored_repo(monkeypatch):
monkeypatch.setattr(embeddings, "active_backend_is_llama", lambda *_a, **_k: True)
+ monkeypatch.setattr(embeddings, "_llama_pooling", lambda *_a, **_k: None)
monkeypatch.setattr(
config,
"effective_gguf_repo_for_embedding_model",