Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: use llama-server for embedding models the installed sentence-transformers cannot load (#13005)

oobabooga184e1fc3e5ce
diff --git a/scripts/risky_loaders_baseline.json b/scripts/risky_loaders_baseline.json
index a3af6c47a53..f9dd93907ed 100644
--- a/scripts/risky_loaders_baseline.json
+++ b/scripts/risky_loaders_baseline.json
@@ -250,6 +250,14 @@
"count": 1,
"reason": "Prefix-bounded to core.inference.<gallery>; the name comes from the fixed _GALLERIES table."
},
+ {
+ "file": "studio/backend/core/rag/embeddings.py",
+ "rule": "dynamic-import",
+ "sink": "import_from_string",
+ "digest": "ad319be07f7da5ad",
+ "count": 1,
+ "reason": "The metadata reference is accepted only with the sentence_transformers. prefix, so it can import installed package code only; non-package refs are skipped and the real loader performs the same lookup."
+ },
{
"file": "studio/backend/core/systemone/laya_runtime.py",
"rule": "dynamic-import",
diff --git a/studio/backend/core/rag/embeddings.py b/studio/backend/core/rag/embeddings.py
index ba393ba407a..a1f6aa78eb5 100644
--- a/studio/backend/core/rag/embeddings.py
+++ b/studio/backend/core/rag/embeddings.py
@@ -24,8 +24,9 @@ import os
import re
import threading
from contextlib import contextmanager
+from contextvars import ContextVar
from functools import lru_cache
-from typing import Callable
+from typing import Any, Callable
from utils.hardware.hardware import DeviceType, get_device
from utils.transformers_dtype import dtype_kwargs
@@ -171,6 +172,38 @@ def _ambient_hf_token() -> str | None:
return None
+def _repo_json(name: str, filename: str, token: str | bool | None) -> Any:
+ """``filename`` parsed from the local folder or Hub repo ``name``, None when the file is absent.
+ Raises whatever reading or parsing raises."""
+ import json
+
+ from utils.paths import is_local_path
+
+ if is_local_path(name):
+ from pathlib import Path
+ from utils.paths import normalize_path
+
+ path = Path(normalize_path(name)).expanduser() / filename
+ if not path.is_file():
+ return None
+ return json.loads(path.read_text(encoding = "utf-8-sig"))
+ from huggingface_hub import hf_hub_download
+ from huggingface_hub.utils import EntryNotFoundError
+ from utils.hf_cache_settings import active_hf_hub_cache
+
+ try:
+ local = hf_hub_download(
+ name,
+ filename,
+ token = account_hf_token(token),
+ cache_dir = active_hf_hub_cache(),
+ )
+ except EntryNotFoundError:
+ return None
+ with open(local, encoding = "utf-8-sig") as f:
+ return json.load(f)
+
+
def _st_module_subdirs(name: str, token: str | None) -> tuple[str, ...]:
"""The module directories a SentenceTransformer load reads weights from, taken from
the repo's ``modules.json`` (each module's non-empty ``path``, e.g. ``0_Transformer``).
@@ -179,33 +212,7 @@ def _st_module_subdirs(name: str, token: str | None) -> tuple[str, ...]:
failure (no modules.json, offline, malformed) so the guard never bricks the embedder.
"""
try:
- import json
-
- from utils.paths import is_local_path
-
- if is_local_path(name):
- from pathlib import Path
- from utils.paths import normalize_path
-
- path = Path(normalize_path(name)).expanduser() / "modules.json"
- if not path.is_file():
- return ()
- data = json.loads(path.read_text(encoding = "utf-8-sig"))
- else:
- from huggingface_hub import hf_hub_download
- from huggingface_hub.utils import EntryNotFoundError
- from utils.hf_cache_settings import active_hf_hub_cache
-
- try:
- local = hf_hub_download(
- name,
- "modules.json",
- token = account_hf_token(token or None),
- cache_dir = active_hf_hub_cache(),
- )
- except EntryNotFoundError:
- return ()
- data = json.loads(open(local, encoding = "utf-8-sig").read())
+ data = _repo_json(name, "modules.json", token)
subdirs = []
for module in data or ():
sub = str((module or {}).get("path", "")).strip().strip("/")
@@ -908,15 +915,106 @@ def _llama_server_runtime_available() -> bool:
return False
-def resolved_backend_for_model(model_name: str) -> str:
+_ST_LOAD_PREFLIGHT_TIMEOUT_S = 5.0
+# Proofs last one Settings resolution: a folder can be fixed, a repo republished, or the next caller lack access.
+_st_unloadable_proofs: ContextVar[set[str] | None] = ContextVar(
+ "st-unloadable-proofs", default = None
+)
+
+
+@contextmanager
+def st_load_proof_scope():
+ """Keep ``sentence_transformers_can_load`` proofs for the enclosed resolution. Nested scopes share one."""
+ marker = _st_unloadable_proofs.set(set()) if _st_unloadable_proofs.get() is None else None
+ try:
+ yield
+ finally:
+ if marker is not None:
+ _st_unloadable_proofs.reset(marker)
+
+
+def sentence_transformers_known_unloadable(model_name: str) -> bool:
+ """Whether a check earlier in this scope proved ``model_name`` unloadable. No network."""
+ proofs = _st_unloadable_proofs.get()
+ return proofs is not None and model_name in proofs
+
+
+def sentence_transformers_can_load(model_name: str, token: str | bool | None = None) -> bool:
+ """False when ``modules.json`` names a sentence_transformers class this install lacks (embeddinggemma-2 names
+ 6.x ``sentence_transformers.base`` modules) or ``config.json`` a ``model_type`` transformers does not know.
+ True when either file cannot be read, so an unreachable repo keeps its plan."""
+ if sentence_transformers_known_unloadable(model_name):
+ return False
+ if _st_load_preflight(model_name, token):
+ return True
+ proofs = _st_unloadable_proofs.get()
+ if proofs is not None:
+ proofs.add(model_name)
+ return False
+
+
+def _st_load_preflight(model_name: str, token: str | bool | None = None) -> bool:
+ try:
+ from sentence_transformers.util import import_from_string
+ from utils.utils import call_with_deadline
+
+ def _metadata():
+ modules = _repo_json(model_name, "modules.json", token)
+ config_files = ["config.json"] if modules is None else []
+ for module in modules or ():
+ ref = str((module or {}).get("type", ""))
+ if ref.rsplit(".", 1)[-1] != "Transformer":
+ continue
+ path = str((module or {}).get("path", "")).strip("/")
+ filename = f"{path}/config.json" if path else "config.json"
+ if filename not in config_files:
+ config_files.append(filename)
+ return modules, {
+ filename: _repo_json(model_name, filename, token) for filename in config_files
+ }
+
+ # bounded with one deadline so nested module metadata cannot multiply the wait
+ modules, model_configs = call_with_deadline(
+ _metadata,
+ _ST_LOAD_PREFLIGHT_TIMEOUT_S,
+ name = "embed-st-load-preflight",
+ )
+ for module in modules or ():
+ ref = str((module or {}).get("type", ""))
+ # Other classes are third-party code, which _gate_st_custom_modules refuses on its own.
+ if not ref.startswith(_ST_PACKAGE_PREFIX):
+ continue
+ try:
+ # How ST's own loader resolves sentence_transformers.* refs.
+ import_from_string(ref)
+ except ImportError:
+ return False
+ for model_config in model_configs.values():
+ model_type = (model_config or {}).get("model_type")
+ if isinstance(model_type, str):
+ # CONFIG_MAPPING, not CONFIG_MAPPING_NAMES, so types registered at runtime count. An auto_map does not
+ # help an unknown type: the loader never passes trust_remote_code.
+ from transformers import CONFIG_MAPPING
+ if model_type not in CONFIG_MAPPING:
+ return False
+ except Exception as exc: # noqa: BLE001 - offline, gated, missing or malformed: no proof either way
+ logger.debug("sentence-transformers load preflight for %s failed: %s", model_name, exc)
+ return True
+
+
+def resolved_backend_for_model(model_name: str, token: str | bool | None = None) -> str:
"""Backend a fresh operation for ``model_name`` would actually select."""
raw = _raw_backend()
forced = _forced_backends.get(model_name)
key = forced or (_resolve_auto_for_model(model_name) if raw in _AUTO_ALIASES else raw)
- if key in _ST_ALIASES and not sentence_transformers_runtime_available():
- # Without a real llama binary ST is the only possible plan, and its eventual error is more useful
- # than a fabricated GGUF destination.
- if _llama_server_runtime_available():
+ # Without a llama binary ST is the only plan; its error beats a fabricated GGUF destination. Stat before Hub.
+ if key in _ST_ALIASES and _llama_server_runtime_available():
+ st_unusable = not sentence_transformers_runtime_available() or (
+ # An ST plan for it never loads, and its pending marker blocks the llama fallback's GGUF.
+ # Auto only: an explicit ST policy ignores the stored backend.
+ raw in _AUTO_ALIASES and not sentence_transformers_can_load(model_name, token)
+ )
+ if st_unusable:
key = "llama-server"
if key in _LLAMA_ALIASES:
return "llama-server"
diff --git a/studio/backend/routes/settings.py b/studio/backend/routes/settings.py
index 04be54c20c3..b328924fbae 100644
--- a/studio/backend/routes/settings.py
+++ b/studio/backend/routes/settings.py
@@ -44,7 +44,12 @@ from utils.account_context import (
is_owner_context,
reset_account,
)
-from hub.utils.hf_tokens import cache_reads_authorized, cached_read_refused, hf_token_arg
+from hub.utils.hf_tokens import (
+ HfTokenArg,
+ cache_reads_authorized,
+ cached_read_refused,
+ hf_token_arg,
+)
from routes.provider_credentials import current_credential_write, require_ui_session
@@ -2810,7 +2815,7 @@ def _llama_runtime_available() -> bool:
return True
-def _llama_backend_active(model: str | None = None) -> bool:
+def _llama_backend_active(model: str | None = None, token: HfTokenArg = None) -> bool:
"""Whether llama serves the active model, or would serve ``model`` if supplied. Delegates to the embeddings
module so a runtime fallback from sentence-transformers to llama-server is honored: in that state the
process loads only inert GGUF, so the ST pickle gate below must not hard-block a repo whose GGUF
@@ -2818,7 +2823,7 @@ def _llama_backend_active(model: str | None = None) -> bool:
from core.rag import embeddings
try:
if model is not None:
- return embeddings.resolved_backend_for_model(model) == "llama-server"
+ return embeddings.resolved_backend_for_model(model, token) == "llama-server"
return embeddings.active_backend_is_llama()
except Exception: # noqa: BLE001 - backend probe must never block saving
return False
@@ -2888,7 +2893,6 @@ def _hf_gguf_backend_error(model: str, hf_token: Optional[str]) -> str | None:
def _no_embedding_weights_error(candidates: list[str]) -> str:
- """Error after the caller has already exhausted GGUF and ST resolution."""
checked = " or ".join(repr(c) for c in candidates)
return (
f"No GGUF weights found in {checked}, and no safetensors to fall back to. "
@@ -2896,6 +2900,14 @@ def _no_embedding_weights_error(candidates: list[str]) -> str:
)
+def _st_cannot_load_error(model: str, candidates: list[str]) -> str:
+ checked = " or ".join(repr(c) for c in candidates)
+ return (
+ f"No GGUF weights found in {checked}, and {model!r} needs a newer sentence-transformers or "
+ "transformers than this install has."
+ )
+
+
@_owner_settings_router.get("/embedding-model", response_model = EmbeddingModelResponse)
def get_embedding_model(
current_subject: str = Depends(get_current_subject),
@@ -2906,8 +2918,7 @@ def get_embedding_model(
class EmbeddingModelResolveResponse(BaseModel):
embedding_model: str
backend: Literal["llama", "sentence-transformers"]
- # Repo the picker hands the download manager, and the files to take from it. Split GGUF plans contain
- # every shard in the selected family; both None when nothing needs fetching or when ``error`` is set.
+ # download source; split plans include every shard; both fields are None on no fetch or error
download_repo: Optional[str] = None
files: Optional[list[str]] = None
cached: bool = False
@@ -2941,7 +2952,7 @@ _EMBEDDING_RESOLVE_DEADLINE: ContextVar[float | None] = ContextVar(
def _call_with_embedding_resolve_budget(fn, *, name: str):
- """Run one remote probe inside the resolution's single time budget."""
+ """run one remote probe within the shared resolution deadline."""
deadline = _EMBEDDING_RESOLVE_DEADLINE.get()
timeout = (
_GGUF_LIST_DEADLINE_S
@@ -2956,15 +2967,18 @@ def _call_with_embedding_resolve_budget(fn, *, name: str):
def _with_embedding_resolve_budget(fn):
- """Give one GET/PUT resolution a deadline shared by every Hub fallback."""
+ """share one deadline and ST proof scope so timeout skips cannot erase earlier results."""
@functools.wraps(fn)
def _wrapped(*args, **kwargs):
if _EMBEDDING_RESOLVE_DEADLINE.get() is not None:
return fn(*args, **kwargs)
+ from core.rag.embeddings import st_load_proof_scope
+
marker = _EMBEDDING_RESOLVE_DEADLINE.set(time.monotonic() + _GGUF_LIST_DEADLINE_S)
try:
- return fn(*args, **kwargs)
+ with st_load_proof_scope():
+ return fn(*args, **kwargs)
finally:
_EMBEDDING_RESOLVE_DEADLINE.reset(marker)
@@ -2972,7 +2986,7 @@ def _with_embedding_resolve_budget(fn):
def _list_repo_files_bounded(repo: str, hf_token: Optional[str]) -> list[str]:
- """List a Hub repo without letting a blackholed route pin Settings forever."""
+ """bound Hub listing time so a blackholed route cannot pin Settings."""
from huggingface_hub import list_repo_files
return _call_with_embedding_resolve_budget(
lambda: list_repo_files(repo, token = hf_token),
@@ -3221,7 +3235,6 @@ def _safetensors_plan(model: str, hf_token: Optional[str]) -> Optional[tuple[str
def _sentence_transformers_fallback_allowed(model: str) -> bool:
- """Whether a newly selected model can actually be served by ST in this process."""
try:
from core.rag import embeddings
return embeddings.sentence_transformers_fallback_allowed(model)
@@ -3229,8 +3242,25 @@ def _sentence_transformers_fallback_allowed(model: str) -> bool:
return False
+def _sentence_transformers_can_load(model: str, token: HfTokenArg = None) -> bool:
+ try:
+ from core.rag import embeddings
+ except Exception: # noqa: BLE001 - unimportable embedder: no proof either way
+ return True
+ # check known failures before the deadline so a timeout cannot erase earlier proof
+ if embeddings.sentence_transformers_known_unloadable(model):
+ return False
+ try:
+ return _call_with_embedding_resolve_budget(
+ lambda: embeddings.sentence_transformers_can_load(model, token),
+ name = "embed-settings-st-load-check",
+ )
+ except Exception: # noqa: BLE001 - spent budget: no proof either way
+ return True
+
+
def _hf_files_size(repo: str, files: list[str], hf_token: Optional[str]) -> Optional[int]:
- """Total bytes of ``files`` in ``repo``, for the confirm dialog. None when the hub does not say."""
+ """return total file bytes for confirmation, or None when Hub omits the sizes."""
try:
from huggingface_hub import model_info
@@ -3296,7 +3326,7 @@ def _local_sentence_transformer_is_present(model: str) -> bool:
@_with_embedding_resolve_budget
def _resolve_embedding_model_plan(
- resolved: str, token: Optional[str]
+ resolved: str, token: HfTokenArg
) -> EmbeddingModelResolveResponse:
"""Server-owned artifact/backend plan shared by GET and PUT.
@@ -3331,7 +3361,7 @@ def _resolve_embedding_model_plan(
# resolver's deadline before a miss. It was also the wrong question, per the note above.
# Resolve for the model being selected.
- on_llama = _llama_backend_active(resolved)
+ on_llama = _llama_backend_active(resolved, token)
backend: Literal["llama", "sentence-transformers"] = (
"llama" if on_llama else "sentence-transformers"
)
@@ -3427,8 +3457,7 @@ def _resolve_embedding_model_plan(
)
plan = _remote_embedding_gguf_plan(candidates, token) or _search_hub_for_gguf(resolved, token)
if plan is None:
- # The loader's offline fallback accepts any complete cached quant from
- # any candidate only after its bounded online listing fails.
+ # the offline fallback accepts any complete cached quant after bounded online listing fails
cached_repo = _cached_embedding_gguf(candidates, require_variant = False)
if cached_repo and not _authorized(cached_repo):
cached_repo = None
@@ -3439,40 +3468,34 @@ def _resolve_embedding_model_plan(
download_repo = cached_repo,
cached = True,
)
- # No GGUF from this publisher: run it on its own safetensors only when
- # configuration/runtime policy can actually select ST for this model.
- st_plan = (
- _safetensors_plan(resolved, token)
- if _sentence_transformers_fallback_allowed(resolved)
- else None
- )
- # The GGUF branches above are gated and this one was not. The plan answers with the
- # repo the snapshot is FILED under, which for a slashless alias is not the name the
- # caller typed, and the response then reports it cached: that is how a denied caller
- # discovers the operator's private weights and force-saves them as the embedder.
- # Reading our own disk to find the repo is fine; naming it back is what is gated.
+ # use publisher safetensors only when this process can select ST
+ st_allowed = _sentence_transformers_fallback_allowed(resolved)
+ st_unloadable = st_allowed and not _sentence_transformers_can_load(resolved, token)
+ st_plan = _safetensors_plan(resolved, token) if st_allowed and not st_unloadable else None
+ # authorize the plan's repo to avoid exposing a cached private repo through an alias
if st_plan is not None and not _authorized(st_plan[0]):
st_plan = None
if st_plan is None:
return EmbeddingModelResolveResponse(
embedding_model = resolved,
backend = backend,
- error = _no_embedding_weights_error(candidates),
+ error = (
+ _st_cannot_load_error(resolved, candidates)
+ if st_unloadable
+ else _no_embedding_weights_error(candidates)
+ ),
)
st_repo, _st_files = st_plan
return EmbeddingModelResolveResponse(
embedding_model = resolved,
backend = "sentence-transformers",
download_repo = st_repo,
- # Same alias-aware predicate, asked about the repo the plan named
- # rather than the alias the user typed, which the gate above authorized.
+ # check the authorized plan repo because slashless aliases are cached under sentence-transformers/
cached = _cached_snapshot_has_st_weights(st_repo),
size_bytes = _hf_snapshot_size(st_repo, token),
)
repo, files = plan
- # A gated repo can publish its filenames, so a plan coming back is not authorization to
- # report the operator's copy of it. The repo the plan names need not be the one the
- # caller asked about, so it is authorized in its own right like every other candidate.
+ # authorize the resolved repo before reporting whether the operator has its exact files cached
if _authorized(repo) and _cached_embedding_gguf_files(repo, files):
return EmbeddingModelResolveResponse(
embedding_model = resolved,
diff --git a/studio/backend/tests/test_embedding_model_resolve.py b/studio/backend/tests/test_embedding_model_resolve.py
index bd45b58a95b..8187ec29b08 100644
--- a/studio/backend/tests/test_embedding_model_resolve.py
+++ b/studio/backend/tests/test_embedding_model_resolve.py
@@ -9,6 +9,7 @@ and with the PUT about what is unusable here."""
from pathlib import Path
import sys
+import time
import types as _types
@@ -103,7 +104,7 @@ def test_resolution_selects_the_backend_for_the_new_model_not_the_old_one(client
monkeypatch.setattr(
settings,
"_llama_backend_active",
- lambda model = None: seen.append(model) or False,
+ lambda model = None, token = None: seen.append(model) or False,
)
import utils.utils as utils
@@ -114,7 +115,14 @@ def test_resolution_selects_the_backend_for_the_new_model_not_the_old_one(client
assert body["backend"] == "sentence-transformers"
-def test_runtime_st_failure_is_planned_as_a_managed_gguf_download(client, monkeypatch):
+ "runtime_available, can_load",
+ # ST cannot run at all, or it runs but cannot open this model (a repo saved by a newer major).
+ [(False, True), (True, False)],
+)
+def test_runtime_st_failure_is_planned_as_a_managed_gguf_download(
+ client, monkeypatch, runtime_available, can_load
+):
from core.rag import embeddings
monkeypatch.setattr(embeddings.config, "EMBED_BACKEND", "auto")
@@ -124,7 +132,12 @@ def test_runtime_st_failure_is_planned_as_a_managed_gguf_download(client, monkey
"_resolve_auto_for_model",
lambda model = None: "sentence-transformers",
)
- monkeypatch.setattr(embeddings, "sentence_transformers_runtime_available", lambda: False)
+ monkeypatch.setattr(
+ embeddings, "sentence_transformers_runtime_available", lambda: runtime_available
+ )
+ monkeypatch.setattr(
+ embeddings, "sentence_transformers_can_load", lambda model, token = None: can_load
+ )
monkeypatch.setattr(embeddings, "_llama_server_runtime_available", lambda: True)
monkeypatch.setattr(
settings, "_cached_embedding_gguf", lambda candidates, require_variant: None
@@ -144,6 +157,42 @@ def test_runtime_st_failure_is_planned_as_a_managed_gguf_download(client, monkey
assert body["size_bytes"] == 1234
+def test_request_token_reaches_the_backend_loadability_probe(client, monkeypatch):
+ from core.rag import embeddings
+
+ seen = []
+ monkeypatch.setattr(embeddings.config, "EMBED_BACKEND", "auto")
+ monkeypatch.setattr(embeddings, "_forced_backends", {})
+ monkeypatch.setattr(
+ embeddings, "_resolve_auto_for_model", lambda model = None: "sentence-transformers"
+ )
+ monkeypatch.setattr(embeddings, "sentence_transformers_runtime_available", lambda: True)
+ monkeypatch.setattr(
+ embeddings,
+ "sentence_transformers_can_load",
+ lambda model, token = None: seen.append((model, token)) or False,
+ )
+ monkeypatch.setattr(embeddings, "_llama_server_runtime_available", lambda: True)
+ monkeypatch.setattr(
+ settings, "_cached_embedding_gguf", lambda candidates, require_variant: None
+ )
+ monkeypatch.setattr(
+ settings,
+ "_remote_embedding_gguf_plan",
+ lambda candidates, token: (candidates[0], ["embed-F16.gguf"]),
+ )
+ monkeypatch.setattr(settings, "_hf_files_size", lambda repo, files, token: 1234)
+
+ body = client.get(
+ "/embedding-model/resolve",
+ params = {"model": "org/private-embed"},
+ headers = {"X-Unsloth-HF-Token": "hf_request"},
+ ).json()
+
+ assert seen == [("org/private-embed", "hf_request")]
+ assert body["backend"] == "llama"
+
+
def test_sentence_transformers_local_path_is_already_present(client, monkeypatch, tmp_path):
monkeypatch.setattr(settings, "_llama_backend_active", lambda *_: False)
local = tmp_path / "embedder"
@@ -376,6 +425,11 @@ def _no_gguf_anywhere(monkeypatch):
)
monkeypatch.setattr(settings, "_remote_embedding_gguf_plan", lambda candidates, token: None)
monkeypatch.setattr(settings, "_search_hub_for_gguf", lambda m, token: None)
+ from core.rag import embeddings
+
+ monkeypatch.setattr(
+ embeddings, "sentence_transformers_can_load", lambda model, token = None: True
+ )
def test_no_gguf_falls_back_to_the_models_own_safetensors(client, monkeypatch):
@@ -454,6 +508,86 @@ def test_explicit_llama_policy_does_not_offer_safetensors(client, monkeypatch):
assert body["error"].startswith("No GGUF weights found")
+def test_no_safetensors_fallback_for_a_model_st_cannot_open(client, monkeypatch):
+ """A model planned on llama-server because ST cannot open it must not fall back to the download ST
+ then fails to load."""
+ from core.rag import embeddings
+
+ _no_gguf_anywhere(monkeypatch)
+ monkeypatch.setattr(settings, "_llama_runtime_available", lambda: True)
+ monkeypatch.setattr(settings, "_sentence_transformers_fallback_allowed", lambda model: True)
+ monkeypatch.setattr(
+ embeddings, "sentence_transformers_can_load", lambda model, token = None: False
+ )
+ monkeypatch.setattr(
+ settings,
+ "_safetensors_plan",
+ lambda *a: (_ for _ in ()).throw(AssertionError("ST fallback must not be probed")),
+ )
+ body = _resolve(client, "acme/st6-embedder").json()
+ assert body["backend"] == "llama"
+ assert body["download_repo"] is None
+ assert "needs a newer sentence-transformers or transformers" in body["error"]
+
+
+def test_a_spent_budget_does_not_undo_a_proven_incompatibility(client, monkeypatch):
+ """The backend check proves ST cannot open the model, then the GGUF search spends the whole budget. The
+ fallback check that follows cannot run, and must not answer "loadable" and offer the cached safetensors."""
+ from core.rag import embeddings
+
+ monkeypatch.setattr(embeddings.config, "EMBED_BACKEND", "auto")
+ monkeypatch.setattr(embeddings, "_forced_backends", {})
+ monkeypatch.setattr(
+ embeddings, "_resolve_auto_for_model", lambda model = None: "sentence-transformers"
+ )
+ monkeypatch.setattr(embeddings, "sentence_transformers_runtime_available", lambda: True)
+ monkeypatch.setattr(embeddings, "_llama_server_runtime_available", lambda: True)
+ st6_modules = [{"type": "sentence_transformers.no_such_module.transformer.Transformer"}]
+ monkeypatch.setattr(
+ embeddings,
+ "_repo_json",
+ lambda name, filename, token: st6_modules if filename == "modules.json" else {},
+ )
+ pytest.importorskip("sentence_transformers")
+
+ def _search_spends_the_budget(candidates, token):
+ settings._EMBEDDING_RESOLVE_DEADLINE.set(time.monotonic() - 1)
+ return None
+
+ monkeypatch.setattr(settings, "_llama_runtime_available", lambda: True)
+ monkeypatch.setattr(
+ settings, "_cached_embedding_gguf", lambda candidates, require_variant: None
+ )
+ monkeypatch.setattr(settings, "_remote_embedding_gguf_plan", _search_spends_the_budget)
+ monkeypatch.setattr(settings, "_search_hub_for_gguf", lambda m, token: None)
+ monkeypatch.setattr(settings, "_sentence_transformers_fallback_allowed", lambda model: True)
+ monkeypatch.setattr(settings, "_safetensors_plan", lambda m, token: (m, ["model.safetensors"]))
+
+ body = _resolve(client, "acme/st6-embedder").json()
+ assert body["backend"] == "llama"
+ assert body["download_repo"] is None
+ assert "needs a newer sentence-transformers or transformers" in body["error"]
+ # Kept for that resolution only: a fixed folder or another caller's access must be asked afresh.
+ assert embeddings.sentence_transformers_known_unloadable("acme/st6-embedder") is False
+
+
+def test_st_load_check_keeps_the_plan_once_the_resolve_budget_is_spent(monkeypatch):
+ from core.rag import embeddings
+
+ monkeypatch.setattr(
+ embeddings,
+ "sentence_transformers_can_load",
+ lambda model, token = None: (_ for _ in ()).throw(
+ AssertionError("must not probe past the budget")
+ ),
+ )
+ marker = settings._EMBEDDING_RESOLVE_DEADLINE.set(time.monotonic() - 1)
+ try:
+ assert settings._sentence_transformers_can_load("acme/st6-embedder") is True
+ finally:
+ settings._EMBEDDING_RESOLVE_DEADLINE.reset(marker)
+
+
def test_the_fallback_stays_in_the_models_own_repo(monkeypatch):
"""Same rule as the GGUF search: only the publisher is a source."""
monkeypatch.setattr(settings, "_st_backend_available", lambda: True)
diff --git a/studio/backend/tests/test_rag_embeddings.py b/studio/backend/tests/test_rag_embeddings.py
index a423a1e6843..cb122b580c8 100644
--- a/studio/backend/tests/test_rag_embeddings.py
+++ b/studio/backend/tests/test_rag_embeddings.py
@@ -4,6 +4,7 @@
"""Embedder concurrency tests: the fast tokenizer isn't thread-safe, so encode
and token counting must be serialized (else threads panic "Already borrowed")."""
+import json
import os
import sys
import threading
@@ -747,6 +748,131 @@ def test_runtime_preflight_keeps_st_when_no_fallback_exists(monkeypatch):
assert embeddings.resolved_backend_for_model("org/embedder") == "sentence-transformers"
+def _st_folder(tmp_path, module_type, model_config):
+ import json
+
+ folder = tmp_path / "embedder"
+ folder.mkdir()
+ (folder / "modules.json").write_text(
+ json.dumps([{"idx": 0, "name": "0", "path": "", "type": module_type}])
+ )
+ (folder / "config.json").write_text(json.dumps(model_config))
+ return str(folder)
+
+
+ "module_type, model_config, loadable",
+ [
+ ("sentence_transformers.models.Transformer", {"model_type": "bert"}, True),
+ # How a repo saved by sentence-transformers 6 names its modules (embeddinggemma-2).
+ (
+ "sentence_transformers.no_such_module.transformer.Transformer",
+ {"model_type": "bert"},
+ False,
+ ),
+ ("sentence_transformers.models.Transformer", {"model_type": "no_such_model_type"}, False),
+ # The loader never trusts remote code, so an auto_map does not make an unknown type loadable.
+ (
+ "sentence_transformers.models.Transformer",
+ {"model_type": "no_such_model_type", "auto_map": {}},
+ False,
+ ),
+ ],
+)
+def test_st_can_load_reads_modules_and_model_type(tmp_path, module_type, model_config, loadable):
+ pytest.importorskip("sentence_transformers")
+ pytest.importorskip("transformers")
+ assert (
+ embeddings.sentence_transformers_can_load(_st_folder(tmp_path, module_type, model_config))
+ is loadable
+ )
+
+
+ "root_model_type,module_model_type,loadable",
+ [
+ ("bert", "no_such_model_type", False),
+ ("no_such_model_type", "bert", True),
+ ],
+)
+def test_st_can_load_reads_the_transformer_modules_config(
+ tmp_path, root_model_type, module_model_type, loadable
+):
+ pytest.importorskip("sentence_transformers")
+ pytest.importorskip("transformers")
+ folder = tmp_path / "embedder"
+ module = folder / "0_Transformer"
+ module.mkdir(parents = True)
+ (folder / "modules.json").write_text(
+ json.dumps(
+ [
+ {
+ "idx": 0,
+ "name": "0",
+ "path": "0_Transformer",
+ "type": "sentence_transformers.models.Transformer",
+ }
+ ]
+ )
+ )
+ (folder / "config.json").write_text(json.dumps({"model_type": root_model_type}))
+ (module / "config.json").write_text(json.dumps({"model_type": module_model_type}))
+
+ assert embeddings.sentence_transformers_can_load(str(folder)) is loadable
+
+
+def test_st_can_load_preserves_anonymous_hub_access(monkeypatch):
+ pytest.importorskip("sentence_transformers")
+ seen = []
+
+ def _read(name, filename, token):
+ seen.append((filename, token))
+ return None if filename == "modules.json" else {}
+
+ monkeypatch.setattr(embeddings, "_repo_json", _read)
+
+ assert embeddings.sentence_transformers_can_load("org/public", False) is True
+ assert seen == [("modules.json", False), ("config.json", False)]
+
+
+def test_st_can_load_without_the_files_keeps_the_plan(tmp_path):
+ assert embeddings.sentence_transformers_can_load(str(tmp_path)) is True
+
+
+def test_st_can_load_gives_up_on_a_stalled_hub(monkeypatch):
+ pytest.importorskip("sentence_transformers")
+ monkeypatch.setattr(embeddings, "_ST_LOAD_PREFLIGHT_TIMEOUT_S", 0.05)
+ monkeypatch.setattr(embeddings, "_repo_json", lambda *_: time.sleep(1))
+
+ started = time.monotonic()
+ assert embeddings.sentence_transformers_can_load("org/stalled") is True
+ assert time.monotonic() - started < 0.5
+
+
+def test_model_st_cannot_open_is_planned_on_llama(monkeypatch):
+ monkeypatch.setattr(embeddings.config, "EMBED_BACKEND", "auto")
+ monkeypatch.setattr(embeddings, "_forced_backends", {})
+ _shared_setup_4(monkeypatch)
+ monkeypatch.setattr(embeddings, "sentence_transformers_runtime_available", lambda: True)
+ monkeypatch.setattr(
+ embeddings,
+ "sentence_transformers_can_load",
+ lambda model, token = None: model != "org/st6-model",
+ )
+ monkeypatch.setattr(embeddings, "_llama_server_runtime_available", lambda: True)
+
+ assert embeddings.resolved_backend_for_model("org/st6-model") == "llama-server"
+ assert embeddings.resolved_backend_for_model("org/embedder") == "sentence-transformers"
+
+ # An explicit ST policy makes the runtime ignore a stored llama backend, so the plan must not promise one.
+ monkeypatch.setattr(embeddings.config, "EMBED_BACKEND", "sentence-transformers")
+ assert embeddings.resolved_backend_for_model("org/st6-model") == "sentence-transformers"
+
+ monkeypatch.setattr(embeddings.config, "EMBED_BACKEND", "auto")
+ monkeypatch.setattr(embeddings, "_llama_server_runtime_available", lambda: False)
+ assert embeddings.resolved_backend_for_model("org/st6-model") == "sentence-transformers"
+
+
def test_runtime_preflight_catches_a_fatal_torch_device_mismatch(monkeypatch):
monkeypatch.setattr(
embeddings,