Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: serve /v1/embeddings from the configured embedding model when the loaded GGUF cannot (#10315)

Nilayb8d120ade8cb
diff --git a/studio/backend/core/inference/llama_keepwarm.py b/studio/backend/core/inference/llama_keepwarm.py
index bef73690938..595c5bc9c85 100644
--- a/studio/backend/core/inference/llama_keepwarm.py
+++ b/studio/backend/core/inference/llama_keepwarm.py
@@ -448,6 +448,20 @@ def _note_admitted_end() -> None:
_admitted_inference = max(0, _admitted_inference - 1)
+def untrack_admitted_inference(scope) -> None:
+ """Drop an already-admitted request from the preview busy guard once the route knows it
+ will not run against the resident GGUF after all.
+
+ ``untrack_current_request`` covers only the in-flight counters; the admitted tally is
+ what ``load_model_for_preview`` reads, so a route that admitted at the auto-switch hook
+ and then served the request some other way keeps blocking preview swaps for its whole
+ duration. Pops the marker so the middleware's finally, which balances only a scope that
+ still carries it, cannot decrement a second time. Idempotent."""
+ if not isinstance(scope, dict) or not scope.pop(_ADMITTED_SCOPE_KEY, False):
+ return
+ _note_admitted_end()
+
+
def begin_preview_serializer_wait(scope) -> bool:
"""Move a tracked preview from active to pending while it waits on the route lock."""
global _inflight, _pending, _preview_inflight, _preview_pending
diff --git a/studio/backend/core/rag/embed_llama_server.py b/studio/backend/core/rag/embed_llama_server.py
index 1ac18958cdd..799244d3984 100644
--- a/studio/backend/core/rag/embed_llama_server.py
+++ b/studio/backend/core/rag/embed_llama_server.py
@@ -19,6 +19,7 @@ from __future__ import annotations
import atexit
import logging
import os
+import struct
import subprocess
import sys
import threading
@@ -48,6 +49,52 @@ _TRANSPORT_ERRORS = (
)
+_GGUF_SCALAR_WIDTHS = {0: 1, 1: 1, 2: 2, 3: 2, 4: 4, 5: 4, 6: 4, 7: 1, 10: 8, 11: 8, 12: 8}
+
+# llama.cpp's own -ub default. An embedding prompt is not splittable, so one longer than this
+# is refused with a 500 no matter how large the model's context is.
+_UBATCH_SIZE = 512
+
+
+def _skip_gguf_value(f, vtype: int) -> None:
+ if vtype == 8:
+ f.seek(struct.unpack("<Q", f.read(8))[0], 1)
+ elif vtype == 9:
+ elem_type, count = struct.unpack("<IQ", f.read(12))
+ if elem_type in _GGUF_SCALAR_WIDTHS:
+ f.seek(_GGUF_SCALAR_WIDTHS[elem_type] * count, 1)
+ else:
+ for _ in range(count):
+ _skip_gguf_value(f, elem_type)
+ else:
+ f.seek(_GGUF_SCALAR_WIDTHS[vtype], 1)
+
+
+def _gguf_context_length(path: str) -> int | None:
+ try:
+ with open(path, "rb") as f:
+ if f.read(4) != b"GGUF":
+ return None
+ f.read(4)
+ _, kv_count = struct.unpack("<QQ", f.read(16))
+ arch = None
+ lengths: dict[str, int] = {}
+ for _ in range(kv_count):
+ key = f.read(struct.unpack("<Q", f.read(8))[0]).decode("utf-8")
+ vtype = struct.unpack("<I", f.read(4))[0]
+ if key == "general.architecture" and vtype == 8:
+ arch = f.read(struct.unpack("<Q", f.read(8))[0]).decode("utf-8")
+ elif key.endswith(".context_length") and vtype in (4, 10):
+ lengths[key] = int.from_bytes(f.read(_GGUF_SCALAR_WIDTHS[vtype]), "little")
+ else:
+ _skip_gguf_value(f, vtype)
+ if arch is not None and f"{arch}.context_length" in lengths:
+ return lengths[f"{arch}.context_length"] or None
+ except (OSError, struct.error, UnicodeDecodeError, KeyError, ValueError):
+ return None
+ return None
+
+
def _resolve_entrypoint(binary: str) -> str:
"""The real executable behind a managed shell/symlink entrypoint.
@@ -109,6 +156,7 @@ class LlamaServerBackend:
# dim() takes the same _serve_lock the request path does; reentrant because dim() -> encode() ->
# _ensure_ready() re-enters on one thread.
self._dim: int | None = None
+ self._max_tokens: int | None = None
# One subprocess serves one GGUF, so readiness and the request must be indivisible: a swap between
# them lands A's POST on B's server.
self._serve_lock = threading.RLock()
@@ -468,7 +516,11 @@ class LlamaServerBackend:
# Serve a stand-in rather than fail the index, but leave the marker: retiring on it would pin this
# model to the wrong quant.
return self._adopt_model_path(cached, desired)
- raise RuntimeError(
+ # Typed: the same condition the ST path raises, and /v1/embeddings answers it
+ # with a 409. Bare, it fell into the catch-all as a 502 naming nothing.
+ from .embeddings import EmbeddingModelDownloadRequiredError
+
+ raise EmbeddingModelDownloadRequiredError(
f"Embedding model {model!r} is not downloaded yet. "
"Finish its Settings download before indexing documents."
)
@@ -539,6 +591,7 @@ class LlamaServerBackend:
self._model_path = path
self._model_repo = desired
self._dim = None
+ self._max_tokens = None
return self._model_path
# A listing, not a transfer: a working hub answers well inside a second.
@@ -673,6 +726,11 @@ class LlamaServerBackend:
"--fit",
"off",
]
+ # -ub at llama.cpp's own default, so nothing is allocated that was not already: raising
+ # the batch allocates n_vocab * n_ubatch * 4 up front (~938 MiB at 8192 with a 30k
+ # vocab), against the 1024 MiB free this backend calls enough to offload every layer.
+ # Passed rather than read back because /props publishes n_ctx and no batch field.
+ cmd += ["-ub", str(_UBATCH_SIZE)]
# -1 offloads every layer (matches the chat server); 0 keeps it on CPU.
cmd += ["-ngl", "-1" if use_gpu else "0"]
return cmd
@@ -878,6 +936,9 @@ class LlamaServerBackend:
if self._binary_path_revision != custom_llama_cpp_path_revision():
self._binary = None
self._binary_path_revision = None
+ # Half a server fact (n_ctx, n_ubatch differ between builds), and
+ # _resolve_model_path fast-paths on an unchanged repo, so nothing else clears it.
+ self._max_tokens = None
self._spawn(model_name)
def _restart(self, model_name: str | None = None) -> None:
@@ -1037,6 +1098,72 @@ class LlamaServerBackend:
self._dim = width
return width
+ def _server_props(self) -> dict | None:
+ """``/props``, or None. Advisory only -- it refines a limit that already has a
+ value from the GGUF, so no failure here may reach the caller. That includes a
+ malformed base URL, which is what an un-started server has.
+ """
+ try:
+ data = httpx.get(f"{self._base_url}/props", timeout = 2.0, trust_env = False).json()
+ except Exception: # noqa: BLE001 - see docstring
+ return None
+ return data if isinstance(data, dict) else None
+
+ @staticmethod
+ def _positive(data: dict | None, key: str) -> int | None:
+ """*key* from /props, whether the build reports it at the top level or under
+ ``default_generation_settings``."""
+ for scope in (data, (data or {}).get("default_generation_settings")):
+ if isinstance(scope, dict):
+ value = scope.get(key)
+ if isinstance(value, int) and value > 0:
+ return value
+ return None
+
+ def _server_context(self) -> int | None:
+ return self._positive(self._server_props(), "n_ctx")
+
+ def _server_batch(self) -> int | None:
+ """The physical batch this server actually runs at.
+
+ A non-causal (embedding) prompt longer than it is refused outright, so it bounds
+ what may be advertised no matter how large the model's context is.
+ """
+ # n_ubatch only: n_batch is the logical batch, and llama.cpp derives the physical one as
+ # min(n_batch, n_ubatch or n_batch), so against the -ub we launch with a reported n_batch
+ # is never the limit. No build publishes either today, leaving the launch value; read it
+ # first anyway, so a build that starts reporting it wins over the assumption.
+ found = self._positive(self._server_props(), "n_ubatch")
+ return min(found, _UBATCH_SIZE) if found else _UBATCH_SIZE
+
+ def max_tokens(self, *, model_name = None) -> int | None:
+ with self._operation(), self._serve_lock:
+ self._ensure_ready(model_name)
+ if self._max_tokens is None and self._model_path:
+ limit = _gguf_context_length(self._model_path)
+ running = self._server_context()
+ if limit and running:
+ limit = min(limit, running)
+ elif running:
+ limit = running
+ # Past one batch llama.cpp returns a 500, not the 400 this limit exists to give.
+ batch = self._server_batch()
+ if batch:
+ limit = min(limit, batch) if limit else batch
+ if limit:
+ data = self._post(
+ "/tokenize",
+ {"content": "", "add_special": True},
+ model_name = model_name,
+ )
+ answer = max(1, limit - len(data.get("tokens", [])))
+ if running is None or batch is None:
+ # /props silent: answer with the header guess but do not freeze it,
+ # or a too-large limit outlives the readback that would correct it.
+ return answer
+ self._max_tokens = answer
+ return self._max_tokens
+
def warm(self, *, model_name = None) -> None:
"""Start the server and probe dim off the request path.
diff --git a/studio/backend/core/rag/embeddings.py b/studio/backend/core/rag/embeddings.py
index 7473f71958f..76f42f53971 100644
--- a/studio/backend/core/rag/embeddings.py
+++ b/studio/backend/core/rag/embeddings.py
@@ -199,7 +199,11 @@ def _st_module_subdirs(name: str, token: str | None) -> tuple[str, ...]:
return ()
-def _guard_model_security(name: str, local_only: bool = False) -> None:
+def _guard_model_security(
+ name: str,
+ local_only: bool = False,
+ display: str | None = None,
+) -> None:
"""Refuse to load a repo HF flagged as unsafe: a poisoned pickle deserializes inside
SentenceTransformer regardless of trust_remote_code. Defense in depth behind the
/settings gate (a name can also arrive via env/default); local paths and unreachable
@@ -207,6 +211,11 @@ def _guard_model_security(name: str, local_only: bool = False) -> None:
``local_only`` (offline) inspects the local cache; subdir probes are skipped (they'd hit the
network and hang, and the offline gate walks the whole snapshot anyway).
+
+ ``display`` names the model in the error. The scan runs on ``name``, which the caller has
+ already resolved to an absolute snapshot directory, and that path reaches a user through
+ /v1/embeddings; the configured model is what they can act on and what the rest of that
+ route reports.
"""
try:
from utils.security import evaluate_file_security, security_load_subdirs
@@ -235,7 +244,7 @@ def _guard_model_security(name: str, local_only: bool = False) -> None:
else "is flagged as unsafe by Hugging Face's security scan"
)
raise UnsafeEmbeddingModelError(
- f"Embedding model {name!r} {reason}; refusing to load. "
+ f"Embedding model {(display or name)!r} {reason}; refusing to load. "
"Set a different RAG embedding model."
)
@@ -470,7 +479,7 @@ def _get(model_name: str | None = None):
# Scan after load_target is settled: on the repo id it checked the Hub's current commit while the
# load opened an older cached one. evaluate_file_security recovers the repo and exact commit from
# a snapshot path.
- _guard_model_security(load_target, offline)
+ _guard_model_security(load_target, offline, display = name)
with _quiet_transformers_load() as report:
# Re-emit in finally: a load that raises after transformers wrote its report is exactly when a
# MISSING or MISMATCH line matters.
@@ -540,6 +549,22 @@ def _st_dim(model_name: str | None = None) -> int:
return _get(model_name).get_sentence_embedding_dimension()
+def _st_max_tokens(model_name: str | None = None) -> int | None:
+ with _compute_lock:
+ model = _get(model_name)
+ limit = getattr(model, "max_seq_length", None)
+ if not limit:
+ return None
+ tokenizer = getattr(model, "tokenizer", None)
+ specials = getattr(tokenizer, "num_special_tokens_to_add", None)
+ reserved = int(specials()) if callable(specials) else 0
+ prompts = getattr(model, "prompts", None) or {}
+ prompt = prompts.get(getattr(model, "default_prompt_name", None))
+ if prompt and tokenizer is not None:
+ reserved += len(tokenizer.encode(prompt, add_special_tokens = False))
+ return max(1, int(limit) - reserved)
+
+
def _st_token_counter(model_name: str | None = None) -> Callable[[str], int]:
"""Token counter using the model's tokenizer, under the compute lock (the same
fast tokenizer backs encode and isn't thread-safe), with rayon enabled for the
@@ -602,6 +627,9 @@ class _SentenceTransformersBackend:
def dim(self, *, model_name = None):
return _st_dim(model_name)
+ def max_tokens(self, *, model_name = None):
+ return _st_max_tokens(model_name)
+
def warm(self, *, model_name = None):
_get(model_name)
@@ -1143,6 +1171,10 @@ def dim(model_name: str | None = None) -> int:
return _get_backend(model_name).dim(model_name = model_name)
+def max_tokens(model_name: str | None = None) -> int | None:
+ return _get_backend(model_name).max_tokens(model_name = model_name)
+
+
def token_counter(model_name: str | None = None) -> Callable[[str], int]:
"""Callable counting tokens with the embedder's own tokenizer.
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 766ba68af36..55bad62893a 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -25730,6 +25730,415 @@ async def openai_completions(request: Request, current_subject: str = Depends(ge
# =====================================================================
+_STUDIO_EMBED_CONCURRENCY = 4
+_STUDIO_EMBED_MAX_INPUTS = 2048
+_studio_embed_semaphores: "weakref.WeakKeyDictionary" = weakref.WeakKeyDictionary()
+_studio_embed_semaphores_guard = threading.Lock()
+
+
+def _studio_embed_semaphore() -> asyncio.Semaphore:
+ loop = asyncio.get_running_loop()
+ with _studio_embed_semaphores_guard:
+ semaphore = _studio_embed_semaphores.get(loop)
+ if semaphore is None:
+ semaphore = _studio_embed_semaphores[loop] = asyncio.Semaphore(
+ _STUDIO_EMBED_CONCURRENCY
+ )
+ return semaphore
+
+
+def _resident_serves_embeddings(llama_backend) -> bool:
+ return bool(llama_backend.is_loaded and getattr(llama_backend, "is_embedding_gguf", True))
+
+
+def _embeddings_items(body: dict, *, tokens_ok: bool) -> list:
+ value = body.get("input")
+ if isinstance(value, str):
+ items = [value]
+ # `type(...) is int`, not isinstance: bool subclasses int, so `[true]` would be read
+ # as a token array and could swap the resident GGUF for a body llama-server rejects.
+ elif (
+ tokens_ok
+ and isinstance(value, list)
+ and value
+ and all(type(token) is int for token in value)
+ ):
+ items = [value]
+ elif isinstance(value, (list, tuple)):
+ items = list(value)
+ else:
+ items = []
+ if not items:
+ raise HTTPException(status_code = 400, detail = "'input' is required for embeddings.")
+ if not tokens_ok and len(items) > _STUDIO_EMBED_MAX_INPUTS:
+ raise HTTPException(
+ status_code = 400,
+ detail = f"'input' may hold at most {_STUDIO_EMBED_MAX_INPUTS} items.",
+ )
+
+ def _ok(item) -> bool:
+ if isinstance(item, str):
+ return bool(item)
+ return (
+ tokens_ok
+ and isinstance(item, list)
+ and bool(item)
+ and all(type(token) is int for token in item)
+ )
+
+ if not all(_ok(item) for item in items):
+ raise HTTPException(
+ status_code = 400,
+ detail = "'input' must be a non-empty string or an array of non-empty strings.",
+ )
+ encoding_format = body.get("encoding_format")
+ if encoding_format is not None and encoding_format not in ("float", "base64"):
+ raise HTTPException(
+ status_code = 400, detail = "'encoding_format' must be 'float' or 'base64'."
+ )
+ return items
+
+
+def _embeddings_texts(body: dict) -> list[str]:
+ return _embeddings_items(body, tokens_ok = False)
+
+
+def _public_embedding_name(model_name: str) -> str:
+ """A name for a local checkpoint that leaks no path but still identifies it.
+
+ The basename alone is not enough: /checkpoints/run-a/model and
+ /checkpoints/run-b/model are different embedding spaces, and reporting both as
+ "model" lets a client file vectors from either under one identity -- the exact
+ confusion this route reports the identity to prevent.
+ """
+ from utils.paths import is_local_path
+
+ if not is_local_path(model_name):
+ return model_name
+ import hashlib
+
+ normalized = model_name.replace("\\", "/").rstrip("/")
+ base = normalized.rsplit("/", 1)[-1] or "model"
+ return f"{base}-{hashlib.sha256(normalized.encode('utf-8')).hexdigest()[:8]}"
+
+
+def _public_embedding_identity(identity: str, model_name: str, label: str) -> str:
+ """Redact the model and repo segments of ``backend:model[:gguf_repo]``, nothing else.
+
+ Segment-wise, not a substring replace: a model can be a relative directory named
+ `transformers`, and replacing that everywhere also rewrites the `sentence-transformers`
+ tag, leaving an identity no backend answers to and that _names_studio_embedder cannot
+ match on the next request.
+ """
+ from core.rag.config import (
+ EMBEDDING_IDENTITY_TAGS,
+ _escape_identity_segment,
+ _unescape_identity_segment,
+ effective_gguf_repo_for_embedding_model,
+ )
+ from utils.paths import is_local_path
+
+ tag = next((t for t in EMBEDDING_IDENTITY_TAGS if identity.startswith(f"{t}:")), None)
+ if tag is None:
+ return identity
+ repo = effective_gguf_repo_for_embedding_model(model_name)
+ # Both segments are checked: a hub model id can carry a GGUF companion that is a local path
+ # (RAG_EMBED_GGUF_REPO, or a stored repo), and that path is a segment of the identity too.
+ if not is_local_path(model_name) and not is_local_path(repo):
+ return identity
+ public = {model_name: label, repo: _public_embedding_name(repo)}
+ segments = [_unescape_identity_segment(s) for s in identity[len(tag) + 1 :].split(":")]
+ redacted = [_escape_identity_segment(public.get(s, s)) for s in segments]
+ return ":".join([tag, *redacted])
+
+
+def _embedding_payload(vector, encoding_format: str):
+ if encoding_format == "base64":
+ import base64
+ import numpy as np
+ return base64.b64encode(np.asarray(vector, dtype = np.float32).tobytes()).decode("ascii")
+ return [float(x) for x in vector]
+
+
+def _names_studio_embedder(requested: str) -> Optional[str]:
+ from core.rag import config as rag_config
+ from core.rag import embeddings as rag_embeddings
+ from utils.paths import is_local_path
+
+ model = rag_config.effective_embedding_model()
+ repo = rag_config.effective_gguf_repo_for_embedding_model(model)
+ label = _public_embedding_name(model)
+ wanted = requested.strip()
+ if rag_config.embedding_identity_model(wanted) is not None:
+ current = rag_embeddings.embedding_identity(model)
+ if wanted in (current, _public_embedding_identity(current, model, label)):
+ return model
+ return None
+ for name in (model, repo, label, _public_embedding_name(repo)):
+ if not name:
+ continue
+ if is_local_path(name) or is_local_path(wanted):
+ if wanted == name:
+ return model
+ elif wanted.casefold() == name.casefold():
+ return model
+ return None
+
+
+def _resident_absent(llama_backend) -> bool:
+ return not getattr(llama_backend, "is_loaded", False)
+
+
+def _stashed_gguf_embeds() -> bool:
+ from core.inference.llama_keepwarm import get_last_unloaded_model
+ from core.inference.local_model_resolver import resolve_local_gguf
+
+ last = get_last_unloaded_model()
+ if not last:
+ return False
+ target_id, variant = last[0], last[1]
+ try:
+ resolved = resolve_local_gguf(
+ f"{target_id}:{variant}" if variant else target_id, allow_scan = False
+ )
+ path = resolved[0] if resolved else None
+ if not path:
+ return False
+ probe = _probe_backend()
+ probe._read_gguf_metadata(path)
+ return bool(probe.is_embedding_gguf)
+ except Exception: # noqa: BLE001 - an unreadable stash is not an embedding model
+ return False
+
+
+async def _default_embeddings_request_body(request: Request) -> Optional[dict]:
+ try:
+ body = await request.json()
+ except (json.JSONDecodeError, ValueError):
+ return None
+ if not isinstance(body, dict):
+ return None
+ requested = body.get("model")
+ return body if requested is None or requested == "" else None
+
+
+async def _embeddings_client_gone(request: Request) -> bool:
+ is_disconnected = getattr(request, "is_disconnected", None)
+ if is_disconnected is None:
+ return False
+ try:
+ return bool(await is_disconnected())
+ except Exception: # noqa: BLE001 - a broken transport reads as still connected
+ return False
+
+
+async def _resident_answers_embeddings(llama_backend, requested: str) -> bool:
+ if not _resident_serves_embeddings(llama_backend):
+ return False
+ return await asyncio.to_thread(_loaded_satisfies, requested)
+
+
+def _reference_is_decisive(requested: str) -> bool:
+ """Whether *requested* is evidence the caller meant a model held HERE.
+
+ The same positive-evidence rule `_reject_unservable_model` applies: an explicit
+ GGUF quant label, which no foreign id carries, or a repo actually on disk. A bare
+ vendor id like `text-embedding-3-small` is neither, and must keep falling through
+ rather than 404 -- that is what kept LiteLLM and OpenRouter style names working.
+ """
+ from core.inference.openai_auto_download import (
+ looks_like_gguf_hub_repo_id,
+ looks_like_quant,
+ split_model_ref,
+ )
+
+ from core.rag.config import embedding_identity_model
+
+ base, variant = split_model_ref(requested)
+ # Shape alone, no index needed: an explicit quant label and a GGUF hub repo id are
+ # references no vendor alias carries.
+ if looks_like_quant(variant) or looks_like_gguf_hub_repo_id(base):
+ return True
+ # A tag names one exact vector space, and this runs only once _names_studio_embedder has
+ # declined it -- so it is a space this server no longer holds. Answering from the current
+ # one files two spaces under the identity the client asked us to distinguish.
+ if embedding_identity_model(requested) is not None:
+ return True
+ try:
+ from core.inference.local_model_resolver import (
+ index_is_built,
+ resolve_local_gguf,
+ warm_index_soon,
+ )
+ from utils.paths import is_local_path
+
+ if resolve_local_gguf(requested, allow_scan = False) is not None:
+ return True
+ if not index_is_built():
+ # A cold index is missing evidence, not absence: warm it, and meanwhile do not
+ # wave a name shaped like a local model through to a different embedding space.
+ warm_index_soon()
+ return is_local_path(requested) or "/" in requested
+ return False
+ except Exception: # noqa: BLE001 - a broken index cannot clear the name either
+ return True
+
+
+async def _studio_embedder_request_body(request: Request) -> Optional[tuple[dict, str]]:
+ try:
+ body = await request.json()
+ except (json.JSONDecodeError, ValueError):
+ return None
+ if not isinstance(body, dict):
+ return None
+ requested = body.get("model")
+ if not isinstance(requested, str) or not requested.strip():
+ return None
+ model = await asyncio.to_thread(_names_studio_embedder, requested)
+ return (body, model) if model else None
+
+
+async def _studio_embeddings(
+ request: Request,
+ body: dict,
+ current_subject: str,
+ model_name: Optional[str] = None,
+) -> Response:
+ from core.inference.llama_keepwarm import (
+ untrack_admitted_inference,
+ untrack_current_request,
+ )
+ from core.rag import config as rag_config
+ from core.rag import embeddings as rag_embeddings
+
+ # /embeddings is an _INFERENCE_SUFFIXES path that is not slot-excluded, so untrack before the
+ # encode: a 2xx would otherwise claim the llama slot, block a preview swap for the whole
+ # encode, and stamp the idle-unload TTL for a model that did no work.
+ _scope = getattr(request, "scope", None)
+ untrack_current_request(_scope)
+ # The auto-switch hook admitted this request before the route knew it lands here, and the
+ # preview busy guard reads the admitted tally, not _inflight.
+ untrack_admitted_inference(_scope)
+
+ texts = _embeddings_texts(body)
+ encoding_format = body.get("encoding_format")
+ if encoding_format is None:
+ encoding_format = "float"
+ dimensions = body.get("dimensions")
+ model_name = model_name or rag_config.effective_embedding_model()
+ label = _public_embedding_name(model_name)
+
+ def _embed():
+ # Every helper takes the model captured above: left to default they re-read the live
+ # setting, mixing one model's limit and dimension with another's vectors.
+ limit = rag_embeddings.max_tokens(model_name)
+ count = rag_embeddings.token_counter(model_name)
+ token_counts = [count(text) for text in texts]
+ if limit and max(token_counts) > limit:
+ raise HTTPException(
+ status_code = 400,
+ detail = f"'input' exceeds the {limit}-token limit of {label}.",
+ )
+ if dimensions is not None and dimensions != rag_embeddings.dim(model_name):
+ raise HTTPException(
+ status_code = 400, detail = f"'dimensions' is not supported by {label}."
+ )
+ # encode_with_identity: an ST failure swaps the process to llama-server mid-encode, and
+ # reporting the configured name anyway files two spaces under one label.
+ vectors, identity = rag_embeddings.encode_with_identity(
+ texts, model_name = model_name, normalize = True
+ )
+ return vectors, identity, sum(token_counts), limit
+
+ monitor_id = None
+ if not getattr(request.state, "skip_api_monitor", False):
+ monitor_id = api_monitor.start(
+ endpoint = request.url.path,
+ via_api_key = _request_used_api_key(request),
+ method = request.method,
+ model = label,
+ prompt = _flatten_monitor_prompt(body.get("input", "")),
+ subject = current_subject,
+ )
+ # The worker's lifetime, not the awaiting task's: cancelling does not stop the thread behind
+ # to_thread, so releasing there admits the next request mid-encode and the cap stops meaning
+ # anything. Same as _drain_pending_worker on the blocking-generation path.
+ semaphore = _studio_embed_semaphore()
+ acquire = asyncio.ensure_future(semaphore.acquire())
+ try:
+ while not acquire.done():
+ await asyncio.wait({acquire}, timeout = 0.25)
+ if not acquire.done() and await _embeddings_client_gone(request):
+ raise asyncio.CancelledError()
+ except asyncio.CancelledError:
+ if not acquire.done():
+ acquire.cancel()
+ elif not acquire.cancelled() and acquire.exception() is None:
+ semaphore.release()
+ api_monitor.finish(monitor_id, "cancelled")
+ raise
+ try:
+ gone = await _embeddings_client_gone(request)
+ except asyncio.CancelledError:
+ semaphore.release()
+ api_monitor.finish(monitor_id, "cancelled")
+ raise
+ if gone:
+ semaphore.release()
+ api_monitor.finish(monitor_id, "cancelled")
+ raise asyncio.CancelledError()
+ worker = asyncio.ensure_future(asyncio.to_thread(_embed))
+
+ def _release_embed_permit(finished) -> None:
+ if not finished.cancelled():
+ finished.exception() # retrieved so a cancelled request logs no "never retrieved"
+ semaphore.release()
+
+ worker.add_done_callback(_release_embed_permit)
+ try:
+ vectors, identity, prompt_tokens, limit = await asyncio.shield(worker)
+ except asyncio.CancelledError:
+ api_monitor.finish(monitor_id, "cancelled")
+ raise
+ except HTTPException as exc:
+ api_monitor.fail(monitor_id, str(exc.detail))
+ raise
+ # The two conditions the caller can act on. _friendly_error, written for llama-server
+ # transport faults, flattens both to "An internal error occurred", which behind an agent's
+ # memory search is the whole diagnosis the user gets. 409: not wrong, not ready yet.
+ except (
+ rag_embeddings.EmbeddingModelDownloadRequiredError,
+ rag_embeddings.UnsafeEmbeddingModelError,
+ ) as exc:
+ # The configured model may be a local path, which every other field here hashes.
+ detail = str(exc).replace(repr(model_name), repr(label))
+ api_monitor.fail(monitor_id, detail)
+ raise HTTPException(status_code = 409, detail = detail) from exc
+ except Exception as exc:
+ api_monitor.fail(monitor_id, _friendly_error(exc))
+ raise HTTPException(
+ status_code = 502, detail = f"Embedding model failed: {_friendly_error(exc)}"
+ ) from exc
+ payload = {
+ "object": "list",
+ "data": [
+ {
+ "object": "embedding",
+ "index": index,
+ "embedding": _embedding_payload(vector, encoding_format),
+ }
+ for index, vector in enumerate(vectors)
+ ],
+ "model": _public_embedding_identity(identity, model_name, label),
+ "usage": {"prompt_tokens": prompt_tokens, "total_tokens": prompt_tokens},
+ }
+ # limit is per text, prompt_tokens is the batch sum, so the monitor's gauge only means
+ # anything for a single input; a batch would sit pinned at 100%.
+ _monitor_openai_chunk(monitor_id, payload, limit if len(texts) == 1 else None)
+ api_monitor.finish(monitor_id)
+ return Response(content = json.dumps(payload), media_type = "application/json")
+
+
def _embeddings_input_present(body: dict) -> bool:
"""Whether an embeddings body carries a usable ``input`` (non-empty)."""
inp = body.get("input")
@@ -25742,14 +26151,8 @@ def _embeddings_input_present(body: dict) -> bool:
@router.post("/embeddings")
async def openai_embeddings(request: Request, current_subject: str = Depends(get_current_subject)):
- """
- OpenAI-compatible embeddings endpoint.
-
- Proxies to the running llama-server's ``/v1/embeddings``. Only available
- when a GGUF model is loaded.
- Note: the loaded model must support pooling, else llama-server returns an
- error (expected).
- """
+ """OpenAI-compatible embeddings: the resident embedding GGUF when one is loaded,
+ else Studio's configured embedding model."""
llama_backend = get_llama_cpp_backend()
# Reject a request with no input before any automatic load so an invalid request never
# swaps or reloads the resident model (as chat/responses/messages already validate before
@@ -25770,6 +26173,7 @@ async def openai_embeddings(request: Request, current_subject: str = Depends(get
raise HTTPException(status_code = 400, detail = "'input' must be a string or array.")
if not _embeddings_input_present(_pre):
raise HTTPException(status_code = 400, detail = "'input' is required for embeddings.")
+ _embeddings_items(_pre, tokens_ok = True)
elif _pre is not _UNPARSEABLE_BODY:
# A valid JSON body that is not an object (e.g. [] or null) is rejected below as
# "Request body must be a JSON object"; reject it here, before the switch, so the
@@ -25781,15 +26185,39 @@ async def openai_embeddings(request: Request, current_subject: str = Depends(get
# no reliable pre-load probe -- is_embedding_model keys on a sentence-transformers
# modules.json a bare .gguf never has -- so embeddings auto-switch is best-effort:
# a non-embedding target switches, then llama-server returns a no-pooling error.
+ studio_request = await _studio_embedder_request_body(request)
+ if studio_request is not None and not await _resident_answers_embeddings(
+ llama_backend, studio_request[0]["model"]
+ ):
+ return await _studio_embeddings(
+ request, studio_request[0], current_subject, model_name = studio_request[1]
+ )
+ if studio_request is None and _resident_absent(llama_backend):
+ default_body = await _default_embeddings_request_body(request)
+ if default_body is not None and not await asyncio.to_thread(_stashed_gguf_embeds):
+ return await _studio_embeddings(request, default_body, current_subject)
body = await _auto_switch_from_request_body(request, current_subject, gguf_only = True)
if not llama_backend.is_loaded:
- _status, _detail = await _no_model_loaded_error(
- "No GGUF model loaded. Load a GGUF model first.",
- _raw_body_model(body),
- request,
- status = 503,
- )
- raise HTTPException(status_code = _status, detail = _detail)
+ # With the slot empty _reject_unservable_model defers to _no_model_loaded_error, so
+ # without this the fallback would answer a decisive repo:QUANT this server does not
+ # hold from the Settings embedder: another space under the caller's name (#7454).
+ # `{"model": 123}` gets this far: both pre-switch readers decline a non-string selector,
+ # and the alias lookup below strips whatever it is handed.
+ _named = _raw_body_model(body)
+ if not isinstance(_named, str):
+ _named = None
+ if _named and not await asyncio.to_thread(_names_studio_embedder, _named):
+ _decisive = await asyncio.to_thread(_reference_is_decisive, _named)
+ else:
+ _decisive = False
+ if _decisive or not isinstance(body, dict):
+ _status, _detail = await _no_model_loaded_error(
+ "No GGUF model loaded. Load a GGUF model first.",
+ _named,
+ request,
+ status = 503,
+ )
+ raise HTTPException(status_code = _status, detail = _detail)
if not isinstance(body, dict):
# Re-read to re-raise a malformed-body error (post-503, pre-feature behavior);
# a valid non-dict body such as a list is a clean 400 rather than a 500.
@@ -25797,6 +26225,9 @@ async def openai_embeddings(request: Request, current_subject: str = Depends(get
if not isinstance(body, dict):
raise HTTPException(status_code = 400, detail = "Request body must be a JSON object")
+ if not _resident_serves_embeddings(llama_backend):
+ return await _studio_embeddings(request, body, current_subject)
+
# GGUF is loaded and the body is valid. The middleware claims the slot on a successful
# 2xx, so no claim here: llama-server can still return a non-2xx for a valid body (e.g. a
# no-pooling error on /v1/embeddings against a non-embedding GGUF), so claiming before the
diff --git a/studio/backend/tests/test_offline_embedding_minimal.py b/studio/backend/tests/test_offline_embedding_minimal.py
index 4162d21440a..d79a137988b 100644
--- a/studio/backend/tests/test_offline_embedding_minimal.py
+++ b/studio/backend/tests/test_offline_embedding_minimal.py
@@ -1022,7 +1022,9 @@ def test_get_offline_uncached_uses_local_files_only(tmp_path, monkeypatch):
monkeypatch.setattr(embeddings, "_install_torchao_stub_once", lambda: None)
monkeypatch.setattr(embeddings, "_device", lambda: "cpu")
# No cache -> repo-id load forced cache-only (fails fast offline, not a hang).
- monkeypatch.setattr(embeddings, "_guard_model_security", lambda name, local_only = False: None)
+ monkeypatch.setattr(
+ embeddings, "_guard_model_security", lambda name, local_only = False, display = None: None
+ )
captured = {}
_install_fake_sentence_transformers(monkeypatch, captured)
embeddings._get("org/uncached-xyz")
@@ -1040,7 +1042,9 @@ def test_get_online_omits_local_files_only(monkeypatch):
monkeypatch.setattr(embeddings, "_install_torchao_stub_once", lambda: None)
monkeypatch.setattr(embeddings, "_device", lambda: "cpu")
# Isolate the loader wiring from the online guard's network calls.
- monkeypatch.setattr(embeddings, "_guard_model_security", lambda name, local_only = False: None)
+ monkeypatch.setattr(
+ embeddings, "_guard_model_security", lambda name, local_only = False, display = None: None
+ )
captured = {}
_install_fake_sentence_transformers(monkeypatch, captured)
embeddings._get("org/online")
@@ -1092,7 +1096,9 @@ def test_get_online_loads_the_snapshot_settings_called_cached(hf_cache, monkeypa
monkeypatch.setattr(embeddings, "_name", None, raising = False)
monkeypatch.setattr(embeddings, "_install_torchao_stub_once", lambda: None)
monkeypatch.setattr(embeddings, "_device", lambda: "cpu")
- monkeypatch.setattr(embeddings, "_guard_model_security", lambda name, local_only = False: None)
+ monkeypatch.setattr(
+ embeddings, "_guard_model_security", lambda name, local_only = False, display = None: None
+ )
captured = {}
_install_fake_sentence_transformers(monkeypatch, captured)
@@ -1112,7 +1118,9 @@ def test_an_uncached_model_online_still_loads_by_repo_id(monkeypatch):
monkeypatch.setattr(embeddings, "_name", None, raising = False)
monkeypatch.setattr(embeddings, "_install_torchao_stub_once", lambda: None)
monkeypatch.setattr(embeddings, "_device", lambda: "cpu")
- monkeypatch.setattr(embeddings, "_guard_model_security", lambda name, local_only = False: None)
+ monkeypatch.setattr(
+ embeddings, "_guard_model_security", lambda name, local_only = False, display = None: None
+ )
captured = {}
_install_fake_sentence_transformers(monkeypatch, captured)
@@ -1338,7 +1346,9 @@ def test_a_pending_transfer_does_not_make_the_security_scan_offline(monkeypatch,
monkeypatch.setattr(
embeddings,
"_guard_model_security",
- lambda target, local_only = False: seen.update(target = target, local_only = local_only),
+ lambda target, local_only = False, display = None: seen.update(
+ target = target, local_only = local_only
+ ),
)
# Pending, but the Hub is reachable.
monkeypatch.setattr(ems, "get_stored_download_pending", lambda m: True)
diff --git a/studio/backend/tests/test_openai_embeddings_studio_fallback.py b/studio/backend/tests/test_openai_embeddings_studio_fallback.py
new file mode 100644
index 00000000000..a074c1c0d9e
--- /dev/null
+++ b/studio/backend/tests/test_openai_embeddings_studio_fallback.py
@@ -0,0 +1,1405 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import asyncio
+import base64
+import os
+import sys
+import threading
+from types import SimpleNamespace
+
+import numpy as np
+import pytest
+
+_backend = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+if _backend not in sys.path:
+ sys.path.insert(0, _backend)
+
+inference_route = pytest.importorskip("routes.inference", reason = "inference stack not installed")
+rag_embeddings = pytest.importorskip("core.rag.embeddings", reason = "rag stack not installed")
+rag_config = pytest.importorskip("core.rag.config", reason = "rag stack not installed")
+
+MODEL = "unsloth/bge-small-en-v1.5"
+IDENTITY = f"sentence-transformers:{MODEL}"
+
+
+class _Request:
+ def __init__(self, body):
+ self._body = body
+ self.method = "POST"
+ self.url = SimpleNamespace(path = "/v1/embeddings")
+ self.state = SimpleNamespace(skip_api_monitor = True)
+ self.scope = {"type": "http", "path": "/v1/embeddings"}
+
+ async def json(self):
+ return self._body
+
+ async def is_disconnected(self):
+ return False
+
+
+def _vectors(texts, **_):
+ return np.array([[1.0, 0.0]] * len(texts), dtype = np.float32)
+
+
+def studio_embedder(monkeypatch):
+ async def passthrough(request, current_subject, **_kwargs):
+ return await request.json()
+
+ def no_proxy():
+ raise AssertionError("the llama-server proxy must not run")
+
+ monkeypatch.setattr(inference_route, "_should_validate_before_switch", lambda: False)
+ monkeypatch.setattr(inference_route, "_auto_switch_from_request_body", passthrough)
+ monkeypatch.setattr(inference_route, "_cancelable_nonstreaming_client", no_proxy)
+ monkeypatch.setattr(rag_config, "effective_embedding_model", lambda: MODEL)
+ monkeypatch.setattr(rag_embeddings, "encode", _vectors)
+
+ def _encode_with_identity(
+ texts,
+ *,
+ model_name = None,
+ normalize = True,
+ ):
+ # Delegates to the test's encode, so a blowing-up or blocking one still drives the route.
+ return rag_embeddings.encode(texts, model_name = model_name, normalize = normalize), IDENTITY
+
+ monkeypatch.setattr(rag_embeddings, "encode_with_identity", _encode_with_identity)
+ monkeypatch.setattr(rag_embeddings, "token_counter", lambda model_name = None: len)
+ monkeypatch.setattr(rag_embeddings, "max_tokens", lambda model_name = None: None)
+ monkeypatch.setattr(rag_embeddings, "dim", lambda model_name = None: 2)
+ return monkeypatch
+
+
+def _call(body):
+ response = asyncio.run(inference_route.openai_embeddings(_Request(body), "tester"))
+ assert response.status_code == 200
+ import json
+
+ return json.loads(response.body)
+
+
+def _http_error(body):
+ from fastapi import HTTPException
+ with pytest.raises(HTTPException) as exc:
+ asyncio.run(inference_route.openai_embeddings(_Request(body), "tester"))
+ return exc.value
+
+
+def test_nothing_loaded_serves_from_the_studio_embedder(studio_embedder):
+ studio_embedder.setattr(
+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
+ )
+ payload = _call({"input": ["alpha", "beta"], "model": "text-embedding-3-small"})
+ assert payload["object"] == "list"
+ assert payload["model"] == IDENTITY
+ assert [row["index"] for row in payload["data"]] == [0, 1]
+ assert payload["data"][0]["embedding"] == [1.0, 0.0]
+ assert payload["usage"] == {"prompt_tokens": 9, "total_tokens": 9}
+
+
+def test_chat_model_loaded_serves_from_the_studio_embedder(studio_embedder):
+ started = []
+ studio_embedder.setattr(
+ inference_route,
+ "get_llama_cpp_backend",
+ lambda: SimpleNamespace(is_loaded = True, is_embedding_gguf = False),
+ )
+ studio_embedder.setattr(
+ inference_route, "_direct_llama_request_started", lambda: started.append(1)
+ )
+ payload = _call({"input": "alpha"})
+ assert payload["model"] == IDENTITY
+ assert started == []
+
+
+def test_resident_embedding_gguf_still_uses_the_proxy(studio_embedder):
+ import httpx
+
+ class _Client:
+ async def post(self, *_args, **_kwargs):
+ return httpx.Response(200, json = {"data": [{"embedding": [0.5]}]})
+
+ async def aclose(self):
+ return None
+
+ def boom(*_args, **_kwargs):
+ raise AssertionError("the studio embedder must not run")
+
+ studio_embedder.setattr(rag_embeddings, "encode", boom)
+ studio_embedder.setattr(inference_route, "_cancelable_nonstreaming_client", _Client)
+ studio_embedder.setattr(
+ inference_route,
+ "get_llama_cpp_backend",
+ lambda: SimpleNamespace(
+ is_loaded = True,
+ is_embedding_gguf = True,
+ base_url = "http://llama.test",
+ context_length = 512,
+ model_identifier = "org/E-GGUF",
+ ),
+ )
+ payload = _call({"input": "alpha", "model": "org/E-GGUF"})
+ assert payload == {"data": [{"embedding": [0.5]}]}
+
+
+def test_base64_encoding_format(studio_embedder):
+ studio_embedder.setattr(
+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
+ )
+ payload = _call({"input": "alpha", "encoding_format": "base64"})
+ raw = base64.b64decode(payload["data"][0]["embedding"])
+ assert np.frombuffer(raw, dtype = np.float32).tolist() == [1.0, 0.0]
+
+
+ "body",
+ [
+ {"input": ""},
+ {"input": []},
+ {"input": [[1, 2, 3]]},
+ {"input": ["alpha", ""]},
+ {"input": "alpha", "encoding_format": "int8"},
+ {"input": "alpha", "encoding_format": ""},
+ {"input": "alpha", "encoding_format": False},
+ {"input": "alpha", "encoding_format": 0},
+ {"input": "alpha", "dimensions": 999},
+ ],
+)
+def test_invalid_requests_are_400(studio_embedder, body):
+ studio_embedder.setattr(
+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
+ )
+ assert _http_error(body).status_code == 400
+
+
+def test_over_length_input_is_rejected_not_truncated(studio_embedder):
+ studio_embedder.setattr(
+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
+ )
+ studio_embedder.setattr(rag_embeddings, "max_tokens", lambda model_name = None: 3)
+ error = _http_error({"input": ["ab", "abcd"]})
+ assert error.status_code == 400
+ assert "3-token limit" in error.detail
+
+
+def test_embedder_failure_is_502(studio_embedder):
+ def boom(*_args, **_kwargs):
+ raise RuntimeError("llama-server embedder POST /v1/embeddings -> 500")
+
+ studio_embedder.setattr(
+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
+ )
+ studio_embedder.setattr(rag_embeddings, "encode", boom)
+ assert _http_error({"input": "alpha"}).status_code == 502
+
+
+def test_identity_redaction_leaves_the_backend_tag_alone(tmp_path, monkeypatch):
+ """A local model whose name occurs inside the backend tag.
+
+ `transformers` is a real directory people have lying around, and is_local_path accepts a
+ bare relative name that exists. A substring replace then rewrites the
+ `sentence-transformers` tag too, and the identity we advertise is one no backend claims
+ and _names_studio_embedder cannot match on the next request.
+ """
+ from core.rag import config as rag_config
+
+ (tmp_path / "transformers").mkdir()
+ monkeypatch.chdir(tmp_path)
+ monkeypatch.setattr(
+ inference_route,
+ "_public_embedding_name",
+ lambda name: "transformers-deadbeef" if name == "transformers" else name,
+ )
+ monkeypatch.setattr(
+ rag_config, "effective_gguf_repo_for_embedding_model", lambda model: "transformers-GGUF"
+ )
+ public = inference_route._public_embedding_identity(
+ "sentence-transformers:transformers", "transformers", "transformers-deadbeef"
+ )
+ assert public == "sentence-transformers:transformers-deadbeef"
+ assert rag_config.embedding_identity_model(public) == "transformers-deadbeef"
+
+
+def test_identity_redaction_covers_the_gguf_repo_segment(monkeypatch):
+ from core.rag import config as rag_config
+
+ monkeypatch.setattr(
+ rag_config, "effective_gguf_repo_for_embedding_model", lambda model: "/models/bge-GGUF"
+ )
+ monkeypatch.setattr(
+ inference_route,
+ "_public_embedding_name",
+ lambda name: {"/models/bge": "bge-1111", "/models/bge-GGUF": "bge-GGUF-2222"}.get(
+ name, name
+ ),
+ )
+ public = inference_route._public_embedding_identity(
+ "llama-server:%2Fmodels%2Fbge:%2Fmodels%2Fbge-GGUF".replace("%2F", "/"),
+ "/models/bge",
+ "bge-1111",
+ )
+ assert public == "llama-server:bge-1111:bge-GGUF-2222"
+
+
+def test_identity_redaction_covers_a_local_repo_under_a_hub_model(monkeypatch):
+ """A hub model id whose GGUF companion is a local path.
+
+ RAG_EMBED_GGUF_REPO takes any path, so the model segment can need no redaction while the
+ repo segment beside it is an absolute path on the server.
+ """
+ from core.rag import config as rag_config
+
+ monkeypatch.setattr(
+ rag_config,
+ "effective_gguf_repo_for_embedding_model",
+ lambda model: "/srv/private/embed.gguf",
+ )
+ monkeypatch.setattr(
+ inference_route,
+ "_public_embedding_name",
+ lambda name: "embed.gguf-3333" if name == "/srv/private/embed.gguf" else name,
+ )
+ public = inference_route._public_embedding_identity(
+ f"llama-server:{MODEL}:/srv/private/embed.gguf", MODEL, MODEL
+ )
+ assert public == f"llama-server:{MODEL}:embed.gguf-3333"
+ assert "/srv/private" not in public
+
+
[email protected]("named", [123, 1.5, True, ["a"], {"a": 1}])
+def test_a_non_string_model_is_not_a_server_error(studio_embedder, named):
+ """Both pre-switch readers decline a non-string selector, so the empty-slot branch sees it."""
+ studio_embedder.setattr(
+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
+ )
+ payload = _call({"input": "alpha", "model": named})
+ assert payload["model"] == IDENTITY
+
+
+ "exc,fragment",
+ [
+ (
+ rag_embeddings.EmbeddingModelDownloadRequiredError(
+ "Embedding model 'unsloth/bge-small-en-v1.5' is not downloaded yet. "
+ "Finish its Settings download before indexing documents."
+ ),
+ "not downloaded yet",
+ ),
+ (
+ rag_embeddings.UnsafeEmbeddingModelError(
+ "Embedding model 'evil/model' requires remote code execution."
+ ),
+ "remote code",
+ ),
+ ],
+)
+def test_actionable_embedder_errors_keep_their_message(studio_embedder, exc, fragment):
+ """The caller can fix these two, but only if it is told what they are.
+
+ Picking a model in Settings and querying before its download finishes is the common
+ one, and it reaches this route through `unsloth start openclaw`: the generated
+ openclaw.json points memory search here, so this detail string is the entire
+ diagnosis the user sees. Routed through _friendly_error -- written for llama-server
+ transport faults -- it became a 502 "An internal error occurred", which says nothing
+ and suggests the wrong owner.
+ """
+
+ def boom(*_args, **_kwargs):
+ raise exc
+
+ studio_embedder.setattr(
+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
+ )
+ studio_embedder.setattr(rag_embeddings, "encode", boom)
+ error = _http_error({"input": "alpha"})
+ assert error.status_code == 409
+ assert fragment in error.detail
+ assert "An internal error occurred" not in error.detail
+
+
+def test_pending_gguf_download_is_classified_like_the_st_one(tmp_path, monkeypatch):
+ """The GGUF backend hits the same condition and used to raise a bare RuntimeError.
+
+ Both backends refuse the same way when the picker's download has not finished, so both
+ have to reach the 409; untyped, this one fell into the catch-all and came back as a 502
+ saying only that an internal error occurred.
+ """
+ from core.rag import embed_llama_server
+
+ backend = embed_llama_server.LlamaServerBackend()
+ monkeypatch.setattr(
+ embed_llama_server.config,
+ "effective_gguf_repo_for_embedding_model",
+ lambda model: "unsloth/bge-small-en-v1.5-GGUF",
+ )
+ monkeypatch.setattr(backend, "_resolve_local_gguf", lambda model: None)
+ monkeypatch.setattr(backend, "_planned_family_path", lambda model, repo: None)
+ monkeypatch.setattr(backend, "_resolve_cached_gguf", lambda *a, **k: None)
+ import utils.embedding_model_settings as ems
+
+ monkeypatch.setattr(ems, "get_stored_download_pending", lambda model: True)
+
+ with pytest.raises(rag_embeddings.EmbeddingModelDownloadRequiredError) as excinfo:
+ backend._resolve_model_path("unsloth/bge-small-en-v1.5")
+ assert "not downloaded yet" in str(excinfo.value)
+
+
+def test_actionable_errors_do_not_leak_a_local_path(studio_embedder, monkeypatch):
+ """Passing the message through must not undo the redaction the rest of the route does.
+
+ _get resolves a cached model to its absolute snapshot directory and hands that to
+ _guard_model_security, so the raised text can carry the server's HF cache layout, and a
+ configured model can be a local path in its own right. Both go out over /v1/embeddings,
+ where the identity and the limit errors already report the hashed label instead.
+ """
+ studio_embedder.setattr(
+ rag_config, "effective_embedding_model", lambda: "/srv/models/bge-small"
+ )
+
+ def boom(*_args, **_kwargs):
+ raise rag_embeddings.UnsafeEmbeddingModelError(
+ "Embedding model '/srv/models/bge-small' is flagged as unsafe by Hugging Face's "
+ "security scan; refusing to load. Set a different RAG embedding model."
+ )
+
+ studio_embedder.setattr(
+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
+ )
+ studio_embedder.setattr(rag_embeddings, "encode", boom)
+ error = _http_error({"input": "alpha"})
+ assert error.status_code == 409
+ assert "/srv/models" not in error.detail
+ assert "flagged as unsafe" in error.detail
+ assert "bge-small-" in error.detail # the hashed public label, not the path
+
+
+def test_guard_model_security_names_the_configured_model_not_the_snapshot(monkeypatch):
+ """The scan needs the snapshot path; the message must not carry it."""
+ monkeypatch.setattr(rag_embeddings, "_ambient_hf_token", lambda: None)
+ import utils.security as security
+
+ monkeypatch.setattr(
+ security,
+ "evaluate_file_security",
+ lambda *a, **k: SimpleNamespace(blocked = True),
+ )
+ monkeypatch.setattr(security, "security_load_subdirs", lambda *a, **k: ())
+ monkeypatch.setattr(rag_embeddings, "_st_module_subdirs", lambda *a, **k: ())
+ with pytest.raises(rag_embeddings.UnsafeEmbeddingModelError) as excinfo:
+ rag_embeddings._guard_model_security(
+ "/home/me/.cache/huggingface/hub/models--org--bge/snapshots/abc123",
+ display = "org/bge",
+ )
+ assert "org/bge" in str(excinfo.value)
+ assert ".cache/huggingface" not in str(excinfo.value)
+
+
+def test_studio_embedder_requests_are_admission_limited(studio_embedder):
+ lock = threading.Lock()
+ gate = threading.Event()
+ active = {"now": 0, "peak": 0}
+
+ def encode(texts, **_kwargs):
+ with lock:
+ active["now"] += 1
+ active["peak"] = max(active["peak"], active["now"])
+ gate.wait(timeout = 5)
+ with lock:
+ active["now"] -= 1
+ return _vectors(texts)
+
+ studio_embedder.setattr(
+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
+ )
+ studio_embedder.setattr(rag_embeddings, "encode", encode)
+
+ async def run():
+ tasks = [
+ asyncio.create_task(inference_route.openai_embeddings(_Request({"input": "x"}), "t"))
+ for _ in range(inference_route._STUDIO_EMBED_CONCURRENCY + 2)
+ ]
+ for _ in range(500):
+ await asyncio.sleep(0.01)
+ if active["now"] == inference_route._STUDIO_EMBED_CONCURRENCY:
+ break
+ assert active["now"] == inference_route._STUDIO_EMBED_CONCURRENCY
+ gate.set()
+ responses = await asyncio.gather(*tasks)
+ assert all(response.status_code == 200 for response in responses)
+
+ asyncio.run(run())
+ assert active["peak"] == inference_route._STUDIO_EMBED_CONCURRENCY
+
+
+def test_studio_fallback_untracks_the_request_from_the_llama_slot(studio_embedder):
+ # An _INFERENCE_SUFFIXES path not in _NON_LLM_SLOT_SUFFIXES: a 2xx reaching _finish claims the
+ # llama slot the studio embedder never touched, so it untracks first, as the chat branch does.
+ from core.inference import llama_keepwarm as kw
+
+ studio_embedder.setattr(
+ inference_route,
+ "get_llama_cpp_backend",
+ lambda: SimpleNamespace(is_loaded = True, is_embedding_gguf = False),
+ )
+ request = _Request({"input": "alpha"})
+ before = kw._inflight
+ kw._inflight = before + 1
+ try:
+ response = asyncio.run(inference_route.openai_embeddings(request, "tester"))
+ assert response.status_code == 200
+ # Set => _finish() skips both _claim_non_preview_slot() and the activity stamp.
+ assert request.scope.get(kw._UNTRACKED_SCOPE_KEY) is True
+ assert kw._inflight == before
+ finally:
+ kw._inflight = before
+
+
+def test_resident_embedding_gguf_still_claims_the_slot(studio_embedder):
+ # The proxy path DOES run against the resident GGUF, so it must stay tracked.
+ import httpx
+
+ class _Client:
+ async def post(self, *_args, **_kwargs):
+ return httpx.Response(200, json = {"data": [{"embedding": [0.5]}]})
+
+ async def aclose(self):
+ return None
+
+ from core.inference import llama_keepwarm as kw
+
+ studio_embedder.setattr(inference_route, "_cancelable_nonstreaming_client", _Client)
+ studio_embedder.setattr(
+ inference_route,
+ "get_llama_cpp_backend",
+ lambda: SimpleNamespace(
+ is_loaded = True,
+ is_embedding_gguf = True,
+ base_url = "http://llama.test",
+ context_length = 512,
+ model_identifier = "org/E-GGUF",
+ ),
+ )
+ request = _Request({"input": "alpha", "model": "org/E-GGUF"})
+ asyncio.run(inference_route.openai_embeddings(request, "tester"))
+ assert request.scope.get(kw._UNTRACKED_SCOPE_KEY) is None
+
+
+def test_context_gauge_is_not_pinned_by_a_batch(studio_embedder):
+ # api_monitor divides the batch's summed prompt_tokens by that 3rd arg, so passing the
+ # per-text limit reports 100% context use for a batch that used a fraction per text.
+ seen = []
+ studio_embedder.setattr(
+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
+ )
+ studio_embedder.setattr(rag_embeddings, "max_tokens", lambda model_name = None: 8)
+ studio_embedder.setattr(
+ inference_route,
+ "_monitor_openai_chunk",
+ lambda monitor_id, payload, context_length = None, **_: seen.append(context_length),
+ )
+ request = _Request({"input": ["alpha", "beta", "gamma"]})
+ request.state.skip_api_monitor = False
+ asyncio.run(inference_route.openai_embeddings(request, "tester"))
+ assert seen == [None]
+
+ seen.clear()
+ request = _Request({"input": "alpha"})
+ request.state.skip_api_monitor = False
+ asyncio.run(inference_route.openai_embeddings(request, "tester"))
+ assert seen == [8]
+
+
+def test_cancelled_requests_do_not_leak_admission_permits(studio_embedder):
+ # to_thread cannot cancel the worker, so releasing on the awaiting task's cancellation lets
+ # the next batch in while the old threads still embed and the cap stops holding.
+ lock = threading.Lock()
+ gate = threading.Event()
+ active = {"now": 0, "peak": 0}
+
+ def encode(texts, **_kwargs):
+ with lock:
+ active["now"] += 1
+ active["peak"] = max(active["peak"], active["now"])
+ gate.wait(timeout = 5)
+ with lock:
+ active["now"] -= 1
+ return _vectors(texts)
+
+ studio_embedder.setattr(
+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
+ )
+ studio_embedder.setattr(rag_embeddings, "encode", encode)
+ cap = inference_route._STUDIO_EMBED_CONCURRENCY
+
+ async def run():
+ first = [
+ asyncio.create_task(inference_route.openai_embeddings(_Request({"input": "x"}), "t"))
+ for _ in range(cap)
+ ]
+ for _ in range(500):
+ await asyncio.sleep(0.01)
+ if active["now"] == cap:
+ break
+ assert active["now"] == cap
+
+ # Every in-flight request goes away, but its thread is still inside encode().
+ for task in first:
+ task.cancel()
+ await asyncio.gather(*first, return_exceptions = True)
+
+ second = [
+ asyncio.create_task(inference_route.openai_embeddings(_Request({"input": "y"}), "t"))
+ for _ in range(cap)
+ ]
+ await asyncio.sleep(0.3)
+ # The permits are still held by the running threads, so nothing new started.
+ assert active["peak"] == cap, f"admission cap exceeded: peak={active['peak']}"
+ gate.set()
+ await asyncio.gather(*second, return_exceptions = True)
+
+ asyncio.run(run())
+ assert active["peak"] == cap
+
+
+def test_reported_model_follows_the_backend_that_made_the_vectors(studio_embedder):
+ # An ST failure swaps the process to llama-server, a different space: the response has to
+ # name the space the vectors are in, or a client files two under one label.
+ llama_identity = f"llama-server:{MODEL}:unsloth/bge-small-en-v1.5-GGUF"
+ studio_embedder.setattr(
+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
+ )
+ studio_embedder.setattr(
+ rag_embeddings,
+ "encode_with_identity",
+ lambda texts, **_kwargs: (_vectors(texts), llama_identity),
+ )
+ payload = _call({"input": "alpha"})
+ assert payload["model"] == llama_identity
+
+
+def test_embedding_helpers_are_pinned_to_the_captured_model(studio_embedder):
+ # A Settings change while queued must not mix one model's limit/dim with another's vectors.
+ seen = {"max_tokens": [], "token_counter": [], "dim": [], "encode": []}
+ studio_embedder.setattr(
+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
+ )
+ studio_embedder.setattr(
+ rag_embeddings,
+ "max_tokens",
+ lambda model_name = None: (seen["max_tokens"].append(model_name), None)[1],
+ )
+ studio_embedder.setattr(
+ rag_embeddings,
+ "token_counter",
+ lambda model_name = None: (seen["token_counter"].append(model_name), len)[1],
+ )
+ studio_embedder.setattr(
+ rag_embeddings,
+ "dim",
+ lambda model_name = None: (seen["dim"].append(model_name), 2)[1],
+ )
+ studio_embedder.setattr(
+ rag_embeddings,
+ "encode_with_identity",
+ lambda texts, **kw: (
+ seen["encode"].append(kw.get("model_name")),
+ (_vectors(texts), IDENTITY),
+ )[1],
+ )
+ _call({"input": "alpha", "dimensions": 2})
+ assert seen == {
+ "max_tokens": [MODEL],
+ "token_counter": [MODEL],
+ "dim": [MODEL],
+ "encode": [MODEL],
+ }
+
+
+def test_studio_fallback_releases_the_preview_busy_guard(studio_embedder):
+ # Admission happens before the route decides how to serve, and load_model_for_preview reads
+ # the admitted tally, not _inflight, so a slow encode keeps 503ing preview swaps.
+ from core.inference import llama_keepwarm as kw
+
+ studio_embedder.setattr(
+ inference_route,
+ "get_llama_cpp_backend",
+ lambda: SimpleNamespace(is_loaded = True, is_embedding_gguf = False),
+ )
+ request = _Request({"input": "alpha"})
+ # Relative: the tally is module state shared with the suite, so absolutes need this file alone.
+ before = kw.other_admitted_inference_count()
+ kw.note_admitted_inference(request.scope)
+ assert kw.other_admitted_inference_count() == before + 1
+ assert request.scope.get(kw._ADMITTED_SCOPE_KEY) is True
+ try:
+ asyncio.run(inference_route.openai_embeddings(request, "tester"))
+ assert kw.other_admitted_inference_count() == before
+ # Popped, so the middleware's finally cannot decrement the tally a second time.
+ assert request.scope.get(kw._ADMITTED_SCOPE_KEY) is None
+ finally:
+ kw._admitted_inference = before
+
+
+def test_cancelled_request_closes_its_monitor_row(studio_embedder):
+ # start() runs before admission and running rows dodge retention trimming, so a request
+ # cancelled while queued has to close its own row.
+ closed = []
+ gate = threading.Event()
+ studio_embedder.setattr(
+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
+ )
+ studio_embedder.setattr(inference_route.api_monitor, "start", lambda **_kwargs: "entry-1")
+ studio_embedder.setattr(
+ inference_route.api_monitor,
+ "finish",
+ lambda entry_id, *args, **_kw: closed.append((entry_id, args[0] if args else None)),
+ )
+ studio_embedder.setattr(
+ rag_embeddings,
+ "encode_with_identity",
+ lambda texts, **_kw: (gate.wait(timeout = 5), (_vectors(texts), IDENTITY))[1],
+ )
+
+ async def run():
+ request = _Request({"input": "alpha"})
+ request.state.skip_api_monitor = False
+ task = asyncio.create_task(inference_route.openai_embeddings(request, "tester"))
+ await asyncio.sleep(0.1)
+ task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await task
+ gate.set()
+
+ asyncio.run(run())
+ assert closed == [("entry-1", "cancelled")]
+
+
+def _identity_names(monkeypatch):
+ monkeypatch.setattr(
+ rag_config, "effective_gguf_repo_for_embedding_model", lambda model: f"{model}-GGUF"
+ )
+ monkeypatch.setattr(rag_embeddings, "embedding_identity", lambda model_name = None: IDENTITY)
+
+
[email protected]("requested", [MODEL, f"{MODEL}-GGUF", IDENTITY, MODEL.upper()])
+def test_naming_the_configured_embedder_skips_the_chat_slot_check(studio_embedder, requested):
+ from fastapi import HTTPException
+
+ async def reject(request, current_subject, **_kwargs):
+ raise HTTPException(status_code = 404, detail = "model_not_found")
+
+ _identity_names(studio_embedder)
+ studio_embedder.setattr(inference_route, "_auto_switch_from_request_body", reject)
+ studio_embedder.setattr(
+ inference_route,
+ "get_llama_cpp_backend",
+ lambda: SimpleNamespace(is_loaded = True, is_embedding_gguf = False),
+ )
+ payload = _call({"input": "alpha", "model": requested})
+ assert payload["model"] == IDENTITY
+ assert payload["data"][0]["embedding"] == [1.0, 0.0]
+
+
+def test_other_model_names_still_run_auto_switch(studio_embedder):
+ seen = []
+
+ async def record(request, current_subject, **_kwargs):
+ seen.append(current_subject)
+ return await request.json()
+
+ _identity_names(studio_embedder)
+ studio_embedder.setattr(inference_route, "_auto_switch_from_request_body", record)
+ studio_embedder.setattr(
+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
+ )
+ _call({"input": "alpha", "model": "text-embedding-3-small"})
+ assert seen == ["tester"]
+
+
+def test_st_max_tokens_leaves_room_for_the_special_tokens(monkeypatch):
+ model = SimpleNamespace(
+ max_seq_length = 512,
+ tokenizer = SimpleNamespace(num_special_tokens_to_add = lambda: 2),
+ )
+ monkeypatch.setattr(rag_embeddings, "_get", lambda model_name = None: model)
+ assert rag_embeddings._SentenceTransformersBackend().max_tokens() == 510
+
+
+def _gguf(tmp_path, entries):
+ import io
+ import struct
+
+ buf = io.BytesIO()
+ buf.write(b"GGUF" + struct.pack("<IQQ", 3, 0, len(entries)))
+ for key, vtype, value in entries:
+ raw = key.encode()
+ buf.write(struct.pack("<Q", len(raw)) + raw + struct.pack("<I", vtype))
+ if vtype == 8:
+ raw = value.encode()
+ buf.write(struct.pack("<Q", len(raw)) + raw)
+ elif vtype == 9:
+ elem_type, items = value
+ buf.write(struct.pack("<IQ", elem_type, len(items)))
+ for item in items:
+ raw = item.encode()
+ buf.write(struct.pack("<Q", len(raw)) + raw)
+ elif vtype == 4:
+ buf.write(struct.pack("<I", value))
+ elif vtype == 10:
+ buf.write(struct.pack("<Q", value))
+ path = tmp_path / "embedder.gguf"
+ path.write_bytes(buf.getvalue())
+ return str(path)
+
+
+def test_gguf_context_length_is_read_from_the_header(tmp_path):
+ from core.rag import embed_llama_server
+
+ path = _gguf(
+ tmp_path,
+ [
+ ("general.architecture", 8, "bert"),
+ ("tokenizer.ggml.tokens", 9, (8, ["[PAD]", "[CLS]", "hello"])),
+ ("llama.context_length", 4, 4096),
+ ("bert.context_length", 10, 512),
+ ],
+ )
+ assert embed_llama_server._gguf_context_length(path) == 512
+ assert embed_llama_server._gguf_context_length(str(tmp_path / "missing.gguf")) is None
+
+
+def test_llama_max_tokens_comes_from_the_gguf_minus_its_special_tokens(tmp_path, monkeypatch):
+ from core.rag import embed_llama_server
+
+ backend = embed_llama_server.LlamaServerBackend()
+ backend._model_path = _gguf(
+ tmp_path, [("general.architecture", 8, "bert"), ("bert.context_length", 4, 512)]
+ )
+ monkeypatch.setattr(backend, "_ensure_ready", lambda model_name = None: None)
+ # Larger than the header, so the GGUF is still what binds; confirmed, so it caches.
+ monkeypatch.setattr(backend, "_server_context", lambda: 4096)
+ monkeypatch.setattr(backend, "_server_batch", lambda: 4096)
+ posts = []
+
+ def post(
+ path,
+ payload,
+ model_name = None,
+ ):
+ posts.append((path, payload))
+ return {"tokens": [101, 102]}
+
+ monkeypatch.setattr(backend, "_post", post)
+ assert backend.max_tokens() == 510
+ assert backend.max_tokens() == 510
+ assert posts == [("/tokenize", {"content": "", "add_special": True})]
+ backend._adopt_model_path(backend._model_path, "unsloth/other-GGUF")
+ assert backend._max_tokens is None
+
+
+def test_llama_max_tokens_is_dropped_when_the_binary_is_swapped(tmp_path, monkeypatch):
+ """Changing the custom llama.cpp path respawns onto the same GGUF.
+
+ _current() goes stale on the binary revision, so _ensure_ready kills and respawns, but
+ _resolve_model_path takes its same-repo fast path and never reaches _adopt_model_path.
+ The limit is not a pure model fact -- it clamps the GGUF context by the server's n_ctx
+ and n_ubatch, whose defaults differ between builds -- so it has to go with the binary.
+ """
+ from core.rag import embed_llama_server
+
+ backend = embed_llama_server.LlamaServerBackend()
+ backend._model_path = _gguf(
+ tmp_path, [("general.architecture", 8, "bert"), ("bert.context_length", 4, 512)]
+ )
+ backend._model_repo = "unsloth/bge-small-en-v1.5-GGUF"
+ backend._max_tokens = 510
+ backend._binary = "/opt/old/llama-server"
+ backend._binary_path_revision = 1
+ monkeypatch.setattr(backend, "_process_alive", lambda: True)
+ monkeypatch.setattr(backend, "_kill_process", lambda: None)
+ monkeypatch.setattr(backend, "_spawn", lambda model_name = None: None)
+ monkeypatch.setattr(
+ embed_llama_server.config,
+ "effective_gguf_repo_for_embedding_model",
+ lambda model: "unsloth/bge-small-en-v1.5-GGUF",
+ )
+ monkeypatch.setattr(
+ embed_llama_server.config,
+ "effective_embedding_model",
+ lambda: "unsloth/bge-small-en-v1.5",
+ )
+ import utils.llama_cpp_path_settings as path_settings
+
+ monkeypatch.setattr(path_settings, "custom_llama_cpp_path_revision", lambda: 2)
+
+ backend._ensure_ready()
+
+ assert backend._max_tokens is None
+ assert backend._model_path is not None # the GGUF itself did not change
+
+
+def test_st_max_tokens_reserves_the_default_prompt(monkeypatch):
+ tokenizer = SimpleNamespace(
+ num_special_tokens_to_add = lambda: 2,
+ encode = lambda text, add_special_tokens = False: text.split(),
+ )
+ model = SimpleNamespace(
+ max_seq_length = 512,
+ tokenizer = tokenizer,
+ prompts = {"query": "Represent this sentence for retrieval:"},
+ default_prompt_name = "query",
+ )
+ monkeypatch.setattr(rag_embeddings, "_get", lambda model_name = None: model)
+ assert rag_embeddings._SentenceTransformersBackend().max_tokens() == 512 - 2 - 5
+
+
+def test_llama_max_tokens_is_capped_by_the_running_context(tmp_path, monkeypatch):
+ from core.rag import embed_llama_server
+
+ backend = embed_llama_server.LlamaServerBackend()
+ backend._model_path = _gguf(
+ tmp_path, [("general.architecture", 8, "bert"), ("bert.context_length", 4, 512)]
+ )
+ monkeypatch.setattr(backend, "_ensure_ready", lambda model_name = None: None)
+ monkeypatch.setattr(backend, "_server_context", lambda: 256)
+ monkeypatch.setattr(backend, "_post", lambda *a, **k: {"tokens": [101, 102]})
+ assert backend.max_tokens() == 254
+
+
+def test_llama_max_tokens_is_capped_by_the_ubatch_we_launched_with(tmp_path, monkeypatch):
+ """/props reports n_ctx and no batch field, so the launch value is the only one there is.
+
+ A prompt past one physical batch is refused by llama-server with a 500, which this route
+ turns into a 502; the limit exists to answer 400 before that.
+ """
+ from core.rag import embed_llama_server
+
+ backend = embed_llama_server.LlamaServerBackend()
+ backend._model_path = _gguf(
+ tmp_path, [("general.architecture", 8, "bert"), ("bert.context_length", 4, 8192)]
+ )
+ monkeypatch.setattr(backend, "_ensure_ready", lambda model_name = None: None)
+ monkeypatch.setattr(backend, "_server_context", lambda: 8192)
+ monkeypatch.setattr(
+ backend, "_server_props", lambda: {"default_generation_settings": {"n_ctx": 8192}}
+ )
+ monkeypatch.setattr(backend, "_post", lambda *a, **k: {"tokens": [101, 102]})
+ assert backend.max_tokens() == embed_llama_server._UBATCH_SIZE - 2
+ assert "-ub" in backend._build_cmd("llama-server", "m.gguf", 1, use_gpu = False)
+
+
+def test_only_a_smaller_reported_ubatch_lowers_the_launch_value(monkeypatch):
+ """n_batch is the logical batch; llama.cpp takes min(n_batch, n_ubatch or n_batch), so
+ against the -ub this backend launches with it is never the physical limit."""
+ from core.rag import embed_llama_server
+
+ backend = embed_llama_server.LlamaServerBackend()
+ ub = embed_llama_server._UBATCH_SIZE
+ monkeypatch.setattr(backend, "_server_props", lambda: {"n_ubatch": ub // 2})
+ assert backend._server_batch() == ub // 2
+ monkeypatch.setattr(backend, "_server_props", lambda: {"n_ubatch": ub * 4})
+ assert backend._server_batch() == ub
+ monkeypatch.setattr(backend, "_server_props", lambda: {"n_batch": ub * 4})
+ assert backend._server_batch() == ub
+
+
+def test_a_stale_tagged_identity_is_refused_not_answered(studio_embedder):
+ """The current space is llama-server; the client resubmits the sentence-transformers tag."""
+ from fastapi import HTTPException
+
+ studio_embedder.setattr(
+ rag_config, "effective_gguf_repo_for_embedding_model", lambda model: f"{model}-GGUF"
+ )
+ studio_embedder.setattr(
+ rag_embeddings, "embedding_identity", lambda model_name = None: f"llama-server:{MODEL}"
+ )
+ studio_embedder.setattr(
+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
+ )
+ # A warm index is what makes this reachable: cold, the name falls back to "has a slash".
+ from core.inference import local_model_resolver
+
+ studio_embedder.setattr(local_model_resolver, "index_is_built", lambda: True)
+ studio_embedder.setattr(local_model_resolver, "resolve_local_gguf", lambda *a, **k: None)
+ stale = IDENTITY
+ assert inference_route._names_studio_embedder(stale) is None
+ assert inference_route._reference_is_decisive(stale) is True
+ with pytest.raises(HTTPException) as exc:
+ asyncio.run(
+ inference_route.openai_embeddings(
+ _Request({"input": "alpha", "model": stale}), "tester"
+ )
+ )
+ assert exc.value.status_code == 503
+
+
+ ("body", "switched"),
+ [
+ ({"input": [""], "model": "org/chat-GGUF"}, False),
+ ({"input": ["alpha", 7], "model": "org/chat-GGUF"}, False),
+ ({"input": "alpha", "encoding_format": "int8", "model": "org/chat-GGUF"}, False),
+ ({"input": [[1, 2, 3]], "model": "org/chat-GGUF"}, True),
+ ],
+)
+def test_invalid_fallback_input_is_rejected_before_the_switch(studio_embedder, body, switched):
+ seen = []
+
+ async def record(request, current_subject, **_kwargs):
+ seen.append(current_subject)
+ return await request.json()
+
+ studio_embedder.setattr(inference_route, "_should_validate_before_switch", lambda: True)
+ studio_embedder.setattr(inference_route, "_auto_switch_from_request_body", record)
+ studio_embedder.setattr(
+ inference_route,
+ "get_llama_cpp_backend",
+ lambda: SimpleNamespace(is_loaded = True, is_embedding_gguf = False),
+ )
+ assert _http_error(body).status_code == 400
+ assert bool(seen) is switched
+
+
+def test_local_path_models_are_not_exposed(studio_embedder, tmp_path):
+ from core.rag.config import _escape_identity_segment
+
+ model_dir = str(tmp_path / "bge")
+ studio_embedder.setattr(rag_config, "effective_embedding_model", lambda: model_dir)
+ studio_embedder.setattr(
+ rag_config, "effective_gguf_repo_for_embedding_model", lambda model: f"{model}-GGUF"
+ )
+ identity = f"sentence-transformers:{_escape_identity_segment(model_dir)}"
+ studio_embedder.setattr(
+ rag_embeddings,
+ "encode_with_identity",
+ lambda texts, **_kwargs: (_vectors(texts), identity),
+ )
+ studio_embedder.setattr(
+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
+ )
+ reported = _call({"input": "alpha"})["model"]
+ assert reported.startswith("sentence-transformers:bge-") and str(tmp_path) not in reported
+ studio_embedder.setattr(rag_embeddings, "max_tokens", lambda model_name = None: 3)
+ error = _http_error({"input": "alphabet"})
+ assert error.status_code == 400
+ assert "bge" in error.detail and str(tmp_path) not in error.detail
+
+
[email protected]("answers", [True, False])
+def test_resident_embedding_gguf_answering_the_name_keeps_the_proxy(studio_embedder, answers):
+ import httpx
+
+ class _Client:
+ async def post(self, *_args, **_kwargs):
+ return httpx.Response(200, json = {"data": [{"embedding": [0.5]}], "model": "proxy"})
+
+ async def aclose(self):
+ return None
+
+ _identity_names(studio_embedder)
+ studio_embedder.setattr(inference_route, "_loaded_satisfies", lambda requested: answers)
+ studio_embedder.setattr(inference_route, "_cancelable_nonstreaming_client", _Client)
+ studio_embedder.setattr(
+ inference_route,
+ "get_llama_cpp_backend",
+ lambda: SimpleNamespace(
+ is_loaded = True,
+ is_embedding_gguf = True,
+ base_url = "http://llama.test",
+ context_length = 512,
+ model_identifier = f"{MODEL}-GGUF",
+ ),
+ )
+ payload = _call({"input": "alpha", "model": MODEL})
+ assert payload["model"] == ("proxy" if answers else IDENTITY)
+
+
+def test_public_embedding_name_hides_any_local_path_shape():
+ public = inference_route._public_embedding_name
+ windows = public("C:\\models\\private\\bge")
+ assert windows.startswith("bge-") and len(windows) == len("bge-") + 8
+ assert "models" not in windows and "private" not in windows
+ assert public("/tmp/models/bge/").startswith("bge-")
+ assert public("/checkpoints/run-a/model") != public("/checkpoints/run-b/model")
+ assert public("unsloth/bge-small-en-v1.5") == "unsloth/bge-small-en-v1.5"
+
+
+def test_flat_token_array_passes_the_pre_switch_check(studio_embedder):
+ seen = []
+
+ async def record(request, current_subject, **_kwargs):
+ seen.append(current_subject)
+ return await request.json()
+
+ studio_embedder.setattr(inference_route, "_should_validate_before_switch", lambda: True)
+ studio_embedder.setattr(inference_route, "_auto_switch_from_request_body", record)
+ studio_embedder.setattr(
+ inference_route,
+ "get_llama_cpp_backend",
+ lambda: SimpleNamespace(is_loaded = True, is_embedding_gguf = False),
+ )
+ assert _http_error({"input": [1, 2, 3]}).status_code == 400
+ assert seen == ["tester"]
+
+
+def test_local_path_aliases_match_case_exactly(studio_embedder, tmp_path):
+ model_dir = str(tmp_path / "Foo")
+ studio_embedder.setattr(rag_config, "effective_embedding_model", lambda: model_dir)
+ studio_embedder.setattr(
+ rag_config, "effective_gguf_repo_for_embedding_model", lambda model: f"{model}-GGUF"
+ )
+ assert inference_route._names_studio_embedder(model_dir)
+ assert not inference_route._names_studio_embedder(str(tmp_path / "foo"))
+ assert inference_route._names_studio_embedder(f"sentence-transformers:{model_dir}")
+
+
+def test_untagged_aliases_never_probe_the_backend(studio_embedder):
+ def boom(*_args, **_kwargs):
+ raise AssertionError("embedding_identity must not run for untagged names")
+
+ _identity_names(studio_embedder)
+ studio_embedder.setattr(rag_embeddings, "embedding_identity", boom)
+ assert inference_route._names_studio_embedder(MODEL.upper())
+ assert not inference_route._names_studio_embedder("text-embedding-3-small")
+
+
+def test_tagged_identity_must_match_the_current_space(studio_embedder):
+ _identity_names(studio_embedder)
+ assert inference_route._names_studio_embedder(IDENTITY) == MODEL
+ stale = f"llama-server:{MODEL}:{MODEL}-GGUF"
+ assert inference_route._names_studio_embedder(stale) is None
+
+
+def test_disconnected_client_leaves_the_queue_without_embedding(studio_embedder):
+ lock = threading.Lock()
+ gate = threading.Event()
+ calls = {"n": 0}
+
+ def encode(texts, **_kwargs):
+ with lock:
+ calls["n"] += 1
+ gate.wait(timeout = 5)
+ return _vectors(texts)
+
+ class _Gone(_Request):
+ async def is_disconnected(self):
+ return True
+
+ studio_embedder.setattr(
+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
+ )
+ studio_embedder.setattr(rag_embeddings, "encode", encode)
+ cap = inference_route._STUDIO_EMBED_CONCURRENCY
+
+ async def run():
+ blockers = [
+ asyncio.create_task(inference_route.openai_embeddings(_Request({"input": "x"}), "t"))
+ for _ in range(cap)
+ ]
+ for _ in range(500):
+ await asyncio.sleep(0.01)
+ if calls["n"] == cap:
+ break
+ with pytest.raises(asyncio.CancelledError):
+ await inference_route.openai_embeddings(_Gone({"input": "gone"}), "t")
+ assert calls["n"] == cap
+ gate.set()
+ await asyncio.gather(*blockers)
+ response = await inference_route.openai_embeddings(_Request({"input": "later"}), "t")
+ assert response.status_code == 200
+ assert calls["n"] == cap + 1
+
+ asyncio.run(run())
+
+
+def test_batch_cap_applies_only_to_the_studio_fallback():
+ from fastapi import HTTPException
+
+ body = {"input": ["x"] * (inference_route._STUDIO_EMBED_MAX_INPUTS + 1)}
+ assert len(inference_route._embeddings_items(body, tokens_ok = True)) == len(body["input"])
+ with pytest.raises(HTTPException) as exc:
+ inference_route._embeddings_texts(body)
+ assert exc.value.status_code == 400
+
+
+def test_a_decisively_named_model_is_refused_when_nothing_is_loaded(studio_embedder):
+ # #7454: a reference clearly meant for this server 404s rather than being answered by other
+ # weights. The slot being empty makes _reject_unservable_model defer, so the fallback decides.
+ _identity_names(studio_embedder)
+ studio_embedder.setattr(
+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
+ )
+ studio_embedder.setattr(inference_route, "_reference_is_decisive", lambda name: True)
+ error = _http_error({"input": "alpha", "model": "unsloth/embeddinggemma-300m-GGUF:Q8_0"})
+ assert error.status_code in (404, 503)
+
+
+def test_a_foreign_model_name_still_reaches_the_studio_embedder(studio_embedder):
+ # The other half: a vendor id is no evidence, so it keeps falling through instead of 404ing.
+ _identity_names(studio_embedder)
+ studio_embedder.setattr(
+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
+ )
+ assert _call({"input": "alpha", "model": "text-embedding-3-small"})["model"] == IDENTITY
+
+
+def test_reference_is_decisive_only_on_positive_evidence(monkeypatch):
+ from core.inference import local_model_resolver as _resolver
+
+ monkeypatch.setattr(_resolver, "resolve_local_gguf", lambda ref, allow_scan = False: None)
+ # An explicit quant label is evidence no foreign id carries.
+ assert inference_route._reference_is_decisive("unsloth/gemma-3-270m-it-GGUF:Q8_0") is True
+ # A vendor id is not, so LiteLLM/OpenRouter style names keep working.
+ assert inference_route._reference_is_decisive("text-embedding-3-small") is False
+ assert inference_route._reference_is_decisive("openai/text-embedding-3-large") is False
+
+
+def test_advertised_local_identity_is_accepted_as_an_alias(studio_embedder, tmp_path):
+ model_dir = str(tmp_path / "bge")
+ studio_embedder.setattr(rag_config, "effective_embedding_model", lambda: model_dir)
+ studio_embedder.setattr(
+ rag_config, "effective_gguf_repo_for_embedding_model", lambda model: f"{model}-GGUF"
+ )
+ from core.rag.config import _escape_identity_segment
+
+ studio_embedder.setattr(
+ rag_embeddings,
+ "embedding_identity",
+ lambda model_name = None: f"sentence-transformers:{_escape_identity_segment(model_dir)}",
+ )
+ public = inference_route._public_embedding_name(model_dir)
+ assert inference_route._names_studio_embedder(f"sentence-transformers:{public}") == model_dir
+ assert inference_route._names_studio_embedder(public) == model_dir
+ assert inference_route._names_studio_embedder("bge-00000000") is None
+
+
+def test_disconnected_client_is_dropped_even_with_a_free_permit(studio_embedder):
+ calls = []
+
+ class _Gone(_Request):
+ async def is_disconnected(self):
+ return True
+
+ studio_embedder.setattr(
+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
+ )
+ studio_embedder.setattr(
+ rag_embeddings, "encode", lambda texts, **_: calls.append(texts) or _vectors(texts)
+ )
+
+ async def run():
+ with pytest.raises(asyncio.CancelledError):
+ await inference_route.openai_embeddings(_Gone({"input": "gone"}), "t")
+ assert calls == []
+ response = await inference_route.openai_embeddings(_Request({"input": "later"}), "t")
+ assert response.status_code == 200
+
+ asyncio.run(run())
+
+
+def test_alias_match_pins_the_model_for_the_request(studio_embedder):
+ settings = iter([MODEL, "unsloth/other-model", "unsloth/other-model"])
+ seen = []
+
+ def _encode_with_identity(
+ texts,
+ *,
+ model_name = None,
+ normalize = True,
+ ):
+ seen.append(model_name)
+ return _vectors(texts), IDENTITY
+
+ _identity_names(studio_embedder)
+ studio_embedder.setattr(rag_config, "effective_embedding_model", lambda: next(settings))
+ studio_embedder.setattr(rag_embeddings, "encode_with_identity", _encode_with_identity)
+ studio_embedder.setattr(
+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
+ )
+ _call({"input": "alpha", "model": MODEL})
+ assert seen == [MODEL]
+
+
+def test_llama_max_tokens_never_exceeds_one_physical_batch(tmp_path, monkeypatch):
+ # Embedding is non-causal, so llama.cpp refuses rather than splits: an 8k-context limit on a
+ # 512 batch turned a legitimate 600-token input into a 502 instead of a 400.
+ from core.rag import embed_llama_server
+
+ backend = embed_llama_server.LlamaServerBackend()
+ backend._model_path = _gguf(
+ tmp_path, [("general.architecture", 8, "bert"), ("bert.context_length", 4, 8192)]
+ )
+ monkeypatch.setattr(backend, "_ensure_ready", lambda model_name = None: None)
+ monkeypatch.setattr(backend, "_server_context", lambda: 8192)
+ monkeypatch.setattr(backend, "_server_batch", lambda: 512)
+ monkeypatch.setattr(backend, "_post", lambda *a, **k: {"tokens": [101, 102]})
+ assert backend.max_tokens() == 510
+
+
+def test_the_embed_server_does_not_enlarge_its_batch(tmp_path):
+ # n_vocab * n_ubatch * 4 is allocated at startup, hundreds of MiB, against the 1024 MiB free
+ # this backend calls enough to offload everything: max_tokens bounds the advert instead.
+ from core.rag import embed_llama_server
+
+ backend = embed_llama_server.LlamaServerBackend()
+ model = _gguf(tmp_path, [("general.architecture", 8, "bert"), ("bert.context_length", 4, 8192)])
+ cmd = backend._build_cmd("llama-server", model, 9999, use_gpu = True)
+ assert "-b" not in cmd
+ # Pinned at llama.cpp's own default, so nothing is allocated that was not already, and
+ # max_tokens has a batch to bound the advert with (/props publishes none).
+ assert cmd[cmd.index("-ub") + 1] == str(embed_llama_server._UBATCH_SIZE)
+
+
+def test_an_unconfirmed_context_limit_is_not_cached(tmp_path, monkeypatch):
+ # A header context can exceed what the server runs at, so freezing it while /props is silent
+ # outlives the readback that would have corrected it.
+ from core.rag import embed_llama_server
+
+ backend = embed_llama_server.LlamaServerBackend()
+ backend._model_path = _gguf(
+ tmp_path, [("general.architecture", 8, "bert"), ("bert.context_length", 4, 8192)]
+ )
+ monkeypatch.setattr(backend, "_ensure_ready", lambda model_name = None: None)
+ monkeypatch.setattr(backend, "_server_batch", lambda: None)
+ monkeypatch.setattr(backend, "_post", lambda *a, **k: {"tokens": [101, 102]})
+ monkeypatch.setattr(backend, "_server_context", lambda: None)
+ assert backend.max_tokens() == 8190
+ assert backend._max_tokens is None
+ # Once /props answers with both bounds, the smaller real context is what sticks.
+ monkeypatch.setattr(backend, "_server_context", lambda: 512)
+ monkeypatch.setattr(backend, "_server_batch", lambda: 512)
+ assert backend.max_tokens() == 510
+ assert backend._max_tokens == 510
+
+
+def test_props_probe_never_raises_before_the_server_is_up():
+ from core.rag import embed_llama_server
+
+ # An un-started server has no port, so the URL itself is invalid: the probe must swallow
+ # that and fall back to the batch we launch with rather than propagate.
+ assert (
+ embed_llama_server.LlamaServerBackend()._server_batch() == embed_llama_server._UBATCH_SIZE
+ )
+
+
+def test_cancel_during_the_final_disconnect_probe_releases_the_permit(studio_embedder):
+ lock = threading.Lock()
+ gate = threading.Event()
+ active = {"now": 0, "peak": 0}
+
+ def encode(texts, **_kwargs):
+ with lock:
+ active["now"] += 1
+ active["peak"] = max(active["peak"], active["now"])
+ gate.wait(timeout = 5)
+ with lock:
+ active["now"] -= 1
+ return _vectors(texts)
+
+ class _Stuck(_Request):
+ async def is_disconnected(self):
+ await asyncio.sleep(3600)
+ return False
+
+ studio_embedder.setattr(
+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
+ )
+ studio_embedder.setattr(rag_embeddings, "encode", encode)
+ cap = inference_route._STUDIO_EMBED_CONCURRENCY
+
+ async def run():
+ stuck = asyncio.create_task(inference_route.openai_embeddings(_Stuck({"input": "x"}), "t"))
+ await asyncio.sleep(0.05)
+ stuck.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await stuck
+ tasks = [
+ asyncio.create_task(inference_route.openai_embeddings(_Request({"input": "y"}), "t"))
+ for _ in range(cap)
+ ]
+ for _ in range(500):
+ await asyncio.sleep(0.01)
+ if active["now"] == cap:
+ break
+ assert active["now"] == cap
+ gate.set()
+ await asyncio.gather(*tasks)
+
+ asyncio.run(run())
+
+
+def test_a_boolean_is_not_a_token_id():
+ # bool subclasses int, so `[true]` read as a token array could swap the resident GGUF.
+ from fastapi import HTTPException
+
+ with pytest.raises(HTTPException) as exc:
+ inference_route._embeddings_items({"input": [True]}, tokens_ok = True)
+ assert exc.value.status_code == 400
+ with pytest.raises(HTTPException):
+ inference_route._embeddings_items({"input": [[1, True, 3]]}, tokens_ok = True)
+ # A real token array is still one text.
+ assert inference_route._embeddings_items({"input": [1, 2, 3]}, tokens_ok = True) == [[1, 2, 3]]
+
+
+def test_a_cold_index_does_not_make_a_local_name_foreign(monkeypatch):
+ # Before the first scan resolve_local_gguf answers None for a model that IS downloaded, and
+ # reading that as foreign served another embedding space under the requested name.
+ from core.inference import local_model_resolver as _resolver
+
+ monkeypatch.setattr(_resolver, "resolve_local_gguf", lambda ref, allow_scan = False: None)
+ monkeypatch.setattr(_resolver, "index_is_built", lambda: False)
+ monkeypatch.setattr(_resolver, "warm_index_soon", lambda: None)
+ assert inference_route._reference_is_decisive("org/my-local-model") is True
+ assert inference_route._reference_is_decisive("/models/local.gguf") is True
+ # A bare vendor alias is still not evidence, cold index or not.
+ assert inference_route._reference_is_decisive("text-embedding-3-small") is False
+
+ # Once the index is built, absence really is evidence.
+ monkeypatch.setattr(_resolver, "index_is_built", lambda: True)
+ assert inference_route._reference_is_decisive("org/not-here") is False
+
+
[email protected]("stash_embeds", [False, True])
+def test_default_request_restores_the_chat_slot_only_for_an_embedding_stash(
+ studio_embedder, stash_embeds
+):
+ seen = []
+
+ async def record(request, current_subject, **_kwargs):
+ seen.append(current_subject)
+ return await request.json()
+
+ studio_embedder.setattr(inference_route, "_auto_switch_from_request_body", record)
+ studio_embedder.setattr(inference_route, "_stashed_gguf_embeds", lambda: stash_embeds)
+ studio_embedder.setattr(
+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
+ )
+ payload = _call({"input": "alpha"})
+ assert payload["model"] == IDENTITY
+ assert seen == (["tester"] if stash_embeds else [])
+
+
+def test_stashed_gguf_embeds_reads_the_stashed_model_header(monkeypatch, tmp_path):
+ import core.inference.llama_keepwarm as keepwarm
+ import core.inference.local_model_resolver as resolver
+
+ monkeypatch.setattr(keepwarm, "get_last_unloaded_model", lambda: None)
+ assert inference_route._stashed_gguf_embeds() is False
+ monkeypatch.setattr(
+ keepwarm, "get_last_unloaded_model", lambda: ("org/E-GGUF", "Q8_0", "org/E-GGUF")
+ )
+ monkeypatch.setattr(
+ resolver, "resolve_local_gguf", lambda ref, **_: ("/tmp/e.gguf", "Q8_0", "org/E-GGUF")
+ )
+
+ class _Probe:
+ is_embedding_gguf = True
+
+ def _read_gguf_metadata(self, path):
+ self.path = path
+
+ monkeypatch.setattr(inference_route, "_probe_backend", _Probe)
+ assert inference_route._stashed_gguf_embeds() is True
diff --git a/studio/backend/tests/test_rag_embeddings.py b/studio/backend/tests/test_rag_embeddings.py
index e88ee3fdd65..0efb005e00a 100644
--- a/studio/backend/tests/test_rag_embeddings.py
+++ b/studio/backend/tests/test_rag_embeddings.py
@@ -947,7 +947,9 @@ def test_the_security_gate_scans_the_snapshot_that_is_actually_loaded(monkeypatc
(snapshot / "model.safetensors").write_bytes(b"ST")
scanned = []
monkeypatch.setattr(
- embeddings, "_guard_model_security", lambda target, local_only = False: scanned.append(target)
+ embeddings,
+ "_guard_model_security",
+ lambda target, local_only = False, display = None: scanned.append(target),
)
class _ST: