Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: serve /v1/embeddings from the configured embedding model when the loaded GGUF cannot (#10315)
Nilayb8d120ade8cb
diff --git a/studio/backend/core/inference/llama_keepwarm.py b/studio/backend/core/inference/llama_keepwarm.pyindex bef73690938..595c5bc9c85 100644--- a/studio/backend/core/inference/llama_keepwarm.py+++ b/studio/backend/core/inference/llama_keepwarm.py@@ -448,6 +448,20 @@ def _note_admitted_end() -> None:_admitted_inference = max(0, _admitted_inference - 1)+def untrack_admitted_inference(scope) -> None:+ """Drop an already-admitted request from the preview busy guard once the route knows it+ will not run against the resident GGUF after all.++ ``untrack_current_request`` covers only the in-flight counters; the admitted tally is+ what ``load_model_for_preview`` reads, so a route that admitted at the auto-switch hook+ and then served the request some other way keeps blocking preview swaps for its whole+ duration. Pops the marker so the middleware's finally, which balances only a scope that+ still carries it, cannot decrement a second time. Idempotent."""+ if not isinstance(scope, dict) or not scope.pop(_ADMITTED_SCOPE_KEY, False):+ return+ _note_admitted_end()++def begin_preview_serializer_wait(scope) -> bool:"""Move a tracked preview from active to pending while it waits on the route lock."""global _inflight, _pending, _preview_inflight, _preview_pendingdiff --git a/studio/backend/core/rag/embed_llama_server.py b/studio/backend/core/rag/embed_llama_server.pyindex 1ac18958cdd..799244d3984 100644--- a/studio/backend/core/rag/embed_llama_server.py+++ b/studio/backend/core/rag/embed_llama_server.py@@ -19,6 +19,7 @@ from __future__ import annotationsimport atexitimport loggingimport os+import structimport subprocessimport sysimport threading@@ -48,6 +49,52 @@ _TRANSPORT_ERRORS = ()+_GGUF_SCALAR_WIDTHS = {0: 1, 1: 1, 2: 2, 3: 2, 4: 4, 5: 4, 6: 4, 7: 1, 10: 8, 11: 8, 12: 8}++# llama.cpp's own -ub default. An embedding prompt is not splittable, so one longer than this+# is refused with a 500 no matter how large the model's context is.+_UBATCH_SIZE = 512+++def _skip_gguf_value(f, vtype: int) -> None:+ if vtype == 8:+ f.seek(struct.unpack("<Q", f.read(8))[0], 1)+ elif vtype == 9:+ elem_type, count = struct.unpack("<IQ", f.read(12))+ if elem_type in _GGUF_SCALAR_WIDTHS:+ f.seek(_GGUF_SCALAR_WIDTHS[elem_type] * count, 1)+ else:+ for _ in range(count):+ _skip_gguf_value(f, elem_type)+ else:+ f.seek(_GGUF_SCALAR_WIDTHS[vtype], 1)+++def _gguf_context_length(path: str) -> int | None:+ try:+ with open(path, "rb") as f:+ if f.read(4) != b"GGUF":+ return None+ f.read(4)+ _, kv_count = struct.unpack("<QQ", f.read(16))+ arch = None+ lengths: dict[str, int] = {}+ for _ in range(kv_count):+ key = f.read(struct.unpack("<Q", f.read(8))[0]).decode("utf-8")+ vtype = struct.unpack("<I", f.read(4))[0]+ if key == "general.architecture" and vtype == 8:+ arch = f.read(struct.unpack("<Q", f.read(8))[0]).decode("utf-8")+ elif key.endswith(".context_length") and vtype in (4, 10):+ lengths[key] = int.from_bytes(f.read(_GGUF_SCALAR_WIDTHS[vtype]), "little")+ else:+ _skip_gguf_value(f, vtype)+ if arch is not None and f"{arch}.context_length" in lengths:+ return lengths[f"{arch}.context_length"] or None+ except (OSError, struct.error, UnicodeDecodeError, KeyError, ValueError):+ return None+ return None++def _resolve_entrypoint(binary: str) -> str:"""The real executable behind a managed shell/symlink entrypoint.@@ -109,6 +156,7 @@ class LlamaServerBackend:# dim() takes the same _serve_lock the request path does; reentrant because dim() -> encode() -># _ensure_ready() re-enters on one thread.self._dim: int | None = None+ self._max_tokens: int | None = None# One subprocess serves one GGUF, so readiness and the request must be indivisible: a swap between# them lands A's POST on B's server.self._serve_lock = threading.RLock()@@ -468,7 +516,11 @@ class LlamaServerBackend:# Serve a stand-in rather than fail the index, but leave the marker: retiring on it would pin this# model to the wrong quant.return self._adopt_model_path(cached, desired)- raise RuntimeError(+ # Typed: the same condition the ST path raises, and /v1/embeddings answers it+ # with a 409. Bare, it fell into the catch-all as a 502 naming nothing.+ from .embeddings import EmbeddingModelDownloadRequiredError++ raise EmbeddingModelDownloadRequiredError(f"Embedding model {model!r} is not downloaded yet. ""Finish its Settings download before indexing documents.")@@ -539,6 +591,7 @@ class LlamaServerBackend:self._model_path = pathself._model_repo = desiredself._dim = None+ self._max_tokens = Nonereturn self._model_path# A listing, not a transfer: a working hub answers well inside a second.@@ -673,6 +726,11 @@ class LlamaServerBackend:"--fit","off",]+ # -ub at llama.cpp's own default, so nothing is allocated that was not already: raising+ # the batch allocates n_vocab * n_ubatch * 4 up front (~938 MiB at 8192 with a 30k+ # vocab), against the 1024 MiB free this backend calls enough to offload every layer.+ # Passed rather than read back because /props publishes n_ctx and no batch field.+ cmd += ["-ub", str(_UBATCH_SIZE)]# -1 offloads every layer (matches the chat server); 0 keeps it on CPU.cmd += ["-ngl", "-1" if use_gpu else "0"]return cmd@@ -878,6 +936,9 @@ class LlamaServerBackend:if self._binary_path_revision != custom_llama_cpp_path_revision():self._binary = Noneself._binary_path_revision = None+ # Half a server fact (n_ctx, n_ubatch differ between builds), and+ # _resolve_model_path fast-paths on an unchanged repo, so nothing else clears it.+ self._max_tokens = Noneself._spawn(model_name)def _restart(self, model_name: str | None = None) -> None:@@ -1037,6 +1098,72 @@ class LlamaServerBackend:self._dim = widthreturn width+ def _server_props(self) -> dict | None:+ """``/props``, or None. Advisory only -- it refines a limit that already has a+ value from the GGUF, so no failure here may reach the caller. That includes a+ malformed base URL, which is what an un-started server has.+ """+ try:+ data = httpx.get(f"{self._base_url}/props", timeout = 2.0, trust_env = False).json()+ except Exception: # noqa: BLE001 - see docstring+ return None+ return data if isinstance(data, dict) else None++ @staticmethod+ def _positive(data: dict | None, key: str) -> int | None:+ """*key* from /props, whether the build reports it at the top level or under+ ``default_generation_settings``."""+ for scope in (data, (data or {}).get("default_generation_settings")):+ if isinstance(scope, dict):+ value = scope.get(key)+ if isinstance(value, int) and value > 0:+ return value+ return None++ def _server_context(self) -> int | None:+ return self._positive(self._server_props(), "n_ctx")++ def _server_batch(self) -> int | None:+ """The physical batch this server actually runs at.++ A non-causal (embedding) prompt longer than it is refused outright, so it bounds+ what may be advertised no matter how large the model's context is.+ """+ # n_ubatch only: n_batch is the logical batch, and llama.cpp derives the physical one as+ # min(n_batch, n_ubatch or n_batch), so against the -ub we launch with a reported n_batch+ # is never the limit. No build publishes either today, leaving the launch value; read it+ # first anyway, so a build that starts reporting it wins over the assumption.+ found = self._positive(self._server_props(), "n_ubatch")+ return min(found, _UBATCH_SIZE) if found else _UBATCH_SIZE++ def max_tokens(self, *, model_name = None) -> int | None:+ with self._operation(), self._serve_lock:+ self._ensure_ready(model_name)+ if self._max_tokens is None and self._model_path:+ limit = _gguf_context_length(self._model_path)+ running = self._server_context()+ if limit and running:+ limit = min(limit, running)+ elif running:+ limit = running+ # Past one batch llama.cpp returns a 500, not the 400 this limit exists to give.+ batch = self._server_batch()+ if batch:+ limit = min(limit, batch) if limit else batch+ if limit:+ data = self._post(+ "/tokenize",+ {"content": "", "add_special": True},+ model_name = model_name,+ )+ answer = max(1, limit - len(data.get("tokens", [])))+ if running is None or batch is None:+ # /props silent: answer with the header guess but do not freeze it,+ # or a too-large limit outlives the readback that would correct it.+ return answer+ self._max_tokens = answer+ return self._max_tokens+def warm(self, *, model_name = None) -> None:"""Start the server and probe dim off the request path.diff --git a/studio/backend/core/rag/embeddings.py b/studio/backend/core/rag/embeddings.pyindex 7473f71958f..76f42f53971 100644--- a/studio/backend/core/rag/embeddings.py+++ b/studio/backend/core/rag/embeddings.py@@ -199,7 +199,11 @@ def _st_module_subdirs(name: str, token: str | None) -> tuple[str, ...]:return ()-def _guard_model_security(name: str, local_only: bool = False) -> None:+def _guard_model_security(+ name: str,+ local_only: bool = False,+ display: str | None = None,+) -> None:"""Refuse to load a repo HF flagged as unsafe: a poisoned pickle deserializes insideSentenceTransformer regardless of trust_remote_code. Defense in depth behind the/settings gate (a name can also arrive via env/default); local paths and unreachable@@ -207,6 +211,11 @@ def _guard_model_security(name: str, local_only: bool = False) -> None:``local_only`` (offline) inspects the local cache; subdir probes are skipped (they'd hit thenetwork and hang, and the offline gate walks the whole snapshot anyway).++ ``display`` names the model in the error. The scan runs on ``name``, which the caller has+ already resolved to an absolute snapshot directory, and that path reaches a user through+ /v1/embeddings; the configured model is what they can act on and what the rest of that+ route reports."""try:from utils.security import evaluate_file_security, security_load_subdirs@@ -235,7 +244,7 @@ def _guard_model_security(name: str, local_only: bool = False) -> None:else "is flagged as unsafe by Hugging Face's security scan")raise UnsafeEmbeddingModelError(- f"Embedding model {name!r} {reason}; refusing to load. "+ f"Embedding model {(display or name)!r} {reason}; refusing to load. ""Set a different RAG embedding model.")@@ -470,7 +479,7 @@ def _get(model_name: str | None = None):# Scan after load_target is settled: on the repo id it checked the Hub's current commit while the# load opened an older cached one. evaluate_file_security recovers the repo and exact commit from# a snapshot path.- _guard_model_security(load_target, offline)+ _guard_model_security(load_target, offline, display = name)with _quiet_transformers_load() as report:# Re-emit in finally: a load that raises after transformers wrote its report is exactly when a# MISSING or MISMATCH line matters.@@ -540,6 +549,22 @@ def _st_dim(model_name: str | None = None) -> int:return _get(model_name).get_sentence_embedding_dimension()+def _st_max_tokens(model_name: str | None = None) -> int | None:+ with _compute_lock:+ model = _get(model_name)+ limit = getattr(model, "max_seq_length", None)+ if not limit:+ return None+ tokenizer = getattr(model, "tokenizer", None)+ specials = getattr(tokenizer, "num_special_tokens_to_add", None)+ reserved = int(specials()) if callable(specials) else 0+ prompts = getattr(model, "prompts", None) or {}+ prompt = prompts.get(getattr(model, "default_prompt_name", None))+ if prompt and tokenizer is not None:+ reserved += len(tokenizer.encode(prompt, add_special_tokens = False))+ return max(1, int(limit) - reserved)++def _st_token_counter(model_name: str | None = None) -> Callable[[str], int]:"""Token counter using the model's tokenizer, under the compute lock (the samefast tokenizer backs encode and isn't thread-safe), with rayon enabled for the@@ -602,6 +627,9 @@ class _SentenceTransformersBackend:def dim(self, *, model_name = None):return _st_dim(model_name)+ def max_tokens(self, *, model_name = None):+ return _st_max_tokens(model_name)+def warm(self, *, model_name = None):_get(model_name)@@ -1143,6 +1171,10 @@ def dim(model_name: str | None = None) -> int:return _get_backend(model_name).dim(model_name = model_name)+def max_tokens(model_name: str | None = None) -> int | None:+ return _get_backend(model_name).max_tokens(model_name = model_name)++def token_counter(model_name: str | None = None) -> Callable[[str], int]:"""Callable counting tokens with the embedder's own tokenizer.diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 766ba68af36..55bad62893a 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -25730,6 +25730,415 @@ async def openai_completions(request: Request, current_subject: str = Depends(ge# =====================================================================+_STUDIO_EMBED_CONCURRENCY = 4+_STUDIO_EMBED_MAX_INPUTS = 2048+_studio_embed_semaphores: "weakref.WeakKeyDictionary" = weakref.WeakKeyDictionary()+_studio_embed_semaphores_guard = threading.Lock()+++def _studio_embed_semaphore() -> asyncio.Semaphore:+ loop = asyncio.get_running_loop()+ with _studio_embed_semaphores_guard:+ semaphore = _studio_embed_semaphores.get(loop)+ if semaphore is None:+ semaphore = _studio_embed_semaphores[loop] = asyncio.Semaphore(+ _STUDIO_EMBED_CONCURRENCY+ )+ return semaphore+++def _resident_serves_embeddings(llama_backend) -> bool:+ return bool(llama_backend.is_loaded and getattr(llama_backend, "is_embedding_gguf", True))+++def _embeddings_items(body: dict, *, tokens_ok: bool) -> list:+ value = body.get("input")+ if isinstance(value, str):+ items = [value]+ # `type(...) is int`, not isinstance: bool subclasses int, so `[true]` would be read+ # as a token array and could swap the resident GGUF for a body llama-server rejects.+ elif (+ tokens_ok+ and isinstance(value, list)+ and value+ and all(type(token) is int for token in value)+ ):+ items = [value]+ elif isinstance(value, (list, tuple)):+ items = list(value)+ else:+ items = []+ if not items:+ raise HTTPException(status_code = 400, detail = "'input' is required for embeddings.")+ if not tokens_ok and len(items) > _STUDIO_EMBED_MAX_INPUTS:+ raise HTTPException(+ status_code = 400,+ detail = f"'input' may hold at most {_STUDIO_EMBED_MAX_INPUTS} items.",+ )++ def _ok(item) -> bool:+ if isinstance(item, str):+ return bool(item)+ return (+ tokens_ok+ and isinstance(item, list)+ and bool(item)+ and all(type(token) is int for token in item)+ )++ if not all(_ok(item) for item in items):+ raise HTTPException(+ status_code = 400,+ detail = "'input' must be a non-empty string or an array of non-empty strings.",+ )+ encoding_format = body.get("encoding_format")+ if encoding_format is not None and encoding_format not in ("float", "base64"):+ raise HTTPException(+ status_code = 400, detail = "'encoding_format' must be 'float' or 'base64'."+ )+ return items+++def _embeddings_texts(body: dict) -> list[str]:+ return _embeddings_items(body, tokens_ok = False)+++def _public_embedding_name(model_name: str) -> str:+ """A name for a local checkpoint that leaks no path but still identifies it.++ The basename alone is not enough: /checkpoints/run-a/model and+ /checkpoints/run-b/model are different embedding spaces, and reporting both as+ "model" lets a client file vectors from either under one identity -- the exact+ confusion this route reports the identity to prevent.+ """+ from utils.paths import is_local_path++ if not is_local_path(model_name):+ return model_name+ import hashlib++ normalized = model_name.replace("\\", "/").rstrip("/")+ base = normalized.rsplit("/", 1)[-1] or "model"+ return f"{base}-{hashlib.sha256(normalized.encode('utf-8')).hexdigest()[:8]}"+++def _public_embedding_identity(identity: str, model_name: str, label: str) -> str:+ """Redact the model and repo segments of ``backend:model[:gguf_repo]``, nothing else.++ Segment-wise, not a substring replace: a model can be a relative directory named+ `transformers`, and replacing that everywhere also rewrites the `sentence-transformers`+ tag, leaving an identity no backend answers to and that _names_studio_embedder cannot+ match on the next request.+ """+ from core.rag.config import (+ EMBEDDING_IDENTITY_TAGS,+ _escape_identity_segment,+ _unescape_identity_segment,+ effective_gguf_repo_for_embedding_model,+ )+ from utils.paths import is_local_path++ tag = next((t for t in EMBEDDING_IDENTITY_TAGS if identity.startswith(f"{t}:")), None)+ if tag is None:+ return identity+ repo = effective_gguf_repo_for_embedding_model(model_name)+ # Both segments are checked: a hub model id can carry a GGUF companion that is a local path+ # (RAG_EMBED_GGUF_REPO, or a stored repo), and that path is a segment of the identity too.+ if not is_local_path(model_name) and not is_local_path(repo):+ return identity+ public = {model_name: label, repo: _public_embedding_name(repo)}+ segments = [_unescape_identity_segment(s) for s in identity[len(tag) + 1 :].split(":")]+ redacted = [_escape_identity_segment(public.get(s, s)) for s in segments]+ return ":".join([tag, *redacted])+++def _embedding_payload(vector, encoding_format: str):+ if encoding_format == "base64":+ import base64+ import numpy as np+ return base64.b64encode(np.asarray(vector, dtype = np.float32).tobytes()).decode("ascii")+ return [float(x) for x in vector]+++def _names_studio_embedder(requested: str) -> Optional[str]:+ from core.rag import config as rag_config+ from core.rag import embeddings as rag_embeddings+ from utils.paths import is_local_path++ model = rag_config.effective_embedding_model()+ repo = rag_config.effective_gguf_repo_for_embedding_model(model)+ label = _public_embedding_name(model)+ wanted = requested.strip()+ if rag_config.embedding_identity_model(wanted) is not None:+ current = rag_embeddings.embedding_identity(model)+ if wanted in (current, _public_embedding_identity(current, model, label)):+ return model+ return None+ for name in (model, repo, label, _public_embedding_name(repo)):+ if not name:+ continue+ if is_local_path(name) or is_local_path(wanted):+ if wanted == name:+ return model+ elif wanted.casefold() == name.casefold():+ return model+ return None+++def _resident_absent(llama_backend) -> bool:+ return not getattr(llama_backend, "is_loaded", False)+++def _stashed_gguf_embeds() -> bool:+ from core.inference.llama_keepwarm import get_last_unloaded_model+ from core.inference.local_model_resolver import resolve_local_gguf++ last = get_last_unloaded_model()+ if not last:+ return False+ target_id, variant = last[0], last[1]+ try:+ resolved = resolve_local_gguf(+ f"{target_id}:{variant}" if variant else target_id, allow_scan = False+ )+ path = resolved[0] if resolved else None+ if not path:+ return False+ probe = _probe_backend()+ probe._read_gguf_metadata(path)+ return bool(probe.is_embedding_gguf)+ except Exception: # noqa: BLE001 - an unreadable stash is not an embedding model+ return False+++async def _default_embeddings_request_body(request: Request) -> Optional[dict]:+ try:+ body = await request.json()+ except (json.JSONDecodeError, ValueError):+ return None+ if not isinstance(body, dict):+ return None+ requested = body.get("model")+ return body if requested is None or requested == "" else None+++async def _embeddings_client_gone(request: Request) -> bool:+ is_disconnected = getattr(request, "is_disconnected", None)+ if is_disconnected is None:+ return False+ try:+ return bool(await is_disconnected())+ except Exception: # noqa: BLE001 - a broken transport reads as still connected+ return False+++async def _resident_answers_embeddings(llama_backend, requested: str) -> bool:+ if not _resident_serves_embeddings(llama_backend):+ return False+ return await asyncio.to_thread(_loaded_satisfies, requested)+++def _reference_is_decisive(requested: str) -> bool:+ """Whether *requested* is evidence the caller meant a model held HERE.++ The same positive-evidence rule `_reject_unservable_model` applies: an explicit+ GGUF quant label, which no foreign id carries, or a repo actually on disk. A bare+ vendor id like `text-embedding-3-small` is neither, and must keep falling through+ rather than 404 -- that is what kept LiteLLM and OpenRouter style names working.+ """+ from core.inference.openai_auto_download import (+ looks_like_gguf_hub_repo_id,+ looks_like_quant,+ split_model_ref,+ )++ from core.rag.config import embedding_identity_model++ base, variant = split_model_ref(requested)+ # Shape alone, no index needed: an explicit quant label and a GGUF hub repo id are+ # references no vendor alias carries.+ if looks_like_quant(variant) or looks_like_gguf_hub_repo_id(base):+ return True+ # A tag names one exact vector space, and this runs only once _names_studio_embedder has+ # declined it -- so it is a space this server no longer holds. Answering from the current+ # one files two spaces under the identity the client asked us to distinguish.+ if embedding_identity_model(requested) is not None:+ return True+ try:+ from core.inference.local_model_resolver import (+ index_is_built,+ resolve_local_gguf,+ warm_index_soon,+ )+ from utils.paths import is_local_path++ if resolve_local_gguf(requested, allow_scan = False) is not None:+ return True+ if not index_is_built():+ # A cold index is missing evidence, not absence: warm it, and meanwhile do not+ # wave a name shaped like a local model through to a different embedding space.+ warm_index_soon()+ return is_local_path(requested) or "/" in requested+ return False+ except Exception: # noqa: BLE001 - a broken index cannot clear the name either+ return True+++async def _studio_embedder_request_body(request: Request) -> Optional[tuple[dict, str]]:+ try:+ body = await request.json()+ except (json.JSONDecodeError, ValueError):+ return None+ if not isinstance(body, dict):+ return None+ requested = body.get("model")+ if not isinstance(requested, str) or not requested.strip():+ return None+ model = await asyncio.to_thread(_names_studio_embedder, requested)+ return (body, model) if model else None+++async def _studio_embeddings(+ request: Request,+ body: dict,+ current_subject: str,+ model_name: Optional[str] = None,+) -> Response:+ from core.inference.llama_keepwarm import (+ untrack_admitted_inference,+ untrack_current_request,+ )+ from core.rag import config as rag_config+ from core.rag import embeddings as rag_embeddings++ # /embeddings is an _INFERENCE_SUFFIXES path that is not slot-excluded, so untrack before the+ # encode: a 2xx would otherwise claim the llama slot, block a preview swap for the whole+ # encode, and stamp the idle-unload TTL for a model that did no work.+ _scope = getattr(request, "scope", None)+ untrack_current_request(_scope)+ # The auto-switch hook admitted this request before the route knew it lands here, and the+ # preview busy guard reads the admitted tally, not _inflight.+ untrack_admitted_inference(_scope)++ texts = _embeddings_texts(body)+ encoding_format = body.get("encoding_format")+ if encoding_format is None:+ encoding_format = "float"+ dimensions = body.get("dimensions")+ model_name = model_name or rag_config.effective_embedding_model()+ label = _public_embedding_name(model_name)++ def _embed():+ # Every helper takes the model captured above: left to default they re-read the live+ # setting, mixing one model's limit and dimension with another's vectors.+ limit = rag_embeddings.max_tokens(model_name)+ count = rag_embeddings.token_counter(model_name)+ token_counts = [count(text) for text in texts]+ if limit and max(token_counts) > limit:+ raise HTTPException(+ status_code = 400,+ detail = f"'input' exceeds the {limit}-token limit of {label}.",+ )+ if dimensions is not None and dimensions != rag_embeddings.dim(model_name):+ raise HTTPException(+ status_code = 400, detail = f"'dimensions' is not supported by {label}."+ )+ # encode_with_identity: an ST failure swaps the process to llama-server mid-encode, and+ # reporting the configured name anyway files two spaces under one label.+ vectors, identity = rag_embeddings.encode_with_identity(+ texts, model_name = model_name, normalize = True+ )+ return vectors, identity, sum(token_counts), limit++ monitor_id = None+ if not getattr(request.state, "skip_api_monitor", False):+ monitor_id = api_monitor.start(+ endpoint = request.url.path,+ via_api_key = _request_used_api_key(request),+ method = request.method,+ model = label,+ prompt = _flatten_monitor_prompt(body.get("input", "")),+ subject = current_subject,+ )+ # The worker's lifetime, not the awaiting task's: cancelling does not stop the thread behind+ # to_thread, so releasing there admits the next request mid-encode and the cap stops meaning+ # anything. Same as _drain_pending_worker on the blocking-generation path.+ semaphore = _studio_embed_semaphore()+ acquire = asyncio.ensure_future(semaphore.acquire())+ try:+ while not acquire.done():+ await asyncio.wait({acquire}, timeout = 0.25)+ if not acquire.done() and await _embeddings_client_gone(request):+ raise asyncio.CancelledError()+ except asyncio.CancelledError:+ if not acquire.done():+ acquire.cancel()+ elif not acquire.cancelled() and acquire.exception() is None:+ semaphore.release()+ api_monitor.finish(monitor_id, "cancelled")+ raise+ try:+ gone = await _embeddings_client_gone(request)+ except asyncio.CancelledError:+ semaphore.release()+ api_monitor.finish(monitor_id, "cancelled")+ raise+ if gone:+ semaphore.release()+ api_monitor.finish(monitor_id, "cancelled")+ raise asyncio.CancelledError()+ worker = asyncio.ensure_future(asyncio.to_thread(_embed))++ def _release_embed_permit(finished) -> None:+ if not finished.cancelled():+ finished.exception() # retrieved so a cancelled request logs no "never retrieved"+ semaphore.release()++ worker.add_done_callback(_release_embed_permit)+ try:+ vectors, identity, prompt_tokens, limit = await asyncio.shield(worker)+ except asyncio.CancelledError:+ api_monitor.finish(monitor_id, "cancelled")+ raise+ except HTTPException as exc:+ api_monitor.fail(monitor_id, str(exc.detail))+ raise+ # The two conditions the caller can act on. _friendly_error, written for llama-server+ # transport faults, flattens both to "An internal error occurred", which behind an agent's+ # memory search is the whole diagnosis the user gets. 409: not wrong, not ready yet.+ except (+ rag_embeddings.EmbeddingModelDownloadRequiredError,+ rag_embeddings.UnsafeEmbeddingModelError,+ ) as exc:+ # The configured model may be a local path, which every other field here hashes.+ detail = str(exc).replace(repr(model_name), repr(label))+ api_monitor.fail(monitor_id, detail)+ raise HTTPException(status_code = 409, detail = detail) from exc+ except Exception as exc:+ api_monitor.fail(monitor_id, _friendly_error(exc))+ raise HTTPException(+ status_code = 502, detail = f"Embedding model failed: {_friendly_error(exc)}"+ ) from exc+ payload = {+ "object": "list",+ "data": [+ {+ "object": "embedding",+ "index": index,+ "embedding": _embedding_payload(vector, encoding_format),+ }+ for index, vector in enumerate(vectors)+ ],+ "model": _public_embedding_identity(identity, model_name, label),+ "usage": {"prompt_tokens": prompt_tokens, "total_tokens": prompt_tokens},+ }+ # limit is per text, prompt_tokens is the batch sum, so the monitor's gauge only means+ # anything for a single input; a batch would sit pinned at 100%.+ _monitor_openai_chunk(monitor_id, payload, limit if len(texts) == 1 else None)+ api_monitor.finish(monitor_id)+ return Response(content = json.dumps(payload), media_type = "application/json")++def _embeddings_input_present(body: dict) -> bool:"""Whether an embeddings body carries a usable ``input`` (non-empty)."""inp = body.get("input")@@ -25742,14 +26151,8 @@ def _embeddings_input_present(body: dict) -> bool:@router.post("/embeddings")async def openai_embeddings(request: Request, current_subject: str = Depends(get_current_subject)):- """- OpenAI-compatible embeddings endpoint.-- Proxies to the running llama-server's ``/v1/embeddings``. Only available- when a GGUF model is loaded.- Note: the loaded model must support pooling, else llama-server returns an- error (expected).- """+ """OpenAI-compatible embeddings: the resident embedding GGUF when one is loaded,+ else Studio's configured embedding model."""llama_backend = get_llama_cpp_backend()# Reject a request with no input before any automatic load so an invalid request never# swaps or reloads the resident model (as chat/responses/messages already validate before@@ -25770,6 +26173,7 @@ async def openai_embeddings(request: Request, current_subject: str = Depends(getraise HTTPException(status_code = 400, detail = "'input' must be a string or array.")if not _embeddings_input_present(_pre):raise HTTPException(status_code = 400, detail = "'input' is required for embeddings.")+ _embeddings_items(_pre, tokens_ok = True)elif _pre is not _UNPARSEABLE_BODY:# A valid JSON body that is not an object (e.g. [] or null) is rejected below as# "Request body must be a JSON object"; reject it here, before the switch, so the@@ -25781,15 +26185,39 @@ async def openai_embeddings(request: Request, current_subject: str = Depends(get# no reliable pre-load probe -- is_embedding_model keys on a sentence-transformers# modules.json a bare .gguf never has -- so embeddings auto-switch is best-effort:# a non-embedding target switches, then llama-server returns a no-pooling error.+ studio_request = await _studio_embedder_request_body(request)+ if studio_request is not None and not await _resident_answers_embeddings(+ llama_backend, studio_request[0]["model"]+ ):+ return await _studio_embeddings(+ request, studio_request[0], current_subject, model_name = studio_request[1]+ )+ if studio_request is None and _resident_absent(llama_backend):+ default_body = await _default_embeddings_request_body(request)+ if default_body is not None and not await asyncio.to_thread(_stashed_gguf_embeds):+ return await _studio_embeddings(request, default_body, current_subject)body = await _auto_switch_from_request_body(request, current_subject, gguf_only = True)if not llama_backend.is_loaded:- _status, _detail = await _no_model_loaded_error(- "No GGUF model loaded. Load a GGUF model first.",- _raw_body_model(body),- request,- status = 503,- )- raise HTTPException(status_code = _status, detail = _detail)+ # With the slot empty _reject_unservable_model defers to _no_model_loaded_error, so+ # without this the fallback would answer a decisive repo:QUANT this server does not+ # hold from the Settings embedder: another space under the caller's name (#7454).+ # `{"model": 123}` gets this far: both pre-switch readers decline a non-string selector,+ # and the alias lookup below strips whatever it is handed.+ _named = _raw_body_model(body)+ if not isinstance(_named, str):+ _named = None+ if _named and not await asyncio.to_thread(_names_studio_embedder, _named):+ _decisive = await asyncio.to_thread(_reference_is_decisive, _named)+ else:+ _decisive = False+ if _decisive or not isinstance(body, dict):+ _status, _detail = await _no_model_loaded_error(+ "No GGUF model loaded. Load a GGUF model first.",+ _named,+ request,+ status = 503,+ )+ raise HTTPException(status_code = _status, detail = _detail)if not isinstance(body, dict):# Re-read to re-raise a malformed-body error (post-503, pre-feature behavior);# a valid non-dict body such as a list is a clean 400 rather than a 500.@@ -25797,6 +26225,9 @@ async def openai_embeddings(request: Request, current_subject: str = Depends(getif not isinstance(body, dict):raise HTTPException(status_code = 400, detail = "Request body must be a JSON object")+ if not _resident_serves_embeddings(llama_backend):+ return await _studio_embeddings(request, body, current_subject)+# GGUF is loaded and the body is valid. The middleware claims the slot on a successful# 2xx, so no claim here: llama-server can still return a non-2xx for a valid body (e.g. a# no-pooling error on /v1/embeddings against a non-embedding GGUF), so claiming before thediff --git a/studio/backend/tests/test_offline_embedding_minimal.py b/studio/backend/tests/test_offline_embedding_minimal.pyindex 4162d21440a..d79a137988b 100644--- a/studio/backend/tests/test_offline_embedding_minimal.py+++ b/studio/backend/tests/test_offline_embedding_minimal.py@@ -1022,7 +1022,9 @@ def test_get_offline_uncached_uses_local_files_only(tmp_path, monkeypatch):monkeypatch.setattr(embeddings, "_install_torchao_stub_once", lambda: None)monkeypatch.setattr(embeddings, "_device", lambda: "cpu")# No cache -> repo-id load forced cache-only (fails fast offline, not a hang).- monkeypatch.setattr(embeddings, "_guard_model_security", lambda name, local_only = False: None)+ monkeypatch.setattr(+ embeddings, "_guard_model_security", lambda name, local_only = False, display = None: None+ )captured = {}_install_fake_sentence_transformers(monkeypatch, captured)embeddings._get("org/uncached-xyz")@@ -1040,7 +1042,9 @@ def test_get_online_omits_local_files_only(monkeypatch):monkeypatch.setattr(embeddings, "_install_torchao_stub_once", lambda: None)monkeypatch.setattr(embeddings, "_device", lambda: "cpu")# Isolate the loader wiring from the online guard's network calls.- monkeypatch.setattr(embeddings, "_guard_model_security", lambda name, local_only = False: None)+ monkeypatch.setattr(+ embeddings, "_guard_model_security", lambda name, local_only = False, display = None: None+ )captured = {}_install_fake_sentence_transformers(monkeypatch, captured)embeddings._get("org/online")@@ -1092,7 +1096,9 @@ def test_get_online_loads_the_snapshot_settings_called_cached(hf_cache, monkeypamonkeypatch.setattr(embeddings, "_name", None, raising = False)monkeypatch.setattr(embeddings, "_install_torchao_stub_once", lambda: None)monkeypatch.setattr(embeddings, "_device", lambda: "cpu")- monkeypatch.setattr(embeddings, "_guard_model_security", lambda name, local_only = False: None)+ monkeypatch.setattr(+ embeddings, "_guard_model_security", lambda name, local_only = False, display = None: None+ )captured = {}_install_fake_sentence_transformers(monkeypatch, captured)@@ -1112,7 +1118,9 @@ def test_an_uncached_model_online_still_loads_by_repo_id(monkeypatch):monkeypatch.setattr(embeddings, "_name", None, raising = False)monkeypatch.setattr(embeddings, "_install_torchao_stub_once", lambda: None)monkeypatch.setattr(embeddings, "_device", lambda: "cpu")- monkeypatch.setattr(embeddings, "_guard_model_security", lambda name, local_only = False: None)+ monkeypatch.setattr(+ embeddings, "_guard_model_security", lambda name, local_only = False, display = None: None+ )captured = {}_install_fake_sentence_transformers(monkeypatch, captured)@@ -1338,7 +1346,9 @@ def test_a_pending_transfer_does_not_make_the_security_scan_offline(monkeypatch,monkeypatch.setattr(embeddings,"_guard_model_security",- lambda target, local_only = False: seen.update(target = target, local_only = local_only),+ lambda target, local_only = False, display = None: seen.update(+ target = target, local_only = local_only+ ),)# Pending, but the Hub is reachable.monkeypatch.setattr(ems, "get_stored_download_pending", lambda m: True)diff --git a/studio/backend/tests/test_openai_embeddings_studio_fallback.py b/studio/backend/tests/test_openai_embeddings_studio_fallback.pynew file mode 100644index 00000000000..a074c1c0d9e--- /dev/null+++ b/studio/backend/tests/test_openai_embeddings_studio_fallback.py@@ -0,0 +1,1405 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import asyncio+import base64+import os+import sys+import threading+from types import SimpleNamespace++import numpy as np+import pytest++_backend = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+if _backend not in sys.path:+ sys.path.insert(0, _backend)++inference_route = pytest.importorskip("routes.inference", reason = "inference stack not installed")+rag_embeddings = pytest.importorskip("core.rag.embeddings", reason = "rag stack not installed")+rag_config = pytest.importorskip("core.rag.config", reason = "rag stack not installed")++MODEL = "unsloth/bge-small-en-v1.5"+IDENTITY = f"sentence-transformers:{MODEL}"+++class _Request:+ def __init__(self, body):+ self._body = body+ self.method = "POST"+ self.url = SimpleNamespace(path = "/v1/embeddings")+ self.state = SimpleNamespace(skip_api_monitor = True)+ self.scope = {"type": "http", "path": "/v1/embeddings"}++ async def json(self):+ return self._body++ async def is_disconnected(self):+ return False+++def _vectors(texts, **_):+ return np.array([[1.0, 0.0]] * len(texts), dtype = np.float32)+++def studio_embedder(monkeypatch):+ async def passthrough(request, current_subject, **_kwargs):+ return await request.json()++ def no_proxy():+ raise AssertionError("the llama-server proxy must not run")++ monkeypatch.setattr(inference_route, "_should_validate_before_switch", lambda: False)+ monkeypatch.setattr(inference_route, "_auto_switch_from_request_body", passthrough)+ monkeypatch.setattr(inference_route, "_cancelable_nonstreaming_client", no_proxy)+ monkeypatch.setattr(rag_config, "effective_embedding_model", lambda: MODEL)+ monkeypatch.setattr(rag_embeddings, "encode", _vectors)++ def _encode_with_identity(+ texts,+ *,+ model_name = None,+ normalize = True,+ ):+ # Delegates to the test's encode, so a blowing-up or blocking one still drives the route.+ return rag_embeddings.encode(texts, model_name = model_name, normalize = normalize), IDENTITY++ monkeypatch.setattr(rag_embeddings, "encode_with_identity", _encode_with_identity)+ monkeypatch.setattr(rag_embeddings, "token_counter", lambda model_name = None: len)+ monkeypatch.setattr(rag_embeddings, "max_tokens", lambda model_name = None: None)+ monkeypatch.setattr(rag_embeddings, "dim", lambda model_name = None: 2)+ return monkeypatch+++def _call(body):+ response = asyncio.run(inference_route.openai_embeddings(_Request(body), "tester"))+ assert response.status_code == 200+ import json++ return json.loads(response.body)+++def _http_error(body):+ from fastapi import HTTPException+ with pytest.raises(HTTPException) as exc:+ asyncio.run(inference_route.openai_embeddings(_Request(body), "tester"))+ return exc.value+++def test_nothing_loaded_serves_from_the_studio_embedder(studio_embedder):+ studio_embedder.setattr(+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)+ )+ payload = _call({"input": ["alpha", "beta"], "model": "text-embedding-3-small"})+ assert payload["object"] == "list"+ assert payload["model"] == IDENTITY+ assert [row["index"] for row in payload["data"]] == [0, 1]+ assert payload["data"][0]["embedding"] == [1.0, 0.0]+ assert payload["usage"] == {"prompt_tokens": 9, "total_tokens": 9}+++def test_chat_model_loaded_serves_from_the_studio_embedder(studio_embedder):+ started = []+ studio_embedder.setattr(+ inference_route,+ "get_llama_cpp_backend",+ lambda: SimpleNamespace(is_loaded = True, is_embedding_gguf = False),+ )+ studio_embedder.setattr(+ inference_route, "_direct_llama_request_started", lambda: started.append(1)+ )+ payload = _call({"input": "alpha"})+ assert payload["model"] == IDENTITY+ assert started == []+++def test_resident_embedding_gguf_still_uses_the_proxy(studio_embedder):+ import httpx++ class _Client:+ async def post(self, *_args, **_kwargs):+ return httpx.Response(200, json = {"data": [{"embedding": [0.5]}]})++ async def aclose(self):+ return None++ def boom(*_args, **_kwargs):+ raise AssertionError("the studio embedder must not run")++ studio_embedder.setattr(rag_embeddings, "encode", boom)+ studio_embedder.setattr(inference_route, "_cancelable_nonstreaming_client", _Client)+ studio_embedder.setattr(+ inference_route,+ "get_llama_cpp_backend",+ lambda: SimpleNamespace(+ is_loaded = True,+ is_embedding_gguf = True,+ base_url = "http://llama.test",+ context_length = 512,+ model_identifier = "org/E-GGUF",+ ),+ )+ payload = _call({"input": "alpha", "model": "org/E-GGUF"})+ assert payload == {"data": [{"embedding": [0.5]}]}+++def test_base64_encoding_format(studio_embedder):+ studio_embedder.setattr(+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)+ )+ payload = _call({"input": "alpha", "encoding_format": "base64"})+ raw = base64.b64decode(payload["data"][0]["embedding"])+ assert np.frombuffer(raw, dtype = np.float32).tolist() == [1.0, 0.0]+++ "body",+ [+ {"input": ""},+ {"input": []},+ {"input": [[1, 2, 3]]},+ {"input": ["alpha", ""]},+ {"input": "alpha", "encoding_format": "int8"},+ {"input": "alpha", "encoding_format": ""},+ {"input": "alpha", "encoding_format": False},+ {"input": "alpha", "encoding_format": 0},+ {"input": "alpha", "dimensions": 999},+ ],+)+def test_invalid_requests_are_400(studio_embedder, body):+ studio_embedder.setattr(+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)+ )+ assert _http_error(body).status_code == 400+++def test_over_length_input_is_rejected_not_truncated(studio_embedder):+ studio_embedder.setattr(+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)+ )+ studio_embedder.setattr(rag_embeddings, "max_tokens", lambda model_name = None: 3)+ error = _http_error({"input": ["ab", "abcd"]})+ assert error.status_code == 400+ assert "3-token limit" in error.detail+++def test_embedder_failure_is_502(studio_embedder):+ def boom(*_args, **_kwargs):+ raise RuntimeError("llama-server embedder POST /v1/embeddings -> 500")++ studio_embedder.setattr(+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)+ )+ studio_embedder.setattr(rag_embeddings, "encode", boom)+ assert _http_error({"input": "alpha"}).status_code == 502+++def test_identity_redaction_leaves_the_backend_tag_alone(tmp_path, monkeypatch):+ """A local model whose name occurs inside the backend tag.++ `transformers` is a real directory people have lying around, and is_local_path accepts a+ bare relative name that exists. A substring replace then rewrites the+ `sentence-transformers` tag too, and the identity we advertise is one no backend claims+ and _names_studio_embedder cannot match on the next request.+ """+ from core.rag import config as rag_config++ (tmp_path / "transformers").mkdir()+ monkeypatch.chdir(tmp_path)+ monkeypatch.setattr(+ inference_route,+ "_public_embedding_name",+ lambda name: "transformers-deadbeef" if name == "transformers" else name,+ )+ monkeypatch.setattr(+ rag_config, "effective_gguf_repo_for_embedding_model", lambda model: "transformers-GGUF"+ )+ public = inference_route._public_embedding_identity(+ "sentence-transformers:transformers", "transformers", "transformers-deadbeef"+ )+ assert public == "sentence-transformers:transformers-deadbeef"+ assert rag_config.embedding_identity_model(public) == "transformers-deadbeef"+++def test_identity_redaction_covers_the_gguf_repo_segment(monkeypatch):+ from core.rag import config as rag_config++ monkeypatch.setattr(+ rag_config, "effective_gguf_repo_for_embedding_model", lambda model: "/models/bge-GGUF"+ )+ monkeypatch.setattr(+ inference_route,+ "_public_embedding_name",+ lambda name: {"/models/bge": "bge-1111", "/models/bge-GGUF": "bge-GGUF-2222"}.get(+ name, name+ ),+ )+ public = inference_route._public_embedding_identity(+ "llama-server:%2Fmodels%2Fbge:%2Fmodels%2Fbge-GGUF".replace("%2F", "/"),+ "/models/bge",+ "bge-1111",+ )+ assert public == "llama-server:bge-1111:bge-GGUF-2222"+++def test_identity_redaction_covers_a_local_repo_under_a_hub_model(monkeypatch):+ """A hub model id whose GGUF companion is a local path.++ RAG_EMBED_GGUF_REPO takes any path, so the model segment can need no redaction while the+ repo segment beside it is an absolute path on the server.+ """+ from core.rag import config as rag_config++ monkeypatch.setattr(+ rag_config,+ "effective_gguf_repo_for_embedding_model",+ lambda model: "/srv/private/embed.gguf",+ )+ monkeypatch.setattr(+ inference_route,+ "_public_embedding_name",+ lambda name: "embed.gguf-3333" if name == "/srv/private/embed.gguf" else name,+ )+ public = inference_route._public_embedding_identity(+ f"llama-server:{MODEL}:/srv/private/embed.gguf", MODEL, MODEL+ )+ assert public == f"llama-server:{MODEL}:embed.gguf-3333"+ assert "/srv/private" not in public++[email protected]("named", [123, 1.5, True, ["a"], {"a": 1}])+def test_a_non_string_model_is_not_a_server_error(studio_embedder, named):+ """Both pre-switch readers decline a non-string selector, so the empty-slot branch sees it."""+ studio_embedder.setattr(+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)+ )+ payload = _call({"input": "alpha", "model": named})+ assert payload["model"] == IDENTITY+++ "exc,fragment",+ [+ (+ rag_embeddings.EmbeddingModelDownloadRequiredError(+ "Embedding model 'unsloth/bge-small-en-v1.5' is not downloaded yet. "+ "Finish its Settings download before indexing documents."+ ),+ "not downloaded yet",+ ),+ (+ rag_embeddings.UnsafeEmbeddingModelError(+ "Embedding model 'evil/model' requires remote code execution."+ ),+ "remote code",+ ),+ ],+)+def test_actionable_embedder_errors_keep_their_message(studio_embedder, exc, fragment):+ """The caller can fix these two, but only if it is told what they are.++ Picking a model in Settings and querying before its download finishes is the common+ one, and it reaches this route through `unsloth start openclaw`: the generated+ openclaw.json points memory search here, so this detail string is the entire+ diagnosis the user sees. Routed through _friendly_error -- written for llama-server+ transport faults -- it became a 502 "An internal error occurred", which says nothing+ and suggests the wrong owner.+ """++ def boom(*_args, **_kwargs):+ raise exc++ studio_embedder.setattr(+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)+ )+ studio_embedder.setattr(rag_embeddings, "encode", boom)+ error = _http_error({"input": "alpha"})+ assert error.status_code == 409+ assert fragment in error.detail+ assert "An internal error occurred" not in error.detail+++def test_pending_gguf_download_is_classified_like_the_st_one(tmp_path, monkeypatch):+ """The GGUF backend hits the same condition and used to raise a bare RuntimeError.++ Both backends refuse the same way when the picker's download has not finished, so both+ have to reach the 409; untyped, this one fell into the catch-all and came back as a 502+ saying only that an internal error occurred.+ """+ from core.rag import embed_llama_server++ backend = embed_llama_server.LlamaServerBackend()+ monkeypatch.setattr(+ embed_llama_server.config,+ "effective_gguf_repo_for_embedding_model",+ lambda model: "unsloth/bge-small-en-v1.5-GGUF",+ )+ monkeypatch.setattr(backend, "_resolve_local_gguf", lambda model: None)+ monkeypatch.setattr(backend, "_planned_family_path", lambda model, repo: None)+ monkeypatch.setattr(backend, "_resolve_cached_gguf", lambda *a, **k: None)+ import utils.embedding_model_settings as ems++ monkeypatch.setattr(ems, "get_stored_download_pending", lambda model: True)++ with pytest.raises(rag_embeddings.EmbeddingModelDownloadRequiredError) as excinfo:+ backend._resolve_model_path("unsloth/bge-small-en-v1.5")+ assert "not downloaded yet" in str(excinfo.value)+++def test_actionable_errors_do_not_leak_a_local_path(studio_embedder, monkeypatch):+ """Passing the message through must not undo the redaction the rest of the route does.++ _get resolves a cached model to its absolute snapshot directory and hands that to+ _guard_model_security, so the raised text can carry the server's HF cache layout, and a+ configured model can be a local path in its own right. Both go out over /v1/embeddings,+ where the identity and the limit errors already report the hashed label instead.+ """+ studio_embedder.setattr(+ rag_config, "effective_embedding_model", lambda: "/srv/models/bge-small"+ )++ def boom(*_args, **_kwargs):+ raise rag_embeddings.UnsafeEmbeddingModelError(+ "Embedding model '/srv/models/bge-small' is flagged as unsafe by Hugging Face's "+ "security scan; refusing to load. Set a different RAG embedding model."+ )++ studio_embedder.setattr(+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)+ )+ studio_embedder.setattr(rag_embeddings, "encode", boom)+ error = _http_error({"input": "alpha"})+ assert error.status_code == 409+ assert "/srv/models" not in error.detail+ assert "flagged as unsafe" in error.detail+ assert "bge-small-" in error.detail # the hashed public label, not the path+++def test_guard_model_security_names_the_configured_model_not_the_snapshot(monkeypatch):+ """The scan needs the snapshot path; the message must not carry it."""+ monkeypatch.setattr(rag_embeddings, "_ambient_hf_token", lambda: None)+ import utils.security as security++ monkeypatch.setattr(+ security,+ "evaluate_file_security",+ lambda *a, **k: SimpleNamespace(blocked = True),+ )+ monkeypatch.setattr(security, "security_load_subdirs", lambda *a, **k: ())+ monkeypatch.setattr(rag_embeddings, "_st_module_subdirs", lambda *a, **k: ())+ with pytest.raises(rag_embeddings.UnsafeEmbeddingModelError) as excinfo:+ rag_embeddings._guard_model_security(+ "/home/me/.cache/huggingface/hub/models--org--bge/snapshots/abc123",+ display = "org/bge",+ )+ assert "org/bge" in str(excinfo.value)+ assert ".cache/huggingface" not in str(excinfo.value)+++def test_studio_embedder_requests_are_admission_limited(studio_embedder):+ lock = threading.Lock()+ gate = threading.Event()+ active = {"now": 0, "peak": 0}++ def encode(texts, **_kwargs):+ with lock:+ active["now"] += 1+ active["peak"] = max(active["peak"], active["now"])+ gate.wait(timeout = 5)+ with lock:+ active["now"] -= 1+ return _vectors(texts)++ studio_embedder.setattr(+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)+ )+ studio_embedder.setattr(rag_embeddings, "encode", encode)++ async def run():+ tasks = [+ asyncio.create_task(inference_route.openai_embeddings(_Request({"input": "x"}), "t"))+ for _ in range(inference_route._STUDIO_EMBED_CONCURRENCY + 2)+ ]+ for _ in range(500):+ await asyncio.sleep(0.01)+ if active["now"] == inference_route._STUDIO_EMBED_CONCURRENCY:+ break+ assert active["now"] == inference_route._STUDIO_EMBED_CONCURRENCY+ gate.set()+ responses = await asyncio.gather(*tasks)+ assert all(response.status_code == 200 for response in responses)++ asyncio.run(run())+ assert active["peak"] == inference_route._STUDIO_EMBED_CONCURRENCY+++def test_studio_fallback_untracks_the_request_from_the_llama_slot(studio_embedder):+ # An _INFERENCE_SUFFIXES path not in _NON_LLM_SLOT_SUFFIXES: a 2xx reaching _finish claims the+ # llama slot the studio embedder never touched, so it untracks first, as the chat branch does.+ from core.inference import llama_keepwarm as kw++ studio_embedder.setattr(+ inference_route,+ "get_llama_cpp_backend",+ lambda: SimpleNamespace(is_loaded = True, is_embedding_gguf = False),+ )+ request = _Request({"input": "alpha"})+ before = kw._inflight+ kw._inflight = before + 1+ try:+ response = asyncio.run(inference_route.openai_embeddings(request, "tester"))+ assert response.status_code == 200+ # Set => _finish() skips both _claim_non_preview_slot() and the activity stamp.+ assert request.scope.get(kw._UNTRACKED_SCOPE_KEY) is True+ assert kw._inflight == before+ finally:+ kw._inflight = before+++def test_resident_embedding_gguf_still_claims_the_slot(studio_embedder):+ # The proxy path DOES run against the resident GGUF, so it must stay tracked.+ import httpx++ class _Client:+ async def post(self, *_args, **_kwargs):+ return httpx.Response(200, json = {"data": [{"embedding": [0.5]}]})++ async def aclose(self):+ return None++ from core.inference import llama_keepwarm as kw++ studio_embedder.setattr(inference_route, "_cancelable_nonstreaming_client", _Client)+ studio_embedder.setattr(+ inference_route,+ "get_llama_cpp_backend",+ lambda: SimpleNamespace(+ is_loaded = True,+ is_embedding_gguf = True,+ base_url = "http://llama.test",+ context_length = 512,+ model_identifier = "org/E-GGUF",+ ),+ )+ request = _Request({"input": "alpha", "model": "org/E-GGUF"})+ asyncio.run(inference_route.openai_embeddings(request, "tester"))+ assert request.scope.get(kw._UNTRACKED_SCOPE_KEY) is None+++def test_context_gauge_is_not_pinned_by_a_batch(studio_embedder):+ # api_monitor divides the batch's summed prompt_tokens by that 3rd arg, so passing the+ # per-text limit reports 100% context use for a batch that used a fraction per text.+ seen = []+ studio_embedder.setattr(+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)+ )+ studio_embedder.setattr(rag_embeddings, "max_tokens", lambda model_name = None: 8)+ studio_embedder.setattr(+ inference_route,+ "_monitor_openai_chunk",+ lambda monitor_id, payload, context_length = None, **_: seen.append(context_length),+ )+ request = _Request({"input": ["alpha", "beta", "gamma"]})+ request.state.skip_api_monitor = False+ asyncio.run(inference_route.openai_embeddings(request, "tester"))+ assert seen == [None]++ seen.clear()+ request = _Request({"input": "alpha"})+ request.state.skip_api_monitor = False+ asyncio.run(inference_route.openai_embeddings(request, "tester"))+ assert seen == [8]+++def test_cancelled_requests_do_not_leak_admission_permits(studio_embedder):+ # to_thread cannot cancel the worker, so releasing on the awaiting task's cancellation lets+ # the next batch in while the old threads still embed and the cap stops holding.+ lock = threading.Lock()+ gate = threading.Event()+ active = {"now": 0, "peak": 0}++ def encode(texts, **_kwargs):+ with lock:+ active["now"] += 1+ active["peak"] = max(active["peak"], active["now"])+ gate.wait(timeout = 5)+ with lock:+ active["now"] -= 1+ return _vectors(texts)++ studio_embedder.setattr(+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)+ )+ studio_embedder.setattr(rag_embeddings, "encode", encode)+ cap = inference_route._STUDIO_EMBED_CONCURRENCY++ async def run():+ first = [+ asyncio.create_task(inference_route.openai_embeddings(_Request({"input": "x"}), "t"))+ for _ in range(cap)+ ]+ for _ in range(500):+ await asyncio.sleep(0.01)+ if active["now"] == cap:+ break+ assert active["now"] == cap++ # Every in-flight request goes away, but its thread is still inside encode().+ for task in first:+ task.cancel()+ await asyncio.gather(*first, return_exceptions = True)++ second = [+ asyncio.create_task(inference_route.openai_embeddings(_Request({"input": "y"}), "t"))+ for _ in range(cap)+ ]+ await asyncio.sleep(0.3)+ # The permits are still held by the running threads, so nothing new started.+ assert active["peak"] == cap, f"admission cap exceeded: peak={active['peak']}"+ gate.set()+ await asyncio.gather(*second, return_exceptions = True)++ asyncio.run(run())+ assert active["peak"] == cap+++def test_reported_model_follows_the_backend_that_made_the_vectors(studio_embedder):+ # An ST failure swaps the process to llama-server, a different space: the response has to+ # name the space the vectors are in, or a client files two under one label.+ llama_identity = f"llama-server:{MODEL}:unsloth/bge-small-en-v1.5-GGUF"+ studio_embedder.setattr(+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)+ )+ studio_embedder.setattr(+ rag_embeddings,+ "encode_with_identity",+ lambda texts, **_kwargs: (_vectors(texts), llama_identity),+ )+ payload = _call({"input": "alpha"})+ assert payload["model"] == llama_identity+++def test_embedding_helpers_are_pinned_to_the_captured_model(studio_embedder):+ # A Settings change while queued must not mix one model's limit/dim with another's vectors.+ seen = {"max_tokens": [], "token_counter": [], "dim": [], "encode": []}+ studio_embedder.setattr(+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)+ )+ studio_embedder.setattr(+ rag_embeddings,+ "max_tokens",+ lambda model_name = None: (seen["max_tokens"].append(model_name), None)[1],+ )+ studio_embedder.setattr(+ rag_embeddings,+ "token_counter",+ lambda model_name = None: (seen["token_counter"].append(model_name), len)[1],+ )+ studio_embedder.setattr(+ rag_embeddings,+ "dim",+ lambda model_name = None: (seen["dim"].append(model_name), 2)[1],+ )+ studio_embedder.setattr(+ rag_embeddings,+ "encode_with_identity",+ lambda texts, **kw: (+ seen["encode"].append(kw.get("model_name")),+ (_vectors(texts), IDENTITY),+ )[1],+ )+ _call({"input": "alpha", "dimensions": 2})+ assert seen == {+ "max_tokens": [MODEL],+ "token_counter": [MODEL],+ "dim": [MODEL],+ "encode": [MODEL],+ }+++def test_studio_fallback_releases_the_preview_busy_guard(studio_embedder):+ # Admission happens before the route decides how to serve, and load_model_for_preview reads+ # the admitted tally, not _inflight, so a slow encode keeps 503ing preview swaps.+ from core.inference import llama_keepwarm as kw++ studio_embedder.setattr(+ inference_route,+ "get_llama_cpp_backend",+ lambda: SimpleNamespace(is_loaded = True, is_embedding_gguf = False),+ )+ request = _Request({"input": "alpha"})+ # Relative: the tally is module state shared with the suite, so absolutes need this file alone.+ before = kw.other_admitted_inference_count()+ kw.note_admitted_inference(request.scope)+ assert kw.other_admitted_inference_count() == before + 1+ assert request.scope.get(kw._ADMITTED_SCOPE_KEY) is True+ try:+ asyncio.run(inference_route.openai_embeddings(request, "tester"))+ assert kw.other_admitted_inference_count() == before+ # Popped, so the middleware's finally cannot decrement the tally a second time.+ assert request.scope.get(kw._ADMITTED_SCOPE_KEY) is None+ finally:+ kw._admitted_inference = before+++def test_cancelled_request_closes_its_monitor_row(studio_embedder):+ # start() runs before admission and running rows dodge retention trimming, so a request+ # cancelled while queued has to close its own row.+ closed = []+ gate = threading.Event()+ studio_embedder.setattr(+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)+ )+ studio_embedder.setattr(inference_route.api_monitor, "start", lambda **_kwargs: "entry-1")+ studio_embedder.setattr(+ inference_route.api_monitor,+ "finish",+ lambda entry_id, *args, **_kw: closed.append((entry_id, args[0] if args else None)),+ )+ studio_embedder.setattr(+ rag_embeddings,+ "encode_with_identity",+ lambda texts, **_kw: (gate.wait(timeout = 5), (_vectors(texts), IDENTITY))[1],+ )++ async def run():+ request = _Request({"input": "alpha"})+ request.state.skip_api_monitor = False+ task = asyncio.create_task(inference_route.openai_embeddings(request, "tester"))+ await asyncio.sleep(0.1)+ task.cancel()+ with pytest.raises(asyncio.CancelledError):+ await task+ gate.set()++ asyncio.run(run())+ assert closed == [("entry-1", "cancelled")]+++def _identity_names(monkeypatch):+ monkeypatch.setattr(+ rag_config, "effective_gguf_repo_for_embedding_model", lambda model: f"{model}-GGUF"+ )+ monkeypatch.setattr(rag_embeddings, "embedding_identity", lambda model_name = None: IDENTITY)++[email protected]("requested", [MODEL, f"{MODEL}-GGUF", IDENTITY, MODEL.upper()])+def test_naming_the_configured_embedder_skips_the_chat_slot_check(studio_embedder, requested):+ from fastapi import HTTPException++ async def reject(request, current_subject, **_kwargs):+ raise HTTPException(status_code = 404, detail = "model_not_found")++ _identity_names(studio_embedder)+ studio_embedder.setattr(inference_route, "_auto_switch_from_request_body", reject)+ studio_embedder.setattr(+ inference_route,+ "get_llama_cpp_backend",+ lambda: SimpleNamespace(is_loaded = True, is_embedding_gguf = False),+ )+ payload = _call({"input": "alpha", "model": requested})+ assert payload["model"] == IDENTITY+ assert payload["data"][0]["embedding"] == [1.0, 0.0]+++def test_other_model_names_still_run_auto_switch(studio_embedder):+ seen = []++ async def record(request, current_subject, **_kwargs):+ seen.append(current_subject)+ return await request.json()++ _identity_names(studio_embedder)+ studio_embedder.setattr(inference_route, "_auto_switch_from_request_body", record)+ studio_embedder.setattr(+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)+ )+ _call({"input": "alpha", "model": "text-embedding-3-small"})+ assert seen == ["tester"]+++def test_st_max_tokens_leaves_room_for_the_special_tokens(monkeypatch):+ model = SimpleNamespace(+ max_seq_length = 512,+ tokenizer = SimpleNamespace(num_special_tokens_to_add = lambda: 2),+ )+ monkeypatch.setattr(rag_embeddings, "_get", lambda model_name = None: model)+ assert rag_embeddings._SentenceTransformersBackend().max_tokens() == 510+++def _gguf(tmp_path, entries):+ import io+ import struct++ buf = io.BytesIO()+ buf.write(b"GGUF" + struct.pack("<IQQ", 3, 0, len(entries)))+ for key, vtype, value in entries:+ raw = key.encode()+ buf.write(struct.pack("<Q", len(raw)) + raw + struct.pack("<I", vtype))+ if vtype == 8:+ raw = value.encode()+ buf.write(struct.pack("<Q", len(raw)) + raw)+ elif vtype == 9:+ elem_type, items = value+ buf.write(struct.pack("<IQ", elem_type, len(items)))+ for item in items:+ raw = item.encode()+ buf.write(struct.pack("<Q", len(raw)) + raw)+ elif vtype == 4:+ buf.write(struct.pack("<I", value))+ elif vtype == 10:+ buf.write(struct.pack("<Q", value))+ path = tmp_path / "embedder.gguf"+ path.write_bytes(buf.getvalue())+ return str(path)+++def test_gguf_context_length_is_read_from_the_header(tmp_path):+ from core.rag import embed_llama_server++ path = _gguf(+ tmp_path,+ [+ ("general.architecture", 8, "bert"),+ ("tokenizer.ggml.tokens", 9, (8, ["[PAD]", "[CLS]", "hello"])),+ ("llama.context_length", 4, 4096),+ ("bert.context_length", 10, 512),+ ],+ )+ assert embed_llama_server._gguf_context_length(path) == 512+ assert embed_llama_server._gguf_context_length(str(tmp_path / "missing.gguf")) is None+++def test_llama_max_tokens_comes_from_the_gguf_minus_its_special_tokens(tmp_path, monkeypatch):+ from core.rag import embed_llama_server++ backend = embed_llama_server.LlamaServerBackend()+ backend._model_path = _gguf(+ tmp_path, [("general.architecture", 8, "bert"), ("bert.context_length", 4, 512)]+ )+ monkeypatch.setattr(backend, "_ensure_ready", lambda model_name = None: None)+ # Larger than the header, so the GGUF is still what binds; confirmed, so it caches.+ monkeypatch.setattr(backend, "_server_context", lambda: 4096)+ monkeypatch.setattr(backend, "_server_batch", lambda: 4096)+ posts = []++ def post(+ path,+ payload,+ model_name = None,+ ):+ posts.append((path, payload))+ return {"tokens": [101, 102]}++ monkeypatch.setattr(backend, "_post", post)+ assert backend.max_tokens() == 510+ assert backend.max_tokens() == 510+ assert posts == [("/tokenize", {"content": "", "add_special": True})]+ backend._adopt_model_path(backend._model_path, "unsloth/other-GGUF")+ assert backend._max_tokens is None+++def test_llama_max_tokens_is_dropped_when_the_binary_is_swapped(tmp_path, monkeypatch):+ """Changing the custom llama.cpp path respawns onto the same GGUF.++ _current() goes stale on the binary revision, so _ensure_ready kills and respawns, but+ _resolve_model_path takes its same-repo fast path and never reaches _adopt_model_path.+ The limit is not a pure model fact -- it clamps the GGUF context by the server's n_ctx+ and n_ubatch, whose defaults differ between builds -- so it has to go with the binary.+ """+ from core.rag import embed_llama_server++ backend = embed_llama_server.LlamaServerBackend()+ backend._model_path = _gguf(+ tmp_path, [("general.architecture", 8, "bert"), ("bert.context_length", 4, 512)]+ )+ backend._model_repo = "unsloth/bge-small-en-v1.5-GGUF"+ backend._max_tokens = 510+ backend._binary = "/opt/old/llama-server"+ backend._binary_path_revision = 1+ monkeypatch.setattr(backend, "_process_alive", lambda: True)+ monkeypatch.setattr(backend, "_kill_process", lambda: None)+ monkeypatch.setattr(backend, "_spawn", lambda model_name = None: None)+ monkeypatch.setattr(+ embed_llama_server.config,+ "effective_gguf_repo_for_embedding_model",+ lambda model: "unsloth/bge-small-en-v1.5-GGUF",+ )+ monkeypatch.setattr(+ embed_llama_server.config,+ "effective_embedding_model",+ lambda: "unsloth/bge-small-en-v1.5",+ )+ import utils.llama_cpp_path_settings as path_settings++ monkeypatch.setattr(path_settings, "custom_llama_cpp_path_revision", lambda: 2)++ backend._ensure_ready()++ assert backend._max_tokens is None+ assert backend._model_path is not None # the GGUF itself did not change+++def test_st_max_tokens_reserves_the_default_prompt(monkeypatch):+ tokenizer = SimpleNamespace(+ num_special_tokens_to_add = lambda: 2,+ encode = lambda text, add_special_tokens = False: text.split(),+ )+ model = SimpleNamespace(+ max_seq_length = 512,+ tokenizer = tokenizer,+ prompts = {"query": "Represent this sentence for retrieval:"},+ default_prompt_name = "query",+ )+ monkeypatch.setattr(rag_embeddings, "_get", lambda model_name = None: model)+ assert rag_embeddings._SentenceTransformersBackend().max_tokens() == 512 - 2 - 5+++def test_llama_max_tokens_is_capped_by_the_running_context(tmp_path, monkeypatch):+ from core.rag import embed_llama_server++ backend = embed_llama_server.LlamaServerBackend()+ backend._model_path = _gguf(+ tmp_path, [("general.architecture", 8, "bert"), ("bert.context_length", 4, 512)]+ )+ monkeypatch.setattr(backend, "_ensure_ready", lambda model_name = None: None)+ monkeypatch.setattr(backend, "_server_context", lambda: 256)+ monkeypatch.setattr(backend, "_post", lambda *a, **k: {"tokens": [101, 102]})+ assert backend.max_tokens() == 254+++def test_llama_max_tokens_is_capped_by_the_ubatch_we_launched_with(tmp_path, monkeypatch):+ """/props reports n_ctx and no batch field, so the launch value is the only one there is.++ A prompt past one physical batch is refused by llama-server with a 500, which this route+ turns into a 502; the limit exists to answer 400 before that.+ """+ from core.rag import embed_llama_server++ backend = embed_llama_server.LlamaServerBackend()+ backend._model_path = _gguf(+ tmp_path, [("general.architecture", 8, "bert"), ("bert.context_length", 4, 8192)]+ )+ monkeypatch.setattr(backend, "_ensure_ready", lambda model_name = None: None)+ monkeypatch.setattr(backend, "_server_context", lambda: 8192)+ monkeypatch.setattr(+ backend, "_server_props", lambda: {"default_generation_settings": {"n_ctx": 8192}}+ )+ monkeypatch.setattr(backend, "_post", lambda *a, **k: {"tokens": [101, 102]})+ assert backend.max_tokens() == embed_llama_server._UBATCH_SIZE - 2+ assert "-ub" in backend._build_cmd("llama-server", "m.gguf", 1, use_gpu = False)+++def test_only_a_smaller_reported_ubatch_lowers_the_launch_value(monkeypatch):+ """n_batch is the logical batch; llama.cpp takes min(n_batch, n_ubatch or n_batch), so+ against the -ub this backend launches with it is never the physical limit."""+ from core.rag import embed_llama_server++ backend = embed_llama_server.LlamaServerBackend()+ ub = embed_llama_server._UBATCH_SIZE+ monkeypatch.setattr(backend, "_server_props", lambda: {"n_ubatch": ub // 2})+ assert backend._server_batch() == ub // 2+ monkeypatch.setattr(backend, "_server_props", lambda: {"n_ubatch": ub * 4})+ assert backend._server_batch() == ub+ monkeypatch.setattr(backend, "_server_props", lambda: {"n_batch": ub * 4})+ assert backend._server_batch() == ub+++def test_a_stale_tagged_identity_is_refused_not_answered(studio_embedder):+ """The current space is llama-server; the client resubmits the sentence-transformers tag."""+ from fastapi import HTTPException++ studio_embedder.setattr(+ rag_config, "effective_gguf_repo_for_embedding_model", lambda model: f"{model}-GGUF"+ )+ studio_embedder.setattr(+ rag_embeddings, "embedding_identity", lambda model_name = None: f"llama-server:{MODEL}"+ )+ studio_embedder.setattr(+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)+ )+ # A warm index is what makes this reachable: cold, the name falls back to "has a slash".+ from core.inference import local_model_resolver++ studio_embedder.setattr(local_model_resolver, "index_is_built", lambda: True)+ studio_embedder.setattr(local_model_resolver, "resolve_local_gguf", lambda *a, **k: None)+ stale = IDENTITY+ assert inference_route._names_studio_embedder(stale) is None+ assert inference_route._reference_is_decisive(stale) is True+ with pytest.raises(HTTPException) as exc:+ asyncio.run(+ inference_route.openai_embeddings(+ _Request({"input": "alpha", "model": stale}), "tester"+ )+ )+ assert exc.value.status_code == 503+++ ("body", "switched"),+ [+ ({"input": [""], "model": "org/chat-GGUF"}, False),+ ({"input": ["alpha", 7], "model": "org/chat-GGUF"}, False),+ ({"input": "alpha", "encoding_format": "int8", "model": "org/chat-GGUF"}, False),+ ({"input": [[1, 2, 3]], "model": "org/chat-GGUF"}, True),+ ],+)+def test_invalid_fallback_input_is_rejected_before_the_switch(studio_embedder, body, switched):+ seen = []++ async def record(request, current_subject, **_kwargs):+ seen.append(current_subject)+ return await request.json()++ studio_embedder.setattr(inference_route, "_should_validate_before_switch", lambda: True)+ studio_embedder.setattr(inference_route, "_auto_switch_from_request_body", record)+ studio_embedder.setattr(+ inference_route,+ "get_llama_cpp_backend",+ lambda: SimpleNamespace(is_loaded = True, is_embedding_gguf = False),+ )+ assert _http_error(body).status_code == 400+ assert bool(seen) is switched+++def test_local_path_models_are_not_exposed(studio_embedder, tmp_path):+ from core.rag.config import _escape_identity_segment++ model_dir = str(tmp_path / "bge")+ studio_embedder.setattr(rag_config, "effective_embedding_model", lambda: model_dir)+ studio_embedder.setattr(+ rag_config, "effective_gguf_repo_for_embedding_model", lambda model: f"{model}-GGUF"+ )+ identity = f"sentence-transformers:{_escape_identity_segment(model_dir)}"+ studio_embedder.setattr(+ rag_embeddings,+ "encode_with_identity",+ lambda texts, **_kwargs: (_vectors(texts), identity),+ )+ studio_embedder.setattr(+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)+ )+ reported = _call({"input": "alpha"})["model"]+ assert reported.startswith("sentence-transformers:bge-") and str(tmp_path) not in reported+ studio_embedder.setattr(rag_embeddings, "max_tokens", lambda model_name = None: 3)+ error = _http_error({"input": "alphabet"})+ assert error.status_code == 400+ assert "bge" in error.detail and str(tmp_path) not in error.detail++[email protected]("answers", [True, False])+def test_resident_embedding_gguf_answering_the_name_keeps_the_proxy(studio_embedder, answers):+ import httpx++ class _Client:+ async def post(self, *_args, **_kwargs):+ return httpx.Response(200, json = {"data": [{"embedding": [0.5]}], "model": "proxy"})++ async def aclose(self):+ return None++ _identity_names(studio_embedder)+ studio_embedder.setattr(inference_route, "_loaded_satisfies", lambda requested: answers)+ studio_embedder.setattr(inference_route, "_cancelable_nonstreaming_client", _Client)+ studio_embedder.setattr(+ inference_route,+ "get_llama_cpp_backend",+ lambda: SimpleNamespace(+ is_loaded = True,+ is_embedding_gguf = True,+ base_url = "http://llama.test",+ context_length = 512,+ model_identifier = f"{MODEL}-GGUF",+ ),+ )+ payload = _call({"input": "alpha", "model": MODEL})+ assert payload["model"] == ("proxy" if answers else IDENTITY)+++def test_public_embedding_name_hides_any_local_path_shape():+ public = inference_route._public_embedding_name+ windows = public("C:\\models\\private\\bge")+ assert windows.startswith("bge-") and len(windows) == len("bge-") + 8+ assert "models" not in windows and "private" not in windows+ assert public("/tmp/models/bge/").startswith("bge-")+ assert public("/checkpoints/run-a/model") != public("/checkpoints/run-b/model")+ assert public("unsloth/bge-small-en-v1.5") == "unsloth/bge-small-en-v1.5"+++def test_flat_token_array_passes_the_pre_switch_check(studio_embedder):+ seen = []++ async def record(request, current_subject, **_kwargs):+ seen.append(current_subject)+ return await request.json()++ studio_embedder.setattr(inference_route, "_should_validate_before_switch", lambda: True)+ studio_embedder.setattr(inference_route, "_auto_switch_from_request_body", record)+ studio_embedder.setattr(+ inference_route,+ "get_llama_cpp_backend",+ lambda: SimpleNamespace(is_loaded = True, is_embedding_gguf = False),+ )+ assert _http_error({"input": [1, 2, 3]}).status_code == 400+ assert seen == ["tester"]+++def test_local_path_aliases_match_case_exactly(studio_embedder, tmp_path):+ model_dir = str(tmp_path / "Foo")+ studio_embedder.setattr(rag_config, "effective_embedding_model", lambda: model_dir)+ studio_embedder.setattr(+ rag_config, "effective_gguf_repo_for_embedding_model", lambda model: f"{model}-GGUF"+ )+ assert inference_route._names_studio_embedder(model_dir)+ assert not inference_route._names_studio_embedder(str(tmp_path / "foo"))+ assert inference_route._names_studio_embedder(f"sentence-transformers:{model_dir}")+++def test_untagged_aliases_never_probe_the_backend(studio_embedder):+ def boom(*_args, **_kwargs):+ raise AssertionError("embedding_identity must not run for untagged names")++ _identity_names(studio_embedder)+ studio_embedder.setattr(rag_embeddings, "embedding_identity", boom)+ assert inference_route._names_studio_embedder(MODEL.upper())+ assert not inference_route._names_studio_embedder("text-embedding-3-small")+++def test_tagged_identity_must_match_the_current_space(studio_embedder):+ _identity_names(studio_embedder)+ assert inference_route._names_studio_embedder(IDENTITY) == MODEL+ stale = f"llama-server:{MODEL}:{MODEL}-GGUF"+ assert inference_route._names_studio_embedder(stale) is None+++def test_disconnected_client_leaves_the_queue_without_embedding(studio_embedder):+ lock = threading.Lock()+ gate = threading.Event()+ calls = {"n": 0}++ def encode(texts, **_kwargs):+ with lock:+ calls["n"] += 1+ gate.wait(timeout = 5)+ return _vectors(texts)++ class _Gone(_Request):+ async def is_disconnected(self):+ return True++ studio_embedder.setattr(+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)+ )+ studio_embedder.setattr(rag_embeddings, "encode", encode)+ cap = inference_route._STUDIO_EMBED_CONCURRENCY++ async def run():+ blockers = [+ asyncio.create_task(inference_route.openai_embeddings(_Request({"input": "x"}), "t"))+ for _ in range(cap)+ ]+ for _ in range(500):+ await asyncio.sleep(0.01)+ if calls["n"] == cap:+ break+ with pytest.raises(asyncio.CancelledError):+ await inference_route.openai_embeddings(_Gone({"input": "gone"}), "t")+ assert calls["n"] == cap+ gate.set()+ await asyncio.gather(*blockers)+ response = await inference_route.openai_embeddings(_Request({"input": "later"}), "t")+ assert response.status_code == 200+ assert calls["n"] == cap + 1++ asyncio.run(run())+++def test_batch_cap_applies_only_to_the_studio_fallback():+ from fastapi import HTTPException++ body = {"input": ["x"] * (inference_route._STUDIO_EMBED_MAX_INPUTS + 1)}+ assert len(inference_route._embeddings_items(body, tokens_ok = True)) == len(body["input"])+ with pytest.raises(HTTPException) as exc:+ inference_route._embeddings_texts(body)+ assert exc.value.status_code == 400+++def test_a_decisively_named_model_is_refused_when_nothing_is_loaded(studio_embedder):+ # #7454: a reference clearly meant for this server 404s rather than being answered by other+ # weights. The slot being empty makes _reject_unservable_model defer, so the fallback decides.+ _identity_names(studio_embedder)+ studio_embedder.setattr(+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)+ )+ studio_embedder.setattr(inference_route, "_reference_is_decisive", lambda name: True)+ error = _http_error({"input": "alpha", "model": "unsloth/embeddinggemma-300m-GGUF:Q8_0"})+ assert error.status_code in (404, 503)+++def test_a_foreign_model_name_still_reaches_the_studio_embedder(studio_embedder):+ # The other half: a vendor id is no evidence, so it keeps falling through instead of 404ing.+ _identity_names(studio_embedder)+ studio_embedder.setattr(+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)+ )+ assert _call({"input": "alpha", "model": "text-embedding-3-small"})["model"] == IDENTITY+++def test_reference_is_decisive_only_on_positive_evidence(monkeypatch):+ from core.inference import local_model_resolver as _resolver++ monkeypatch.setattr(_resolver, "resolve_local_gguf", lambda ref, allow_scan = False: None)+ # An explicit quant label is evidence no foreign id carries.+ assert inference_route._reference_is_decisive("unsloth/gemma-3-270m-it-GGUF:Q8_0") is True+ # A vendor id is not, so LiteLLM/OpenRouter style names keep working.+ assert inference_route._reference_is_decisive("text-embedding-3-small") is False+ assert inference_route._reference_is_decisive("openai/text-embedding-3-large") is False+++def test_advertised_local_identity_is_accepted_as_an_alias(studio_embedder, tmp_path):+ model_dir = str(tmp_path / "bge")+ studio_embedder.setattr(rag_config, "effective_embedding_model", lambda: model_dir)+ studio_embedder.setattr(+ rag_config, "effective_gguf_repo_for_embedding_model", lambda model: f"{model}-GGUF"+ )+ from core.rag.config import _escape_identity_segment++ studio_embedder.setattr(+ rag_embeddings,+ "embedding_identity",+ lambda model_name = None: f"sentence-transformers:{_escape_identity_segment(model_dir)}",+ )+ public = inference_route._public_embedding_name(model_dir)+ assert inference_route._names_studio_embedder(f"sentence-transformers:{public}") == model_dir+ assert inference_route._names_studio_embedder(public) == model_dir+ assert inference_route._names_studio_embedder("bge-00000000") is None+++def test_disconnected_client_is_dropped_even_with_a_free_permit(studio_embedder):+ calls = []++ class _Gone(_Request):+ async def is_disconnected(self):+ return True++ studio_embedder.setattr(+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)+ )+ studio_embedder.setattr(+ rag_embeddings, "encode", lambda texts, **_: calls.append(texts) or _vectors(texts)+ )++ async def run():+ with pytest.raises(asyncio.CancelledError):+ await inference_route.openai_embeddings(_Gone({"input": "gone"}), "t")+ assert calls == []+ response = await inference_route.openai_embeddings(_Request({"input": "later"}), "t")+ assert response.status_code == 200++ asyncio.run(run())+++def test_alias_match_pins_the_model_for_the_request(studio_embedder):+ settings = iter([MODEL, "unsloth/other-model", "unsloth/other-model"])+ seen = []++ def _encode_with_identity(+ texts,+ *,+ model_name = None,+ normalize = True,+ ):+ seen.append(model_name)+ return _vectors(texts), IDENTITY++ _identity_names(studio_embedder)+ studio_embedder.setattr(rag_config, "effective_embedding_model", lambda: next(settings))+ studio_embedder.setattr(rag_embeddings, "encode_with_identity", _encode_with_identity)+ studio_embedder.setattr(+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)+ )+ _call({"input": "alpha", "model": MODEL})+ assert seen == [MODEL]+++def test_llama_max_tokens_never_exceeds_one_physical_batch(tmp_path, monkeypatch):+ # Embedding is non-causal, so llama.cpp refuses rather than splits: an 8k-context limit on a+ # 512 batch turned a legitimate 600-token input into a 502 instead of a 400.+ from core.rag import embed_llama_server++ backend = embed_llama_server.LlamaServerBackend()+ backend._model_path = _gguf(+ tmp_path, [("general.architecture", 8, "bert"), ("bert.context_length", 4, 8192)]+ )+ monkeypatch.setattr(backend, "_ensure_ready", lambda model_name = None: None)+ monkeypatch.setattr(backend, "_server_context", lambda: 8192)+ monkeypatch.setattr(backend, "_server_batch", lambda: 512)+ monkeypatch.setattr(backend, "_post", lambda *a, **k: {"tokens": [101, 102]})+ assert backend.max_tokens() == 510+++def test_the_embed_server_does_not_enlarge_its_batch(tmp_path):+ # n_vocab * n_ubatch * 4 is allocated at startup, hundreds of MiB, against the 1024 MiB free+ # this backend calls enough to offload everything: max_tokens bounds the advert instead.+ from core.rag import embed_llama_server++ backend = embed_llama_server.LlamaServerBackend()+ model = _gguf(tmp_path, [("general.architecture", 8, "bert"), ("bert.context_length", 4, 8192)])+ cmd = backend._build_cmd("llama-server", model, 9999, use_gpu = True)+ assert "-b" not in cmd+ # Pinned at llama.cpp's own default, so nothing is allocated that was not already, and+ # max_tokens has a batch to bound the advert with (/props publishes none).+ assert cmd[cmd.index("-ub") + 1] == str(embed_llama_server._UBATCH_SIZE)+++def test_an_unconfirmed_context_limit_is_not_cached(tmp_path, monkeypatch):+ # A header context can exceed what the server runs at, so freezing it while /props is silent+ # outlives the readback that would have corrected it.+ from core.rag import embed_llama_server++ backend = embed_llama_server.LlamaServerBackend()+ backend._model_path = _gguf(+ tmp_path, [("general.architecture", 8, "bert"), ("bert.context_length", 4, 8192)]+ )+ monkeypatch.setattr(backend, "_ensure_ready", lambda model_name = None: None)+ monkeypatch.setattr(backend, "_server_batch", lambda: None)+ monkeypatch.setattr(backend, "_post", lambda *a, **k: {"tokens": [101, 102]})+ monkeypatch.setattr(backend, "_server_context", lambda: None)+ assert backend.max_tokens() == 8190+ assert backend._max_tokens is None+ # Once /props answers with both bounds, the smaller real context is what sticks.+ monkeypatch.setattr(backend, "_server_context", lambda: 512)+ monkeypatch.setattr(backend, "_server_batch", lambda: 512)+ assert backend.max_tokens() == 510+ assert backend._max_tokens == 510+++def test_props_probe_never_raises_before_the_server_is_up():+ from core.rag import embed_llama_server++ # An un-started server has no port, so the URL itself is invalid: the probe must swallow+ # that and fall back to the batch we launch with rather than propagate.+ assert (+ embed_llama_server.LlamaServerBackend()._server_batch() == embed_llama_server._UBATCH_SIZE+ )+++def test_cancel_during_the_final_disconnect_probe_releases_the_permit(studio_embedder):+ lock = threading.Lock()+ gate = threading.Event()+ active = {"now": 0, "peak": 0}++ def encode(texts, **_kwargs):+ with lock:+ active["now"] += 1+ active["peak"] = max(active["peak"], active["now"])+ gate.wait(timeout = 5)+ with lock:+ active["now"] -= 1+ return _vectors(texts)++ class _Stuck(_Request):+ async def is_disconnected(self):+ await asyncio.sleep(3600)+ return False++ studio_embedder.setattr(+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)+ )+ studio_embedder.setattr(rag_embeddings, "encode", encode)+ cap = inference_route._STUDIO_EMBED_CONCURRENCY++ async def run():+ stuck = asyncio.create_task(inference_route.openai_embeddings(_Stuck({"input": "x"}), "t"))+ await asyncio.sleep(0.05)+ stuck.cancel()+ with pytest.raises(asyncio.CancelledError):+ await stuck+ tasks = [+ asyncio.create_task(inference_route.openai_embeddings(_Request({"input": "y"}), "t"))+ for _ in range(cap)+ ]+ for _ in range(500):+ await asyncio.sleep(0.01)+ if active["now"] == cap:+ break+ assert active["now"] == cap+ gate.set()+ await asyncio.gather(*tasks)++ asyncio.run(run())+++def test_a_boolean_is_not_a_token_id():+ # bool subclasses int, so `[true]` read as a token array could swap the resident GGUF.+ from fastapi import HTTPException++ with pytest.raises(HTTPException) as exc:+ inference_route._embeddings_items({"input": [True]}, tokens_ok = True)+ assert exc.value.status_code == 400+ with pytest.raises(HTTPException):+ inference_route._embeddings_items({"input": [[1, True, 3]]}, tokens_ok = True)+ # A real token array is still one text.+ assert inference_route._embeddings_items({"input": [1, 2, 3]}, tokens_ok = True) == [[1, 2, 3]]+++def test_a_cold_index_does_not_make_a_local_name_foreign(monkeypatch):+ # Before the first scan resolve_local_gguf answers None for a model that IS downloaded, and+ # reading that as foreign served another embedding space under the requested name.+ from core.inference import local_model_resolver as _resolver++ monkeypatch.setattr(_resolver, "resolve_local_gguf", lambda ref, allow_scan = False: None)+ monkeypatch.setattr(_resolver, "index_is_built", lambda: False)+ monkeypatch.setattr(_resolver, "warm_index_soon", lambda: None)+ assert inference_route._reference_is_decisive("org/my-local-model") is True+ assert inference_route._reference_is_decisive("/models/local.gguf") is True+ # A bare vendor alias is still not evidence, cold index or not.+ assert inference_route._reference_is_decisive("text-embedding-3-small") is False++ # Once the index is built, absence really is evidence.+ monkeypatch.setattr(_resolver, "index_is_built", lambda: True)+ assert inference_route._reference_is_decisive("org/not-here") is False++[email protected]("stash_embeds", [False, True])+def test_default_request_restores_the_chat_slot_only_for_an_embedding_stash(+ studio_embedder, stash_embeds+):+ seen = []++ async def record(request, current_subject, **_kwargs):+ seen.append(current_subject)+ return await request.json()++ studio_embedder.setattr(inference_route, "_auto_switch_from_request_body", record)+ studio_embedder.setattr(inference_route, "_stashed_gguf_embeds", lambda: stash_embeds)+ studio_embedder.setattr(+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)+ )+ payload = _call({"input": "alpha"})+ assert payload["model"] == IDENTITY+ assert seen == (["tester"] if stash_embeds else [])+++def test_stashed_gguf_embeds_reads_the_stashed_model_header(monkeypatch, tmp_path):+ import core.inference.llama_keepwarm as keepwarm+ import core.inference.local_model_resolver as resolver++ monkeypatch.setattr(keepwarm, "get_last_unloaded_model", lambda: None)+ assert inference_route._stashed_gguf_embeds() is False+ monkeypatch.setattr(+ keepwarm, "get_last_unloaded_model", lambda: ("org/E-GGUF", "Q8_0", "org/E-GGUF")+ )+ monkeypatch.setattr(+ resolver, "resolve_local_gguf", lambda ref, **_: ("/tmp/e.gguf", "Q8_0", "org/E-GGUF")+ )++ class _Probe:+ is_embedding_gguf = True++ def _read_gguf_metadata(self, path):+ self.path = path++ monkeypatch.setattr(inference_route, "_probe_backend", _Probe)+ assert inference_route._stashed_gguf_embeds() is Truediff --git a/studio/backend/tests/test_rag_embeddings.py b/studio/backend/tests/test_rag_embeddings.pyindex e88ee3fdd65..0efb005e00a 100644--- a/studio/backend/tests/test_rag_embeddings.py+++ b/studio/backend/tests/test_rag_embeddings.py@@ -947,7 +947,9 @@ def test_the_security_gate_scans_the_snapshot_that_is_actually_loaded(monkeypatc(snapshot / "model.safetensors").write_bytes(b"ST")scanned = []monkeypatch.setattr(- embeddings, "_guard_model_security", lambda target, local_only = False: scanned.append(target)+ embeddings,+ "_guard_model_security",+ lambda target, local_only = False, display = None: scanned.append(target),)class _ST: