Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Dense embedder facade over config.EMBED_BACKEND (sentence-transformers or llama-server); off GPU
unless asked; backends differ, so switching needs reindex; ST failure falls back to llama-server;
_load_device probes in a child and falls back to CPU without changing the embedding space."""
from __future__ import annotations
from core.training.account_jobs import account_hf_token, account_path, managed_account
import logging
import os
import re
import threading
import weakref
from contextlib import contextmanager
from contextvars import ContextVar
from functools import lru_cache
from typing import Any, Callable
from utils.cpu_threads import fit_threads_to_affinity, runtime_capped_openblas_loaded
from utils.hardware.hardware import DeviceType, get_device
from utils.transformers_dtype import dtype_kwargs
from utils.utils import hf_env_offline
from . import config
logger = logging.getLogger(__name__)
# "false" silences the fast tokenizer's fork warning; encode() flips it only during a batch tokenize.
os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
_lock = threading.Lock()
# HF fast tokenizer is not thread-safe; separate from _lock so encode never blocks reload.
_compute_lock = threading.Lock()
_model = None
_name: str | None = None
# Table lookup: a GPU request with no GPU maps to cpu, not to a device string torch cannot open.
_served_by = threading.local()
# Unsloth device -> torch device string. Apple has no torch device -> CPU.
_TORCH_DEVICE = {DeviceType.CUDA: "cuda", DeviceType.XPU: "xpu"}
def _device() -> str:
"""Torch device for the in-process embedder. CPU unless asked otherwise.
Defaulting a GPU machine to CPU is deliberate: this embedder runs inside the backend process,
and the first CUDA allocation there creates a primary context never returned while the process
lives (712 MiB on a B200 against 74 MiB for bge-small's own weights), so ingesting one document
cost most of a gigabyte of VRAM for the session and no amount of unloading gets it back.
The trade is small at these sizes: one 128-token chunk takes 18.7ms on CPU against 5.2ms on
CUDA. Bulk indexing is where it shows (batch 64: 445 chunks/s on CPU against 3174/s), and
Settings (GPU) or ``RAG_EMBED_DEVICE=gpu`` opts back in for a large corpus.
This reads the same setting as the llama-server backend but resolves ``auto`` differently, which
is intended: that backend offloads inside its own subprocess, where the context dies with the
child.
"""
if config.embed_device_preference() != "gpu":
return "cpu"
# Still a table lookup: asking for a GPU on a host without one lands on CPU rather than on a device
# string torch cannot open.
return _TORCH_DEVICE.get(get_device(), "cpu")
class TorchDeviceUnusableError(RuntimeError):
"""Raised when torch cannot allocate safely on the accelerator or CPU."""
def _load_device() -> str:
"""Falls back to CPU to keep the embedding space; re-raises only if CPU also crashes."""
device = _device()
if device == "cpu":
return device
from utils.torch_device_probe import device_can_allocate
if device_can_allocate(device):
return device
if device_can_allocate("cpu"):
logger.warning(
"torch cannot allocate on %s without crashing; loading the embedding model "
"on CPU instead. This install's torch build does not match this machine.",
device,
)
return "cpu"
raise TorchDeviceUnusableError(
f"torch crashes when allocating on {device}; this install's torch build does "
"not match this machine"
)
# Matched as a substring of the lowercased model name. EmbeddingGemma's activations overflow float16 and every
# vector comes back NaN; its model card says to use float32 or bfloat16.
_FLOAT16_UNSAFE_MODELS = ("embeddinggemma",)
def _load_dtype(device: str, name: str) -> str:
"""float32 on CPU, where fp16 BERT raises "not implemented for Half" and encode() answers by swapping the
whole process to llama-server. float16 on an accelerator, except for models that cannot run in it: those
get bf16 where the device has it natively, else float32."""
if device == "cpu":
return "float32"
if not any(m in name.lower() for m in _FLOAT16_UNSAFE_MODELS):
return "float16"
from core.training.diffusion_train_common import (
native_bf16_supported,
native_bf16_supported_xpu,
)
native = native_bf16_supported_xpu() if device == "xpu" else native_bf16_supported()
return "bfloat16" if native else "float32"
_torchao_stub_done = False
# Its own lock, not _lock: that one is held across a whole model construction, so borrowing it made
# a preflight probe wait out someone else's download.
_stub_lock = threading.Lock()
def _install_torchao_stub_once() -> None:
"""Stub torchao before sentence-transformers loads: on Windows ROCm its absent c10d backend aborts."""
global _torchao_stub_done
with _stub_lock:
if _torchao_stub_done:
return
_torchao_stub_done = True
from core._torchao_stub import install_torchao_windows_rocm_stub
install_torchao_windows_rocm_stub()
class UnsafeEmbeddingModelError(RuntimeError):
"""Raised when the embedding model repo is flagged unsafe. A distinct type so the
llama-server fallback paths re-raise it instead of masking a security block as a
routine ST failure."""
class EmbeddingModelDownloadRequiredError(RuntimeError):
"""The picker activated a model whose explicit transfer is still pending."""
# Shared by the raise sites and is_download_required_error: a document job keeps only the error text.
DOWNLOAD_REQUIRED_HINT = "Finish its Settings download before indexing documents."
def download_required_message(name: str) -> str:
return f"Embedding model {name!r} is not downloaded yet. {DOWNLOAD_REQUIRED_HINT}"
def is_download_required_error(error: str | None) -> bool:
"""A stored error text from EmbeddingModelDownloadRequiredError: a wait, not a bad file."""
return bool(error) and DOWNLOAD_REQUIRED_HINT in error
def download_required(model_name: str | None = None) -> bool:
"""True while ``model_name``'s Settings transfer is pending and the cache-only loader still cannot load it."""
name = model_name or config.effective_embedding_model()
try:
from utils.embedding_model_settings import get_stored_download_pending
if not get_stored_download_pending(name):
return False
except Exception: # noqa: BLE001 - old/unavailable settings store
return False
try:
warm(name)
except EmbeddingModelDownloadRequiredError:
return True
except Exception: # noqa: BLE001 - any other load failure stays the per-document error it was
logger.debug("embedding model %s failed to warm", name, exc_info = True)
return False
def _ambient_hf_token() -> str | None:
"""The HF token the loader itself would use (HF_TOKEN env or the cached login), so
the scan can reach a gated/private repo instead of failing open. None if unavailable."""
if managed_account():
return False
try:
from huggingface_hub import get_token
return get_token()
except Exception:
return None
def _repo_json(name: str, filename: str, token: str | bool | None) -> Any:
"""``filename`` parsed from the local folder or Hub repo ``name``, None when the file is absent.
Raises whatever reading or parsing raises."""
import json
from utils.paths import is_local_path
if is_local_path(name):
from pathlib import Path
from utils.paths import normalize_path
path = Path(normalize_path(name)).expanduser() / filename
if not path.is_file():
return None
return json.loads(path.read_text(encoding = "utf-8-sig"))
from huggingface_hub import hf_hub_download
from huggingface_hub.utils import EntryNotFoundError
from utils.hf_cache_settings import active_hf_hub_cache
try:
local = hf_hub_download(
name,
filename,
token = account_hf_token(token),
cache_dir = active_hf_hub_cache(),
)
except EntryNotFoundError:
return None
with open(local, encoding = "utf-8-sig") as f:
return json.load(f)
def _st_module_subdirs(name: str, token: str | None) -> tuple[str, ...]:
"""Module subdirs from modules.json, the load roots for the security scan; () on any failure."""
try:
data = _repo_json(name, "modules.json", token)
subdirs = []
for module in data or ():
sub = str((module or {}).get("path", "")).strip().strip("/")
if sub:
subdirs.append(sub)
return tuple(dict.fromkeys(subdirs))
except Exception:
return ()
def _guard_model_security(
name: str,
local_only: bool = False,
display: str | None = None,
) -> None:
"""Refuses repos HF flags unsafe: a poisoned pickle loads regardless of trust_remote_code."""
try:
from utils.security import evaluate_file_security, security_load_subdirs
token = _ambient_hf_token()
if local_only:
load_subdirs = ()
else:
# Union of load roots, so a flagged pickle under a module dir still blocks.
load_subdirs = tuple(
dict.fromkeys(
(*security_load_subdirs(name, token), *_st_module_subdirs(name, token))
)
)
blocked = evaluate_file_security(
name, hf_token = token, load_subdirs = load_subdirs, local_only_load = local_only
).blocked
except Exception:
return
if blocked:
reason = (
"has cached pickle weights that cannot be security-scanned offline and no "
"safetensors alternative"
if local_only
else "is flagged as unsafe by Hugging Face's security scan"
)
raise UnsafeEmbeddingModelError(
f"Embedding model {(display or name)!r} {reason}; refusing to load. "
"Set a different RAG embedding model."
)
_ANSI_RE = re.compile(r"\x1b\[[0-9;]*m")
class _CaptureLoadReport(logging.Filter):
"""Swallow transformers' multi-line "<Model> LOAD REPORT" table, keeping the text.
transformers >= 5 emits the report through ``logger.warning`` with embedded ANSI colour codes,
so it lands in the server log as ~7 unstructured lines that break every JSON consumer. It fires
on every boot for the RAG embedder because bge-small-en-v1.5 ships a legacy
``embeddings.position_ids`` key the current BertModel does not expect, which is benign and
identical every time.
Nothing is lost: the caller re-emits the report (see ``_quiet_transformers_load``) at debug when
it only reports that known legacy key, and at warning when it mentions anything that could
change the model's behaviour.
"""
_SERIOUS = ("MISSING", "MISMATCH", "CONVERSION")
# Only this legacy BERT buffer is benign; other discarded weights can change retrieval.
_KNOWN_BENIGN_UNEXPECTED = "embeddings.position_ids"
def __init__(self) -> None:
super().__init__()
self.reports: list[str] = []
# These filters sit on process-global loggers, so capture only what the thread that opened the context emits.
self.thread_id = threading.get_ident()
def filter(self, record: logging.LogRecord) -> bool:
if threading.get_ident() != self.thread_id:
return True
try:
msg = record.getMessage()
except Exception: # noqa: BLE001 - a broken record must not break loading
return True
if "LOAD REPORT" not in msg:
return True
self.reports.append(msg)
return False
def is_serious(self) -> bool:
for report in self.reports:
if any(tag in report for tag in self._SERIOUS):
return True
# Skip transformers' trailing Notes: section, whose lines look like key rows.
table = report.split("Notes:", 1)[0]
for row in table.splitlines():
if "UNEXPECTED" not in row:
continue
key = _ANSI_RE.sub("", row).split("|", 1)[0].strip()
if key != self._KNOWN_BENIGN_UNEXPECTED and not key.endswith(
"." + self._KNOWN_BENIGN_UNEXPECTED
):
return True
return False
_LOAD_REPORT_LOGGERS = (
"transformers.utils.loading_report",
"transformers.modeling_utils",
# An adapter-backed embedding model reports through the PEFT integration's own logger, not a
# descendant of either above.
"transformers.integrations.peft",
)
@contextmanager
def _quiet_transformers_load():
"""Keeps transformers load progress and ANSI off stdout; restores the progress-bar setting as found."""
capture = _CaptureLoadReport()
attached = []
for name in _LOAD_REPORT_LOGGERS:
log = logging.getLogger(name)
log.addFilter(capture)
attached.append(log)
# Progress-bar probe name differs across transformers versions; restore Hub state separately.
hub_bars_off = None
try:
from huggingface_hub.utils import are_progress_bars_disabled
hub_bars_off = bool(are_progress_bars_disabled())
except Exception: # noqa: BLE001 - no Hub, or a version without the probe
hub_bars_off = None
reenable = False
hf_logging = None
try:
from transformers.utils import logging as hf_logging
if hasattr(hf_logging, "is_progress_bar_enabled"):
was_on = bool(hf_logging.is_progress_bar_enabled())
elif hasattr(hf_logging, "are_progress_bars_disabled"):
was_on = not bool(hf_logging.are_progress_bars_disabled())
else:
was_on = False
if was_on:
hf_logging.disable_progress_bar()
reenable = True
except Exception: # noqa: BLE001 - older/absent transformers: nothing to disable
hf_logging = None
try:
yield capture
finally:
for log in attached:
log.removeFilter(capture)
if reenable and hf_logging is not None:
try:
hf_logging.enable_progress_bar()
except Exception: # noqa: BLE001
pass
if hub_bars_off:
try:
from huggingface_hub.utils import disable_progress_bars
disable_progress_bars()
except Exception: # noqa: BLE001
pass
def _one_line(text: str) -> str:
"""Flattens the report to one plain line, since the stdlib logger would echo ANSI and newlines."""
plain = _ANSI_RE.sub("", text)
return " | ".join(part.strip() for part in plain.splitlines() if part.strip())
def _emit_load_reports(report) -> None:
"""Re-emit what the filter swallowed, as one record on our own logger: debug for
the expected legacy-key notice, warning for anything that could change the
embeddings. Drains the list so a retry does not report the same lines twice."""
serious = report.is_serious()
for text in report.reports:
if serious:
logger.warning("embedding model load report: %s", _one_line(text))
else:
logger.debug("embedding model load report: %s", _one_line(text))
report.reports.clear()
def _st_accepts_local_files_only(st_cls) -> bool:
"""Whether this SentenceTransformer version accepts local_files_only; passing it to an
older constructor raises, so gate on the signature."""
try:
import inspect
return "local_files_only" in inspect.signature(st_cls.__init__).parameters
except Exception:
return False
_ST_PACKAGE_PREFIX = "sentence_transformers."
_ST_GATE_MARKER = "_unsloth_custom_module_gate"
def _refuse_custom_module(
class_ref,
model_name_or_path,
trust_remote_code,
prefix = _ST_PACKAGE_PREFIX,
) -> None:
if (
isinstance(class_ref, str)
and not class_ref.startswith(prefix)
and model_name_or_path is not None
and not trust_remote_code
):
raise ValueError(
f"The model {model_name_or_path} references the module class {class_ref!r}, which is not "
"part of Sentence Transformers. Importing it executes third-party code, so Unsloth refuses "
"to load it as an embedding model."
)
def _gate_st_custom_modules() -> None:
"""Backport sentence-transformers 6.0's trust gate (CVE-2026-68770): before 6.0 a cached
model's modules.json could import repo-hosted classes without trust_remote_code."""
try:
import inspect
import sentence_transformers as st
from packaging.version import Version
if Version(st.__version__).major >= 6:
return
from sentence_transformers import SentenceTransformer
except Exception:
return
# 5.0-5.4 resolve on the model class; 5.5+ also via util.misc.import_module_class.
owner = next(
(c for c in SentenceTransformer.__mro__ if "_load_module_class_from_ref" in vars(c)),
None,
)
if owner is not None:
original = vars(owner)["_load_module_class_from_ref"]
if not getattr(original, _ST_GATE_MARKER, False):
signature = inspect.signature(original)
def _load_module_class_from_ref(self, *args, **kwargs):
bound = signature.bind_partial(self, *args, **kwargs).arguments
_refuse_custom_module(
bound.get("class_ref"),
bound.get("model_name_or_path"),
bound.get("trust_remote_code"),
)
return original(self, *args, **kwargs)
setattr(_load_module_class_from_ref, _ST_GATE_MARKER, True)
_load_module_class_from_ref.__wrapped__ = original
setattr(owner, "_load_module_class_from_ref", _load_module_class_from_ref)
# Below 6.0 these module loaders import a dotted path out of their own config via
# import_from_string, past both resolvers: Router its types, Dense its
# activation_function (and calls it), WordEmbeddings its tokenizer_class.
for module_path, prefix, where in (
("sentence_transformers.models.Router", _ST_PACKAGE_PREFIX, "behind this Router"),
("sentence_transformers.models.Dense", "torch.", "in a Dense module"),
(
"sentence_transformers.models.WordEmbeddings",
_ST_PACKAGE_PREFIX,
"in a WordEmbeddings module",
),
):
try:
import importlib
module = importlib.import_module(module_path)
original_from_string = getattr(module, "import_from_string", None)
if original_from_string is None or getattr(
original_from_string, _ST_GATE_MARKER, False
):
continue
def import_from_string(
dotted_path,
*args,
_original = original_from_string,
_prefix = prefix,
_where = where,
**kwargs,
):
_refuse_custom_module(dotted_path, _where, False, prefix = _prefix)
return _original(dotted_path, *args, **kwargs)
setattr(import_from_string, _ST_GATE_MARKER, True)
import_from_string.__wrapped__ = original_from_string
module.import_from_string = import_from_string
except Exception:
pass
try:
import sys
from sentence_transformers.util import misc
except Exception:
return
original_import = getattr(misc, "import_module_class", None)
if original_import is None or getattr(original_import, _ST_GATE_MARKER, False):
return
def import_module_class(
class_ref,
model_name_or_path = None,
*args,
**kwargs,
):
_refuse_custom_module(class_ref, model_name_or_path, kwargs.get("trust_remote_code"))
return original_import(class_ref, model_name_or_path, *args, **kwargs)
setattr(import_module_class, _ST_GATE_MARKER, True)
import_module_class.__wrapped__ = original_import
# Rebind every module that imported the function by name.
for module in list(sys.modules.values()):
if (
getattr(module, "__name__", "").startswith("sentence_transformers")
and getattr(module, "import_module_class", None) is original_import
):
setattr(module, "import_module_class", import_module_class)
def _route_linears_through_onednn(model) -> int:
"""Run a CPU model's fp32 Linear layers in oneDNN when torch's BLAS is Windows ROCm's OpenBLAS.
That DLL does ~4 GFLOPS per core against MKL's ~130, so bge-small indexed ~7x slower on all cores
(#14313). oneDNN ships in the same wheel; the vectors match to ~2e-7, so the index stays valid.
Probed on the model's own first layer; any doubt keeps the stock path. Returns layers rerouted.
"""
if not runtime_capped_openblas_loaded():
return 0
try:
import torch
if not getattr(torch.version, "hip", None) or not torch.backends.mkldnn.is_available():
return 0
linears = [
m
for m in model.modules()
if type(m) is torch.nn.Linear
and m.weight.device.type == "cpu"
and m.weight.dtype == torch.float32
]
if not linears:
return 0
class OneDnnLinear(torch.nn.Linear):
def forward(self, x):
return torch.ops.mkldnn._linear_pointwise(x, self.weight, self.bias, "none", [], "")
probe = linears[0]
with torch.inference_mode():
x = torch.randn(2, 5, probe.in_features)
expected = torch.nn.functional.linear(x, probe.weight, probe.bias)
got = OneDnnLinear.forward(probe, x)
if got.shape != expected.shape or not torch.allclose(got, expected, rtol = 1e-4, atol = 1e-5):
return 0
except Exception: # noqa: BLE001 -- the stock path is always correct
return 0
for module in linears:
module.__class__ = OneDnnLinear
logger.info(
"embedding model: %d Linear layers run in oneDNN (ROCm OpenBLAS is slow on CPU)",
len(linears),
)
return len(linears)
def _get(model_name: str | None = None):
"""Cached SentenceTransformer, (re)loading on a name change. Loaded in fp16 on an
accelerator for a ~1.5x speedup at negligible accuracy loss, fp32 on CPU; see ``_load_dtype``
for the models that cannot run in fp16."""
account_path(model_name, reference = True)
global _model, _name
name = model_name or config.effective_embedding_model()
# Capture offline state once so the gate and the load agree.
try:
from utils.embedding_model_settings import get_stored_download_pending
download_pending = get_stored_download_pending(name)
except Exception: # noqa: BLE001 - old/unavailable settings store
download_pending = False
offline = hf_env_offline()
# local_only means "load from cache"; offline is what the security gate needs, and claiming offline
# while online rejects a .bin-only repo the resolver just scanned.
local_only = offline or download_pending
with _lock:
if _model is None or _name != name:
device = _load_device()
_install_torchao_stub_once()
from sentence_transformers import SentenceTransformer
from utils.hf_cache_settings import active_hf_hub_cache
_gate_st_custom_modules()
logger.info("loading embedding model %s on %s", name, device)
st_kwargs = dict(
device = device,
cache_folder = active_hf_hub_cache(),
model_kwargs = dtype_kwargs(_load_dtype(device, name)),
)
if managed_account():
st_kwargs["token"] = False
load_target = name
from utils.paths import is_local_path
from utils.utils import cached_st_source, hf_cache_snapshot_dir
# Repo ids only: a local folder with a Hub-like name must not load the Hub's weights.
st_source = None if is_local_path(name) else cached_st_source(name)
if not local_only and st_source is not None:
# Load the cached snapshot path, not the repo id, so ST cannot fetch a newer revision.
load_target = str(st_source[1])
if local_only:
# ST-specific AND complete: a hybrid repo's cached GGUF, or a transfer that finalized only its first
# shard, would otherwise retire the marker.
if download_pending and st_source is None:
# Defensive: a loadable check and snapshot lookup share no lock, so eviction between them is
# still a pending model.
raise EmbeddingModelDownloadRequiredError(download_required_message(name))
snapshot = st_source[1] if st_source else hf_cache_snapshot_dir(name)
if snapshot is not None:
# Re-emit in finally: MISSING or MISMATCH lines matter most when the load raises.
load_target = str(snapshot)
elif download_pending:
raise EmbeddingModelDownloadRequiredError(download_required_message(name))
elif _st_accepts_local_files_only(SentenceTransformer):
st_kwargs["local_files_only"] = True
# Scan only after load_target is settled, so the scanned commit is the loaded one.
_guard_model_security(load_target, offline, display = name)
with _quiet_transformers_load() as report:
# Retire only after construction succeeds, or a failed load falls to llama-server with no marker.
try:
try:
_model = SentenceTransformer(load_target, **st_kwargs)
except Exception as exc: # noqa: BLE001 - e.g. CUDA OOM beside a resident chat model
# Retry on CPU before encode()'s llama-server fallback, which changes the embedding space.
if device == "cpu" or config.embed_device_requires_gpu():
raise
logger.warning(
"loading embedding model %s on %s failed (%s); retrying on CPU",
name,
device,
exc,
)
st_kwargs["device"] = "cpu"
st_kwargs["model_kwargs"] = dtype_kwargs(_load_dtype("cpu", name))
_model = SentenceTransformer(load_target, **st_kwargs)
finally:
_emit_load_reports(report)
_route_linears_through_onednn(_model)
_name = name
if download_pending:
# Retire only once the model is constructed: retiring earlier let a failed construction fall through
# to llama-server with no marker, freeing the fallback to fetch the GGUF companion.
try:
from utils.embedding_model_settings import clear_stored_download_pending
clear_stored_download_pending(name)
except Exception: # noqa: BLE001 - a settings write must not fail a load
pass
return _model
@lru_cache(maxsize = 1)
def _inference_ctx_factory():
"""``torch.inference_mode`` if torch imports, else ``nullcontext``. Returns the
factory so each call gets a fresh single-use guard."""
try:
import torch
return torch.inference_mode
except Exception: # noqa: BLE001 - torch may be missing or broken
from contextlib import nullcontext
return nullcontext
def _inference_ctx():
return _inference_ctx_factory()()
def _st_encode(
texts: list[str],
*,
model_name: str | None = None,
normalize: bool = True,
):
"""ST encode -> (N, dim) float32. Serialized (fast-tokenizer borrow check),
under inference_mode when torch is present, with rayon enabled for the call."""
with _compute_lock:
# ST encode failed after load: switch to llama-server so later encodes share one vector space.
model = _get(model_name)
# Affinity narrowed in Task Manager mid-index must reach this thread's pool (#14313).
fit_threads_to_affinity()
os.environ["TOKENIZERS_PARALLELISM"] = "true"
try:
with _inference_ctx():
out = model.encode(
texts,
normalize_embeddings = normalize,
convert_to_numpy = True,
show_progress_bar = False,
)
finally:
os.environ["TOKENIZERS_PARALLELISM"] = "false"
# fp16 weights yield fp16 output; store float32 for sqlite-vec + stable cosine.
if hasattr(out, "astype"):
out = out.astype("float32", copy = False)
return out
def _st_dim(model_name: str | None = None) -> int:
with _compute_lock:
return _get(model_name).get_sentence_embedding_dimension()
def _st_max_tokens(model_name: str | None = None) -> int | None:
with _compute_lock:
model = _get(model_name)
limit = getattr(model, "max_seq_length", None)
if not limit:
return None
tokenizer = getattr(model, "tokenizer", None)
specials = getattr(tokenizer, "num_special_tokens_to_add", None)
reserved = int(specials()) if callable(specials) else 0
prompts = getattr(model, "prompts", None) or {}
prompt = prompts.get(getattr(model, "default_prompt_name", None))
if prompt and tokenizer is not None:
reserved += len(tokenizer.encode(prompt, add_special_tokens = False))
return max(1, int(limit) - reserved)
_ST_COUNT_CACHE_SIZE = 4096
def _st_token_counter(model_name: str | None = None) -> Callable[[str], int]:
"""Reads the tokenizer per call under _compute_lock; caching it would outlive an unload.
The count memo is tied to that tokenizer by a weakref: a switch empties it, an unload frees it."""
counts: dict[str, int] = {}
counted_by: list = [None]
def _count_many(texts) -> list[int]:
texts = list(texts)
out = [0] * len(texts)
# The lock is released between batches so a query encode never waits out a whole document.
for start in range(0, len(texts), _ST_COUNT_BATCH):
with _compute_lock:
tok = _get(model_name).tokenizer
owner = counted_by[0]
if owner is None or owner() is not tok:
counts.clear()
try:
counted_by[0] = weakref.ref(tok)
except TypeError:
counted_by[0] = None
memoize = counted_by[0] is not None
misses: dict[str, list[int]] = {}
loose: list[int] = []
for i in range(start, min(start + _ST_COUNT_BATCH, len(texts))):
t = texts[i]
memo = memoize and type(t) is str
known = counts.get(t) if memo else None
if known is not None:
out[i] = known
elif memo:
misses.setdefault(t, []).append(i)
else:
loose.append(i)
if not misses and not loose:
# Leave the environment a tokenize would.
if os.environ.get("TOKENIZERS_PARALLELISM") != "false":
os.environ["TOKENIZERS_PARALLELISM"] = "false"
continue
keys = list(misses)
os.environ["TOKENIZERS_PARALLELISM"] = "true"
try:
lengths = _encoded_lengths(tok, keys + [texts[i] for i in loose])
finally:
os.environ["TOKENIZERS_PARALLELISM"] = "false"
for key, n in zip(keys, lengths):
for i in misses[key]:
out[i] = n
if len(counts) >= _ST_COUNT_CACHE_SIZE:
counts.pop(next(iter(counts)))
counts[key] = n
for i, n in zip(loose, lengths[len(keys) :]):
out[i] = n
return out
def _count(t: str) -> int:
return _count_many((t,))[0]
_count.many = _count_many
return _count
_ST_COUNT_BATCH = 256
def _encoded_lengths(tok, texts: list) -> list[int]:
"""``len(tok.encode(t, add_special_tokens=False))`` per text; a fast tokenizer's batch call is the path encode wraps."""
if len(texts) > 1 and getattr(tok, "is_fast", False) is True:
ids = tok(
texts,
add_special_tokens = False,
return_attention_mask = False,
return_token_type_ids = False,
)["input_ids"]
if len(ids) == len(texts):
return [len(row) for row in ids]
return [len(tok.encode(t, add_special_tokens = False)) for t in texts]
def _release_st_model() -> bool:
"""Drop the module-level SentenceTransformer without racing an encode."""
global _model, _name
with _compute_lock:
with _lock:
released = _model is not None
_model = None
_name = None
if released:
# Dropping the reference leaves its blocks reserved; gated, since before detection clear_gpu_cache() runs it.
try:
from utils import hardware
if hardware.hardware.DEVICE is not None:
hardware.clear_gpu_cache()
except Exception as e: # noqa: BLE001 - freeing is best effort
logger.debug("GPU cache clear after embedder release failed: %s", e)
return released
class _SentenceTransformersBackend:
"""Default backend; delegates to the module-level ST helpers so the ``_get``
monkeypatch in tests keeps working."""
def encode(
self,
texts,
*,
model_name = None,
normalize = True,
):
try:
return _st_encode(texts, model_name = model_name, normalize = normalize)
except (UnsafeEmbeddingModelError, EmbeddingModelDownloadRequiredError):
raise
except Exception as st_err: # noqa: BLE001 - runtime ST/CUDA encode failure
# Directory names prove nothing; only the filesystem shows that a dir holding safetensors is ST.
fallback = _switch_to_llama_fallback(st_err, model_name)
if fallback is None:
raise
_served_by.backend = fallback
return fallback.encode(texts, model_name = model_name, normalize = normalize)
def token_counter(self, *, model_name = None):
return _st_token_counter(model_name)
def dim(self, *, model_name = None):
return _st_dim(model_name)
def max_tokens(self, *, model_name = None):
return _st_max_tokens(model_name)
def warm(self, *, model_name = None):
_get(model_name)
_backend_lock = threading.Lock()
_backend = None
_backend_key: str | None = None
# Read without _backend_lock (held across model loads); dict.get is atomic.
_forced_backends: dict[str, str] = {}
_ST_ALIASES = frozenset({"sentence-transformers", "sentence_transformers", "st"})
_LLAMA_ALIASES = frozenset(
{"llama-server", "llama_server", "llama", "llama.cpp", "llamacpp", "gguf"}
)
_AUTO_ALIASES = frozenset({"auto", ""})
def _resolve_auto() -> str:
"""Auto picks sentence-transformers on a CUDA/ROCm GPU, else llama-server; GPU detection uses smi."""
from core.inference.llama_cpp import LlamaCppBackend
# Unfiltered probe: the llama.cpp ROCm arch gate (#7624) does not apply to PyTorch.
if LlamaCppBackend._get_gpu_free_memory():
return "sentence-transformers"
if LlamaCppBackend._find_llama_server_binary():
return "llama-server"
return "sentence-transformers"
# Cached: the auto GPU probe is a subprocess (#10390).
_resident_hardware: tuple[object, str] | None = None
_hardware_probe = threading.local()
def _resident_hardware_choice() -> str:
"""_resolve_auto, asked once for the backend that is published."""
held = _resident_hardware
if held is not None and _backend is not None and held[0] is _backend:
return held[1]
_hardware_probe.choice = _resolve_auto()
return _hardware_probe.choice
def _keep_hardware_choice(backend) -> None:
"""Keep the answer only when THIS call resolved it; see the clear in _get_backend."""
global _resident_hardware
choice = getattr(_hardware_probe, "choice", None)
if choice is not None:
_resident_hardware = (backend, choice)
def _model_is_local_gguf(model: str | None) -> bool:
"""True for a local .gguf file or folder; the cheap is_local_path check runs first on the hot path."""
if not model:
return False
try:
from utils.paths import is_local_path
if not is_local_path(model):
return False
from core.rag.embed_llama_server import LlamaServerBackend
return LlamaServerBackend._resolve_local_gguf(model) is not None
except Exception: # noqa: BLE001 - filesystem oddity is not a llama signal
return False
def _model_names_gguf_repo(model: str | None) -> bool:
"""Name-only test for a -GGUF repo: it has no safetensors for sentence-transformers to open."""
if not model:
return False
try:
from utils.paths import is_local_path
# Before the stored record: asks the filesystem, not a guess; RAG_EMBED_BACKEND still overrides.
if is_local_path(model):
return False
except Exception: # noqa: BLE001 - unparseable path is not a repo id either
return False
# Must agree with config's predicate used by gguf_repo_candidates.
return config._names_gguf(model.strip().rstrip("/").rsplit("/", 1)[-1])
# ST under auto stays on the CPU in float32 (_device); llama-server offloads. Exact ids: a fine-tune may lack the -GGUF.
_LLAMA_SERVER_PREFERRED_MODELS = frozenset(
{"unsloth/embeddinggemma-300m", "unsloth/embeddinggemma-2"}
)
def _plan_prefers_llama_server(model: str) -> bool:
"""Picker plan only: the runtime follows the saved record, so an env-configured model keeps its embedding space."""
if model.strip().rstrip("/").lower() not in _LLAMA_SERVER_PREFERRED_MODELS:
return False
# The runtime serves the unrecorded model in effect on the hardware default; another plan would be a dead download.
if model == config.effective_embedding_model():
return False
# A pinned GGUF repo serves other weights; an explicit device already chose where torch runs.
if config.gguf_repo_is_explicit() or config.embed_device_preference() != "auto":
return False
try:
from utils.embedding_model_settings import get_stored_backend
return not get_stored_backend(model)
except Exception: # noqa: BLE001 - store unavailable: keep the hardware plan
return False
def _resolve_auto_for_model(model_name: str | None = None) -> str:
"""Auto, but honouring the saved backend: an embedder with no GGUF stays on sentence-transformers."""
model = model_name or config.effective_embedding_model()
# Ahead of the stored record, since the filesystem was asked rather than guessed at; only auto
# consults this, so an explicit RAG_EMBED_BACKEND still wins.
if _model_is_local_gguf(model):
return "llama-server"
try:
from utils.embedding_model_settings import get_stored_backend
stored = get_stored_backend(model)
except Exception: # noqa: BLE001 - store unavailable: fall back to hardware
stored = None
if stored:
key = stored.strip().lower()
if key in _ST_ALIASES or key in _LLAMA_ALIASES:
return key
# Below the stored record, since a name is only a guess: a repo with a torn GGUF family and usable
# safetensors has a validated ST plan.
if _model_names_gguf_repo(model):
return "llama-server"
return _resident_hardware_choice()
def sentence_transformers_runtime_available() -> bool:
"""Checks torch, sentence-transformers and the device, without building a model that may download."""
try:
_load_device()
# Not under _lock: _get holds it across a whole model download/construction.
_install_torchao_stub_once()
from sentence_transformers import SentenceTransformer
return callable(SentenceTransformer)
except Exception as exc: # noqa: BLE001 - any failed runtime import selects the fallback
logger.debug("sentence-transformers runtime preflight failed: %s", exc)
return False
def _llama_server_runtime_available() -> bool:
try:
from core.inference.llama_cpp import LlamaCppBackend
return bool(LlamaCppBackend._find_llama_server_binary())
except Exception: # noqa: BLE001 - an unavailable fallback cannot be planned
return False
_ST_LOAD_PREFLIGHT_TIMEOUT_S = 5.0
# Proofs last one Settings resolution: a folder can be fixed, a repo republished, or the next caller lack access.
_st_unloadable_proofs: ContextVar[set[str] | None] = ContextVar(
"st-unloadable-proofs", default = None
)
@contextmanager
def st_load_proof_scope():
"""Keep ``sentence_transformers_can_load`` proofs for the enclosed resolution. Nested scopes share one."""
marker = _st_unloadable_proofs.set(set()) if _st_unloadable_proofs.get() is None else None
try:
yield
finally:
if marker is not None:
_st_unloadable_proofs.reset(marker)
def sentence_transformers_known_unloadable(model_name: str) -> bool:
"""Whether a check earlier in this scope proved ``model_name`` unloadable. No network."""
proofs = _st_unloadable_proofs.get()
return proofs is not None and model_name in proofs
def sentence_transformers_can_load(model_name: str, token: str | bool | None = None) -> bool:
"""False when ``modules.json`` names a sentence_transformers class this install lacks (embeddinggemma-2 names
6.x ``sentence_transformers.base`` modules) or ``config.json`` a ``model_type`` transformers does not know.
True when either file cannot be read, so an unreachable repo keeps its plan."""
if sentence_transformers_known_unloadable(model_name):
return False
if _st_load_preflight(model_name, token):
return True
proofs = _st_unloadable_proofs.get()
if proofs is not None:
proofs.add(model_name)
return False
def _st_load_preflight(model_name: str, token: str | bool | None = None) -> bool:
try:
from sentence_transformers.util import import_from_string
from utils.utils import call_with_deadline
def _metadata():
modules = _repo_json(model_name, "modules.json", token)
config_files = ["config.json"] if modules is None else []
for module in modules or ():
ref = str((module or {}).get("type", ""))
if ref.rsplit(".", 1)[-1] != "Transformer":
continue
path = str((module or {}).get("path", "")).strip("/")
filename = f"{path}/config.json" if path else "config.json"
if filename not in config_files:
config_files.append(filename)
return modules, {
filename: _repo_json(model_name, filename, token) for filename in config_files
}
# bounded with one deadline so nested module metadata cannot multiply the wait
modules, model_configs = call_with_deadline(
_metadata,
_ST_LOAD_PREFLIGHT_TIMEOUT_S,
name = "embed-st-load-preflight",
)
for module in modules or ():
ref = str((module or {}).get("type", ""))
# Other classes are third-party code, which _gate_st_custom_modules refuses on its own.
if not ref.startswith(_ST_PACKAGE_PREFIX):
continue
try:
# How ST's own loader resolves sentence_transformers.* refs.
import_from_string(ref)
except ImportError:
return False
for model_config in model_configs.values():
model_type = (model_config or {}).get("model_type")
if isinstance(model_type, str):
# CONFIG_MAPPING, not CONFIG_MAPPING_NAMES, so types registered at runtime count. An auto_map does not
# help an unknown type: the loader never passes trust_remote_code.
from transformers import CONFIG_MAPPING
if model_type not in CONFIG_MAPPING:
return False
except Exception as exc: # noqa: BLE001 - offline, gated, missing or malformed: no proof either way
logger.debug("sentence-transformers load preflight for %s failed: %s", model_name, exc)
return True
def resolved_backend_for_model(model_name: str, token: str | bool | None = None) -> str:
"""Backend a fresh operation for ``model_name`` would actually select."""
raw = _raw_backend()
forced = _forced_backends.get(model_name)
key = forced or (_resolve_auto_for_model(model_name) if raw in _AUTO_ALIASES else raw)
# Without a llama binary ST is the only plan; its error beats a fabricated GGUF destination. Stat before Hub.
if key in _ST_ALIASES and _llama_server_runtime_available():
use_llama = (
# Auto only: an explicit ST policy ignores both the preference and the stored backend.
(raw in _AUTO_ALIASES and _plan_prefers_llama_server(model_name))
or not sentence_transformers_runtime_available()
# An ST plan for it never loads, and its pending marker blocks the llama fallback's GGUF.
or (raw in _AUTO_ALIASES and not sentence_transformers_can_load(model_name, token))
)
if use_llama:
key = "llama-server"
if key in _LLAMA_ALIASES:
return "llama-server"
if key in _ST_ALIASES:
return "sentence-transformers"
raise ValueError(
f"Unknown RAG_EMBED_BACKEND={config.EMBED_BACKEND!r}; expected "
"'auto', 'sentence-transformers' or 'llama-server'"
)
def _try_make_llama_backend():
"""A llama-server GGUF embedding backend if its binary is present, else None.
Construction is lazy -- no server starts until warm."""
from core.inference.llama_cpp import LlamaCppBackend
if not LlamaCppBackend._find_llama_server_binary():
return None
from .embed_llama_server import LlamaServerBackend
return LlamaServerBackend()
def _build_st_backend_or_fallback(model_name: str | None = None):
"""Warms the pinned model_name, not the live setting; a failed ST probe falls back to llama-server."""
backend = _SentenceTransformersBackend()
try:
backend.warm(model_name = model_name)
return backend
except (UnsafeEmbeddingModelError, EmbeddingModelDownloadRequiredError):
raise
except Exception as st_err: # noqa: BLE001 - any ST/torch import or load failure
fallback = _try_make_llama_backend()
if fallback is None:
raise
logger.warning(
"sentence-transformers embedder unavailable (%s); falling back to the "
"llama-server GGUF embedder",
st_err,
)
return fallback
def _switch_to_llama_fallback(err, model_name: str | None = None):
"""After a runtime ST encode failure, swaps to llama-server; earlier ST vectors need a reindex."""
global _backend, _backend_key
failed_model = model_name or config.effective_embedding_model()
old = None
with _backend_lock:
if not isinstance(_backend, _SentenceTransformersBackend):
return _backend
fallback = _try_make_llama_backend()
if fallback is None:
return None
logger.warning(
"sentence-transformers encode failed (%s); switching to the llama-server "
"embedder for the rest of this process. Reindex any knowledge base that "
"was already embedded with sentence-transformers.",
err,
)
old, _backend = _backend, fallback
_forced_backends[failed_model] = "llama-server"
_backend_key = _backend_cache_key(_raw_backend(), "llama-server")
# ST wrappers share one module-level model; the warmed replacement would lose it if cleared here.
_dispose_replaced_backend(old, fallback)
return fallback
def _raw_backend() -> str:
return (config.EMBED_BACKEND or "auto").strip().lower()
def sentence_transformers_fallback_allowed(model_name: str | None = None) -> bool:
"""Under an explicit llama config or a runtime ST failure pin, ST cannot be offered for a new model."""
raw = _raw_backend()
model = model_name or config.effective_embedding_model()
if _forced_backends.get(model) in _LLAMA_ALIASES:
return False
return raw in _AUTO_ALIASES or raw in _ST_ALIASES
def _current_backend_key() -> str:
"""The cache key the backend in use should carry right now. Tests that install a
stub backend set ``_backend_key`` from this so it is not rebuilt under them."""
raw = _raw_backend()
forced = _forced_backends.get(config.effective_embedding_model())
if forced:
return _backend_cache_key(raw, forced)
key = _resolve_auto_for_model() if raw in _AUTO_ALIASES else raw
return _backend_cache_key(raw, key)
def _backend_cache_key(raw: str, key: str) -> str:
"""Cache key for a built backend. It carries the RESOLVED choice, not just the
raw config, so saving a model that needs the other backend rebuilds instead of
serving the one already built for the previous model."""
return f"{raw}\x00{key}"
def _dispose_replaced_backend(old, new = None) -> None:
if old is None or old is new:
return
if isinstance(old, _SentenceTransformersBackend):
# Two ST wrappers share the module-level model and the replacement is already warmed, so clearing it
# here would discard the model just selected.
if not isinstance(new, _SentenceTransformersBackend):
_release_st_model()
return
shutdown = getattr(old, "_shutdown", None)
if callable(shutdown):
try:
shutdown()
except Exception: # noqa: BLE001 - replacement is already selected
logger.warning("replaced embedding backend shutdown failed", exc_info = True)
def _get_backend(model_name: str | None = None):
"""Uses the job's pinned model_name, not the live setting; a Settings change would mislabel vectors."""
global _backend, _backend_key
raw = _raw_backend()
old = None
new = None
with _backend_lock:
model = model_name or config.effective_embedding_model()
forced = _forced_backends.get(model)
# Load-bearing: other probes resolve auto outside this lock and would keep a stale answer.
_hardware_probe.choice = None
key = forced or (_resolve_auto_for_model(model) if raw in _AUTO_ALIASES else raw)
if _backend is not None and _backend_key == _backend_cache_key(raw, key):
_keep_hardware_choice(_backend)
return _backend
old = _backend
if key in _ST_ALIASES:
new = _build_st_backend_or_fallback(model)
elif key in _LLAMA_ALIASES:
# Imported lazily so the ST path never imports llama plumbing.
from .embed_llama_server import LlamaServerBackend
new = LlamaServerBackend()
else:
raise ValueError(
f"Unknown RAG_EMBED_BACKEND={config.EMBED_BACKEND!r}; expected "
"'auto', 'sentence-transformers' or 'llama-server'"
)
_backend = new
_keep_hardware_choice(new)
if key in _ST_ALIASES and _is_llama_backend(new):
# Pin the backend the warm probe actually fell back to, but let a different model retry ST.
key = "llama-server"
_forced_backends[model] = key
_backend_key = _backend_cache_key(raw, key)
# A llama shutdown can wait for an in-flight encode, so keep that wait out of the global publication lock.
_dispose_replaced_backend(old, new)
return new
def _reset_backend() -> None:
"""Drop the cached backend (test teardown / re-init)."""
global _backend, _backend_key, _resident_hardware
with _backend_lock:
_forced_backends.clear()
_backend = None
_backend_key = None
_resident_hardware = None
def backend_is_loaded(model_name: str | None = None) -> bool:
"""Lock-free on purpose: the load locks are held across whole model loads, which would stall status."""
backend = _backend
if backend is None:
# No published backend does not mean nothing is loaded: answering False stranded module-level
# weights, since release_backend returns on the same test.
if model_name is None:
return _model is not None
return _model is not None and _name == model_name
if model_name is None:
# A llama backend whose process is gone is not resident, whichever model was asked about.
if _is_llama_backend(backend):
try:
return bool(backend._process_alive())
except Exception: # noqa: BLE001 - a status probe must never block settings
return False
return True
if isinstance(backend, _SentenceTransformersBackend):
return _model is not None and _name == model_name
if _is_llama_backend(backend):
try:
# Unload is a fresh start: clearing _forced_backends lets the saved model pick its backend again.
if not backend._process_alive():
return False
return backend._model_repo == config.effective_gguf_repo_for_embedding_model(model_name)
except Exception: # noqa: BLE001 - a status probe must never block settings
return False
return False
def release_backend() -> bool:
"""Drops the embedder, stopping its llama-server; safe mid-ingestion as the next embed rebuilds it."""
global _backend, _backend_key, _resident_hardware
with _backend_lock:
# Unload is an explicit fresh start, so a past runtime fallback stops pinning the choice and the saved
# model picks its backend again.
_forced_backends.clear()
backend, _backend, _backend_key = _backend, None, None
_resident_hardware = None
if backend is None:
# Live servers keep spawn-time pooling; a stopped backend re-reads the file, so dedup must match.
return _release_st_model()
_dispose_replaced_backend(backend)
return True
def active_backend_is_llama(model_name: str | None = None) -> bool:
"""True when the built backend is llama-server, including runtime fallbacks; never raises."""
try:
with _backend_lock:
backend = _backend
if backend is not None:
# Report the actual backend, so a concrete ST backend's pickle stays gated.
try:
from .embed_llama_server import LlamaServerBackend
except Exception: # noqa: BLE001 - llama plumbing import must never block
return False
return isinstance(backend, LlamaServerBackend)
raw = (config.EMBED_BACKEND or "auto").strip().lower()
key = _resolve_auto_for_model(model_name) if raw in _AUTO_ALIASES else raw
return key in _LLAMA_ALIASES
except Exception: # noqa: BLE001 - a backend probe must never block saving
return False
def batches_across_documents(model_name: str | None = None) -> bool:
"""Whether concurrent documents should share encode calls (#16715). Not for sentence-transformers on
CPU: it pads each batch to its longest text, and on CPU that padding costs more than the calls saved."""
if active_backend_is_llama(model_name):
return True
model = _model
# A loaded model's device counts: a GPU choice can still have fallen back to CPU at load.
device = getattr(model, "device", None) if model is not None else None
return (str(device) if device is not None else _device()).split(":")[0] != "cpu"
def _llama_pooling(name: str, served = None) -> str | None:
try:
from .embed_llama_server import LlamaServerBackend, _gguf_pooling
except Exception: # noqa: BLE001 - llama plumbing import must never block
return None
# The file the running server loaded wins over a cache search: moving the HF cache leaves it serving the old path.
backend = served if served is not None else _backend
desired = config.effective_gguf_repo_for_embedding_model(name)
if isinstance(backend, LlamaServerBackend):
path, repo, captured, alive = backend.pooling_identity_snapshot()
else:
path, repo, captured, alive = None, None, None, False
if path and repo == desired:
# AppleDouble files keep the extension, so a text parser would embed and cite them as chunks.
if served is not None or alive:
pooling = captured or _gguf_pooling(path)
elif os.path.isfile(path):
pooling = _gguf_pooling(path)
else:
pooling = LlamaServerBackend.cached_pooling(name)
else:
pooling = LlamaServerBackend.cached_pooling(name)
if pooling is None:
# Nothing on disk to read, so match no stored row: pre-encode dedupe must not take forced-CLS vectors as current.
return "unresolved"
return None if pooling == "cls" else pooling
def _identity(
is_llama: bool,
name: str,
served = None,
served_identity: tuple[str | None, str | None] | None = None,
) -> str:
if is_llama:
if served_identity is None:
repo = config.effective_gguf_repo_for_embedding_model(name)
pooling = _llama_pooling(name, served)
else:
repo, captured_pooling = served_identity
pooling = None if captured_pooling == "cls" else captured_pooling
return config.embedding_identity(
"llama-server",
name,
gguf_repo = repo,
pooling = pooling,
)
return config.embedding_identity("sentence-transformers", name)
def _identity_backend_is_llama(name: str) -> bool:
"""Predicts the backend the next encode for name uses, including any replacement it will trigger."""
try:
raw = _raw_backend()
with _backend_lock:
backend = _backend
cached_key = _backend_key
forced = _forced_backends.get(name)
resolved = forced or (_resolve_auto_for_model(name) if raw in _AUTO_ALIASES else raw)
expected_key = _backend_cache_key(raw, resolved)
if backend is None or cached_key == expected_key:
return active_backend_is_llama(name)
return resolved in _LLAMA_ALIASES
except Exception: # noqa: BLE001 - identity prediction must not block ingestion
return active_backend_is_llama(name)
def embedding_identity(model_name: str | None = None) -> str:
"""Identity of the vector space, not the model name: llama-server uses the GGUF's own pooling."""
name = model_name or config.effective_embedding_model()
return _identity(_identity_backend_is_llama(name), name)
def _is_llama_backend(backend) -> bool:
try:
from .embed_llama_server import LlamaServerBackend
except Exception: # noqa: BLE001 - llama plumbing import must never block
return False
return isinstance(backend, LlamaServerBackend)
def encode_with_identity(
texts: list[str],
*,
model_name: str | None = None,
normalize: bool = True,
):
"""Takes identity from the encode that made the vectors; a concurrent ST swap could mislabel them."""
_served_by.backend = None
_served_by.llama_identity = None
vectors = encode(texts, model_name = model_name, normalize = normalize)
served = getattr(_served_by, "backend", None)
served_identity = getattr(_served_by, "llama_identity", None)
name = model_name or config.effective_embedding_model()
if served is None:
return vectors, embedding_identity(name)
return vectors, _identity(_is_llama_backend(served), name, served, served_identity)
def warm(model_name: str | None = None) -> None:
"""Eagerly load the embedder so the first real request isn't slow."""
_get_backend(model_name).warm(model_name = model_name)
def encode(
texts: list[str],
*,
model_name: str | None = None,
normalize: bool = True,
):
"""An explicit unload retires the backend; reacquire the new one rather than fail the document."""
backend = _get_backend(model_name)
_served_by.backend = backend
_served_by.llama_identity = None
try:
vectors = backend.encode(texts, model_name = model_name, normalize = normalize)
served = getattr(_served_by, "backend", None) or backend
_served_by.llama_identity = getattr(served, "served_embedding_identity", lambda: None)()
return vectors
except RuntimeError:
if not (_is_llama_backend(backend) and getattr(backend, "_closed", False)):
raise
replacement = _get_backend(model_name)
if replacement is backend:
raise RuntimeError("llama-server embedding backend was unloaded")
_served_by.backend = replacement
vectors = replacement.encode(texts, model_name = model_name, normalize = normalize)
_served_by.llama_identity = getattr(replacement, "served_embedding_identity", lambda: None)()
return vectors
def dim(model_name: str | None = None) -> int:
return _get_backend(model_name).dim(model_name = model_name)
def max_tokens(model_name: str | None = None) -> int | None:
return _get_backend(model_name).max_tokens(model_name = model_name)
def clip_to_limit(text: str, count, limit: int | None) -> tuple[str, bool]:
"""The longest prefix of ``text`` within ``limit`` tokens by ``count``, and whether it was cut."""
if not limit or count(text) <= limit:
return text, False
lo, hi = 0, len(text)
while lo < hi:
mid = (lo + hi + 1) // 2
if count(text[:mid]) <= limit:
lo = mid
else:
hi = mid - 1
return text[:lo], True
def clip_many_to_limit(texts, count, limit: int | None) -> list[tuple[str, bool]]:
"""``clip_to_limit`` per text, bisecting every over-limit text in lockstep: one ``count.many`` per round."""
texts = list(texts)
if not limit or getattr(count, "many", None) is None or len(texts) < 2:
return [clip_to_limit(t, count, limit) for t in texts]
out = [(t, False) for t in texts]
bounds = {
i: [0, len(t)] for i, (t, n) in enumerate(zip(texts, _many(count, texts))) if n > limit
}
active = list(bounds)
while active:
mids = [(bounds[i][0] + bounds[i][1] + 1) // 2 for i in active]
for i, mid, n in zip(
active, mids, _many(count, [texts[i][:m] for i, m in zip(active, mids)])
):
if n <= limit:
bounds[i][0] = mid
else:
bounds[i][1] = mid - 1
active = [i for i in active if bounds[i][0] < bounds[i][1]]
for i, (lo, _) in bounds.items():
out[i] = (texts[i][:lo], True)
return out
def chunk_tokens(model_name: str | None = None) -> int:
limit = max_tokens(model_name)
return min(config.CHUNK_TOKENS, limit) if limit else config.CHUNK_TOKENS
def token_counter(model_name: str | None = None) -> Callable[[str], int]:
"""Reacquires the backend only for an unload's lifecycle failure; other tokenizer errors propagate."""
backend = _get_backend(model_name)
state = (backend, backend.token_counter(model_name = model_name))
counter_lock = threading.Lock()
def _served(use):
nonlocal state
served_backend, served_count = state
try:
return use(served_count)
except RuntimeError:
if not (
_is_llama_backend(served_backend) and getattr(served_backend, "_closed", False)
):
raise
with counter_lock:
# Another counting thread may already have replaced the retired counter.
if state[0] is served_backend:
replacement = _get_backend(model_name)
if replacement is served_backend:
raise RuntimeError("llama-server embedding backend was unloaded")
state = (
replacement,
replacement.token_counter(model_name = model_name),
)
retry = state[1]
return use(retry)
def _count(text: str) -> int:
return _served(lambda count: count(text))
def _count_many(texts) -> list[int]:
texts = list(texts)
return _served(lambda count: _many(count, texts))
# Only a batching backend (sentence-transformers, llama-server) changes how chunking asks for counts.
if getattr(state[1], "many", None) is not None:
_count.many = _count_many
return _count
def _many(count, texts: list) -> list[int]:
many = getattr(count, "many", None)
return list(many(texts)) if many is not None else [count(t) for t in texts]