~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""One resident local embedding backend; callers own weight-free handles.
The cache contains model weights/tokenizer only, never user settings, documents,
query results, or authorization decisions. Provider admission runs before this
module is called. Inference is serial: one call at a time uses the resident
model, and model replacement waits until every admitted call finishes.
"""
from __future__ import annotations
import atexit
from copy import deepcopy
import gc
import hashlib
from itertools import islice
from os import scandir
from pathlib import Path
import re
from stat import S_ISDIR
from threading import Lock, local
from typing import Any, Callable, TypeVar
from langchain_core.embeddings import Embeddings
from .model_loading import MODEL_LOAD_LOCK, load_sentence_transformer
from ..security.secure_logging import logger
_Result = TypeVar("_Result")
_Factory = Callable[..., Embeddings]
_CacheKey = tuple[tuple[_Factory, Any], str]
_MAX_ARTIFACT_STAMP_ENTRIES = 512
_ARTIFACT_STAMP_LIMIT_MESSAGE = (
f"Local embedding model artifact tree exceeds "
f"{_MAX_ARTIFACT_STAMP_ENTRIES} entries; remove unrelated files "
"or directory links"
)
def _freeze(value: Any) -> Any:
if isinstance(value, dict):
return tuple(
sorted((key, _freeze(item)) for key, item in value.items())
)
if isinstance(value, (list, tuple)):
return tuple(_freeze(item) for item in value)
return value
def _artifact_stamp(kwargs: dict) -> str:
"""Notice local replacements and floating Hub ref changes without I/O
of model contents or contacting the network.
Hub snapshots consist of links to immutable blobs. Stat follows those links
so an operator replacing a blob also changes the key. Local module directory
links are followed too, visiting each directory once to stop cycles. A handle
cannot reload changed files after eviction and silently use different weights.
"""
name = kwargs["model_name"]
# nosemgrep: semgrep.rules.path-traversal-risk, reason: Provider admission confines absolute model paths to the app models directory; the stamp only stats and lists entries
path = Path(name)
references = []
if path.is_absolute():
roots = [path]
else:
# nosemgrep: semgrep.rules.path-traversal-risk, reason: Operator model cache root; admitted repository ids have slashes replaced and revisions are catalog or cached commit ids
cache = Path(kwargs["cache_folder"])
repositories = [name]
if "/" not in name:
repositories.append(f"sentence-transformers/{name}")
roots = []
revision = kwargs.get("model_kwargs", {}).get("revision")
for repository in repositories:
cached_repo = cache / ("models--" + repository.replace("/", "--"))
snapshot = revision
if not revision or not re.fullmatch(r"[0-9a-f]{40}", revision):
reference = cached_repo / "refs" / (revision or "main")
references.append(reference)
try:
# nosemgrep: semgrep.rules.path-traversal-risk, reason: Reads refs/main of an admitted repository inside the operator model cache; repository ids have slashes replaced and provider revisions are commit ids
snapshot = reference.read_text(encoding="ascii").strip()
except FileNotFoundError:
snapshot = None
if snapshot and re.fullmatch(r"[0-9a-f]{40}", snapshot):
roots.append(cached_repo / "snapshots" / snapshot)
# Do not scan all snapshots/blobs or .no_exist markers: warming an
# unrelated revision must not invalidate handles pinned to old bytes.
roots.extend(references)
digest = hashlib.sha256()
for root in roots:
# Missing files are part of the identity too (initial download/repair).
digest.update(str(root).encode())
paths = [root]
seen_directories = set()
visited = 0
while paths:
if visited >= _MAX_ARTIFACT_STAMP_ENTRIES:
raise ValueError(_ARTIFACT_STAMP_LIMIT_MESSAGE)
path = paths.pop()
visited += 1
try:
stat = path.stat()
except FileNotFoundError:
continue
digest.update(str(path).encode())
digest.update(
repr(
(
stat.st_dev,
stat.st_ino,
stat.st_size,
stat.st_mtime_ns,
stat.st_ctime_ns,
)
).encode()
)
if S_ISDIR(stat.st_mode):
# rglob does not descend into directory symlinks. Stamp the
# target and its files, while preventing alias/cycle expansion.
directory = (stat.st_dev, stat.st_ino)
if directory in seen_directories:
continue
seen_directories.add(directory)
# Path.iterdir() materializes the whole directory before a
# caller can stop, so use lazy scandir and consume at most
# one entry beyond the remaining budget. Never return a
# partial stamp: unseen model bytes could change silently.
remaining = _MAX_ARTIFACT_STAMP_ENTRIES - visited
# nosemgrep: semgrep.rules.path-traversal-risk, reason: Lists an admitted model root and linked module directories only within a bounded budget; no file contents are read or written
with scandir(path) as entries:
children = [
path / entry.name
for entry in islice(entries, remaining + 1)
]
if len(children) > remaining:
raise ValueError(_ARTIFACT_STAMP_LIMIT_MESSAGE)
paths.extend(sorted(children, reverse=True))
return digest.hexdigest()
class _ModelCache:
def __init__(self) -> None:
self.key: _CacheKey | None = None
self.backend: Embeddings | None = None
# Tokenization and forward passes mutate model and tokenizer state,
# so admitted calls take turns on the one resident backend.
self._inference = Lock()
self._inference_state = local()
def _clear_locked(self) -> None:
had_model = self.backend is not None
self.backend = None
self.key = None
if had_model:
# SentenceTransformer -> model_card_data -> SentenceTransformer
# is a strong cycle. Dropping the wrapper alone retains hundreds
# of MiB until a generation-2 collection eventually happens.
gc.collect()
def clear(self, *, wait: bool = True) -> bool:
if MODEL_LOAD_LOCK.acquire(blocking=wait):
try:
self._clear_locked()
return True
finally:
MODEL_LOAD_LOCK.release()
return False
def prepare(
self, factory: _Factory, kwargs: dict[str, Any]
) -> SharedSentenceTransformerEmbeddings:
# A warm handle for an identical configuration is created without
# waiting for active calls; its own calls then wait for their turn.
with MODEL_LOAD_LOCK.inference():
options, identity = self._options(factory, kwargs)
key = (identity, _artifact_stamp(kwargs))
if self.key == key:
return SharedSentenceTransformerEmbeddings(
self, factory, options, key
)
with MODEL_LOAD_LOCK:
return self._prepare_locked(factory, kwargs)
@staticmethod
def _options(factory: _Factory, kwargs: dict[str, Any]):
local_options = deepcopy(kwargs)
local_options.setdefault("model_kwargs", {})["local_files_only"] = True
return local_options, (factory, _freeze(local_options))
def _load_locked(self, factory: _Factory, kwargs: dict[str, Any]) -> None:
self.backend = load_sentence_transformer(factory, **kwargs)
def _prepare_locked(
self, factory: _Factory, kwargs: dict[str, Any]
) -> SharedSentenceTransformerEmbeddings:
options = deepcopy(kwargs)
# Network permission is checked by the provider on EVERY request.
# It is not a property of the resident weights. After initial
# construction, an evicted handle may only reload local files.
local_options, identity = self._options(factory, options)
stamp = _artifact_stamp(options)
key = (identity, stamp)
if self.key != key:
self._clear_locked()
logger.info("Loading resident local embedding model")
try:
self._load_locked(factory, options)
loaded = _artifact_stamp(options)
if loaded != stamp:
# An authorized download/repair, or a replacement, changed
# the files while the constructor read them, so neither
# stamp need describe the loaded bytes. Load once more
# from local files and keep it only if they stay intact.
self._clear_locked()
logger.info(
"Reloading local embedding model after its files changed"
)
self._load_locked(factory, local_options)
stamp, loaded = loaded, _artifact_stamp(options)
except BaseException:
self._clear_locked()
raise
if loaded != stamp:
self._clear_locked()
raise ValueError(
"Local embedding model files changed; create a new embedding client"
)
key = (identity, stamp)
self.key = key
return SharedSentenceTransformerEmbeddings(
self, factory, local_options, key
)
def embed(
self,
handle: SharedSentenceTransformerEmbeddings,
operation: Callable[[Embeddings], _Result],
) -> _Result:
while True:
with MODEL_LOAD_LOCK.inference():
if handle._closed:
raise RuntimeError("Embedding client is closed")
if self.key == handle._key:
return self._run_serially(handle, operation)
# Never upgrade shared access in place: another reader may also
# need a different model. Recheck after obtaining exclusive access.
with MODEL_LOAD_LOCK:
self._reload_locked(handle)
def _reload_locked(
self, handle: SharedSentenceTransformerEmbeddings
) -> None:
if handle._closed:
raise RuntimeError("Embedding client is closed")
if self.key != handle._key:
if _artifact_stamp(handle._kwargs) != handle._key[1]:
raise ValueError(
"Local embedding model files changed; create a new embedding client"
)
self._clear_locked()
logger.info("Reloading evicted local embedding model")
try:
self._load_locked(handle._factory, handle._kwargs)
# An operator or another process can replace artifacts while
# the constructor reads them. Never give an old handle the
# resulting weights under its original configuration identity.
stamp = _artifact_stamp(handle._kwargs)
except BaseException:
self._clear_locked()
raise
if stamp != handle._key[1]:
self._clear_locked()
raise ValueError(
"Local embedding model files changed; create a new embedding client"
)
self.key = handle._key
def _run_serially(
self,
handle: SharedSentenceTransformerEmbeddings,
operation: Callable[[Embeddings], _Result],
) -> _Result:
# The caller holds shared access, so no writer can replace the backend
# while this call waits for its turn or runs. The returned vectors are
# ordinary lists; no caller keeps a model reference that could defeat
# eviction or the one-model cap.
backend = self.backend
if backend is None:
raise RuntimeError("Local embedding model was not initialized")
if getattr(self._inference_state, "active", False):
# The inference lock is not reentrant; waiting would deadlock.
raise RuntimeError(
"Recursive local embedding inference is not supported"
)
with self._inference:
try:
self._inference_state.active = True
if handle._closed:
raise RuntimeError("Embedding client is closed")
return operation(backend)
finally:
self._inference_state.active = False
class SharedSentenceTransformerEmbeddings(Embeddings):
"""Per-caller lifecycle/configuration, with no ownership of model weights."""
def __init__(
self,
cache: _ModelCache,
factory: _Factory,
kwargs: dict[str, Any],
key: _CacheKey,
) -> None:
self._cache = cache
self._factory = factory
self._kwargs = kwargs
self._key = key
self._closed = False
def embed_documents(self, texts: list[str]) -> list[list[float]]:
return self._cache.embed(
self, lambda backend: backend.embed_documents(texts)
)
def embed_query(self, text: str) -> list[float]:
return self._cache.embed(
self, lambda backend: backend.embed_query(text)
)
def close(self) -> None:
# Closing one search must not invalidate another search's handle.
# Already-started inference may finish; future calls are refused.
self._closed = True
_MODEL_CACHE = _ModelCache()
def shared_sentence_transformer_embeddings(
factory: Callable[..., Embeddings], **kwargs: Any
) -> Embeddings:
"""Prepare a checked provider configuration and return a lightweight handle."""
return _MODEL_CACHE.prepare(factory, kwargs)
def clear_sentence_transformer_cache(*, wait: bool = True) -> bool:
"""Release weights, returning False if shutdown finds inference busy.
Shutdown must not wait indefinitely for a daemon research worker. In that
case the process exit reclaims its active model along with the worker.
"""
return _MODEL_CACHE.clear(wait=wait)
atexit.register(clear_sentence_transformer_cache, wait=False)