Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

14.5 KB
Raw Blame
"""One resident local embedding backend; callers own weight-free handles.

The cache contains model weights/tokenizer only, never user settings, documents,
query results, or authorization decisions. Provider admission runs before this
module is called. Inference is serial: one call at a time uses the resident
model, and model replacement waits until every admitted call finishes.
"""

from __future__ import annotations

import atexit
from copy import deepcopy
import gc
import hashlib
from itertools import islice
from os import scandir
from pathlib import Path
import re
from stat import S_ISDIR
from threading import Lock, local
from typing import Any, Callable, TypeVar

from langchain_core.embeddings import Embeddings

from .model_loading import MODEL_LOAD_LOCK, load_sentence_transformer
from ..security.secure_logging import logger

_Result = TypeVar("_Result")
_Factory = Callable[..., Embeddings]
_CacheKey = tuple[tuple[_Factory, Any], str]
_MAX_ARTIFACT_STAMP_ENTRIES = 512
_ARTIFACT_STAMP_LIMIT_MESSAGE = (
    f"Local embedding model artifact tree exceeds "
    f"{_MAX_ARTIFACT_STAMP_ENTRIES} entries; remove unrelated files "
    "or directory links"
)


def _freeze(value: Any) -> Any:
    if isinstance(value, dict):
        return tuple(
            sorted((key, _freeze(item)) for key, item in value.items())
        )
    if isinstance(value, (list, tuple)):
        return tuple(_freeze(item) for item in value)
    return value


def _artifact_stamp(kwargs: dict) -> str:
    """Notice local replacements and floating Hub ref changes without I/O
    of model contents or contacting the network.

    Hub snapshots consist of links to immutable blobs. Stat follows those links
    so an operator replacing a blob also changes the key. Local module directory
    links are followed too, visiting each directory once to stop cycles. A handle
    cannot reload changed files after eviction and silently use different weights.
    """
    name = kwargs["model_name"]
    # nosemgrep: semgrep.rules.path-traversal-risk, reason: Provider admission confines absolute model paths to the app models directory; the stamp only stats and lists entries
    path = Path(name)
    references = []
    if path.is_absolute():
        roots = [path]
    else:
        # nosemgrep: semgrep.rules.path-traversal-risk, reason: Operator model cache root; admitted repository ids have slashes replaced and revisions are catalog or cached commit ids
        cache = Path(kwargs["cache_folder"])
        repositories = [name]
        if "/" not in name:
            repositories.append(f"sentence-transformers/{name}")
        roots = []
        revision = kwargs.get("model_kwargs", {}).get("revision")
        for repository in repositories:
            cached_repo = cache / ("models--" + repository.replace("/", "--"))
            snapshot = revision
            if not revision or not re.fullmatch(r"[0-9a-f]{40}", revision):
                reference = cached_repo / "refs" / (revision or "main")
                references.append(reference)
                try:
                    # nosemgrep: semgrep.rules.path-traversal-risk, reason: Reads refs/main of an admitted repository inside the operator model cache; repository ids have slashes replaced and provider revisions are commit ids
                    snapshot = reference.read_text(encoding="ascii").strip()
                except FileNotFoundError:
                    snapshot = None
            if snapshot and re.fullmatch(r"[0-9a-f]{40}", snapshot):
                roots.append(cached_repo / "snapshots" / snapshot)
        # Do not scan all snapshots/blobs or .no_exist markers: warming an
        # unrelated revision must not invalidate handles pinned to old bytes.
        roots.extend(references)
    digest = hashlib.sha256()
    for root in roots:
        # Missing files are part of the identity too (initial download/repair).
        digest.update(str(root).encode())
        paths = [root]
        seen_directories = set()
        visited = 0
        while paths:
            if visited >= _MAX_ARTIFACT_STAMP_ENTRIES:
                raise ValueError(_ARTIFACT_STAMP_LIMIT_MESSAGE)
            path = paths.pop()
            visited += 1
            try:
                stat = path.stat()
            except FileNotFoundError:
                continue
            digest.update(str(path).encode())
            digest.update(
                repr(
                    (
                        stat.st_dev,
                        stat.st_ino,
                        stat.st_size,
                        stat.st_mtime_ns,
                        stat.st_ctime_ns,
                    )
                ).encode()
            )
            if S_ISDIR(stat.st_mode):
                # rglob does not descend into directory symlinks. Stamp the
                # target and its files, while preventing alias/cycle expansion.
                directory = (stat.st_dev, stat.st_ino)
                if directory in seen_directories:
                    continue
                seen_directories.add(directory)
                # Path.iterdir() materializes the whole directory before a
                # caller can stop, so use lazy scandir and consume at most
                # one entry beyond the remaining budget. Never return a
                # partial stamp: unseen model bytes could change silently.
                remaining = _MAX_ARTIFACT_STAMP_ENTRIES - visited
                # nosemgrep: semgrep.rules.path-traversal-risk, reason: Lists an admitted model root and linked module directories only within a bounded budget; no file contents are read or written
                with scandir(path) as entries:
                    children = [
                        path / entry.name
                        for entry in islice(entries, remaining + 1)
                    ]
                if len(children) > remaining:
                    raise ValueError(_ARTIFACT_STAMP_LIMIT_MESSAGE)
                paths.extend(sorted(children, reverse=True))
    return digest.hexdigest()


class _ModelCache:
    def __init__(self) -> None:
        self.key: _CacheKey | None = None
        self.backend: Embeddings | None = None
        # Tokenization and forward passes mutate model and tokenizer state,
        # so admitted calls take turns on the one resident backend.
        self._inference = Lock()
        self._inference_state = local()

    def _clear_locked(self) -> None:
        had_model = self.backend is not None
        self.backend = None
        self.key = None
        if had_model:
            # SentenceTransformer -> model_card_data -> SentenceTransformer
            # is a strong cycle. Dropping the wrapper alone retains hundreds
            # of MiB until a generation-2 collection eventually happens.
            gc.collect()

    def clear(self, *, wait: bool = True) -> bool:
        if MODEL_LOAD_LOCK.acquire(blocking=wait):
            try:
                self._clear_locked()
                return True
            finally:
                MODEL_LOAD_LOCK.release()
        return False

    def prepare(
        self, factory: _Factory, kwargs: dict[str, Any]
    ) -> SharedSentenceTransformerEmbeddings:
        # A warm handle for an identical configuration is created without
        # waiting for active calls; its own calls then wait for their turn.
        with MODEL_LOAD_LOCK.inference():
            options, identity = self._options(factory, kwargs)
            key = (identity, _artifact_stamp(kwargs))
            if self.key == key:
                return SharedSentenceTransformerEmbeddings(
                    self, factory, options, key
                )
        with MODEL_LOAD_LOCK:
            return self._prepare_locked(factory, kwargs)

    @staticmethod
    def _options(factory: _Factory, kwargs: dict[str, Any]):
        local_options = deepcopy(kwargs)
        local_options.setdefault("model_kwargs", {})["local_files_only"] = True
        return local_options, (factory, _freeze(local_options))

    def _load_locked(self, factory: _Factory, kwargs: dict[str, Any]) -> None:
        self.backend = load_sentence_transformer(factory, **kwargs)

    def _prepare_locked(
        self, factory: _Factory, kwargs: dict[str, Any]
    ) -> SharedSentenceTransformerEmbeddings:
        options = deepcopy(kwargs)
        # Network permission is checked by the provider on EVERY request.
        # It is not a property of the resident weights. After initial
        # construction, an evicted handle may only reload local files.
        local_options, identity = self._options(factory, options)
        stamp = _artifact_stamp(options)
        key = (identity, stamp)
        if self.key != key:
            self._clear_locked()
            logger.info("Loading resident local embedding model")
            try:
                self._load_locked(factory, options)
                loaded = _artifact_stamp(options)
                if loaded != stamp:
                    # An authorized download/repair, or a replacement, changed
                    # the files while the constructor read them, so neither
                    # stamp need describe the loaded bytes. Load once more
                    # from local files and keep it only if they stay intact.
                    self._clear_locked()
                    logger.info(
                        "Reloading local embedding model after its files changed"
                    )
                    self._load_locked(factory, local_options)
                    stamp, loaded = loaded, _artifact_stamp(options)
            except BaseException:
                self._clear_locked()
                raise
            if loaded != stamp:
                self._clear_locked()
                raise ValueError(
                    "Local embedding model files changed; create a new embedding client"
                )
            key = (identity, stamp)
            self.key = key
        return SharedSentenceTransformerEmbeddings(
            self, factory, local_options, key
        )

    def embed(
        self,
        handle: SharedSentenceTransformerEmbeddings,
        operation: Callable[[Embeddings], _Result],
    ) -> _Result:
        while True:
            with MODEL_LOAD_LOCK.inference():
                if handle._closed:
                    raise RuntimeError("Embedding client is closed")
                if self.key == handle._key:
                    return self._run_serially(handle, operation)
            # Never upgrade shared access in place: another reader may also
            # need a different model. Recheck after obtaining exclusive access.
            with MODEL_LOAD_LOCK:
                self._reload_locked(handle)

    def _reload_locked(
        self, handle: SharedSentenceTransformerEmbeddings
    ) -> None:
        if handle._closed:
            raise RuntimeError("Embedding client is closed")
        if self.key != handle._key:
            if _artifact_stamp(handle._kwargs) != handle._key[1]:
                raise ValueError(
                    "Local embedding model files changed; create a new embedding client"
                )
            self._clear_locked()
            logger.info("Reloading evicted local embedding model")
            try:
                self._load_locked(handle._factory, handle._kwargs)
                # An operator or another process can replace artifacts while
                # the constructor reads them. Never give an old handle the
                # resulting weights under its original configuration identity.
                stamp = _artifact_stamp(handle._kwargs)
            except BaseException:
                self._clear_locked()
                raise
            if stamp != handle._key[1]:
                self._clear_locked()
                raise ValueError(
                    "Local embedding model files changed; create a new embedding client"
                )
            self.key = handle._key

    def _run_serially(
        self,
        handle: SharedSentenceTransformerEmbeddings,
        operation: Callable[[Embeddings], _Result],
    ) -> _Result:
        # The caller holds shared access, so no writer can replace the backend
        # while this call waits for its turn or runs. The returned vectors are
        # ordinary lists; no caller keeps a model reference that could defeat
        # eviction or the one-model cap.
        backend = self.backend
        if backend is None:
            raise RuntimeError("Local embedding model was not initialized")
        if getattr(self._inference_state, "active", False):
            # The inference lock is not reentrant; waiting would deadlock.
            raise RuntimeError(
                "Recursive local embedding inference is not supported"
            )
        with self._inference:
            try:
                self._inference_state.active = True
                if handle._closed:
                    raise RuntimeError("Embedding client is closed")
                return operation(backend)
            finally:
                self._inference_state.active = False


class SharedSentenceTransformerEmbeddings(Embeddings):
    """Per-caller lifecycle/configuration, with no ownership of model weights."""

    def __init__(
        self,
        cache: _ModelCache,
        factory: _Factory,
        kwargs: dict[str, Any],
        key: _CacheKey,
    ) -> None:
        self._cache = cache
        self._factory = factory
        self._kwargs = kwargs
        self._key = key
        self._closed = False

    def embed_documents(self, texts: list[str]) -> list[list[float]]:
        return self._cache.embed(
            self, lambda backend: backend.embed_documents(texts)
        )

    def embed_query(self, text: str) -> list[float]:
        return self._cache.embed(
            self, lambda backend: backend.embed_query(text)
        )

    def close(self) -> None:
        # Closing one search must not invalidate another search's handle.
        # Already-started inference may finish; future calls are refused.
        self._closed = True


_MODEL_CACHE = _ModelCache()


def shared_sentence_transformer_embeddings(
    factory: Callable[..., Embeddings], **kwargs: Any
) -> Embeddings:
    """Prepare a checked provider configuration and return a lightweight handle."""
    return _MODEL_CACHE.prepare(factory, kwargs)


def clear_sentence_transformer_cache(*, wait: bool = True) -> bool:
    """Release weights, returning False if shutdown finds inference busy.

    Shutdown must not wait indefinitely for a daemon research worker. In that
    case the process exit reclaims its active model along with the worker.
    """
    return _MODEL_CACHE.clear(wait=wait)


atexit.register(clear_sentence_transformer_cache, wait=False)