Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

3.5 KB
Raw Blame
"""Keep model loading exclusive while allowing resident-model inference."""

from contextlib import contextmanager
from threading import Condition, RLock, get_ident
from typing import Any, Callable, TypeVar

_Model = TypeVar("_Model")


class ModelAccessLock:
    """Reentrant writer lock with shared inference access.

    Shared holders create handles for the warm model and run admitted
    embedding calls; the model cache lets those calls use the model one at a
    time. Writers take priority over new readers, so configuration changes and
    cleanup cannot starve behind a stream of inference requests. The cache's
    model constructors also use this gate: Transformers temporarily changes
    process-wide dtype and initialization state, which must not overlap
    resident inference.
    """

    def __init__(self) -> None:
        self._condition = Condition(RLock())
        self._writer: int | None = None
        self._depth = 0
        self._readers: dict[int, int] = {}
        self._waiting_writers = 0

    def acquire(self, blocking: bool = True) -> bool:
        if not self._condition.acquire(blocking=blocking):
            return False
        try:
            owner = get_ident()
            if self._writer == owner:
                self._depth += 1
                return True
            if not blocking and (self._writer is not None or self._readers):
                return False
            if owner in self._readers:
                raise RuntimeError(
                    "Cannot load or clear a model during inference"
                )
            self._waiting_writers += 1
            try:
                while self._writer is not None or self._readers:
                    self._condition.wait()
                self._writer = owner
                self._depth = 1
                return True
            finally:
                self._waiting_writers -= 1
                self._condition.notify_all()
        finally:
            self._condition.release()

    def release(self) -> None:
        with self._condition:
            if self._writer != get_ident():
                raise RuntimeError(
                    "Model loading lock is not owned by this thread"
                )
            self._depth -= 1
            if not self._depth:
                self._writer = None
                self._condition.notify_all()

    def __enter__(self):
        self.acquire()
        return self

    def __exit__(self, *_exc):
        self.release()

    @contextmanager
    def inference(self):
        owner = get_ident()
        with self._condition:
            while self._writer != owner and (
                self._writer is not None
                or (self._waiting_writers and owner not in self._readers)
            ):
                self._condition.wait()
            self._readers[owner] = self._readers.get(owner, 0) + 1
        try:
            yield
        finally:
            with self._condition:
                self._readers[owner] -= 1
                if not self._readers[owner]:
                    del self._readers[owner]
                self._condition.notify_all()


MODEL_LOAD_LOCK = ModelAccessLock()


def load_sentence_transformer(
    factory: Callable[..., _Model], **kwargs: Any
) -> _Model:
    """Keep Transformers' dtype and initialization helpers out of races.

    The gate is held for the whole constructor call and blocks every local
    embedding call, so a constructor that downloads holds it for the whole
    download.
    """
    with MODEL_LOAD_LOCK:
        return factory(**kwargs)