~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Keep model loading exclusive while allowing resident-model inference."""
from contextlib import contextmanager
from threading import Condition, RLock, get_ident
from typing import Any, Callable, TypeVar
_Model = TypeVar("_Model")
class ModelAccessLock:
"""Reentrant writer lock with shared inference access.
Shared holders create handles for the warm model and run admitted
embedding calls; the model cache lets those calls use the model one at a
time. Writers take priority over new readers, so configuration changes and
cleanup cannot starve behind a stream of inference requests. The cache's
model constructors also use this gate: Transformers temporarily changes
process-wide dtype and initialization state, which must not overlap
resident inference.
"""
def __init__(self) -> None:
self._condition = Condition(RLock())
self._writer: int | None = None
self._depth = 0
self._readers: dict[int, int] = {}
self._waiting_writers = 0
def acquire(self, blocking: bool = True) -> bool:
if not self._condition.acquire(blocking=blocking):
return False
try:
owner = get_ident()
if self._writer == owner:
self._depth += 1
return True
if not blocking and (self._writer is not None or self._readers):
return False
if owner in self._readers:
raise RuntimeError(
"Cannot load or clear a model during inference"
)
self._waiting_writers += 1
try:
while self._writer is not None or self._readers:
self._condition.wait()
self._writer = owner
self._depth = 1
return True
finally:
self._waiting_writers -= 1
self._condition.notify_all()
finally:
self._condition.release()
def release(self) -> None:
with self._condition:
if self._writer != get_ident():
raise RuntimeError(
"Model loading lock is not owned by this thread"
)
self._depth -= 1
if not self._depth:
self._writer = None
self._condition.notify_all()
def __enter__(self):
self.acquire()
return self
def __exit__(self, *_exc):
self.release()
@contextmanager
def inference(self):
owner = get_ident()
with self._condition:
while self._writer != owner and (
self._writer is not None
or (self._waiting_writers and owner not in self._readers)
):
self._condition.wait()
self._readers[owner] = self._readers.get(owner, 0) + 1
try:
yield
finally:
with self._condition:
self._readers[owner] -= 1
if not self._readers[owner]:
del self._readers[owner]
self._condition.notify_all()
MODEL_LOAD_LOCK = ModelAccessLock()
def load_sentence_transformer(
factory: Callable[..., _Model], **kwargs: Any
) -> _Model:
"""Keep Transformers' dtype and initialization helpers out of races.
The gate is held for the whole constructor call and blocks every local
embedding call, so a constructor that downloads holds it for the whole
download.
"""
with MODEL_LOAD_LOCK:
return factory(**kwargs)