~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Exercise real, tiny offline models: one resident model, serial inference."""
import asyncio
from concurrent.futures import ThreadPoolExecutor
import gc
from threading import Event, Lock
import time
import weakref
import pytest
from local_deep_research.embeddings import model_cache
from local_deep_research.embeddings.model_loading import (
MODEL_LOAD_LOCK,
ModelAccessLock,
load_sentence_transformer,
)
@pytest.fixture(scope="module", params=["bert", "modernbert"])
def tiny_model_path(request, tmp_path_factory):
"""Random weights and a six-word tokenizer; never download a real model."""
import torch
from tokenizers import Tokenizer, models, pre_tokenizers
from transformers import (
BertConfig,
BertModel,
ModernBertConfig,
ModernBertModel,
PreTrainedTokenizerFast,
)
path = tmp_path_factory.mktemp(request.param)
tokenizer = Tokenizer(
models.WordLevel(
{"[PAD]": 0, "[UNK]": 1, "a": 2, "b": 3, "c": 4, "d": 5},
unk_token="[UNK]",
)
)
tokenizer.pre_tokenizer = pre_tokenizers.Whitespace()
PreTrainedTokenizerFast(
tokenizer_object=tokenizer,
pad_token="[PAD]",
unk_token="[UNK]",
model_max_length=64,
).save_pretrained(path)
options = dict(
vocab_size=6,
hidden_size=16,
intermediate_size=32,
num_hidden_layers=2,
num_attention_heads=2,
max_position_embeddings=64,
pad_token_id=0,
)
if request.param == "bert":
model = BertModel(BertConfig(**options))
else:
model = ModernBertModel(
ModernBertConfig(
**options,
local_attention=8,
global_attn_every_n_layers=2,
cls_token_id=2,
sep_token_id=3,
)
)
model.save_pretrained(path)
del model
old_threads = torch.get_num_threads()
torch.set_num_threads(1)
try:
yield str(path)
finally:
torch.set_num_threads(old_threads)
@pytest.fixture
def real_backend(tiny_model_path, monkeypatch):
from langchain_community.embeddings.huggingface import HuggingFaceEmbeddings
monkeypatch.setenv("HF_HUB_OFFLINE", "1")
monkeypatch.setenv("TRANSFORMERS_OFFLINE", "1")
return HuggingFaceEmbeddings(
model_name=tiny_model_path,
model_kwargs={
"device": "cpu",
"local_files_only": True,
"model_kwargs": {"attn_implementation": "eager"},
},
encode_kwargs={"normalize_embeddings": True},
)
class _ObservedLock:
"""Delegate to the cache's inference lock; report calls taking a turn.
``waiting`` is set once ``after`` calls have asked for a turn, before the
last of them blocks on the real lock.
"""
def __init__(self, lock, waiting, after=1):
self._lock = lock
self._waiting = waiting
self._after = after
self._count = Lock()
self.waits = 0
def __enter__(self):
with self._count:
self.waits += 1
if self.waits >= self._after:
self._waiting.set()
return self._lock.__enter__()
def __exit__(self, *exc):
return self._lock.__exit__(*exc)
def test_calls_take_turns_on_one_model_and_block_loaders_and_cleanup(
real_backend, tiny_model_path, monkeypatch
):
client = model_cache.shared_sentence_transformer_embeddings(
lambda **_: real_backend, model_name=tiny_model_path
)
transformer_type = type(real_backend.client[0].auto_model)
forward = transformer_type.forward
entered, release, loader_requested, loaded = (Event() for _ in range(4))
mutex = Lock()
active = peak = 0
modules = set()
def blocking_forward(self, *args, **kwargs):
nonlocal active, peak
with mutex:
active += 1
peak = max(peak, active)
modules.add(id(self))
entered.set()
try:
assert release.wait(timeout=10)
return forward(self, *args, **kwargs)
finally:
with mutex:
active -= 1
def load_other():
loader_requested.set()
return load_sentence_transformer(lambda: loaded.set())
monkeypatch.setattr(transformer_type, "forward", blocking_forward)
with ThreadPoolExecutor(max_workers=4) as executor:
futures = [executor.submit(client.embed_query, "a b") for _ in range(3)]
try:
assert entered.wait(timeout=10)
loading = executor.submit(load_other)
assert loader_requested.wait(timeout=10)
assert (
model_cache.clear_sentence_transformer_cache(wait=False)
is False
)
assert not loaded.is_set()
assert peak == 1
finally:
release.set()
for future in futures:
assert len(future.result(timeout=15)) == 16
loading.result(timeout=15)
assert peak == 1
# Every call used the one resident model; no copy was made for a call.
assert modules == {id(real_backend.client[0].auto_model)}
assert loaded.is_set()
@pytest.mark.asyncio
async def test_async_calls_take_turns_on_the_resident_model(
real_backend, tiny_model_path, monkeypatch
):
client = model_cache.shared_sentence_transformer_embeddings(
lambda **_: real_backend, model_name=tiny_model_path
)
transformer_type = type(real_backend.client[0].auto_model)
forward = transformer_type.forward
cache = model_cache._MODEL_CACHE
both_waiting = Event()
observed = _ObservedLock(cache._inference, both_waiting, after=2)
monkeypatch.setattr(cache, "_inference", observed)
mutex = Lock()
active = peak = 0
def blocking_forward(self, *args, **kwargs):
nonlocal active, peak
with mutex:
active += 1
peak = max(peak, active)
try:
# Hold the model until the other call waits for its turn.
assert both_waiting.wait(timeout=10)
return forward(self, *args, **kwargs)
finally:
with mutex:
active -= 1
monkeypatch.setattr(transformer_type, "forward", blocking_forward)
results = await asyncio.wait_for(
asyncio.gather(
client.aembed_query("a"), client.aembed_documents(["b c"])
),
timeout=15,
)
assert len(results[0]) == len(results[1][0]) == 16
assert observed.waits == 2
assert peak == 1
def test_warm_handle_is_created_while_busy_and_closes_while_queued(
real_backend, tiny_model_path, monkeypatch
):
loads = []
def factory(**_kwargs):
loads.append(True)
return real_backend
def create():
return model_cache.shared_sentence_transformer_embeddings(
factory, model_name=tiny_model_path
)
client = create()
entered, release, queued = Event(), Event(), Event()
transformer_type = type(real_backend.client[0].auto_model)
forward = transformer_type.forward
lock = Lock()
forwards = 0
def blocking_forward(self, *args, **kwargs):
nonlocal forwards
with lock:
forwards += 1
entered.set()
assert release.wait(timeout=10)
return forward(self, *args, **kwargs)
monkeypatch.setattr(transformer_type, "forward", blocking_forward)
cache = model_cache._MODEL_CACHE
with ThreadPoolExecutor(max_workers=2) as executor:
first = executor.submit(client.embed_query, "a")
try:
assert entered.wait(timeout=10)
# Shared access: creating a warm handle needs no turn on the model.
late = executor.submit(create).result(timeout=5)
assert len(loads) == 1
monkeypatch.setattr(
cache, "_inference", _ObservedLock(cache._inference, queued)
)
third = executor.submit(late.embed_query, "c")
assert queued.wait(timeout=5)
late.close()
finally:
release.set()
assert len(first.result(timeout=10)) == 16
with pytest.raises(RuntimeError, match="closed"):
third.result(timeout=10)
assert forwards == 1
assert len(client.embed_query("d")) == 16
def test_configuration_change_waits_for_active_and_queued_calls(
real_backend, tiny_model_path, monkeypatch
):
loads = []
def factory(**kwargs):
loads.append(kwargs["encode_kwargs"])
return real_backend
def create(normalize):
return model_cache.shared_sentence_transformer_embeddings(
factory,
model_name=tiny_model_path,
encode_kwargs={"normalize_embeddings": normalize},
)
client = create(True)
(
entered_first,
entered_second,
release_first,
release_second,
queued,
requested,
) = (Event() for _ in range(6))
transformer_type = type(real_backend.client[0].auto_model)
forward = transformer_type.forward
lock = Lock()
forwards = 0
def blocking_forward(self, *args, **kwargs):
nonlocal forwards
with lock:
index = forwards
forwards += 1
(entered_first if index == 0 else entered_second).set()
assert (release_first if index == 0 else release_second).wait(
timeout=10
)
return forward(self, *args, **kwargs)
def switch():
requested.set()
return create(False)
monkeypatch.setattr(transformer_type, "forward", blocking_forward)
cache = model_cache._MODEL_CACHE
with ThreadPoolExecutor(max_workers=3) as executor:
first = executor.submit(client.embed_query, "a")
try:
assert entered_first.wait(timeout=10)
monkeypatch.setattr(
cache, "_inference", _ObservedLock(cache._inference, queued)
)
second = executor.submit(client.embed_query, "b c")
# Admitted before the switch, so the switch must wait for it too.
assert queued.wait(timeout=5)
switching = executor.submit(switch)
assert requested.wait(timeout=5)
release_first.set()
assert len(first.result(timeout=10)) == 16
# The queued call takes its turn on the old model; the switch waits.
assert entered_second.wait(timeout=10)
assert not switching.done()
assert len(loads) == 1
assert cache.key == client._key
finally:
release_first.set()
release_second.set()
assert len(second.result(timeout=10)) == 16
switching.result(timeout=10)
assert loads == [
{"normalize_embeddings": True},
{"normalize_embeddings": False},
]
def test_failed_forward_releases_inference_turn_and_gate(
real_backend, tiny_model_path, monkeypatch
):
client = model_cache.shared_sentence_transformer_embeddings(
lambda **_: real_backend, model_name=tiny_model_path
)
transformer_type = type(real_backend.client[0].auto_model)
forward = transformer_type.forward
def fail(self, *args, **kwargs):
raise RuntimeError("synthetic inference failure")
monkeypatch.setattr(transformer_type, "forward", fail)
for _ in range(3):
with pytest.raises(RuntimeError, match="synthetic inference failure"):
client.embed_query("a")
monkeypatch.setattr(transformer_type, "forward", forward)
assert len(client.embed_query("b")) == 16
assert model_cache.clear_sentence_transformer_cache(wait=False) is True
def test_cache_clear_collects_the_real_model_cycle_with_live_handles(
tiny_model_path, monkeypatch
):
from langchain_community.embeddings.huggingface import HuggingFaceEmbeddings
monkeypatch.setenv("HF_HUB_OFFLINE", "1")
monkeypatch.setenv("TRANSFORMERS_OFFLINE", "1")
# As with provider handles, the cache is the model's only owner.
client = model_cache.shared_sentence_transformer_embeddings(
HuggingFaceEmbeddings,
model_name=tiny_model_path,
model_kwargs={
"device": "cpu",
"model_kwargs": {"attn_implementation": "eager"},
},
)
assert len(client.embed_query("a")) == 16
model = weakref.ref(model_cache._MODEL_CACHE.backend.client)
# SentenceTransformer -> model_card_data -> SentenceTransformer.
assert model().model_card_data.model is model()
was_enabled = gc.isenabled()
gc.disable()
try:
assert model_cache.clear_sentence_transformer_cache(wait=False)
assert model() is None
assert not client._closed
finally:
if was_enabled:
gc.enable()
def test_reader_gate_rejects_unsafe_upgrade_and_remains_usable():
with MODEL_LOAD_LOCK.inference():
assert MODEL_LOAD_LOCK.acquire(blocking=False) is False
with pytest.raises(RuntimeError, match="during inference"):
load_sentence_transformer(lambda: None)
with MODEL_LOAD_LOCK.inference():
pass
with MODEL_LOAD_LOCK:
with MODEL_LOAD_LOCK:
with MODEL_LOAD_LOCK.inference():
pass
assert model_cache.clear_sentence_transformer_cache(wait=False)
def test_blocked_writer_is_served_before_new_readers():
lock = ModelAccessLock()
reader_holding, release_reader = Event(), Event()
writer_requested, writer_holding, release_writer = (
Event(),
Event(),
Event(),
)
late_reader_holding = Event()
order = []
order_lock = Lock()
def record(step):
with order_lock:
order.append(step)
def hold_shared():
with lock.inference():
record("reader")
reader_holding.set()
assert release_reader.wait(timeout=10)
def write_exclusively():
assert reader_holding.wait(timeout=10)
writer_requested.set()
with lock:
record("writer")
writer_holding.set()
assert release_writer.wait(timeout=10)
def read_late():
with lock.inference():
record("late-reader")
late_reader_holding.set()
with ThreadPoolExecutor(max_workers=3) as executor:
reader = executor.submit(hold_shared)
assert reader_holding.wait(timeout=10)
writer = executor.submit(write_exclusively)
assert writer_requested.wait(timeout=10)
# Wait until the writer is blocked behind the shared holder so the
# late reader arrives while a writer is already waiting. The lock
# exposes no public "a writer is waiting" probe, so poll the waiter
# count its writer priority is built on.
deadline = time.monotonic() + 10
while lock._waiting_writers == 0 and time.monotonic() < deadline:
time.sleep(0.01)
assert lock._waiting_writers == 1
late = executor.submit(read_late)
try:
# New readers must not be admitted ahead of the waiting writer.
assert not late_reader_holding.wait(timeout=0.2)
release_reader.set()
assert writer_holding.wait(timeout=10)
assert not late_reader_holding.wait(timeout=0.2)
finally:
release_reader.set()
release_writer.set()
reader.result(timeout=10)
writer.result(timeout=10)
late.result(timeout=10)
assert order == ["reader", "writer", "late-reader"]