~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Memory ownership and concurrent use of the resident embedding backend."""
from concurrent.futures import ThreadPoolExecutor
import gc
from pathlib import Path
from threading import Barrier, Event
from types import SimpleNamespace
import weakref
import pytest
from local_deep_research.embeddings import model_cache
from local_deep_research.embeddings.model_cache import (
clear_sentence_transformer_cache,
shared_sentence_transformer_embeddings,
)
class CyclicBackend:
def __init__(self, model_name, **kwargs):
self.name = model_name
self.cycle = self
def embed_query(self, text):
return [float(len(text))]
def embed_documents(self, texts):
return [self.embed_query(text) for text in texts]
@pytest.fixture
def models(tmp_path):
refs = []
calls = []
def factory(**kwargs):
calls.append(kwargs)
backend = CyclicBackend(**kwargs)
refs.append(weakref.ref(backend))
return backend
def create(name="synthetic/model", **options):
return shared_sentence_transformer_embeddings(
factory,
model_name=name,
cache_folder=str(tmp_path),
model_kwargs=options,
)
return create, calls, refs
def test_finished_searches_reuse_weights_and_do_not_share_results(models):
create, calls, refs = models
for i in range(12):
client = create(device="cpu", revision="one")
result = client.embed_query("query" + "x" * i)
assert result == [float(5 + i)]
result[0] = -1 # A caller cannot poison a subsequent result.
client.close()
assert len(calls) == 1
assert sum(ref() is not None for ref in refs) == 1
def test_handles_do_not_pin_evicted_models_or_their_cycles(models):
create, calls, refs = models
was_enabled = gc.isenabled()
gc.disable() # Prove eviction does not depend on automatic collection.
try:
first = create("synthetic/first")
second = create("synthetic/second")
assert refs[0]() is None
assert refs[1]() is not None
assert first.embed_query("reloaded") == [8.0]
assert refs[1]() is None
assert refs[2]() is not None
assert len(calls) == 3
assert calls[-1]["model_kwargs"]["local_files_only"] is True
clear_sentence_transformer_cache()
assert all(ref() is None for ref in refs)
# Both lightweight handles still exist, but neither owns weights.
assert not first._closed and not second._closed
finally:
if was_enabled:
gc.enable()
@pytest.mark.parametrize(
"first,second",
[
({"device": "cpu"}, {"device": "cuda"}),
({"revision": "one"}, {"revision": "two"}),
(
{"model_kwargs": {"torch_dtype": "float16"}},
{"model_kwargs": {"torch_dtype": "float32"}},
),
],
)
def test_model_options_are_part_of_the_identity(models, first, second):
create, calls, refs = models
a = create(**first)
b = create(**second)
assert len(calls) == 2
assert refs[0]() is None
assert a is not b
def test_close_is_per_handle_and_idempotent(models):
create, calls, refs = models
a, b = create(), create()
a.close()
a.close()
with pytest.raises(RuntimeError, match="closed"):
a.embed_query("closed")
assert b.embed_documents(["a", "bb"]) == [[1.0], [2.0]]
assert len(calls) == 1 and refs[0]() is not None
def test_simultaneous_first_users_load_one_model(models):
create, calls, _ = models
barrier = Barrier(6)
def search(i):
barrier.wait(timeout=10)
client = create()
try:
return client.embed_query("x" * i)
finally:
client.close()
with ThreadPoolExecutor(max_workers=6) as executor:
results = list(executor.map(search, range(6)))
assert results == [[float(i)] for i in range(6)]
assert len(calls) == 1
def test_eviction_waits_for_inference(tmp_path):
entered, release, requested = Event(), Event(), Event()
refs = []
class BlockingBackend(CyclicBackend):
def embed_query(self, text):
entered.set()
assert release.wait(timeout=10)
assert refs[0]() is self
return super().embed_query(text)
def factory(**kwargs):
backend = BlockingBackend(**kwargs)
refs.append(weakref.ref(backend))
return backend
def create(name):
return shared_sentence_transformer_embeddings(
factory,
model_name=name,
cache_folder=str(tmp_path),
model_kwargs={},
)
def replace():
requested.set()
return create("synthetic/second")
first = create("synthetic/first")
with ThreadPoolExecutor(max_workers=2) as executor:
inference = executor.submit(first.embed_query, "busy")
assert entered.wait(timeout=10)
replacement = executor.submit(replace)
assert requested.wait(timeout=10)
try:
assert clear_sentence_transformer_cache(wait=False) is False
assert not replacement.done()
assert len(refs) == 1
finally:
release.set()
assert inference.result(timeout=10) == [4.0]
replacement.result(timeout=10)
assert refs[0]() is None and refs[1]() is not None
def test_failed_load_is_retryable_and_never_poisoned(tmp_path):
attempts = []
def factory(**kwargs):
attempts.append(kwargs)
if len(attempts) == 1:
raise OSError("synthetic missing weights")
return CyclicBackend(**kwargs)
kwargs = dict(
model_name="synthetic/model",
cache_folder=str(tmp_path),
model_kwargs={},
)
with pytest.raises(OSError):
shared_sentence_transformer_embeddings(factory, **kwargs)
client = shared_sentence_transformer_embeddings(factory, **kwargs)
assert client.embed_query("retry") == [5.0]
assert len(attempts) == 2
def test_failed_reload_is_retryable_and_never_poisoned(tmp_path):
attempts = []
failing = False
def factory(**kwargs):
attempts.append(kwargs)
if failing:
raise OSError("synthetic reload failure")
return CyclicBackend(**kwargs)
def create(name="synthetic/model"):
return shared_sentence_transformer_embeddings(
factory, model_name=name, cache_folder=str(tmp_path)
)
client = create()
assert client.embed_query("resident") == [8.0]
create("synthetic/other") # Evict without closing the first handle.
failing = True
with pytest.raises(OSError, match="synthetic reload failure"):
client.embed_query("must fail")
# A failed reload must leave nothing behind that an old handle could
# mistake for its own weights.
assert model_cache._MODEL_CACHE.key is None
assert model_cache._MODEL_CACHE.backend is None
failing = False
assert client.embed_query("retry") == [5.0]
assert model_cache._MODEL_CACHE.key == client._key
assert len(attempts) == 4
def test_shutdown_cache_clear_is_registered_without_waiting(monkeypatch):
import atexit
import importlib
registrations = []
monkeypatch.setattr(
atexit,
"register",
lambda func, **kwargs: registrations.append((func, kwargs)),
)
try:
importlib.reload(model_cache)
assert registrations == [
(model_cache.clear_sentence_transformer_cache, {"wait": False})
]
finally:
# undo() puts atexit.register back; reloading again here would only add a
# duplicate handler for the real one.
monkeypatch.undo()
def test_encoding_options_are_part_of_the_configuration(tmp_path):
calls = []
def factory(**kwargs):
calls.append(kwargs)
return CyclicBackend(**kwargs)
def create(**encode_kwargs):
return shared_sentence_transformer_embeddings(
factory,
model_name="synthetic/model",
cache_folder=str(tmp_path),
encode_kwargs=encode_kwargs,
)
create(normalize_embeddings=True, prompt="query: ")
create(prompt="query: ", normalize_embeddings=True)
assert len(calls) == 1
create(normalize_embeddings=False, prompt="query: ")
create(normalize_embeddings=False, prompt="document: ")
assert len(calls) == 3
def test_recursive_serial_backend_fails_instead_of_waiting_for_its_own_slot(
tmp_path,
):
class RecursiveBackend(CyclicBackend):
def embed_query(self, text):
if text == "recursive":
return client.embed_query("inner")
return super().embed_query(text)
client = shared_sentence_transformer_embeddings(
RecursiveBackend,
model_name="synthetic/model",
cache_folder=str(tmp_path),
)
with pytest.raises(RuntimeError, match="Recursive local embedding"):
client.embed_query("recursive")
assert client.embed_query("recovered") == [9.0]
def test_local_replacement_invalidates_cache_and_old_handles(models, tmp_path):
create, calls, refs = models
model_dir = tmp_path / "local-model"
model_dir.mkdir()
weight = model_dir / "model.safetensors"
weight.write_bytes(b"synthetic-v1")
old = create(str(model_dir))
# A same-path replacement has a different file identity, even if size and
# timestamps have low resolution on the platform running this test.
replacement = model_dir / "replacement"
replacement.write_bytes(b"synthetic-v2-longer")
replacement.replace(weight)
new = create(str(model_dir))
assert len(calls) == 2 and refs[0]() is None
assert new.embed_query("new") == [3.0]
with pytest.raises(ValueError, match="files changed"):
old.embed_query("old")
assert len(calls) == 2
class ArtifactBackend(CyclicBackend):
def __init__(self, value, **kwargs):
super().__init__(**kwargs)
self.value = value
def embed_query(self, _text):
return [self.value]
def _model_artifacts(tmp_path, layout):
"""Synthetic weights "1"; each replacement toggles them between 1 and 2.
``local`` rewrites a file in an explicit model directory. ``hub_ref``
moves a Hub cache's floating ``refs/main`` between two snapshots.
Neither change may depend on the clock. The artifact stamp sees an
in-place rewrite through size and timestamps only, and a rewrite
microseconds after the previous write can keep the same timestamps on
coarse filesystems. So the ``local`` rewrite also changes the size. The
``hub_ref`` move changes the stamped snapshot path itself.
"""
cache_dir = tmp_path / "hub"
if layout == "local":
model = tmp_path / "local-model"
model.mkdir()
weights = model / "model.safetensors"
weights.write_text("1")
def current_weights():
return float(weights.read_text())
def replace_weights():
weights.write_text("2.0" if current_weights() == 1.0 else "1")
return str(model), cache_dir, current_weights, replace_weights
repository = cache_dir / "models--synthetic--model"
for revision, value in (("a" * 40, "1"), ("b" * 40, "2")):
snapshot = repository / "snapshots" / revision
snapshot.mkdir(parents=True)
(snapshot / "model.safetensors").write_text(value)
(repository / "refs").mkdir()
reference = repository / "refs/main"
reference.write_text("a" * 40)
def current_weights():
return float(
(
repository
/ "snapshots"
/ reference.read_text()
/ "model.safetensors"
).read_text()
)
def replace_weights():
reference.write_text("b" * 40 if current_weights() == 1.0 else "a" * 40)
return "synthetic/model", cache_dir, current_weights, replace_weights
@pytest.fixture
def coarse_clock(monkeypatch):
"""Hide file timestamps from the artifact stamp, as one tick would.
Tests using ``_model_artifacts`` then pass or fail the same way on every
filesystem, however fine its timestamps.
"""
class CoarseClockPath(type(Path())):
def stat(self, *, follow_symlinks=True):
real = super().stat(follow_symlinks=follow_symlinks)
return SimpleNamespace(
st_mode=real.st_mode,
st_dev=real.st_dev,
st_ino=real.st_ino,
st_size=real.st_size,
st_mtime_ns=0,
st_ctime_ns=0,
)
monkeypatch.setattr(model_cache, "Path", CoarseClockPath)
@pytest.mark.usefixtures("coarse_clock")
@pytest.mark.parametrize("layout", ["local", "hub_ref"])
@pytest.mark.parametrize("replace_during_load", [False, True])
def test_evicted_reload_rechecks_artifacts_after_loading(
tmp_path, layout, replace_during_load
):
"""An old handle must never adopt weights replaced during its reload."""
model_name, cache_dir, current_weights, replace_weights = _model_artifacts(
tmp_path, layout
)
calls = []
refs = []
replacing = False
def factory(**kwargs):
calls.append(kwargs)
if kwargs["model_name"] == model_name:
if replacing:
# Runs after pre-load validation, before the loader reads the
# artifacts. No timing sleeps or model download are needed.
replace_weights()
value = current_weights()
else:
value = 9.0
backend = ArtifactBackend(value, **kwargs)
refs.append(weakref.ref(backend))
return backend
def create(name):
return shared_sentence_transformer_embeddings(
factory, model_name=name, cache_folder=str(cache_dir)
)
old = create(model_name)
assert old.embed_query("original") == [1.0]
assert create("synthetic/other").embed_query("other") == [9.0]
replacing = replace_during_load
if replace_during_load:
with pytest.raises(ValueError, match="files changed"):
old.embed_query("must not use different weights")
assert refs[-1]() is None # Failed reload releases its model cycle.
replacing = False
fresh = create(model_name)
assert fresh.embed_query("new handle") == [2.0]
with pytest.raises(ValueError, match="files changed"):
old.embed_query("old handle stays invalid")
assert fresh.embed_query("still usable") == [2.0]
assert len(calls) == 4
else:
assert old.embed_query("unchanged weights") == [1.0]
assert len(calls) == 3
assert calls[2]["model_kwargs"]["local_files_only"] is True
@pytest.mark.usefixtures("coarse_clock")
@pytest.mark.parametrize("layout", ["local", "hub_ref"])
@pytest.mark.parametrize("replaced", ["before_read", "after_read"])
def test_first_load_rechecks_artifacts_after_loading(
tmp_path, layout, replaced
):
"""A first load must not adopt a key describing bytes it did not load.
A download or repair populates files before the constructor reads them;
an operator replacement can also land after the read. Either way the
handle must use the bytes now on disk, before and after an eviction.
"""
model_name, cache_dir, current_weights, replace_weights = _model_artifacts(
tmp_path, layout
)
calls = []
replacing = True
def factory(**kwargs):
nonlocal replacing
calls.append(kwargs)
if kwargs["model_name"] != model_name:
return ArtifactBackend(9.0, **kwargs)
if replacing and replaced == "before_read":
replace_weights()
value = current_weights()
if replacing and replaced == "after_read":
replace_weights()
replacing = False
return ArtifactBackend(value, **kwargs)
def create(name):
return shared_sentence_transformer_embeddings(
factory, model_name=name, cache_folder=str(cache_dir)
)
client = create(model_name)
assert client.embed_query("first call") == [2.0]
assert create("synthetic/other").embed_query("other") == [9.0]
assert client.embed_query("same handle after reload") == [2.0]
# The changed first load is retried once from local files, never by
# another download, and the reloaded handle keeps that key.
assert len(calls) == 4
assert calls[1]["model_name"] == model_name
assert calls[1]["model_kwargs"]["local_files_only"] is True
@pytest.mark.usefixtures("coarse_clock")
@pytest.mark.parametrize("layout", ["local", "hub_ref"])
def test_first_load_gives_up_when_files_keep_changing(tmp_path, layout):
model_name, cache_dir, current_weights, replace_weights = _model_artifacts(
tmp_path, layout
)
calls = []
refs = []
changing = True
def factory(**kwargs):
calls.append(kwargs)
if changing:
replace_weights()
backend = ArtifactBackend(current_weights(), **kwargs)
refs.append(weakref.ref(backend))
return backend
def create():
return shared_sentence_transformer_embeddings(
factory, model_name=model_name, cache_folder=str(cache_dir)
)
with pytest.raises(ValueError, match="files changed"):
create()
assert len(calls) == 2 # One bounded retry, never a loop.
assert all(ref() is None for ref in refs) # Neither model stays resident.
changing = False
assert create().embed_query("stable files") == [current_weights()]
assert len(calls) == 3
def _model_symlink(link, target, *, directory=False):
try:
link.symlink_to(target, target_is_directory=directory)
except (OSError, NotImplementedError): # pragma: no cover - platform
pytest.skip("symlinks not supported on this platform")
@pytest.mark.parametrize("layout", ["directory", "file_link", "directory_link"])
def test_evicted_handle_rejects_changed_linked_artifacts(
models, tmp_path, layout
):
create, calls, _ = models
model = tmp_path / "local-model"
backing = tmp_path / "shared-module"
model.mkdir()
backing.mkdir()
module = model / "0_Transformer"
weight = backing / "model.safetensors"
weight.write_bytes(b"original")
if layout == "directory_link":
_model_symlink(module, backing, directory=True)
else:
module.mkdir()
if layout == "file_link":
_model_symlink(module / weight.name, weight)
else:
weight = module / weight.name
weight.write_bytes(b"original")
old = create(str(model))
assert create(str(model)).embed_query("warm") == [4.0]
assert len(calls) == 1
create("synthetic/other") # Evict without closing the first handle.
weight.write_bytes(b"replacement weights")
with pytest.raises(ValueError, match="files changed"):
old.embed_query("must not reload changed weights")
assert len(calls) == 2
assert create(str(model)).embed_query("new") == [3.0]
assert len(calls) == 3
@pytest.mark.parametrize("cycle", [False, True])
@pytest.mark.timeout(10)
def test_shared_directory_links_are_stamped_without_following_cycles(
models, tmp_path, cycle
):
create, calls, _ = models
model = tmp_path / "local-model"
backing = tmp_path / "shared-module"
model.mkdir()
backing.mkdir()
_model_symlink(model / "first", backing, directory=True)
_model_symlink(model / "second", backing, directory=True)
if cycle:
_model_symlink(backing / "parent", model, directory=True)
weight = backing / "model.safetensors"
weight.write_bytes(b"original")
old = create(str(model))
create(str(model))
assert len(calls) == 1
weight.write_bytes(b"replacement weights")
create(str(model))
assert len(calls) == 2
with pytest.raises(ValueError, match="files changed"):
old.embed_query("old")
def test_retargeting_a_module_directory_invalidates_old_handles(
models, tmp_path
):
create, calls, _ = models
model = tmp_path / "local-model"
model.mkdir()
for name in ("original", "replacement"):
backing = tmp_path / name
backing.mkdir()
(backing / "model.safetensors").write_text(name)
module = model / "0_Transformer"
_model_symlink(module, tmp_path / "original", directory=True)
old = create(str(model))
module.unlink()
_model_symlink(module, tmp_path / "replacement", directory=True)
create(str(model))
assert len(calls) == 2
with pytest.raises(ValueError, match="files changed"):
old.embed_query("old")
@pytest.mark.parametrize("linked", [False, True])
def test_large_artifact_tree_is_bounded(tmp_path, linked):
model = tmp_path / "local-model"
model.mkdir()
tree = (
tmp_path / "unrelated-large-tree" if linked else model / "0_Transformer"
)
tree.mkdir()
(model / "model.safetensors").write_bytes(b"weights")
if linked:
_model_symlink(model / "0_Transformer", tree, directory=True)
for number in range(600):
(tree / f"unrelated-{number:04d}").write_bytes(b"x")
with pytest.raises(ValueError, match="artifact tree exceeds 512 entries"):
model_cache._artifact_stamp(
{"model_name": str(model), "cache_folder": str(tmp_path)}
)
def test_caller_mutation_cannot_change_evicted_reload(models):
create, calls, _ = models
options = {"torch_dtype": "original"}
first = create(model_kwargs=options)
options["torch_dtype"] = "modified"
create("synthetic/other")
first.embed_query("reload")
assert calls[-1]["model_kwargs"]["model_kwargs"] == {
"torch_dtype": "original"
}
def test_pinned_model_ignores_other_hub_revisions(models, tmp_path):
create, calls, _ = models
cache = tmp_path / "models--synthetic--model"
revision = "a" * 40
original = cache / "snapshots" / revision
original.mkdir(parents=True)
(original / "config.json").write_text("original")
first = create(revision=revision)
other = cache / "snapshots" / ("b" * 40)
other.mkdir()
(other / "config.json").write_text("other revision")
(cache / "refs").mkdir()
(cache / "refs/main").write_text("b" * 40)
second = create(revision=revision)
assert first.embed_query("one") == second.embed_query("two") == [3.0]
assert len(calls) == 1
def test_floating_ref_update_invalidates_old_handle(models, tmp_path):
create, calls, _ = models
cache = tmp_path / "models--synthetic--model"
(cache / "refs").mkdir(parents=True)
reference = cache / "refs/main"
reference.write_text("a" * 40)
first = create()
reference.write_text("b" * 40)
create()
assert len(calls) == 2
with pytest.raises(ValueError, match="files changed"):
first.embed_query("cannot change weights silently")
@pytest.mark.asyncio
async def test_async_embeddings_use_the_same_resident_model(models):
create, calls, _ = models
client = create()
assert await client.aembed_query("async") == [5.0]
assert await client.aembed_documents(["x", "yy"]) == [[1.0], [2.0]]
assert len(calls) == 1
def test_real_managers_reuse_backend_across_users_without_retaining_settings(
monkeypatch, tmp_path
):
from langchain_community import embeddings
from local_deep_research.embeddings.providers.implementations.sentence_transformers import (
SentenceTransformersProvider as Provider,
)
from local_deep_research.web_search_engines.engines.local_embedding_manager import (
LocalEmbeddingManager,
)
calls = []
def factory(**kwargs):
calls.append(kwargs)
return CyclicBackend(**kwargs)
monkeypatch.setattr(embeddings, "SentenceTransformerEmbeddings", factory)
monkeypatch.setattr(
Provider, "_model_cache_directory", lambda: str(tmp_path)
)
monkeypatch.setattr(
Provider, "_is_model_cached_locally", lambda *a, **kw: True
)
for username in ("synthetic-alice", "synthetic-bob", "synthetic-alice"):
with LocalEmbeddingManager(
settings_snapshot={
"_username": username,
"policy.egress_scope": "public_only",
"search.tool": "arxiv",
}
) as manager:
handle = manager.embeddings
assert handle.embed_query(username) == [float(len(username))]
with pytest.raises(RuntimeError, match="closed"):
handle.embed_query("after close")
assert len(calls) == 1
assert "_username" not in repr(calls)
assert "synthetic-alice" not in repr(vars(handle))
def test_warm_weights_do_not_bypass_provider_admission(monkeypatch, tmp_path):
from langchain_community import embeddings
from local_deep_research.embeddings.providers.implementations.sentence_transformers import (
SentenceTransformersProvider as Provider,
)
from local_deep_research.security.egress.policy import PolicyDeniedError
calls = []
def factory(**kwargs):
calls.append(kwargs)
return CyclicBackend(**kwargs)
monkeypatch.setattr(embeddings, "SentenceTransformerEmbeddings", factory)
monkeypatch.setattr(
Provider, "_model_cache_directory", lambda: str(tmp_path)
)
monkeypatch.setattr(
Provider, "_is_model_cached_locally", lambda *a, **kw: True
)
Provider.create_embeddings(settings_snapshot={}).embed_query("warm")
monkeypatch.setattr(
Provider, "_is_model_cached_locally", lambda *a, **kw: False
)
with pytest.raises(PolicyDeniedError):
Provider.create_embeddings(settings_snapshot={})
assert len(calls) == 1