Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

25.2 KB
Raw Blame
"""Memory ownership and concurrent use of the resident embedding backend."""

from concurrent.futures import ThreadPoolExecutor
import gc
from pathlib import Path
from threading import Barrier, Event
from types import SimpleNamespace
import weakref

import pytest

from local_deep_research.embeddings import model_cache
from local_deep_research.embeddings.model_cache import (
    clear_sentence_transformer_cache,
    shared_sentence_transformer_embeddings,
)


class CyclicBackend:
    def __init__(self, model_name, **kwargs):
        self.name = model_name
        self.cycle = self

    def embed_query(self, text):
        return [float(len(text))]

    def embed_documents(self, texts):
        return [self.embed_query(text) for text in texts]


@pytest.fixture
def models(tmp_path):
    refs = []
    calls = []

    def factory(**kwargs):
        calls.append(kwargs)
        backend = CyclicBackend(**kwargs)
        refs.append(weakref.ref(backend))
        return backend

    def create(name="synthetic/model", **options):
        return shared_sentence_transformer_embeddings(
            factory,
            model_name=name,
            cache_folder=str(tmp_path),
            model_kwargs=options,
        )

    return create, calls, refs


def test_finished_searches_reuse_weights_and_do_not_share_results(models):
    create, calls, refs = models
    for i in range(12):
        client = create(device="cpu", revision="one")
        result = client.embed_query("query" + "x" * i)
        assert result == [float(5 + i)]
        result[0] = -1  # A caller cannot poison a subsequent result.
        client.close()
    assert len(calls) == 1
    assert sum(ref() is not None for ref in refs) == 1


def test_handles_do_not_pin_evicted_models_or_their_cycles(models):
    create, calls, refs = models
    was_enabled = gc.isenabled()
    gc.disable()  # Prove eviction does not depend on automatic collection.
    try:
        first = create("synthetic/first")
        second = create("synthetic/second")
        assert refs[0]() is None
        assert refs[1]() is not None
        assert first.embed_query("reloaded") == [8.0]
        assert refs[1]() is None
        assert refs[2]() is not None
        assert len(calls) == 3
        assert calls[-1]["model_kwargs"]["local_files_only"] is True
        clear_sentence_transformer_cache()
        assert all(ref() is None for ref in refs)
        # Both lightweight handles still exist, but neither owns weights.
        assert not first._closed and not second._closed
    finally:
        if was_enabled:
            gc.enable()


@pytest.mark.parametrize(
    "first,second",
    [
        ({"device": "cpu"}, {"device": "cuda"}),
        ({"revision": "one"}, {"revision": "two"}),
        (
            {"model_kwargs": {"torch_dtype": "float16"}},
            {"model_kwargs": {"torch_dtype": "float32"}},
        ),
    ],
)
def test_model_options_are_part_of_the_identity(models, first, second):
    create, calls, refs = models
    a = create(**first)
    b = create(**second)
    assert len(calls) == 2
    assert refs[0]() is None
    assert a is not b


def test_close_is_per_handle_and_idempotent(models):
    create, calls, refs = models
    a, b = create(), create()
    a.close()
    a.close()
    with pytest.raises(RuntimeError, match="closed"):
        a.embed_query("closed")
    assert b.embed_documents(["a", "bb"]) == [[1.0], [2.0]]
    assert len(calls) == 1 and refs[0]() is not None


def test_simultaneous_first_users_load_one_model(models):
    create, calls, _ = models
    barrier = Barrier(6)

    def search(i):
        barrier.wait(timeout=10)
        client = create()
        try:
            return client.embed_query("x" * i)
        finally:
            client.close()

    with ThreadPoolExecutor(max_workers=6) as executor:
        results = list(executor.map(search, range(6)))
    assert results == [[float(i)] for i in range(6)]
    assert len(calls) == 1


def test_eviction_waits_for_inference(tmp_path):
    entered, release, requested = Event(), Event(), Event()
    refs = []

    class BlockingBackend(CyclicBackend):
        def embed_query(self, text):
            entered.set()
            assert release.wait(timeout=10)
            assert refs[0]() is self
            return super().embed_query(text)

    def factory(**kwargs):
        backend = BlockingBackend(**kwargs)
        refs.append(weakref.ref(backend))
        return backend

    def create(name):
        return shared_sentence_transformer_embeddings(
            factory,
            model_name=name,
            cache_folder=str(tmp_path),
            model_kwargs={},
        )

    def replace():
        requested.set()
        return create("synthetic/second")

    first = create("synthetic/first")
    with ThreadPoolExecutor(max_workers=2) as executor:
        inference = executor.submit(first.embed_query, "busy")
        assert entered.wait(timeout=10)
        replacement = executor.submit(replace)
        assert requested.wait(timeout=10)
        try:
            assert clear_sentence_transformer_cache(wait=False) is False
            assert not replacement.done()
            assert len(refs) == 1
        finally:
            release.set()
        assert inference.result(timeout=10) == [4.0]
        replacement.result(timeout=10)
    assert refs[0]() is None and refs[1]() is not None


def test_failed_load_is_retryable_and_never_poisoned(tmp_path):
    attempts = []

    def factory(**kwargs):
        attempts.append(kwargs)
        if len(attempts) == 1:
            raise OSError("synthetic missing weights")
        return CyclicBackend(**kwargs)

    kwargs = dict(
        model_name="synthetic/model",
        cache_folder=str(tmp_path),
        model_kwargs={},
    )
    with pytest.raises(OSError):
        shared_sentence_transformer_embeddings(factory, **kwargs)
    client = shared_sentence_transformer_embeddings(factory, **kwargs)
    assert client.embed_query("retry") == [5.0]
    assert len(attempts) == 2


def test_failed_reload_is_retryable_and_never_poisoned(tmp_path):
    attempts = []
    failing = False

    def factory(**kwargs):
        attempts.append(kwargs)
        if failing:
            raise OSError("synthetic reload failure")
        return CyclicBackend(**kwargs)

    def create(name="synthetic/model"):
        return shared_sentence_transformer_embeddings(
            factory, model_name=name, cache_folder=str(tmp_path)
        )

    client = create()
    assert client.embed_query("resident") == [8.0]
    create("synthetic/other")  # Evict without closing the first handle.
    failing = True
    with pytest.raises(OSError, match="synthetic reload failure"):
        client.embed_query("must fail")
    # A failed reload must leave nothing behind that an old handle could
    # mistake for its own weights.
    assert model_cache._MODEL_CACHE.key is None
    assert model_cache._MODEL_CACHE.backend is None
    failing = False
    assert client.embed_query("retry") == [5.0]
    assert model_cache._MODEL_CACHE.key == client._key
    assert len(attempts) == 4


def test_shutdown_cache_clear_is_registered_without_waiting(monkeypatch):
    import atexit
    import importlib

    registrations = []
    monkeypatch.setattr(
        atexit,
        "register",
        lambda func, **kwargs: registrations.append((func, kwargs)),
    )
    try:
        importlib.reload(model_cache)
        assert registrations == [
            (model_cache.clear_sentence_transformer_cache, {"wait": False})
        ]
    finally:
        # undo() puts atexit.register back; reloading again here would only add a
        # duplicate handler for the real one.
        monkeypatch.undo()


def test_encoding_options_are_part_of_the_configuration(tmp_path):
    calls = []

    def factory(**kwargs):
        calls.append(kwargs)
        return CyclicBackend(**kwargs)

    def create(**encode_kwargs):
        return shared_sentence_transformer_embeddings(
            factory,
            model_name="synthetic/model",
            cache_folder=str(tmp_path),
            encode_kwargs=encode_kwargs,
        )

    create(normalize_embeddings=True, prompt="query: ")
    create(prompt="query: ", normalize_embeddings=True)
    assert len(calls) == 1
    create(normalize_embeddings=False, prompt="query: ")
    create(normalize_embeddings=False, prompt="document: ")
    assert len(calls) == 3


def test_recursive_serial_backend_fails_instead_of_waiting_for_its_own_slot(
    tmp_path,
):
    class RecursiveBackend(CyclicBackend):
        def embed_query(self, text):
            if text == "recursive":
                return client.embed_query("inner")
            return super().embed_query(text)

    client = shared_sentence_transformer_embeddings(
        RecursiveBackend,
        model_name="synthetic/model",
        cache_folder=str(tmp_path),
    )
    with pytest.raises(RuntimeError, match="Recursive local embedding"):
        client.embed_query("recursive")
    assert client.embed_query("recovered") == [9.0]


def test_local_replacement_invalidates_cache_and_old_handles(models, tmp_path):
    create, calls, refs = models
    model_dir = tmp_path / "local-model"
    model_dir.mkdir()
    weight = model_dir / "model.safetensors"
    weight.write_bytes(b"synthetic-v1")
    old = create(str(model_dir))
    # A same-path replacement has a different file identity, even if size and
    # timestamps have low resolution on the platform running this test.
    replacement = model_dir / "replacement"
    replacement.write_bytes(b"synthetic-v2-longer")
    replacement.replace(weight)
    new = create(str(model_dir))
    assert len(calls) == 2 and refs[0]() is None
    assert new.embed_query("new") == [3.0]
    with pytest.raises(ValueError, match="files changed"):
        old.embed_query("old")
    assert len(calls) == 2


class ArtifactBackend(CyclicBackend):
    def __init__(self, value, **kwargs):
        super().__init__(**kwargs)
        self.value = value

    def embed_query(self, _text):
        return [self.value]


def _model_artifacts(tmp_path, layout):
    """Synthetic weights "1"; each replacement toggles them between 1 and 2.

    ``local`` rewrites a file in an explicit model directory. ``hub_ref``
    moves a Hub cache's floating ``refs/main`` between two snapshots.

    Neither change may depend on the clock. The artifact stamp sees an
    in-place rewrite through size and timestamps only, and a rewrite
    microseconds after the previous write can keep the same timestamps on
    coarse filesystems. So the ``local`` rewrite also changes the size. The
    ``hub_ref`` move changes the stamped snapshot path itself.
    """
    cache_dir = tmp_path / "hub"
    if layout == "local":
        model = tmp_path / "local-model"
        model.mkdir()
        weights = model / "model.safetensors"
        weights.write_text("1")

        def current_weights():
            return float(weights.read_text())

        def replace_weights():
            weights.write_text("2.0" if current_weights() == 1.0 else "1")

        return str(model), cache_dir, current_weights, replace_weights

    repository = cache_dir / "models--synthetic--model"
    for revision, value in (("a" * 40, "1"), ("b" * 40, "2")):
        snapshot = repository / "snapshots" / revision
        snapshot.mkdir(parents=True)
        (snapshot / "model.safetensors").write_text(value)
    (repository / "refs").mkdir()
    reference = repository / "refs/main"
    reference.write_text("a" * 40)

    def current_weights():
        return float(
            (
                repository
                / "snapshots"
                / reference.read_text()
                / "model.safetensors"
            ).read_text()
        )

    def replace_weights():
        reference.write_text("b" * 40 if current_weights() == 1.0 else "a" * 40)

    return "synthetic/model", cache_dir, current_weights, replace_weights


@pytest.fixture
def coarse_clock(monkeypatch):
    """Hide file timestamps from the artifact stamp, as one tick would.

    Tests using ``_model_artifacts`` then pass or fail the same way on every
    filesystem, however fine its timestamps.
    """

    class CoarseClockPath(type(Path())):
        def stat(self, *, follow_symlinks=True):
            real = super().stat(follow_symlinks=follow_symlinks)
            return SimpleNamespace(
                st_mode=real.st_mode,
                st_dev=real.st_dev,
                st_ino=real.st_ino,
                st_size=real.st_size,
                st_mtime_ns=0,
                st_ctime_ns=0,
            )

    monkeypatch.setattr(model_cache, "Path", CoarseClockPath)


@pytest.mark.usefixtures("coarse_clock")
@pytest.mark.parametrize("layout", ["local", "hub_ref"])
@pytest.mark.parametrize("replace_during_load", [False, True])
def test_evicted_reload_rechecks_artifacts_after_loading(
    tmp_path, layout, replace_during_load
):
    """An old handle must never adopt weights replaced during its reload."""
    model_name, cache_dir, current_weights, replace_weights = _model_artifacts(
        tmp_path, layout
    )
    calls = []
    refs = []
    replacing = False

    def factory(**kwargs):
        calls.append(kwargs)
        if kwargs["model_name"] == model_name:
            if replacing:
                # Runs after pre-load validation, before the loader reads the
                # artifacts. No timing sleeps or model download are needed.
                replace_weights()
            value = current_weights()
        else:
            value = 9.0
        backend = ArtifactBackend(value, **kwargs)
        refs.append(weakref.ref(backend))
        return backend

    def create(name):
        return shared_sentence_transformer_embeddings(
            factory, model_name=name, cache_folder=str(cache_dir)
        )

    old = create(model_name)
    assert old.embed_query("original") == [1.0]
    assert create("synthetic/other").embed_query("other") == [9.0]
    replacing = replace_during_load
    if replace_during_load:
        with pytest.raises(ValueError, match="files changed"):
            old.embed_query("must not use different weights")
        assert refs[-1]() is None  # Failed reload releases its model cycle.
        replacing = False
        fresh = create(model_name)
        assert fresh.embed_query("new handle") == [2.0]
        with pytest.raises(ValueError, match="files changed"):
            old.embed_query("old handle stays invalid")
        assert fresh.embed_query("still usable") == [2.0]
        assert len(calls) == 4
    else:
        assert old.embed_query("unchanged weights") == [1.0]
        assert len(calls) == 3
    assert calls[2]["model_kwargs"]["local_files_only"] is True


@pytest.mark.usefixtures("coarse_clock")
@pytest.mark.parametrize("layout", ["local", "hub_ref"])
@pytest.mark.parametrize("replaced", ["before_read", "after_read"])
def test_first_load_rechecks_artifacts_after_loading(
    tmp_path, layout, replaced
):
    """A first load must not adopt a key describing bytes it did not load.

    A download or repair populates files before the constructor reads them;
    an operator replacement can also land after the read. Either way the
    handle must use the bytes now on disk, before and after an eviction.
    """
    model_name, cache_dir, current_weights, replace_weights = _model_artifacts(
        tmp_path, layout
    )
    calls = []
    replacing = True

    def factory(**kwargs):
        nonlocal replacing
        calls.append(kwargs)
        if kwargs["model_name"] != model_name:
            return ArtifactBackend(9.0, **kwargs)
        if replacing and replaced == "before_read":
            replace_weights()
        value = current_weights()
        if replacing and replaced == "after_read":
            replace_weights()
        replacing = False
        return ArtifactBackend(value, **kwargs)

    def create(name):
        return shared_sentence_transformer_embeddings(
            factory, model_name=name, cache_folder=str(cache_dir)
        )

    client = create(model_name)
    assert client.embed_query("first call") == [2.0]
    assert create("synthetic/other").embed_query("other") == [9.0]
    assert client.embed_query("same handle after reload") == [2.0]
    # The changed first load is retried once from local files, never by
    # another download, and the reloaded handle keeps that key.
    assert len(calls) == 4
    assert calls[1]["model_name"] == model_name
    assert calls[1]["model_kwargs"]["local_files_only"] is True


@pytest.mark.usefixtures("coarse_clock")
@pytest.mark.parametrize("layout", ["local", "hub_ref"])
def test_first_load_gives_up_when_files_keep_changing(tmp_path, layout):
    model_name, cache_dir, current_weights, replace_weights = _model_artifacts(
        tmp_path, layout
    )
    calls = []
    refs = []
    changing = True

    def factory(**kwargs):
        calls.append(kwargs)
        if changing:
            replace_weights()
        backend = ArtifactBackend(current_weights(), **kwargs)
        refs.append(weakref.ref(backend))
        return backend

    def create():
        return shared_sentence_transformer_embeddings(
            factory, model_name=model_name, cache_folder=str(cache_dir)
        )

    with pytest.raises(ValueError, match="files changed"):
        create()
    assert len(calls) == 2  # One bounded retry, never a loop.
    assert all(ref() is None for ref in refs)  # Neither model stays resident.
    changing = False
    assert create().embed_query("stable files") == [current_weights()]
    assert len(calls) == 3


def _model_symlink(link, target, *, directory=False):
    try:
        link.symlink_to(target, target_is_directory=directory)
    except (OSError, NotImplementedError):  # pragma: no cover - platform
        pytest.skip("symlinks not supported on this platform")


@pytest.mark.parametrize("layout", ["directory", "file_link", "directory_link"])
def test_evicted_handle_rejects_changed_linked_artifacts(
    models, tmp_path, layout
):
    create, calls, _ = models
    model = tmp_path / "local-model"
    backing = tmp_path / "shared-module"
    model.mkdir()
    backing.mkdir()
    module = model / "0_Transformer"
    weight = backing / "model.safetensors"
    weight.write_bytes(b"original")
    if layout == "directory_link":
        _model_symlink(module, backing, directory=True)
    else:
        module.mkdir()
        if layout == "file_link":
            _model_symlink(module / weight.name, weight)
        else:
            weight = module / weight.name
            weight.write_bytes(b"original")

    old = create(str(model))
    assert create(str(model)).embed_query("warm") == [4.0]
    assert len(calls) == 1
    create("synthetic/other")  # Evict without closing the first handle.
    weight.write_bytes(b"replacement weights")
    with pytest.raises(ValueError, match="files changed"):
        old.embed_query("must not reload changed weights")
    assert len(calls) == 2
    assert create(str(model)).embed_query("new") == [3.0]
    assert len(calls) == 3


@pytest.mark.parametrize("cycle", [False, True])
@pytest.mark.timeout(10)
def test_shared_directory_links_are_stamped_without_following_cycles(
    models, tmp_path, cycle
):
    create, calls, _ = models
    model = tmp_path / "local-model"
    backing = tmp_path / "shared-module"
    model.mkdir()
    backing.mkdir()
    _model_symlink(model / "first", backing, directory=True)
    _model_symlink(model / "second", backing, directory=True)
    if cycle:
        _model_symlink(backing / "parent", model, directory=True)
    weight = backing / "model.safetensors"
    weight.write_bytes(b"original")
    old = create(str(model))
    create(str(model))
    assert len(calls) == 1
    weight.write_bytes(b"replacement weights")
    create(str(model))
    assert len(calls) == 2
    with pytest.raises(ValueError, match="files changed"):
        old.embed_query("old")


def test_retargeting_a_module_directory_invalidates_old_handles(
    models, tmp_path
):
    create, calls, _ = models
    model = tmp_path / "local-model"
    model.mkdir()
    for name in ("original", "replacement"):
        backing = tmp_path / name
        backing.mkdir()
        (backing / "model.safetensors").write_text(name)
    module = model / "0_Transformer"
    _model_symlink(module, tmp_path / "original", directory=True)
    old = create(str(model))
    module.unlink()
    _model_symlink(module, tmp_path / "replacement", directory=True)
    create(str(model))
    assert len(calls) == 2
    with pytest.raises(ValueError, match="files changed"):
        old.embed_query("old")


@pytest.mark.parametrize("linked", [False, True])
def test_large_artifact_tree_is_bounded(tmp_path, linked):
    model = tmp_path / "local-model"
    model.mkdir()
    tree = (
        tmp_path / "unrelated-large-tree" if linked else model / "0_Transformer"
    )
    tree.mkdir()
    (model / "model.safetensors").write_bytes(b"weights")
    if linked:
        _model_symlink(model / "0_Transformer", tree, directory=True)
    for number in range(600):
        (tree / f"unrelated-{number:04d}").write_bytes(b"x")

    with pytest.raises(ValueError, match="artifact tree exceeds 512 entries"):
        model_cache._artifact_stamp(
            {"model_name": str(model), "cache_folder": str(tmp_path)}
        )


def test_caller_mutation_cannot_change_evicted_reload(models):
    create, calls, _ = models
    options = {"torch_dtype": "original"}
    first = create(model_kwargs=options)
    options["torch_dtype"] = "modified"
    create("synthetic/other")
    first.embed_query("reload")
    assert calls[-1]["model_kwargs"]["model_kwargs"] == {
        "torch_dtype": "original"
    }


def test_pinned_model_ignores_other_hub_revisions(models, tmp_path):
    create, calls, _ = models
    cache = tmp_path / "models--synthetic--model"
    revision = "a" * 40
    original = cache / "snapshots" / revision
    original.mkdir(parents=True)
    (original / "config.json").write_text("original")
    first = create(revision=revision)
    other = cache / "snapshots" / ("b" * 40)
    other.mkdir()
    (other / "config.json").write_text("other revision")
    (cache / "refs").mkdir()
    (cache / "refs/main").write_text("b" * 40)
    second = create(revision=revision)
    assert first.embed_query("one") == second.embed_query("two") == [3.0]
    assert len(calls) == 1


def test_floating_ref_update_invalidates_old_handle(models, tmp_path):
    create, calls, _ = models
    cache = tmp_path / "models--synthetic--model"
    (cache / "refs").mkdir(parents=True)
    reference = cache / "refs/main"
    reference.write_text("a" * 40)
    first = create()
    reference.write_text("b" * 40)
    create()
    assert len(calls) == 2
    with pytest.raises(ValueError, match="files changed"):
        first.embed_query("cannot change weights silently")


@pytest.mark.asyncio
async def test_async_embeddings_use_the_same_resident_model(models):
    create, calls, _ = models
    client = create()
    assert await client.aembed_query("async") == [5.0]
    assert await client.aembed_documents(["x", "yy"]) == [[1.0], [2.0]]
    assert len(calls) == 1


def test_real_managers_reuse_backend_across_users_without_retaining_settings(
    monkeypatch, tmp_path
):
    from langchain_community import embeddings
    from local_deep_research.embeddings.providers.implementations.sentence_transformers import (
        SentenceTransformersProvider as Provider,
    )
    from local_deep_research.web_search_engines.engines.local_embedding_manager import (
        LocalEmbeddingManager,
    )

    calls = []

    def factory(**kwargs):
        calls.append(kwargs)
        return CyclicBackend(**kwargs)

    monkeypatch.setattr(embeddings, "SentenceTransformerEmbeddings", factory)
    monkeypatch.setattr(
        Provider, "_model_cache_directory", lambda: str(tmp_path)
    )
    monkeypatch.setattr(
        Provider, "_is_model_cached_locally", lambda *a, **kw: True
    )
    for username in ("synthetic-alice", "synthetic-bob", "synthetic-alice"):
        with LocalEmbeddingManager(
            settings_snapshot={
                "_username": username,
                "policy.egress_scope": "public_only",
                "search.tool": "arxiv",
            }
        ) as manager:
            handle = manager.embeddings
            assert handle.embed_query(username) == [float(len(username))]
        with pytest.raises(RuntimeError, match="closed"):
            handle.embed_query("after close")
    assert len(calls) == 1
    assert "_username" not in repr(calls)
    assert "synthetic-alice" not in repr(vars(handle))


def test_warm_weights_do_not_bypass_provider_admission(monkeypatch, tmp_path):
    from langchain_community import embeddings
    from local_deep_research.embeddings.providers.implementations.sentence_transformers import (
        SentenceTransformersProvider as Provider,
    )
    from local_deep_research.security.egress.policy import PolicyDeniedError

    calls = []

    def factory(**kwargs):
        calls.append(kwargs)
        return CyclicBackend(**kwargs)

    monkeypatch.setattr(embeddings, "SentenceTransformerEmbeddings", factory)
    monkeypatch.setattr(
        Provider, "_model_cache_directory", lambda: str(tmp_path)
    )
    monkeypatch.setattr(
        Provider, "_is_model_cached_locally", lambda *a, **kw: True
    )
    Provider.create_embeddings(settings_snapshot={}).embed_query("warm")
    monkeypatch.setattr(
        Provider, "_is_model_cached_locally", lambda *a, **kw: False
    )
    with pytest.raises(PolicyDeniedError):
        Provider.create_embeddings(settings_snapshot={})
    assert len(calls) == 1