Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

15.2 KB
Raw Blame
"""Exercise real, tiny offline models: one resident model, serial inference."""

import asyncio
from concurrent.futures import ThreadPoolExecutor
import gc
from threading import Event, Lock
import time
import weakref

import pytest

from local_deep_research.embeddings import model_cache
from local_deep_research.embeddings.model_loading import (
    MODEL_LOAD_LOCK,
    ModelAccessLock,
    load_sentence_transformer,
)


@pytest.fixture(scope="module", params=["bert", "modernbert"])
def tiny_model_path(request, tmp_path_factory):
    """Random weights and a six-word tokenizer; never download a real model."""
    import torch
    from tokenizers import Tokenizer, models, pre_tokenizers
    from transformers import (
        BertConfig,
        BertModel,
        ModernBertConfig,
        ModernBertModel,
        PreTrainedTokenizerFast,
    )

    path = tmp_path_factory.mktemp(request.param)
    tokenizer = Tokenizer(
        models.WordLevel(
            {"[PAD]": 0, "[UNK]": 1, "a": 2, "b": 3, "c": 4, "d": 5},
            unk_token="[UNK]",
        )
    )
    tokenizer.pre_tokenizer = pre_tokenizers.Whitespace()
    PreTrainedTokenizerFast(
        tokenizer_object=tokenizer,
        pad_token="[PAD]",
        unk_token="[UNK]",
        model_max_length=64,
    ).save_pretrained(path)
    options = dict(
        vocab_size=6,
        hidden_size=16,
        intermediate_size=32,
        num_hidden_layers=2,
        num_attention_heads=2,
        max_position_embeddings=64,
        pad_token_id=0,
    )
    if request.param == "bert":
        model = BertModel(BertConfig(**options))
    else:
        model = ModernBertModel(
            ModernBertConfig(
                **options,
                local_attention=8,
                global_attn_every_n_layers=2,
                cls_token_id=2,
                sep_token_id=3,
            )
        )
    model.save_pretrained(path)
    del model
    old_threads = torch.get_num_threads()
    torch.set_num_threads(1)
    try:
        yield str(path)
    finally:
        torch.set_num_threads(old_threads)


@pytest.fixture
def real_backend(tiny_model_path, monkeypatch):
    from langchain_community.embeddings.huggingface import HuggingFaceEmbeddings

    monkeypatch.setenv("HF_HUB_OFFLINE", "1")
    monkeypatch.setenv("TRANSFORMERS_OFFLINE", "1")
    return HuggingFaceEmbeddings(
        model_name=tiny_model_path,
        model_kwargs={
            "device": "cpu",
            "local_files_only": True,
            "model_kwargs": {"attn_implementation": "eager"},
        },
        encode_kwargs={"normalize_embeddings": True},
    )


class _ObservedLock:
    """Delegate to the cache's inference lock; report calls taking a turn.

    ``waiting`` is set once ``after`` calls have asked for a turn, before the
    last of them blocks on the real lock.
    """

    def __init__(self, lock, waiting, after=1):
        self._lock = lock
        self._waiting = waiting
        self._after = after
        self._count = Lock()
        self.waits = 0

    def __enter__(self):
        with self._count:
            self.waits += 1
            if self.waits >= self._after:
                self._waiting.set()
        return self._lock.__enter__()

    def __exit__(self, *exc):
        return self._lock.__exit__(*exc)


def test_calls_take_turns_on_one_model_and_block_loaders_and_cleanup(
    real_backend, tiny_model_path, monkeypatch
):
    client = model_cache.shared_sentence_transformer_embeddings(
        lambda **_: real_backend, model_name=tiny_model_path
    )
    transformer_type = type(real_backend.client[0].auto_model)
    forward = transformer_type.forward
    entered, release, loader_requested, loaded = (Event() for _ in range(4))
    mutex = Lock()
    active = peak = 0
    modules = set()

    def blocking_forward(self, *args, **kwargs):
        nonlocal active, peak
        with mutex:
            active += 1
            peak = max(peak, active)
            modules.add(id(self))
        entered.set()
        try:
            assert release.wait(timeout=10)
            return forward(self, *args, **kwargs)
        finally:
            with mutex:
                active -= 1

    def load_other():
        loader_requested.set()
        return load_sentence_transformer(lambda: loaded.set())

    monkeypatch.setattr(transformer_type, "forward", blocking_forward)
    with ThreadPoolExecutor(max_workers=4) as executor:
        futures = [executor.submit(client.embed_query, "a b") for _ in range(3)]
        try:
            assert entered.wait(timeout=10)
            loading = executor.submit(load_other)
            assert loader_requested.wait(timeout=10)
            assert (
                model_cache.clear_sentence_transformer_cache(wait=False)
                is False
            )
            assert not loaded.is_set()
            assert peak == 1
        finally:
            release.set()
        for future in futures:
            assert len(future.result(timeout=15)) == 16
        loading.result(timeout=15)
    assert peak == 1
    # Every call used the one resident model; no copy was made for a call.
    assert modules == {id(real_backend.client[0].auto_model)}
    assert loaded.is_set()


@pytest.mark.asyncio
async def test_async_calls_take_turns_on_the_resident_model(
    real_backend, tiny_model_path, monkeypatch
):
    client = model_cache.shared_sentence_transformer_embeddings(
        lambda **_: real_backend, model_name=tiny_model_path
    )
    transformer_type = type(real_backend.client[0].auto_model)
    forward = transformer_type.forward
    cache = model_cache._MODEL_CACHE
    both_waiting = Event()
    observed = _ObservedLock(cache._inference, both_waiting, after=2)
    monkeypatch.setattr(cache, "_inference", observed)
    mutex = Lock()
    active = peak = 0

    def blocking_forward(self, *args, **kwargs):
        nonlocal active, peak
        with mutex:
            active += 1
            peak = max(peak, active)
        try:
            # Hold the model until the other call waits for its turn.
            assert both_waiting.wait(timeout=10)
            return forward(self, *args, **kwargs)
        finally:
            with mutex:
                active -= 1

    monkeypatch.setattr(transformer_type, "forward", blocking_forward)
    results = await asyncio.wait_for(
        asyncio.gather(
            client.aembed_query("a"), client.aembed_documents(["b c"])
        ),
        timeout=15,
    )
    assert len(results[0]) == len(results[1][0]) == 16
    assert observed.waits == 2
    assert peak == 1


def test_warm_handle_is_created_while_busy_and_closes_while_queued(
    real_backend, tiny_model_path, monkeypatch
):
    loads = []

    def factory(**_kwargs):
        loads.append(True)
        return real_backend

    def create():
        return model_cache.shared_sentence_transformer_embeddings(
            factory, model_name=tiny_model_path
        )

    client = create()
    entered, release, queued = Event(), Event(), Event()
    transformer_type = type(real_backend.client[0].auto_model)
    forward = transformer_type.forward
    lock = Lock()
    forwards = 0

    def blocking_forward(self, *args, **kwargs):
        nonlocal forwards
        with lock:
            forwards += 1
        entered.set()
        assert release.wait(timeout=10)
        return forward(self, *args, **kwargs)

    monkeypatch.setattr(transformer_type, "forward", blocking_forward)
    cache = model_cache._MODEL_CACHE
    with ThreadPoolExecutor(max_workers=2) as executor:
        first = executor.submit(client.embed_query, "a")
        try:
            assert entered.wait(timeout=10)
            # Shared access: creating a warm handle needs no turn on the model.
            late = executor.submit(create).result(timeout=5)
            assert len(loads) == 1
            monkeypatch.setattr(
                cache, "_inference", _ObservedLock(cache._inference, queued)
            )
            third = executor.submit(late.embed_query, "c")
            assert queued.wait(timeout=5)
            late.close()
        finally:
            release.set()
        assert len(first.result(timeout=10)) == 16
        with pytest.raises(RuntimeError, match="closed"):
            third.result(timeout=10)
    assert forwards == 1
    assert len(client.embed_query("d")) == 16


def test_configuration_change_waits_for_active_and_queued_calls(
    real_backend, tiny_model_path, monkeypatch
):
    loads = []

    def factory(**kwargs):
        loads.append(kwargs["encode_kwargs"])
        return real_backend

    def create(normalize):
        return model_cache.shared_sentence_transformer_embeddings(
            factory,
            model_name=tiny_model_path,
            encode_kwargs={"normalize_embeddings": normalize},
        )

    client = create(True)
    (
        entered_first,
        entered_second,
        release_first,
        release_second,
        queued,
        requested,
    ) = (Event() for _ in range(6))
    transformer_type = type(real_backend.client[0].auto_model)
    forward = transformer_type.forward
    lock = Lock()
    forwards = 0

    def blocking_forward(self, *args, **kwargs):
        nonlocal forwards
        with lock:
            index = forwards
            forwards += 1
        (entered_first if index == 0 else entered_second).set()
        assert (release_first if index == 0 else release_second).wait(
            timeout=10
        )
        return forward(self, *args, **kwargs)

    def switch():
        requested.set()
        return create(False)

    monkeypatch.setattr(transformer_type, "forward", blocking_forward)
    cache = model_cache._MODEL_CACHE
    with ThreadPoolExecutor(max_workers=3) as executor:
        first = executor.submit(client.embed_query, "a")
        try:
            assert entered_first.wait(timeout=10)
            monkeypatch.setattr(
                cache, "_inference", _ObservedLock(cache._inference, queued)
            )
            second = executor.submit(client.embed_query, "b c")
            # Admitted before the switch, so the switch must wait for it too.
            assert queued.wait(timeout=5)
            switching = executor.submit(switch)
            assert requested.wait(timeout=5)
            release_first.set()
            assert len(first.result(timeout=10)) == 16
            # The queued call takes its turn on the old model; the switch waits.
            assert entered_second.wait(timeout=10)
            assert not switching.done()
            assert len(loads) == 1
            assert cache.key == client._key
        finally:
            release_first.set()
            release_second.set()
        assert len(second.result(timeout=10)) == 16
        switching.result(timeout=10)
    assert loads == [
        {"normalize_embeddings": True},
        {"normalize_embeddings": False},
    ]


def test_failed_forward_releases_inference_turn_and_gate(
    real_backend, tiny_model_path, monkeypatch
):
    client = model_cache.shared_sentence_transformer_embeddings(
        lambda **_: real_backend, model_name=tiny_model_path
    )
    transformer_type = type(real_backend.client[0].auto_model)
    forward = transformer_type.forward

    def fail(self, *args, **kwargs):
        raise RuntimeError("synthetic inference failure")

    monkeypatch.setattr(transformer_type, "forward", fail)
    for _ in range(3):
        with pytest.raises(RuntimeError, match="synthetic inference failure"):
            client.embed_query("a")
    monkeypatch.setattr(transformer_type, "forward", forward)
    assert len(client.embed_query("b")) == 16
    assert model_cache.clear_sentence_transformer_cache(wait=False) is True


def test_cache_clear_collects_the_real_model_cycle_with_live_handles(
    tiny_model_path, monkeypatch
):
    from langchain_community.embeddings.huggingface import HuggingFaceEmbeddings

    monkeypatch.setenv("HF_HUB_OFFLINE", "1")
    monkeypatch.setenv("TRANSFORMERS_OFFLINE", "1")
    # As with provider handles, the cache is the model's only owner.
    client = model_cache.shared_sentence_transformer_embeddings(
        HuggingFaceEmbeddings,
        model_name=tiny_model_path,
        model_kwargs={
            "device": "cpu",
            "model_kwargs": {"attn_implementation": "eager"},
        },
    )
    assert len(client.embed_query("a")) == 16
    model = weakref.ref(model_cache._MODEL_CACHE.backend.client)
    # SentenceTransformer -> model_card_data -> SentenceTransformer.
    assert model().model_card_data.model is model()
    was_enabled = gc.isenabled()
    gc.disable()
    try:
        assert model_cache.clear_sentence_transformer_cache(wait=False)
        assert model() is None
        assert not client._closed
    finally:
        if was_enabled:
            gc.enable()


def test_reader_gate_rejects_unsafe_upgrade_and_remains_usable():
    with MODEL_LOAD_LOCK.inference():
        assert MODEL_LOAD_LOCK.acquire(blocking=False) is False
        with pytest.raises(RuntimeError, match="during inference"):
            load_sentence_transformer(lambda: None)
        with MODEL_LOAD_LOCK.inference():
            pass
    with MODEL_LOAD_LOCK:
        with MODEL_LOAD_LOCK:
            with MODEL_LOAD_LOCK.inference():
                pass
    assert model_cache.clear_sentence_transformer_cache(wait=False)


def test_blocked_writer_is_served_before_new_readers():
    lock = ModelAccessLock()
    reader_holding, release_reader = Event(), Event()
    writer_requested, writer_holding, release_writer = (
        Event(),
        Event(),
        Event(),
    )
    late_reader_holding = Event()
    order = []
    order_lock = Lock()

    def record(step):
        with order_lock:
            order.append(step)

    def hold_shared():
        with lock.inference():
            record("reader")
            reader_holding.set()
            assert release_reader.wait(timeout=10)

    def write_exclusively():
        assert reader_holding.wait(timeout=10)
        writer_requested.set()
        with lock:
            record("writer")
            writer_holding.set()
            assert release_writer.wait(timeout=10)

    def read_late():
        with lock.inference():
            record("late-reader")
            late_reader_holding.set()

    with ThreadPoolExecutor(max_workers=3) as executor:
        reader = executor.submit(hold_shared)
        assert reader_holding.wait(timeout=10)
        writer = executor.submit(write_exclusively)
        assert writer_requested.wait(timeout=10)
        # Wait until the writer is blocked behind the shared holder so the
        # late reader arrives while a writer is already waiting. The lock
        # exposes no public "a writer is waiting" probe, so poll the waiter
        # count its writer priority is built on.
        deadline = time.monotonic() + 10
        while lock._waiting_writers == 0 and time.monotonic() < deadline:
            time.sleep(0.01)
        assert lock._waiting_writers == 1
        late = executor.submit(read_late)
        try:
            # New readers must not be admitted ahead of the waiting writer.
            assert not late_reader_holding.wait(timeout=0.2)
            release_reader.set()
            assert writer_holding.wait(timeout=10)
            assert not late_reader_holding.wait(timeout=0.2)
        finally:
            release_reader.set()
            release_writer.set()
        reader.result(timeout=10)
        writer.result(timeout=10)
        late.result(timeout=10)
    assert order == ["reader", "writer", "late-reader"]