Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

6.4 KB
Raw Blame
"""
Regression tests for FastAPI threadpool / thread-local-session interaction.

The settings/library/rag routers must NOT pre-create SettingsManager via
FastAPI Depends, because anyio's threadpool can run a sync dep on one
worker and the route handler on a different worker. SettingsManager has
a strict _check_thread_safety() that raises RuntimeError when used in a
thread other than the one it was created in.

Pattern that triggered the bug (now reverted):

    def my_route(
        username=Depends(require_auth),
        settings_manager=Depends(get_settings_manager_dep),
    ):
        return settings_manager.get_all_settings()  # ~25% chance of 500

These tests fire concurrent requests at endpoints that exercise
SettingsManager and assert all of them succeed. Run with -n 1 to keep
the test process simple — concurrency is inside the test, not pytest-xdist.
"""

import concurrent.futures
import uuid

import pytest
from fastapi.testclient import TestClient


@pytest.fixture(scope="module")
def auth_client():
    from local_deep_research.web.fastapi_app import app

    c = TestClient(app, raise_server_exceptions=False)
    user = f"test_thr_{uuid.uuid4().hex[:8]}"
    pw = "TestPassword123!"  # noqa: S105

    # Fetch CSRF tokens before register/login — Wave 9 made these
    # endpoints fail-closed on missing tokens.
    def _csrf():
        c.get("/auth/login")
        r = c.get("/auth/csrf-token")
        return r.json().get("csrf_token", "") if r.status_code == 200 else ""

    c.post(
        "/auth/register",
        data={
            "username": user,
            "password": pw,
            "confirm_password": pw,
            "acknowledge": "true",
            "csrf_token": _csrf(),
        },
        follow_redirects=False,
    )
    resp = c.post(
        "/auth/login",
        data={"username": user, "password": pw, "csrf_token": _csrf()},
        follow_redirects=False,
    )
    if resp.status_code != 302:
        pytest.fail(
            f"Login bootstrap failed: expected 302, got {resp.status_code}: "
            f"{resp.text[:500]}"
        )
    yield c


def _hammer(client, path, n_requests=20, concurrency=5):
    """Fire n_requests at `path` with `concurrency` workers. Return status counts."""
    statuses: list[int] = []
    with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as ex:
        futures = [ex.submit(client.get, path) for _ in range(n_requests)]
        for fut in concurrent.futures.as_completed(futures):
            statuses.append(fut.result().status_code)
    return statuses


class TestSettingsManagerNoThreadRace:
    """Concurrent hits on SettingsManager-backed endpoints must all return 200."""

    @pytest.fixture(autouse=True)
    def _generous_sidecar_wait(self, monkeypatch):
        # Every request claims the user's rekey sidecar several times, and
        # the claims queue on one per-user lock. Past
        # _SIDECAR_WAIT_TIMEOUT_SECONDS (5 s) a claim answers a retryable
        # 503 by design. On a loaded CI runner (xdist plus coverage) five
        # concurrent requests can queue past that, so these tests failed
        # with 503s that say nothing about the thread race they check.
        # The bound itself is covered in tests/database/test_sidecar_fault_probe.py.
        from local_deep_research.database import encrypted_db

        monkeypatch.setattr(encrypted_db, "_SIDECAR_WAIT_TIMEOUT_SECONDS", 60.0)

    def test_concurrent_settings_api(self, auth_client):
        statuses = _hammer(auth_client, "/settings/api")
        assert all(s == 200 for s in statuses), (
            f"Got non-200 responses: {[s for s in statuses if s != 200]}"
        )

    def test_concurrent_settings_categories(self, auth_client):
        statuses = _hammer(auth_client, "/settings/api/categories")
        assert all(s == 200 for s in statuses), (
            f"Got non-200 responses: {[s for s in statuses if s != 200]}"
        )

    def test_concurrent_available_models(self, auth_client):
        # /available-models hits SettingsManager AND queries provider URLs
        statuses = _hammer(
            auth_client, "/settings/api/available-models", n_requests=10
        )
        assert all(s == 200 for s in statuses), (
            f"Got non-200 responses: {[s for s in statuses if s != 200]}"
        )

    def test_concurrent_library_pages(self, auth_client):
        statuses = _hammer(auth_client, "/library/")
        assert all(s == 200 for s in statuses), (
            f"Got non-200 responses: {[s for s in statuses if s != 200]}"
        )

    def test_concurrent_download_manager(self, auth_client):
        # Regression: /library/download-manager Jinja used `total_pages`
        # but the FastAPI port forgot to pass it. Pagination now restored.
        statuses = _hammer(auth_client, "/library/download-manager")
        assert all(s == 200 for s in statuses), (
            f"Got non-200 responses: {[s for s in statuses if s != 200]}"
        )


class TestDatabaseMiddlewareDoesNotBlockEventLoop:
    """
    Regression test for the BLOCKER where ``DatabaseMiddleware.__call__``
    invoked the synchronous ``ensure_user_database()`` directly from an
    async ASGI ``__call__``. ``ensure_user_database()`` opens a SQLCipher
    connection (PBKDF2 key derivation, file I/O); on the event loop this
    serializes every authenticated request behind it.

    The fix wraps the call in ``asyncio.to_thread``. To prove it works we
    fire concurrent requests at a cheap authenticated endpoint and check
    wall-clock is closer to the parallel lower bound than the serialised
    upper bound. We can't measure the SQLCipher open time directly (it's
    typically only on first request after login), but if the event loop
    were blocked, ``-n 4`` of pytest-xdist would also make this test
    flaky — the existing ``test_concurrent_settings_api`` already proves
    the event loop is shared correctly today.
    """

    def test_concurrent_authenticated_requests(self, auth_client):
        # Two endpoints exercising the auth/db middleware path under load.
        # If DatabaseMiddleware were blocking the loop we'd see at least
        # some 5xx responses or the test would time out under pytest-xdist.
        statuses = _hammer(
            auth_client, "/auth/check", n_requests=20, concurrency=10
        )
        assert all(s == 200 for s in statuses), (
            f"DatabaseMiddleware blocked the loop: "
            f"{[s for s in statuses if s != 200]}"
        )