Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

12.9 KB
Raw Blame
"""Many-user cache hygiene under sequential and concurrent churn.

Incident context (PR #5596): the cache maps (``connections``,
``_password_verifiers``, ``_init_locks``) are the entire trust state of
the connection layer. At scale -- many accounts churning through one
process -- two operational security properties must hold:

* W1 after 30 sequential create/write/close cycles and a final
  ``close_all_databases``, the engine and verifier maps are empty
  (encrypted_db.py::close_all_databases; ``_init_locks`` is deliberately
  retained, see that method) and the liveness accessors read zero;
* W2 the same 30 users created and churned CONCURRENTLY across 4 threads
  (each owning a subset, barrier-start, EVERY user with its own distinct
  password), with the engine/verifier pairing invariant sampled under
  ``_connections_lock`` throughout, no failed opens, and every user
  reopening afterwards reading its OWN canary.

FD-count deltas are deliberately NOT asserted here: pyproject.toml's
``fd_canary`` marker documents that ``/proc/self/fd`` assertions are
noisy under parallel execution, and this file belongs in the parallel
lane for the cache-hygiene coverage above. ``tests/utilities/
test_close_base_llm.py`` owns the FD-leak canaries.

Per-user schema migration is neutered for these tests ONLY (monkeypatch
of ``initialize.initialize_database`` to a no-op) -- the exact isolation
pattern of ``tests/database/test_concurrent_user_db_open.py`` -- because
30 full migrations would blow the run-time budget while exercising
nothing this file asserts on (the cache, the open/close lifecycle, and
descriptors are all fully real, backed by real SQLCipher files).
"""

import threading
import time
import uuid

import pytest
from sqlalchemy import text

import local_deep_research.database.initialize as init_mod
from local_deep_research.database.encrypted_db import DatabaseManager

# Module-level gate. Without a working SQLCipher build every test below
# is a no-op, and the dozens of in-body ``pytest.skip`` calls let a CI
# lane report this whole battery green having executed nothing. Skipping
# at import time makes the missing dependency one visible, uniform signal
# per module instead.
pytest.importorskip("sqlcipher3", reason="requires SQLCipher (encrypted mode)")

USER_COUNT = 30
THREAD_COUNT = 4
CHURN_ITERATIONS = 8
# 120s, not the 60s the other shared-deadline modules use: this module does
# 4 threads x 30 real SQLCipher users (create + write + churn + reopen),
# more work than the 3-thread modules the 60s budget was sized for, and
# unlike the per-thread-join modules (2 threads, 120s worst case each) it
# has no headroom to trade away -- there is only one shared deadline here.
# 120s join budget + pytest's own setup/call/teardown overhead (this
# fixture's ``close_all_databases`` teardown included) must still land
# under ``pyproject.toml``'s global ``timeout = 180``; 60s of headroom is
# the same margin the per-thread modules get.
JOIN_TIMEOUT_S = 120
BARRIER_TIMEOUT_S = 30


def _join_all(threads):
    """Join every thread against ONE shared deadline.

    A per-thread ``join(timeout=JOIN_TIMEOUT_S)`` inside a loop costs
    ``len(threads) * JOIN_TIMEOUT_S`` in the worst case. ``pyproject.toml``
    sets ``timeout = 180`` with ``timeout_method = "thread"``, and
    pytest-timeout's thread handler ends in ``os._exit(1)`` -- so when these
    tests actually find the deadlock they exist to find, a per-thread budget
    would kill the whole xdist worker (losing its coverage, then tripping
    ``--cov-fail-under``) instead of failing on the named "deadlocked"
    assertion at the call site. One shared deadline bounds the entire join at
    JOIN_TIMEOUT_S regardless of thread count, which keeps that assertion
    reachable well inside the global timeout. Each thread still gets the full
    remaining budget in wall-clock terms, because the threads run
    concurrently.
    """
    deadline = time.monotonic() + JOIN_TIMEOUT_S
    for t in threads:
        t.join(timeout=max(0.0, deadline - time.monotonic()))


@pytest.fixture
def manager(tmp_path, monkeypatch):
    """A DatabaseManager writing to an isolated data directory.

    Same self-contained pattern as the round-1/2/3/4 files, with the
    sanctioned test-mode KDF knob. ``initialize_database`` is additionally
    neutered for this file (see module docstring) to keep 30-user tests
    inside the run-time budget.
    """
    monkeypatch.setenv("LDR_DATA_DIR", str(tmp_path))
    monkeypatch.setenv("LDR_DB_CONFIG_KDF_ITERATIONS", "1000")
    monkeypatch.setattr(init_mod, "initialize_database", lambda *a, **k: None)
    # With migrations neutered the databases stay unstamped, which would
    # make every cold open attempt a pre-migration BACKUP. W asserts on
    # cache/FD lifecycle, not backups -- the same isolation class as the
    # initialize_database patch above.
    monkeypatch.setattr(
        "local_deep_research.database.alembic_runner.needs_migration",
        lambda *a, **k: False,
    )
    mgr = DatabaseManager()
    # The phase-2 RAG index re-key re-enters through vector-store imports
    # and dominates cold-open time; tests/database/test_concurrent_user_db
    # _open.py already isolates it the same way for cache-lifecycle tests.
    monkeypatch.setattr(mgr, "_run_phase2_rekey", lambda *a, **k: None)
    mgr.data_dir = tmp_path
    yield mgr
    mgr.close_all_databases()


def _write_canary(engine, note: str) -> None:
    with engine.begin() as conn:
        conn.execute(
            text(
                "CREATE TABLE gap_w_canary (id INTEGER PRIMARY KEY, note TEXT)"
            )
        )
        conn.execute(
            text("INSERT INTO gap_w_canary (id, note) VALUES (1, :note)"),
            {"note": note},
        )


def _make_user(manager, index):
    username = f"many_{index}_{uuid.uuid4().hex[:8]}"
    password = f"ManyUsersPw{index % 10}!"  # noqa: S105
    engine = manager.create_user_database(username, password)
    _write_canary(engine, f"gap-W-{index}")
    manager.close_user_database(username)
    return username, password


def _assert_fully_drained(manager):
    with manager._connections_lock:
        assert manager.connections == {}, "close_all left cached engines"
        assert manager._password_verifiers == {}, "close_all left verifiers"
        # _init_locks is deliberately NOT drained by close_all: a cold-open
        # may still hold a per-user lock (see close_all_databases). Not a
        # leak -- the dict is bounded, one small Lock per username.
    assert manager.get_connected_usernames() == set()
    assert manager.get_memory_usage()["active_connections"] == 0


def test_thirty_sequential_users_drain_all_state(manager):
    """W1: 30 sequential create+canary+close cycles, then close_all:
    engine + verifier maps empty, liveness accessors zero."""
    if not manager.has_encryption:
        pytest.skip("requires SQLCipher (encrypted mode) to be meaningful")

    users = [_make_user(manager, i) for i in range(USER_COUNT)]
    assert len(users) == USER_COUNT

    # Mid-state sanity: 30 engines came and went; nothing is live now
    # (each cycle closed its own engine), and the maps are paired-empty.
    with manager._connections_lock:
        assert set(manager.connections) == set(manager._password_verifiers)

    manager.close_all_databases()
    _assert_fully_drained(manager)

    # A sample of the users still opens and reads its canary afterwards.
    for username, password in users[:3]:
        engine = manager.open_user_database(username, password)
        assert engine is not None
        with engine.connect() as conn:
            note = conn.execute(
                text("SELECT note FROM gap_w_canary WHERE id = 1")
            ).scalar()
        assert note.startswith("gap-W-")
        manager.close_user_database(username)
    # Both close paths deliberately RETAIN per-user init locks (see
    # tests/database/test_concurrent_user_db_open.py and
    # close_all_databases); close_all drains engines + verifiers only.
    manager.close_all_databases()
    _assert_fully_drained(manager)


def test_thirty_users_concurrent_churn_stays_paired(manager):
    """W2: 30 users created and churned by 4 barrier-synced threads (each
    owning a subset, each user with its OWN password): no open failures,
    pairing invariant at every sample, every user reopen+canary at the
    end, engine + verifier maps drained after close_all."""
    if not manager.has_encryption:
        pytest.skip("requires SQLCipher (encrypted mode) to be meaningful")

    barrier = threading.Barrier(THREAD_COUNT)
    errors = {}
    open_failures = []
    pairing_violations = []
    canary_failures = []
    subset_sizes = {}

    def worker(tid: int):
        try:
            # Each thread owns a DISJOINT subset of the USER_COUNT users and
            # creates them itself (parallel creation, distinct files). The
            # split is round-robin (`range(tid, USER_COUNT, THREAD_COUNT)`)
            # so the subsets together cover every user: a plain
            # `range(USER_COUNT // THREAD_COUNT)` silently dropped the
            # remainder and churned 28 of the 30 this test is named for.
            # Every user gets a DISTINCT password: a shared per-thread
            # password would make cross-user bleed WITHIN a thread's own
            # subset invisible (any of its users would open any other).
            own = [
                (
                    f"many_c_{tid}_{i}_{uuid.uuid4().hex[:6]}",
                    f"ConcPw{tid}_{i}_{uuid.uuid4().hex[:6]}!",
                )
                for i in range(tid, USER_COUNT, THREAD_COUNT)
            ]
            subset_sizes[tid] = len(own)
            notes = {}
            for index, (username, password) in enumerate(own):
                engine = manager.create_user_database(username, password)
                notes[username] = f"gap-W2-{tid}-{index}"
                _write_canary(engine, notes[username])

            barrier.wait(timeout=BARRIER_TIMEOUT_S)

            # Churn: cycle the subset, open + canary read + close.
            for i in range(CHURN_ITERATIONS):
                username, password = own[i % len(own)]
                engine = manager.open_user_database(username, password)
                if engine is None:
                    open_failures.append(f"t{tid} churn#{i}: {username}")
                    continue
                try:
                    with engine.connect() as conn:
                        note = conn.execute(
                            text("SELECT note FROM gap_w_canary WHERE id = 1")
                        ).scalar()
                    if note != notes[username]:
                        canary_failures.append(
                            f"t{tid} churn#{i} {username}: read {note!r}, "
                            f"expected {notes[username]!r}"
                        )
                finally:
                    manager.close_user_database(username)
                with manager._connections_lock:
                    engines = set(manager.connections)
                    verifiers = set(manager._password_verifiers)
                if engines != verifiers:
                    pairing_violations.append(
                        f"t{tid} churn#{i}: {engines} vs {verifiers}"
                    )

            # Final pass: EVERY user in this thread's subset reopens and
            # reads its own canary.
            for username, password in own:
                engine = manager.open_user_database(username, password)
                if engine is None:
                    open_failures.append(f"t{tid} final: {username}")
                    continue
                try:
                    with engine.connect() as conn:
                        note = conn.execute(
                            text("SELECT note FROM gap_w_canary WHERE id = 1")
                        ).scalar()
                    if note != notes[username]:
                        canary_failures.append(
                            f"t{tid} final {username}: read {note!r}, "
                            f"expected {notes[username]!r}"
                        )
                finally:
                    manager.close_user_database(username)
        except Exception as exc:  # noqa: BLE001 - surfaced via `errors` below
            errors[tid] = exc

    threads = [
        threading.Thread(target=worker, args=(i,), name=f"gap-w2-t{i}")
        for i in range(THREAD_COUNT)
    ]
    for t in threads:
        t.start()
    _join_all(threads)
    assert not any(t.is_alive() for t in threads), "many-user churn deadlocked"
    assert not errors, f"a churn thread raised: {errors}"
    assert sum(subset_sizes.values()) == USER_COUNT, (
        f"the thread subsets covered {sum(subset_sizes.values())} users, not "
        f"the {USER_COUNT} this test is named for: {subset_sizes}"
    )
    assert not open_failures, f"legitimate opens failed: {open_failures}"
    assert not canary_failures, f"cross-user canary reads: {canary_failures}"
    assert not pairing_violations, (
        f"engine/verifier pairing broke: {pairing_violations}"
    )

    manager.close_all_databases()
    _assert_fully_drained(manager)