Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

13.3 KB
Raw Blame
"""Pool bounds under concurrency and get_session racing close.

Incident context (PR #5596): the shared per-user engine is the trust
boundary -- every Session handed out by ``get_session`` /
``create_thread_safe_session_for_metrics`` binds to the ONE
credential-verified engine (ADR-0004: QueuePool, pool_size=20,
max_overflow=40). Two behavioral properties complete the picture the
incident tests left open:

* T1 concurrent readers through the shared engine: many threads reading
  simultaneously never exceed the pool's configured bounds
  (checked-out <= pool_size + max_overflow, parsed from
  ``engine.pool.status()`` snapshots taken INSIDE the threads) and never
  fail;
* T2 ``get_session`` racing ``close_user_database``: get_session is a
  pure cache lookup under ``_connections_lock``; when it wins the race it
  binds a Session to the live engine, and when the close wins first the
  round-3 resurrection contract covers the Session (the creator closure
  reconnects with the same key) -- so every session the racer obtains
  must still execute real table reads successfully, never raise, and the
  engine/verifier pairing must hold at every sampled instant.

FD-count deltas are deliberately NOT asserted here: pyproject.toml's
``fd_canary`` marker documents that ``/proc/self/fd`` assertions are
noisy under parallel execution (and ``/proc`` is Linux-only), while this
file belongs in the parallel lane for the pool/pairing coverage above.
"""

import re
import threading
import time
import uuid
import warnings

import pytest
from sqlalchemy import text

from local_deep_research.database.encrypted_db import DatabaseManager

# Module-level gate. Without a working SQLCipher build every test below
# is a no-op, and the dozens of in-body ``pytest.skip`` calls let a CI
# lane report this whole battery green having executed nothing. Skipping
# at import time makes the missing dependency one visible, uniform signal
# per module instead.
pytest.importorskip("sqlcipher3", reason="requires SQLCipher (encrypted mode)")

READER_THREADS = 4
READS_PER_THREAD = 8
GET_SESSION_ITERATIONS = 30
OPEN_CLOSE_ITERATIONS = 15
JOIN_TIMEOUT_S = 60
BARRIER_TIMEOUT_S = 30
# Backoff before the single retry of a legitimate operation (see the
# `open(correct)` churn below): absorbs a transient SQLITE_BUSY without
# hiding a repeatable refusal.
RETRY_BACKOFF_S = 0.1


def _join_all(threads):
    """Join every thread against ONE shared deadline.

    A per-thread ``join(timeout=JOIN_TIMEOUT_S)`` inside a loop costs
    ``len(threads) * JOIN_TIMEOUT_S`` in the worst case. ``pyproject.toml``
    sets ``timeout = 180`` with ``timeout_method = "thread"``, and
    pytest-timeout's thread handler ends in ``os._exit(1)`` -- so when these
    tests actually find the deadlock they exist to find, a per-thread budget
    would kill the whole xdist worker (losing its coverage, then tripping
    ``--cov-fail-under``) instead of failing on the named "deadlocked"
    assertion at the call site. One shared deadline bounds the entire join at
    JOIN_TIMEOUT_S regardless of thread count, which keeps that assertion
    reachable well inside the global timeout. Each thread still gets the full
    remaining budget in wall-clock terms, because the threads run
    concurrently.
    """
    deadline = time.monotonic() + JOIN_TIMEOUT_S
    for t in threads:
        t.join(timeout=max(0.0, deadline - time.monotonic()))


STATUS_RE = re.compile(r"Pool size: (\d+).*?Checked out connections: (\d+)")


@pytest.fixture
def manager(tmp_path, monkeypatch):
    """A DatabaseManager writing to an isolated data directory.

    Same self-contained pattern as the round-1/2/3 files, with the
    sanctioned test-mode KDF knob. TESTING is deliberately NOT set, so
    the engine uses the production QueuePool (pool_size=20,
    max_overflow=40) the bounds assertions parse from the live pool.
    Explicitly unset rather than just "never set here": several other
    test modules do a bare ``os.environ.setdefault("TESTING", "1")`` at
    import time (not ``monkeypatch``), which never reverts for the rest
    of the xdist worker process once any of them has been collected --
    so ambient absence cannot be assumed.
    """
    monkeypatch.delenv("TESTING", raising=False)
    monkeypatch.setenv("LDR_DATA_DIR", str(tmp_path))
    monkeypatch.setenv("LDR_DB_CONFIG_KDF_ITERATIONS", "1000")
    mgr = DatabaseManager()
    mgr.data_dir = tmp_path
    yield mgr
    mgr.close_all_databases()


def _write_canary(engine, note: str) -> None:
    with engine.begin() as conn:
        conn.execute(
            text(
                "CREATE TABLE gap_t_canary (id INTEGER PRIMARY KEY, note TEXT)"
            )
        )
        conn.execute(
            text("INSERT INTO gap_t_canary (id, note) VALUES (1, :note)"),
            {"note": note},
        )


def test_concurrent_readers_stay_within_pool_bounds(manager):
    """T1: 4 threads x 8 reads through one warm QueuePool engine; zero
    failures; every in-thread pool.status() snapshot shows checked-out
    connections within pool_size + max_overflow (and trivially <= the
    thread count)."""
    if not manager.has_encryption:
        pytest.skip("requires SQLCipher (encrypted mode) to be meaningful")

    username = f"poolbounds_{uuid.uuid4().hex[:8]}"
    password = "PoolBoundsPassword1!"  # noqa: S105
    engine = manager.create_user_database(username, password)
    _write_canary(engine, "gap-T1")
    # The engine must really be the production-shape pooled engine for
    # the bounds assertions to be meaningful.
    assert hasattr(engine.pool, "size"), (
        "expected a QueuePool-backed engine (TESTING env must be unset)"
    )
    pool_size = engine.pool.size()
    max_overflow = engine.pool._max_overflow
    assert pool_size + max_overflow >= READER_THREADS

    barrier = threading.Barrier(READER_THREADS)
    errors = {}
    failures = []
    statuses = []

    def reader(tid):
        try:
            barrier.wait(timeout=BARRIER_TIMEOUT_S)
            for i in range(READS_PER_THREAD):
                try:
                    with engine.connect() as conn:
                        note = conn.execute(
                            text("SELECT note FROM gap_t_canary WHERE id = 1")
                        ).scalar()
                    if note != "gap-T1":
                        failures.append(f"t{tid}#{i}: canary read {note!r}")
                except Exception as exc:  # noqa: BLE001 - loud assert below
                    failures.append(f"t{tid}#{i}: {exc!r}")
                statuses.append(engine.pool.status())
        except Exception as exc:  # noqa: BLE001 - surfaced via `errors` below
            errors[tid] = exc

    threads = [
        threading.Thread(target=reader, args=(i,), name=f"gap-t1-r{i}")
        for i in range(READER_THREADS)
    ]
    for t in threads:
        t.start()
    _join_all(threads)
    assert not any(t.is_alive() for t in threads), "reader pool deadlocked"
    assert not errors, f"a reader thread raised: {errors}"
    assert not failures, f"concurrent reads failed: {failures}"

    assert statuses, "no pool status snapshots captured"
    worst = 0
    for snapshot in statuses:
        match = STATUS_RE.search(snapshot)
        assert match, f"unparseable pool status: {snapshot!r}"
        checked_out = int(match.group(2))
        assert checked_out <= pool_size + max_overflow, (
            f"pool bounds exceeded: checked-out {checked_out} > "
            f"{pool_size}+{max_overflow} ({snapshot!r})"
        )
        worst = max(worst, checked_out)
    if worst < 1:
        # Four threads doing sub-millisecond SELECTs can serialise
        # completely under '-n auto' plus coverage tracing, so every
        # pool.status() snapshot legitimately reads zero checked-out
        # connections. That makes the run uninformative, not wrong -- but
        # a ``pytest.skip`` here is indistinguishable in the CI summary
        # from "no SQLCipher build", so the lane would report green
        # having bounded nothing. Warn instead: the bounds assertion
        # above still ran (vacuously), and the no-overlap case survives
        # into the run report rather than passing silently.
        warnings.warn(
            "pool-bounds race not witnessed: no snapshot observed a "
            "checked-out connection -- the readers never overlapped on "
            "this runner, so the bounds assertion had nothing to bound",
            stacklevel=2,
        )


def test_get_session_racing_close_never_raises_and_sessions_work(manager):
    """T2: thread A hammers get_session; thread B hammers open+close.
    Every session A obtains must execute a real table read (resurrection
    contract) or, when the cache is momentarily empty, get_session
    returns None -- but NOTHING may raise, the pairing invariant holds at
    every sample, and the post-close_all state is clean."""
    if not manager.has_encryption:
        pytest.skip("requires SQLCipher (encrypted mode) to be meaningful")

    username = f"getsessionrace_{uuid.uuid4().hex[:8]}"
    password = "GetSessionRacePw1!"  # noqa: S105
    engine = manager.create_user_database(username, password)
    _write_canary(engine, "gap-T2")

    barrier = threading.Barrier(2)
    churn_done = threading.Event()
    errors = {}
    violations = []
    pairing_violations = []
    stats = {"sessions": 0, "nones": 0}

    def session_racer():
        try:
            barrier.wait(timeout=BARRIER_TIMEOUT_S)
            iterations = 0
            # Spin until the churn finishes (a fixed count can complete
            # entirely inside one cold-open window and make this test
            # vacuous), bounded by a generous cap.
            while not churn_done.is_set() and iterations < 3000:
                iterations += 1
                session = manager.get_session(username)
                if session is None:
                    stats["nones"] += 1
                    continue
                stats["sessions"] += 1
                try:
                    note = session.execute(
                        text("SELECT note FROM gap_t_canary WHERE id = 1")
                    ).scalar()
                    if note != "gap-T2":
                        violations.append(f"bad canary read: {note!r}")
                except Exception as exc:  # noqa: BLE001 - loud assert below
                    violations.append(f"session execute failed: {exc!r}")
                finally:
                    session.close()
        except Exception as exc:  # noqa: BLE001 - surfaced via `errors` below
            errors["racer"] = exc

    def open_close_churn():
        try:
            barrier.wait(timeout=BARRIER_TIMEOUT_S)
            for i in range(OPEN_CLOSE_ITERATIONS):
                opened = manager.open_user_database(username, password)
                if opened is None:
                    # Retry once after a short backoff. A transient
                    # SQLITE_BUSY on a LEGITIMATE open under contention is
                    # not the property under test, and reddening on it turns
                    # a loaded runner into a security-gate failure. A real,
                    # repeatable refusal still fails on the retry.
                    time.sleep(RETRY_BACKOFF_S)
                    opened = manager.open_user_database(username, password)
                if opened is None:
                    violations.append(f"churn #{i}: open(correct) failed twice")
                    continue
                # Hold the engine briefly before closing: open_user_database
                # returns only AFTER caching, so without this pause the
                # populated window is sub-millisecond and the get_session
                # racer could legitimately never observe it (vacuous test).
                time.sleep(0.01)
                manager.close_user_database(username)
                with manager._connections_lock:
                    engines = set(manager.connections)
                    verifiers = set(manager._password_verifiers)
                if engines != verifiers:
                    pairing_violations.append(
                        f"churn #{i}: {engines} vs {verifiers}"
                    )
        except Exception as exc:  # noqa: BLE001 - surfaced via `errors` below
            errors["churn"] = exc
        finally:
            churn_done.set()

    threads = [
        threading.Thread(target=session_racer, name="gap-t2-a"),
        threading.Thread(target=open_close_churn, name="gap-t2-b"),
    ]
    for t in threads:
        t.start()
    _join_all(threads)
    assert not any(t.is_alive() for t in threads), "get_session race deadlocked"
    assert not errors, f"a racing thread raised: {errors}"
    assert not violations, f"get_session racing close misbehaved: {violations}"
    assert not pairing_violations, (
        f"engine/verifier pairing broke: {pairing_violations}"
    )
    assert stats["sessions"] >= 1, (
        "the racer never obtained a single session -- test is vacuous"
    )

    # Final state is clean after close_all.
    manager.close_all_databases()
    with manager._connections_lock:
        assert manager.connections == {}
        assert manager._password_verifiers == {}

    # And the user still works end-to-end.
    reopened = manager.open_user_database(username, password)
    assert reopened is not None
    with reopened.connect() as conn:
        assert (
            conn.execute(
                text("SELECT note FROM gap_t_canary WHERE id = 1")
            ).scalar()
            == "gap-T2"
        )