~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Pool bounds under concurrency and get_session racing close.
Incident context (PR #5596): the shared per-user engine is the trust
boundary -- every Session handed out by ``get_session`` /
``create_thread_safe_session_for_metrics`` binds to the ONE
credential-verified engine (ADR-0004: QueuePool, pool_size=20,
max_overflow=40). Two behavioral properties complete the picture the
incident tests left open:
* T1 concurrent readers through the shared engine: many threads reading
simultaneously never exceed the pool's configured bounds
(checked-out <= pool_size + max_overflow, parsed from
``engine.pool.status()`` snapshots taken INSIDE the threads) and never
fail;
* T2 ``get_session`` racing ``close_user_database``: get_session is a
pure cache lookup under ``_connections_lock``; when it wins the race it
binds a Session to the live engine, and when the close wins first the
round-3 resurrection contract covers the Session (the creator closure
reconnects with the same key) -- so every session the racer obtains
must still execute real table reads successfully, never raise, and the
engine/verifier pairing must hold at every sampled instant.
FD-count deltas are deliberately NOT asserted here: pyproject.toml's
``fd_canary`` marker documents that ``/proc/self/fd`` assertions are
noisy under parallel execution (and ``/proc`` is Linux-only), while this
file belongs in the parallel lane for the pool/pairing coverage above.
"""
import re
import threading
import time
import uuid
import warnings
import pytest
from sqlalchemy import text
from local_deep_research.database.encrypted_db import DatabaseManager
# Module-level gate. Without a working SQLCipher build every test below
# is a no-op, and the dozens of in-body ``pytest.skip`` calls let a CI
# lane report this whole battery green having executed nothing. Skipping
# at import time makes the missing dependency one visible, uniform signal
# per module instead.
pytest.importorskip("sqlcipher3", reason="requires SQLCipher (encrypted mode)")
READER_THREADS = 4
READS_PER_THREAD = 8
GET_SESSION_ITERATIONS = 30
OPEN_CLOSE_ITERATIONS = 15
JOIN_TIMEOUT_S = 60
BARRIER_TIMEOUT_S = 30
# Backoff before the single retry of a legitimate operation (see the
# `open(correct)` churn below): absorbs a transient SQLITE_BUSY without
# hiding a repeatable refusal.
RETRY_BACKOFF_S = 0.1
def _join_all(threads):
"""Join every thread against ONE shared deadline.
A per-thread ``join(timeout=JOIN_TIMEOUT_S)`` inside a loop costs
``len(threads) * JOIN_TIMEOUT_S`` in the worst case. ``pyproject.toml``
sets ``timeout = 180`` with ``timeout_method = "thread"``, and
pytest-timeout's thread handler ends in ``os._exit(1)`` -- so when these
tests actually find the deadlock they exist to find, a per-thread budget
would kill the whole xdist worker (losing its coverage, then tripping
``--cov-fail-under``) instead of failing on the named "deadlocked"
assertion at the call site. One shared deadline bounds the entire join at
JOIN_TIMEOUT_S regardless of thread count, which keeps that assertion
reachable well inside the global timeout. Each thread still gets the full
remaining budget in wall-clock terms, because the threads run
concurrently.
"""
deadline = time.monotonic() + JOIN_TIMEOUT_S
for t in threads:
t.join(timeout=max(0.0, deadline - time.monotonic()))
STATUS_RE = re.compile(r"Pool size: (\d+).*?Checked out connections: (\d+)")
@pytest.fixture
def manager(tmp_path, monkeypatch):
"""A DatabaseManager writing to an isolated data directory.
Same self-contained pattern as the round-1/2/3 files, with the
sanctioned test-mode KDF knob. TESTING is deliberately NOT set, so
the engine uses the production QueuePool (pool_size=20,
max_overflow=40) the bounds assertions parse from the live pool.
Explicitly unset rather than just "never set here": several other
test modules do a bare ``os.environ.setdefault("TESTING", "1")`` at
import time (not ``monkeypatch``), which never reverts for the rest
of the xdist worker process once any of them has been collected --
so ambient absence cannot be assumed.
"""
monkeypatch.delenv("TESTING", raising=False)
monkeypatch.setenv("LDR_DATA_DIR", str(tmp_path))
monkeypatch.setenv("LDR_DB_CONFIG_KDF_ITERATIONS", "1000")
mgr = DatabaseManager()
mgr.data_dir = tmp_path
yield mgr
mgr.close_all_databases()
def _write_canary(engine, note: str) -> None:
with engine.begin() as conn:
conn.execute(
text(
"CREATE TABLE gap_t_canary (id INTEGER PRIMARY KEY, note TEXT)"
)
)
conn.execute(
text("INSERT INTO gap_t_canary (id, note) VALUES (1, :note)"),
{"note": note},
)
def test_concurrent_readers_stay_within_pool_bounds(manager):
"""T1: 4 threads x 8 reads through one warm QueuePool engine; zero
failures; every in-thread pool.status() snapshot shows checked-out
connections within pool_size + max_overflow (and trivially <= the
thread count)."""
if not manager.has_encryption:
pytest.skip("requires SQLCipher (encrypted mode) to be meaningful")
username = f"poolbounds_{uuid.uuid4().hex[:8]}"
password = "PoolBoundsPassword1!" # noqa: S105
engine = manager.create_user_database(username, password)
_write_canary(engine, "gap-T1")
# The engine must really be the production-shape pooled engine for
# the bounds assertions to be meaningful.
assert hasattr(engine.pool, "size"), (
"expected a QueuePool-backed engine (TESTING env must be unset)"
)
pool_size = engine.pool.size()
max_overflow = engine.pool._max_overflow
assert pool_size + max_overflow >= READER_THREADS
barrier = threading.Barrier(READER_THREADS)
errors = {}
failures = []
statuses = []
def reader(tid):
try:
barrier.wait(timeout=BARRIER_TIMEOUT_S)
for i in range(READS_PER_THREAD):
try:
with engine.connect() as conn:
note = conn.execute(
text("SELECT note FROM gap_t_canary WHERE id = 1")
).scalar()
if note != "gap-T1":
failures.append(f"t{tid}#{i}: canary read {note!r}")
except Exception as exc: # noqa: BLE001 - loud assert below
failures.append(f"t{tid}#{i}: {exc!r}")
statuses.append(engine.pool.status())
except Exception as exc: # noqa: BLE001 - surfaced via `errors` below
errors[tid] = exc
threads = [
threading.Thread(target=reader, args=(i,), name=f"gap-t1-r{i}")
for i in range(READER_THREADS)
]
for t in threads:
t.start()
_join_all(threads)
assert not any(t.is_alive() for t in threads), "reader pool deadlocked"
assert not errors, f"a reader thread raised: {errors}"
assert not failures, f"concurrent reads failed: {failures}"
assert statuses, "no pool status snapshots captured"
worst = 0
for snapshot in statuses:
match = STATUS_RE.search(snapshot)
assert match, f"unparseable pool status: {snapshot!r}"
checked_out = int(match.group(2))
assert checked_out <= pool_size + max_overflow, (
f"pool bounds exceeded: checked-out {checked_out} > "
f"{pool_size}+{max_overflow} ({snapshot!r})"
)
worst = max(worst, checked_out)
if worst < 1:
# Four threads doing sub-millisecond SELECTs can serialise
# completely under '-n auto' plus coverage tracing, so every
# pool.status() snapshot legitimately reads zero checked-out
# connections. That makes the run uninformative, not wrong -- but
# a ``pytest.skip`` here is indistinguishable in the CI summary
# from "no SQLCipher build", so the lane would report green
# having bounded nothing. Warn instead: the bounds assertion
# above still ran (vacuously), and the no-overlap case survives
# into the run report rather than passing silently.
warnings.warn(
"pool-bounds race not witnessed: no snapshot observed a "
"checked-out connection -- the readers never overlapped on "
"this runner, so the bounds assertion had nothing to bound",
stacklevel=2,
)
def test_get_session_racing_close_never_raises_and_sessions_work(manager):
"""T2: thread A hammers get_session; thread B hammers open+close.
Every session A obtains must execute a real table read (resurrection
contract) or, when the cache is momentarily empty, get_session
returns None -- but NOTHING may raise, the pairing invariant holds at
every sample, and the post-close_all state is clean."""
if not manager.has_encryption:
pytest.skip("requires SQLCipher (encrypted mode) to be meaningful")
username = f"getsessionrace_{uuid.uuid4().hex[:8]}"
password = "GetSessionRacePw1!" # noqa: S105
engine = manager.create_user_database(username, password)
_write_canary(engine, "gap-T2")
barrier = threading.Barrier(2)
churn_done = threading.Event()
errors = {}
violations = []
pairing_violations = []
stats = {"sessions": 0, "nones": 0}
def session_racer():
try:
barrier.wait(timeout=BARRIER_TIMEOUT_S)
iterations = 0
# Spin until the churn finishes (a fixed count can complete
# entirely inside one cold-open window and make this test
# vacuous), bounded by a generous cap.
while not churn_done.is_set() and iterations < 3000:
iterations += 1
session = manager.get_session(username)
if session is None:
stats["nones"] += 1
continue
stats["sessions"] += 1
try:
note = session.execute(
text("SELECT note FROM gap_t_canary WHERE id = 1")
).scalar()
if note != "gap-T2":
violations.append(f"bad canary read: {note!r}")
except Exception as exc: # noqa: BLE001 - loud assert below
violations.append(f"session execute failed: {exc!r}")
finally:
session.close()
except Exception as exc: # noqa: BLE001 - surfaced via `errors` below
errors["racer"] = exc
def open_close_churn():
try:
barrier.wait(timeout=BARRIER_TIMEOUT_S)
for i in range(OPEN_CLOSE_ITERATIONS):
opened = manager.open_user_database(username, password)
if opened is None:
# Retry once after a short backoff. A transient
# SQLITE_BUSY on a LEGITIMATE open under contention is
# not the property under test, and reddening on it turns
# a loaded runner into a security-gate failure. A real,
# repeatable refusal still fails on the retry.
time.sleep(RETRY_BACKOFF_S)
opened = manager.open_user_database(username, password)
if opened is None:
violations.append(f"churn #{i}: open(correct) failed twice")
continue
# Hold the engine briefly before closing: open_user_database
# returns only AFTER caching, so without this pause the
# populated window is sub-millisecond and the get_session
# racer could legitimately never observe it (vacuous test).
time.sleep(0.01)
manager.close_user_database(username)
with manager._connections_lock:
engines = set(manager.connections)
verifiers = set(manager._password_verifiers)
if engines != verifiers:
pairing_violations.append(
f"churn #{i}: {engines} vs {verifiers}"
)
except Exception as exc: # noqa: BLE001 - surfaced via `errors` below
errors["churn"] = exc
finally:
churn_done.set()
threads = [
threading.Thread(target=session_racer, name="gap-t2-a"),
threading.Thread(target=open_close_churn, name="gap-t2-b"),
]
for t in threads:
t.start()
_join_all(threads)
assert not any(t.is_alive() for t in threads), "get_session race deadlocked"
assert not errors, f"a racing thread raised: {errors}"
assert not violations, f"get_session racing close misbehaved: {violations}"
assert not pairing_violations, (
f"engine/verifier pairing broke: {pairing_violations}"
)
assert stats["sessions"] >= 1, (
"the racer never obtained a single session -- test is vacuous"
)
# Final state is clean after close_all.
manager.close_all_databases()
with manager._connections_lock:
assert manager.connections == {}
assert manager._password_verifiers == {}
# And the user still works end-to-end.
reopened = manager.open_user_database(username, password)
assert reopened is not None
with reopened.connect() as conn:
assert (
conn.execute(
text("SELECT note FROM gap_t_canary WHERE id = 1")
).scalar()
== "gap-T2"
)