~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Many-user cache hygiene under sequential and concurrent churn.
Incident context (PR #5596): the cache maps (``connections``,
``_password_verifiers``, ``_init_locks``) are the entire trust state of
the connection layer. At scale -- many accounts churning through one
process -- two operational security properties must hold:
* W1 after 30 sequential create/write/close cycles and a final
``close_all_databases``, the engine and verifier maps are empty
(encrypted_db.py::close_all_databases; ``_init_locks`` is deliberately
retained, see that method) and the liveness accessors read zero;
* W2 the same 30 users created and churned CONCURRENTLY across 4 threads
(each owning a subset, barrier-start, EVERY user with its own distinct
password), with the engine/verifier pairing invariant sampled under
``_connections_lock`` throughout, no failed opens, and every user
reopening afterwards reading its OWN canary.
FD-count deltas are deliberately NOT asserted here: pyproject.toml's
``fd_canary`` marker documents that ``/proc/self/fd`` assertions are
noisy under parallel execution, and this file belongs in the parallel
lane for the cache-hygiene coverage above. ``tests/utilities/
test_close_base_llm.py`` owns the FD-leak canaries.
Per-user schema migration is neutered for these tests ONLY (monkeypatch
of ``initialize.initialize_database`` to a no-op) -- the exact isolation
pattern of ``tests/database/test_concurrent_user_db_open.py`` -- because
30 full migrations would blow the run-time budget while exercising
nothing this file asserts on (the cache, the open/close lifecycle, and
descriptors are all fully real, backed by real SQLCipher files).
"""
import threading
import time
import uuid
import pytest
from sqlalchemy import text
import local_deep_research.database.initialize as init_mod
from local_deep_research.database.encrypted_db import DatabaseManager
# Module-level gate. Without a working SQLCipher build every test below
# is a no-op, and the dozens of in-body ``pytest.skip`` calls let a CI
# lane report this whole battery green having executed nothing. Skipping
# at import time makes the missing dependency one visible, uniform signal
# per module instead.
pytest.importorskip("sqlcipher3", reason="requires SQLCipher (encrypted mode)")
USER_COUNT = 30
THREAD_COUNT = 4
CHURN_ITERATIONS = 8
# 120s, not the 60s the other shared-deadline modules use: this module does
# 4 threads x 30 real SQLCipher users (create + write + churn + reopen),
# more work than the 3-thread modules the 60s budget was sized for, and
# unlike the per-thread-join modules (2 threads, 120s worst case each) it
# has no headroom to trade away -- there is only one shared deadline here.
# 120s join budget + pytest's own setup/call/teardown overhead (this
# fixture's ``close_all_databases`` teardown included) must still land
# under ``pyproject.toml``'s global ``timeout = 180``; 60s of headroom is
# the same margin the per-thread modules get.
JOIN_TIMEOUT_S = 120
BARRIER_TIMEOUT_S = 30
def _join_all(threads):
"""Join every thread against ONE shared deadline.
A per-thread ``join(timeout=JOIN_TIMEOUT_S)`` inside a loop costs
``len(threads) * JOIN_TIMEOUT_S`` in the worst case. ``pyproject.toml``
sets ``timeout = 180`` with ``timeout_method = "thread"``, and
pytest-timeout's thread handler ends in ``os._exit(1)`` -- so when these
tests actually find the deadlock they exist to find, a per-thread budget
would kill the whole xdist worker (losing its coverage, then tripping
``--cov-fail-under``) instead of failing on the named "deadlocked"
assertion at the call site. One shared deadline bounds the entire join at
JOIN_TIMEOUT_S regardless of thread count, which keeps that assertion
reachable well inside the global timeout. Each thread still gets the full
remaining budget in wall-clock terms, because the threads run
concurrently.
"""
deadline = time.monotonic() + JOIN_TIMEOUT_S
for t in threads:
t.join(timeout=max(0.0, deadline - time.monotonic()))
@pytest.fixture
def manager(tmp_path, monkeypatch):
"""A DatabaseManager writing to an isolated data directory.
Same self-contained pattern as the round-1/2/3/4 files, with the
sanctioned test-mode KDF knob. ``initialize_database`` is additionally
neutered for this file (see module docstring) to keep 30-user tests
inside the run-time budget.
"""
monkeypatch.setenv("LDR_DATA_DIR", str(tmp_path))
monkeypatch.setenv("LDR_DB_CONFIG_KDF_ITERATIONS", "1000")
monkeypatch.setattr(init_mod, "initialize_database", lambda *a, **k: None)
# With migrations neutered the databases stay unstamped, which would
# make every cold open attempt a pre-migration BACKUP. W asserts on
# cache/FD lifecycle, not backups -- the same isolation class as the
# initialize_database patch above.
monkeypatch.setattr(
"local_deep_research.database.alembic_runner.needs_migration",
lambda *a, **k: False,
)
mgr = DatabaseManager()
# The phase-2 RAG index re-key re-enters through vector-store imports
# and dominates cold-open time; tests/database/test_concurrent_user_db
# _open.py already isolates it the same way for cache-lifecycle tests.
monkeypatch.setattr(mgr, "_run_phase2_rekey", lambda *a, **k: None)
mgr.data_dir = tmp_path
yield mgr
mgr.close_all_databases()
def _write_canary(engine, note: str) -> None:
with engine.begin() as conn:
conn.execute(
text(
"CREATE TABLE gap_w_canary (id INTEGER PRIMARY KEY, note TEXT)"
)
)
conn.execute(
text("INSERT INTO gap_w_canary (id, note) VALUES (1, :note)"),
{"note": note},
)
def _make_user(manager, index):
username = f"many_{index}_{uuid.uuid4().hex[:8]}"
password = f"ManyUsersPw{index % 10}!" # noqa: S105
engine = manager.create_user_database(username, password)
_write_canary(engine, f"gap-W-{index}")
manager.close_user_database(username)
return username, password
def _assert_fully_drained(manager):
with manager._connections_lock:
assert manager.connections == {}, "close_all left cached engines"
assert manager._password_verifiers == {}, "close_all left verifiers"
# _init_locks is deliberately NOT drained by close_all: a cold-open
# may still hold a per-user lock (see close_all_databases). Not a
# leak -- the dict is bounded, one small Lock per username.
assert manager.get_connected_usernames() == set()
assert manager.get_memory_usage()["active_connections"] == 0
def test_thirty_sequential_users_drain_all_state(manager):
"""W1: 30 sequential create+canary+close cycles, then close_all:
engine + verifier maps empty, liveness accessors zero."""
if not manager.has_encryption:
pytest.skip("requires SQLCipher (encrypted mode) to be meaningful")
users = [_make_user(manager, i) for i in range(USER_COUNT)]
assert len(users) == USER_COUNT
# Mid-state sanity: 30 engines came and went; nothing is live now
# (each cycle closed its own engine), and the maps are paired-empty.
with manager._connections_lock:
assert set(manager.connections) == set(manager._password_verifiers)
manager.close_all_databases()
_assert_fully_drained(manager)
# A sample of the users still opens and reads its canary afterwards.
for username, password in users[:3]:
engine = manager.open_user_database(username, password)
assert engine is not None
with engine.connect() as conn:
note = conn.execute(
text("SELECT note FROM gap_w_canary WHERE id = 1")
).scalar()
assert note.startswith("gap-W-")
manager.close_user_database(username)
# Both close paths deliberately RETAIN per-user init locks (see
# tests/database/test_concurrent_user_db_open.py and
# close_all_databases); close_all drains engines + verifiers only.
manager.close_all_databases()
_assert_fully_drained(manager)
def test_thirty_users_concurrent_churn_stays_paired(manager):
"""W2: 30 users created and churned by 4 barrier-synced threads (each
owning a subset, each user with its OWN password): no open failures,
pairing invariant at every sample, every user reopen+canary at the
end, engine + verifier maps drained after close_all."""
if not manager.has_encryption:
pytest.skip("requires SQLCipher (encrypted mode) to be meaningful")
barrier = threading.Barrier(THREAD_COUNT)
errors = {}
open_failures = []
pairing_violations = []
canary_failures = []
subset_sizes = {}
def worker(tid: int):
try:
# Each thread owns a DISJOINT subset of the USER_COUNT users and
# creates them itself (parallel creation, distinct files). The
# split is round-robin (`range(tid, USER_COUNT, THREAD_COUNT)`)
# so the subsets together cover every user: a plain
# `range(USER_COUNT // THREAD_COUNT)` silently dropped the
# remainder and churned 28 of the 30 this test is named for.
# Every user gets a DISTINCT password: a shared per-thread
# password would make cross-user bleed WITHIN a thread's own
# subset invisible (any of its users would open any other).
own = [
(
f"many_c_{tid}_{i}_{uuid.uuid4().hex[:6]}",
f"ConcPw{tid}_{i}_{uuid.uuid4().hex[:6]}!",
)
for i in range(tid, USER_COUNT, THREAD_COUNT)
]
subset_sizes[tid] = len(own)
notes = {}
for index, (username, password) in enumerate(own):
engine = manager.create_user_database(username, password)
notes[username] = f"gap-W2-{tid}-{index}"
_write_canary(engine, notes[username])
barrier.wait(timeout=BARRIER_TIMEOUT_S)
# Churn: cycle the subset, open + canary read + close.
for i in range(CHURN_ITERATIONS):
username, password = own[i % len(own)]
engine = manager.open_user_database(username, password)
if engine is None:
open_failures.append(f"t{tid} churn#{i}: {username}")
continue
try:
with engine.connect() as conn:
note = conn.execute(
text("SELECT note FROM gap_w_canary WHERE id = 1")
).scalar()
if note != notes[username]:
canary_failures.append(
f"t{tid} churn#{i} {username}: read {note!r}, "
f"expected {notes[username]!r}"
)
finally:
manager.close_user_database(username)
with manager._connections_lock:
engines = set(manager.connections)
verifiers = set(manager._password_verifiers)
if engines != verifiers:
pairing_violations.append(
f"t{tid} churn#{i}: {engines} vs {verifiers}"
)
# Final pass: EVERY user in this thread's subset reopens and
# reads its own canary.
for username, password in own:
engine = manager.open_user_database(username, password)
if engine is None:
open_failures.append(f"t{tid} final: {username}")
continue
try:
with engine.connect() as conn:
note = conn.execute(
text("SELECT note FROM gap_w_canary WHERE id = 1")
).scalar()
if note != notes[username]:
canary_failures.append(
f"t{tid} final {username}: read {note!r}, "
f"expected {notes[username]!r}"
)
finally:
manager.close_user_database(username)
except Exception as exc: # noqa: BLE001 - surfaced via `errors` below
errors[tid] = exc
threads = [
threading.Thread(target=worker, args=(i,), name=f"gap-w2-t{i}")
for i in range(THREAD_COUNT)
]
for t in threads:
t.start()
_join_all(threads)
assert not any(t.is_alive() for t in threads), "many-user churn deadlocked"
assert not errors, f"a churn thread raised: {errors}"
assert sum(subset_sizes.values()) == USER_COUNT, (
f"the thread subsets covered {sum(subset_sizes.values())} users, not "
f"the {USER_COUNT} this test is named for: {subset_sizes}"
)
assert not open_failures, f"legitimate opens failed: {open_failures}"
assert not canary_failures, f"cross-user canary reads: {canary_failures}"
assert not pairing_violations, (
f"engine/verifier pairing broke: {pairing_violations}"
)
manager.close_all_databases()
_assert_fully_drained(manager)