~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Cross-process cache isolation: a warm in-process cache gives a separate
OS process nothing.
Incident context (PR #5596): the verifier is a per-PROCESS keyed HMAC --
``self._verifier_key = secrets.token_bytes(32)`` in ``DatabaseManager.
__init__`` -- and both ``connections`` and ``_password_verifiers`` are
plain in-process dicts. The realistic deployment shape is the web app
plus a second process (CLI / tooling / research worker) over the same
data directory. ``test_password_verifier_primitive.py`` pinned the
cross-KEY digest property; ``test_db_connection_multi_manager_isolation.
py`` (round 1) pinned cross-INSTANCE isolation within one process. This
file pins the actual multi-PROCESS property with real spawned children:
* Q1 a child process with its OWN DatabaseManager over the parent's data
dir: wrong password -> None with nothing cached; correct password ->
a working engine that reads the canary; the parent's warm cache,
engine identity and verifier are completely undisturbed by the child's
activity;
* Q2 the cross-process echo of round 4's gap P: the child holds a warm
correct-password engine; the PARENT then rekeys the file via
change_password; the child's next table read must die on the stale key,
and the parent's new password reads the canary.
"""
import multiprocessing
import queue
import re
import time
import uuid
from pathlib import Path
import pytest
from sqlalchemy import text
from local_deep_research.database.encrypted_db import DatabaseManager
# Module-level gate. Without a working SQLCipher build every test below
# is a no-op, and the dozens of in-body ``pytest.skip`` calls let a CI
# lane report this whole battery green having executed nothing. Skipping
# at import time makes the missing dependency one visible, uniform signal
# per module instead.
pytest.importorskip("sqlcipher3", reason="requires SQLCipher (encrypted mode)")
# Deliberately short. The global pytest timeout is 180s (pyproject.toml),
# and a child that dies before its first put() -- or one left blocked
# because an assertion escaped the parent's try -- otherwise burns a full
# timeout per queue get AND another on the join, which can approach the
# global budget on its own. Children do one cold SQLCipher open at
# LDR_DB_CONFIG_KDF_ITERATIONS=1000, so 20s is a very generous ceiling.
CHILD_TIMEOUT_S = 20
# Granularity of the liveness poll while waiting on a child's queue.
CHILD_POLL_S = 0.25
# What a read through a STALE SQLCipher key looks like once the file has
# been rekeyed underneath it. Deliberately excludes contention errors
# ("database is locked"), which would otherwise satisfy a bare
# `select_ok is False` while proving nothing about the key.
STALE_KEY_ERROR_RE = re.compile(
r"file is not a database|not a database|decrypt|HMAC|hmac|"
r"malformed|encrypted",
re.IGNORECASE,
)
@pytest.fixture
def manager(tmp_path, monkeypatch):
"""A DatabaseManager writing to an isolated data directory.
Same self-contained pattern as the round-1/2/3 files, with the
sanctioned test-mode KDF knob (passed explicitly to the children --
spawn inherits the environment, but the children set it themselves
for determinism).
"""
monkeypatch.setenv("LDR_DATA_DIR", str(tmp_path))
monkeypatch.setenv("LDR_DB_CONFIG_KDF_ITERATIONS", "1000")
mgr = DatabaseManager()
mgr.data_dir = tmp_path
yield mgr
mgr.close_all_databases()
def _child_env(data_dir: str) -> None:
"""Point THIS (child) process's environment at the parent's data dir
with the same KDF knob, before any DatabaseManager is built."""
import os
os.environ["LDR_DATA_DIR"] = data_dir
os.environ["LDR_DB_CONFIG_KDF_ITERATIONS"] = "1000"
def _child_open_probe(data_dir, username, wrong_pw, correct_pw, result_q):
"""Q1 child: builds its own manager, tries wrong then correct."""
try:
_child_env(data_dir)
from local_deep_research.database.encrypted_db import (
DatabaseManager as ChildManager,
)
mgr = ChildManager()
mgr.data_dir = Path(data_dir)
try:
wrong_engine = mgr.open_user_database(username, wrong_pw)
result_q.put(
{
"wrong_engine": wrong_engine is not None,
"wrong_cached": (
username in mgr.connections
or username in mgr._password_verifiers
),
}
)
engine = mgr.open_user_database(username, correct_pw)
canary = None
if engine is not None:
with engine.connect() as conn:
canary = conn.execute(
text("SELECT note FROM gap_q_canary WHERE id = 1")
).scalar()
result_q.put(
{
"correct_engine": engine is not None,
"canary": canary,
}
)
finally:
mgr.close_all_databases()
except Exception as exc: # noqa: BLE001 - reported to the parent loudly
result_q.put({"child_error": repr(exc)})
def _child_holds_engine_then_reads(data_dir, username, pw, cmd_q, result_q):
"""Q2 child: opens (correct pw), signals, then SELECTs on command."""
try:
_child_env(data_dir)
from pathlib import Path
from local_deep_research.database.encrypted_db import (
DatabaseManager as ChildManager,
)
mgr = ChildManager()
mgr.data_dir = Path(data_dir)
try:
engine = mgr.open_user_database(username, pw)
if engine is None:
result_q.put({"child_error": "child cold-open failed"})
return
with engine.connect() as conn:
note = conn.execute(
text("SELECT note FROM gap_q_canary WHERE id = 1")
).scalar()
result_q.put({"opened": True, "canary_before": note})
while True:
cmd = cmd_q.get(timeout=CHILD_TIMEOUT_S)
if cmd == "done":
break
if cmd == "select":
try:
with engine.connect() as conn:
note = conn.execute(
text(
"SELECT note FROM gap_q_canary WHERE id = 1"
)
).scalar()
result_q.put({"select_ok": True, "canary": note})
except Exception as exc: # noqa: BLE001 - reported out
result_q.put({"select_ok": False, "error": repr(exc)})
finally:
mgr.close_all_databases()
except Exception as exc: # noqa: BLE001 - reported to the parent loudly
result_q.put({"child_error": repr(exc)})
def _get_result(result_q, what, child):
"""Fetch one result dict from a child, failing loudly on death.
Polls ``child.is_alive()`` between short queue waits so a child that
dies before its first ``put`` fails in seconds instead of stalling for
the whole ``CHILD_TIMEOUT_S``.
"""
deadline = time.monotonic() + CHILD_TIMEOUT_S
while True:
try:
result = result_q.get(timeout=CHILD_POLL_S)
break
except queue.Empty:
if not child.is_alive():
# A child that already exited may still have results in
# flight (the Queue feeder thread flushes on exit), so
# give the pipe one last, generous chance before calling
# it a death.
try:
result = result_q.get(timeout=CHILD_POLL_S * 8)
break
except queue.Empty:
raise AssertionError(
f"child died during {what} without reporting "
f"(exitcode {child.exitcode})"
) from None
if time.monotonic() >= deadline:
raise AssertionError(
f"child produced no result for {what} within "
f"{CHILD_TIMEOUT_S}s"
) from None
assert "child_error" not in result, (
f"child failed during {what}: {result.get('child_error')}"
)
return result
def _reap(child):
"""Join a child, killing it if it is wedged, so a failed assertion in
the parent cannot cost a second full timeout in ``finally``."""
child.join(timeout=CHILD_TIMEOUT_S)
if child.is_alive():
child.terminate()
child.join(timeout=CHILD_POLL_S * 4)
def test_separate_process_gets_nothing_from_parent_cache(manager, tmp_path):
"""Q1: parent cache is warm; a spawned child with its own manager gets
no benefit from it (wrong pw refused, correct pw re-proven against
SQLCipher) and the parent's state is untouched."""
if not manager.has_encryption:
pytest.skip("requires SQLCipher (encrypted mode) to be meaningful")
username = f"xproc_{uuid.uuid4().hex[:8]}"
password = "CrossProcessPw1!" # noqa: S105
parent_engine = manager.create_user_database(username, password)
with parent_engine.begin() as conn:
conn.execute(
text(
"CREATE TABLE gap_q_canary (id INTEGER PRIMARY KEY, note TEXT)"
)
)
conn.execute(
text("INSERT INTO gap_q_canary (id, note) VALUES (1, :note)"),
{"note": "gap-Q"},
)
assert manager.is_user_connected(username), "parent cache must be warm"
ctx = multiprocessing.get_context("spawn")
result_q = ctx.Queue()
child = ctx.Process(
target=_child_open_probe,
args=(
str(manager.data_dir),
username,
"xproc-wrong",
password,
result_q,
),
name="gap-q1-child",
)
child.start()
try:
wrong = _get_result(result_q, "wrong-password probe", child)
assert wrong["wrong_engine"] is False, (
"the child process opened the database with a wrong password"
)
assert wrong["wrong_cached"] is False, (
"the child's refused attempt cached engine or verifier state"
)
correct = _get_result(result_q, "correct-password probe", child)
assert correct["correct_engine"] is True, (
"the child could not open with the correct password"
)
assert correct["canary"] == "gap-Q", (
"the child's engine could not read the canary"
)
finally:
_reap(child)
assert not child.is_alive(), "Q1 child did not exit"
assert child.exitcode == 0, f"Q1 child exitcode {child.exitcode}"
# The child's activity must not have disturbed the parent at all.
assert manager.connections[username] is parent_engine, (
"the child replaced or evicted the parent's cached engine"
)
assert manager._password_matches_cached(username, password) is True
with parent_engine.connect() as conn:
assert (
conn.execute(
text("SELECT note FROM gap_q_canary WHERE id = 1")
).scalar()
== "gap-Q"
)
def test_parent_rekey_kills_child_held_engine(manager, tmp_path):
"""Q2: the child holds a warm correct-password engine; the parent
rekeys the file; the child's next table read dies on the stale key
(gap P's cross-process echo), and the parent's new password reads the
canary."""
if not manager.has_encryption:
pytest.skip("requires SQLCipher (encrypted mode) to be meaningful")
username = f"xprockey_{uuid.uuid4().hex[:8]}"
old = "CrossProcOldPw1!" # noqa: S105
new = "CrossProcNewPw2!" # noqa: S105
engine = manager.create_user_database(username, old)
with engine.begin() as conn:
conn.execute(
text(
"CREATE TABLE gap_q_canary (id INTEGER PRIMARY KEY, note TEXT)"
)
)
conn.execute(
text("INSERT INTO gap_q_canary (id, note) VALUES (1, :note)"),
{"note": "gap-Q2"},
)
manager.close_user_database(username)
ctx = multiprocessing.get_context("spawn")
result_q = ctx.Queue()
cmd_q = ctx.Queue()
child = ctx.Process(
target=_child_holds_engine_then_reads,
args=(str(manager.data_dir), username, old, cmd_q, result_q),
name="gap-q2-child",
)
child.start()
try:
opened = _get_result(result_q, "child warm-open", child)
assert opened["opened"] is True
assert opened["canary_before"] == "gap-Q2"
# Parent-side rekey while the child holds its warm engine.
assert manager.change_password(username, old, new) is True
cmd_q.put("select")
after = _get_result(result_q, "post-rekey child read", child)
assert after["select_ok"] is False, (
"the child's pre-rekey engine still reads data after the "
"parent rekeyed the file -- the stale key outlived the rekey"
)
# The failure must be a KEY failure, not incidental contention:
# "database is locked" would satisfy select_ok is False while
# proving nothing about the stale key.
assert STALE_KEY_ERROR_RE.search(after["error"]), (
"the child's post-rekey read failed for the wrong reason -- "
f"expected a key/HMAC/decrypt failure, got {after['error']!r}"
)
cmd_q.put("done")
finally:
_reap(child)
assert not child.is_alive(), "Q2 child did not exit"
assert child.exitcode == 0, f"Q2 child exitcode {child.exitcode}"
# Parent-side recovery with the new password.
reopened = manager.open_user_database(username, new)
assert reopened is not None
with reopened.connect() as conn:
assert (
conn.execute(
text("SELECT note FROM gap_q_canary WHERE id = 1")
).scalar()
== "gap-Q2"
)
assert manager.open_user_database(username, old) is None