~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(database): never follow a symlink when creating a user database (#6378)
Lukas8f6e336eebc4
diff --git a/changelog.d/6378.security.md b/changelog.d/6378.security.mdnew file mode 100644index 0000000000..d7db735fab--- /dev/null+++ b/changelog.d/6378.security.md@@ -0,0 +1 @@+Creating a user database no longer follows a symlink planted at the database path. The path is refused when it is a link, and the file is created exclusively and without following links before SQLCipher or SQLite opens it; a path replaced while the database is being created is detected and the creation fails, although a replacement undone before the check runs is not caught.diff --git a/src/local_deep_research/database/encrypted_db.py b/src/local_deep_research/database/encrypted_db.pyindex f17c57cca4..40bcdf3929 100644--- a/src/local_deep_research/database/encrypted_db.py+++ b/src/local_deep_research/database/encrypted_db.py@@ -5,7 +5,9 @@ Handles per-user encrypted databases with browser-friendly authentication.import hmacimport secrets+import errnoimport os+import statimport threadingimport timefrom pathlib import Path@@ -100,6 +102,74 @@ def _best_effort_chmod(path, mode: int, *, warn: bool = False) -> None:)+def _create_private_file(path: Path) -> tuple[int, int]:+ """Create ``path`` as an empty, owner-only regular file, never via a link.++ ``O_CREAT | O_EXCL`` fails when anything exists at ``path`` — including a+ symlink, dangling or not, which is exactly the no-follow protection the+ ``.salt`` file already gets in ``create_database_salt``. ``O_NOFOLLOW`` is+ added where the platform has it. SQLite and SQLCipher treat the empty+ file as a fresh database (#6354).++ Returns the ``(st_dev, st_ino)`` of the file it created, read from the+ descriptor rather than the name, so ``_assert_reserved_file`` can tell+ afterwards whether the pathname still refers to this file.+ """+ flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0)+ try:+ fd = os.open(str(path), flags, 0o600)+ except OSError as exc:+ # A platform may report the symlink through O_NOFOLLOW (ELOOP)+ # before O_EXCL gets to say EEXIST; both mean "something is already+ # at this path", so callers see one exception type.+ if exc.errno == errno.ELOOP:+ raise FileExistsError(+ errno.EEXIST, "Path is a symlink", str(path)+ ) from exc+ raise+ try:+ reserved = os.fstat(fd)+ finally:+ os.close(fd)+ return reserved.st_dev, reserved.st_ino+++def _assert_reserved_file(path: Path, reserved: tuple[int, int]) -> None:+ """Fail when ``path`` no longer names the file that was reserved.++ This is partial hardening, not a guarantee. The reservation cannot be held+ across the database open: SQLCipher and SQLite take a pathname, and+ neither ``sqlcipher3`` nor the standard ``sqlite3`` module can ask for+ ``SQLITE_OPEN_NOFOLLOW`` -- it is an open flag of the C API, not a URI+ parameter, and a ``file:...?nofollow=1`` open still follows the link.++ What the check does catch is a pathname that is still replaced when it+ runs -- by a symlink, a hard link or a freshly created file -- and the+ create then fails instead of adopting it (#6378). What it cannot catch is+ a replacement that is undone again before it runs: a swap to another+ file, open, and swap back leaves the connection on the other file while+ the name looks untouched. The protection that does not depend on timing+ is the refusal of a link that is already there and the exclusive,+ no-follow creation in :func:`_create_private_file`.+ """+ try:+ stat_result = os.lstat(path)+ except OSError as exc:+ raise ValueError(+ f"Database path {path} disappeared while the database was "+ "being created; refusing to use it"+ ) from exc++ if (+ stat.S_ISLNK(stat_result.st_mode)+ or (stat_result.st_dev, stat_result.st_ino) != reserved+ ):+ raise ValueError(+ f"Database path {path} was replaced while the database was "+ "being created; refusing to use it"+ )++def _db_path_is_symlinked(db_path: Path, username: str) -> bool:"""True when a user-database pathname is a symlink, which is refused.@@ -595,23 +665,33 @@ class DatabaseManager:# this user then waits here instead of migrating the same file at# the same time as the create.with self._get_init_lock(username):- db_path, engine = self._create_user_database_files(+ db_path, engine, reserved = self._create_user_database_files(username, password)self._initialize_new_user_database(- username, password, db_path, engine+ username, password, db_path, engine, reserved)logger.info(f"Created encrypted database for user {username}")return enginedef _initialize_new_user_database(- self, username: str, password: str, db_path: Path, engine: Engine+ self,+ username: str,+ password: str,+ db_path: Path,+ engine: Engine,+ reserved: tuple[int, int],) -> None:"""Migrate a just-created user database and publish its engine.Called by ``create_user_database`` while holding this user's initlock. Removes the partial database when the migrations fail.++ ``reserved`` identifies the file ``_create_user_database_files``+ materialized, so the first real open below -- the unencrypted+ branch's first, since SQLAlchemy opens lazily -- is checked against it+ before the migrations write anything."""# Initialize database tables using centralized initializationfrom .initialize import initialize_database@@ -627,9 +707,17 @@ class DatabaseManager:# cause (e.g. world-writable migrations dir) gets fixed instead# of producing a permanently-locked-out account.try:+ # SQLAlchemy opens the file lazily, so for the unencrypted branch+ # this is the first real open. It is checked before the migrations+ # write anything, and the pool hands the checked connection back+ # to the session below instead of opening the pathname again.+ with engine.connect():+ _assert_reserved_file(db_path, reserved)Session = sessionmaker(bind=engine)with Session() as session:initialize_database(engine, session)+ # Again afterwards, for a replacement made while they ran.+ _assert_reserved_file(db_path, reserved)except Exception as e:# ``password`` is in scope and was passed into the engine# creator closure above. Drop the traceback to avoid leaking@@ -665,14 +753,26 @@ class DatabaseManager:def _create_user_database_files(self, username: str, password: str- ) -> tuple[Path, Engine]:+ ) -> tuple[Path, Engine, tuple[int, int]]:"""Create the on-disk database and its engine; caller holds the init lock.- Returns the database path and an engine whose tables exist but whose- migrations have not run yet.+ Returns the database path, an engine whose tables exist but whose+ migrations have not run yet, and the identity of the file this method+ materialized — see :func:`_assert_reserved_file`."""db_path = self._get_user_db_path(username)+ # ``Path.exists()`` follows symlinks and is False for a dangling one,+ # so a planted ``ldr_user_<hash>.db -> /elsewhere`` link passed this+ # guard and the create wrote the database at the link target, outside+ # the data dir (#6354). A user database is always a regular file this+ # code created, so refuse any link, dangling or not.+ if _db_path_is_symlinked(db_path, username):+ raise ValueError(+ f"Database path for user {username} is a symlink; "+ "refusing to create"+ )+if db_path.exists():raise ValueError(f"Database already exists for user {username}")@@ -732,6 +832,11 @@ class DatabaseManager:# orphaning the just-created salt file.hex_key = get_key_from_password(password, db_path=db_path).hex()+ # Materialize the .db ourselves, no-follow and exclusive, so+ # a link planted between the guard above and this open cannot+ # redirect SQLCipher's create either.+ reserved = _create_private_file(db_path)+conn = create_sqlcipher_connection(db_path,password=password,@@ -750,6 +855,10 @@ class DatabaseManager:"check_same_thread": False,},)+ # The open above is where a swapped pathname would have+ # taken effect, so this is the first point the reservation+ # can be checked against what SQLCipher actually opened.+ _assert_reserved_file(db_path, reserved)_best_effort_chmod(db_path, 0o600, warn=True)try:# Get the CREATE TABLE statements from SQLAlchemy models@@ -861,23 +970,36 @@ class DatabaseManager:logger.warning(f"SQLCipher not available - creating UNENCRYPTED database for user {username}")- # Fall back to regular SQLite with query cache- engine = create_engine(- f"sqlite:///{db_path}",- connect_args={"check_same_thread": False, "timeout": 30},- poolclass=self._pool_class,- echo=False,- query_cache_size=1000,- **self._get_pool_kwargs(),- )-- # For unencrypted databases, just apply pragmas- event.listen(engine, "connect", self._apply_pragmas)+ # Fall back to regular SQLite with query cache. Materialize the+ # file no-follow and exclusive first, as the encrypted branch+ # does, so SQLite's lazy create cannot follow a planted link.+ reserved = _create_private_file(db_path)+ # The file now exists, so a failure here must clean it up or the+ # retry trips the db_path.exists() guard (same reasoning as the+ # encrypted branch's engine cleanup above).+ try:+ engine = create_engine(+ f"sqlite:///{db_path}",+ connect_args={"check_same_thread": False, "timeout": 30},+ poolclass=self._pool_class,+ echo=False,+ query_cache_size=1000,+ **self._get_pool_kwargs(),+ )+ # For unencrypted databases, just apply pragmas. Registered+ # here, next to the create_engine it belongs to, so the+ # engine never leaves this block unconfigured and the+ # branch-level guard in test_migration_chain_integrity can+ # see the listener for this call.+ event.listen(engine, "connect", self._apply_pragmas)+ except Exception:+ _remove_partial_user_db_files(db_path)+ raise# Tables have already been created using raw SQLCipher above# No need to create them again with SQLAlchemy- return db_path, engine+ return db_path, engine, reserveddef _compute_verifier_digest(self, salt: bytes, password: str) -> bytes:"""Keyed digest binding ``password`` to a per-entry ``salt``.diff --git a/tests/security/test_create_user_database_symlink.py b/tests/security/test_create_user_database_symlink.pynew file mode 100644index 0000000000..ad013cb271--- /dev/null+++ b/tests/security/test_create_user_database_symlink.py@@ -0,0 +1,275 @@+"""``create_user_database`` must never follow a symlink at the DB path.++``Path.exists()`` follows links and is False for a dangling one, so a link+planted at ``data_dir/ldr_user_<hash>.db`` pointing outside the data dir+passed the guard and the database was written at the link target (#6354).+The path is now refused when it is a link, and the file is materialized+with ``O_CREAT | O_EXCL`` (plus ``O_NOFOLLOW``) so a link planted after the+guard cannot redirect the create either.+"""++import errno+import os+from pathlib import Path+from unittest.mock import patch++import pytest++from local_deep_research.database import encrypted_db+from local_deep_research.database.encrypted_db import (+ DatabaseManager,+ _create_private_file,+)+from local_deep_research.database.sqlcipher_utils import get_salt_file_path++PASSWORD = "CorrectHorse1!"+++def manager(tmp_path, monkeypatch):+ monkeypatch.setenv("LDR_DATA_DIR", str(tmp_path / "data"))+ mgr = DatabaseManager()+ mgr.data_dir = tmp_path / "data"+ mgr.data_dir.mkdir(mode=0o700, exist_ok=True)+ yield mgr+ mgr.close_all_databases()+++def victim(tmp_path):+ outside = tmp_path / "outside"+ outside.mkdir()+ return outside / "victim.db"+++def _plant_dangling_link(manager, username, victim):+ db_path = manager._get_user_db_path(username)+ os.symlink(victim, db_path)+ assert db_path.is_symlink() and not db_path.exists()+ return db_path+++def test_dangling_symlink_at_db_path_is_refused(manager, victim):+ db_path = _plant_dangling_link(manager, "linked_user", victim)++ with pytest.raises(ValueError, match="symlink"):+ manager.create_user_database("linked_user", PASSWORD)++ assert not victim.exists()+ assert not get_salt_file_path(db_path).exists()+ assert db_path.is_symlink() # the planted link is left for forensics+++def test_symlink_planted_after_the_guard_is_not_followed(+ manager, victim, monkeypatch+):+ if not manager.has_encryption:+ pytest.skip("SQLCipher not available; the salt hook is encrypted-only")+ username = "raced_user"+ db_path = manager._get_user_db_path(username)+ real_create_salt = encrypted_db.create_database_salt++ def plant_then_create_salt(path):+ # Runs after the is_symlink()/exists() guards and before the+ # SQLCipher open: the classic check-then-create window.+ os.symlink(victim, db_path)+ return real_create_salt(path)++ monkeypatch.setattr(+ encrypted_db, "create_database_salt", plant_then_create_salt+ )++ with pytest.raises(FileExistsError):+ manager.create_user_database(username, PASSWORD)++ assert not victim.exists()+ # The failure path removed this call's partial artifacts (salt + link).+ assert not get_salt_file_path(db_path).exists()+ assert not db_path.is_symlink()+++def test_unencrypted_fallback_refuses_dangling_symlink(+ tmp_path, victim, monkeypatch+):+ monkeypatch.setenv("LDR_BOOTSTRAP_ALLOW_UNENCRYPTED", "true")+ with (+ patch(+ "local_deep_research.database.encrypted_db.get_sqlcipher_module",+ side_effect=ImportError("No module named 'sqlcipher3'"),+ ),+ patch(+ "local_deep_research.database.encrypted_db.get_data_directory",+ return_value=tmp_path,+ ),+ ):+ manager = DatabaseManager()+ assert manager.has_encryption is False+ db_path = _plant_dangling_link(manager, "plain_user", victim)++ with pytest.raises(ValueError, match="symlink"):+ manager.create_user_database("plain_user", PASSWORD)++ assert not victim.exists()+ assert db_path.is_symlink()+++def test_file_swapped_after_reservation_is_refused(+ manager, victim, monkeypatch+):+ """The window between reserving the file and the library's own open.++ _create_private_file cannot hold its descriptor across the database+ open: SQLCipher and SQLite take a pathname, and neither binding can ask+ for SQLITE_OPEN_NOFOLLOW. A pathname swapped in that window is+ therefore caught by the identity of the reserved file rather than+ prevented, and the half-created database is refused instead of adopted.+ """+ username = "swapped_user"+ db_path = manager._get_user_db_path(username)+ real_create_private_file = encrypted_db._create_private_file++ def reserve_then_swap(path):+ reserved = real_create_private_file(path)+ if Path(path) == db_path:+ # The classic swap: the reserved file is replaced by a link to a+ # file outside the data dir, after the reservation and before the+ # database open that uses the pathname.+ os.unlink(path)+ os.symlink(victim, path)+ return reserved++ monkeypatch.setattr(encrypted_db, "_create_private_file", reserve_then_swap)++ with pytest.raises(ValueError, match="was replaced"):+ manager.create_user_database(username, PASSWORD)++ # Nothing of this attempt is left behind for a retry to adopt.+ assert not db_path.exists() and not db_path.is_symlink()+ assert not get_salt_file_path(db_path).exists()+ assert username not in manager.connections++ # Documented residual: the library had already created a database at the+ # link target by the time the swap was detected, so for a target that did+ # not exist this is detection, not prevention -- the attempt is refused+ # and never registered, but the write happened. Preventing it needs the+ # database built under a private name and linked into place, a larger+ # change than this guard. A target that already holds a non-database+ # file is refused by SQLCipher itself ("file is not a database") and+ # stays untouched.+ assert victim.exists()+++def test_unencrypted_swap_is_refused_before_the_migrations_write(+ tmp_path, victim, monkeypatch+):+ """SQLAlchemy opens the unencrypted database lazily, so the first real+ open used to be the migrations' own. A pathname replaced after the+ reservation therefore received the whole schema before the identity check+ ran. The check now runs on that first open, before anything is written.+ """+ import sqlite3++ monkeypatch.setenv("LDR_BOOTSTRAP_ALLOW_UNENCRYPTED", "true")+ real_create_private_file = encrypted_db._create_private_file++ with (+ patch(+ "local_deep_research.database.encrypted_db.get_sqlcipher_module",+ side_effect=ImportError("No module named 'sqlcipher3'"),+ ),+ patch(+ "local_deep_research.database.encrypted_db.get_data_directory",+ return_value=tmp_path,+ ),+ ):+ manager = DatabaseManager()+ username = "plain_swapped_user"+ db_path = manager._get_user_db_path(username)++ def reserve_then_swap(path):+ reserved = real_create_private_file(path)+ if Path(path) == db_path:+ os.unlink(path)+ os.symlink(victim, path)+ return reserved++ monkeypatch.setattr(+ encrypted_db, "_create_private_file", reserve_then_swap+ )++ with pytest.raises(ValueError, match="was replaced"):+ manager.create_user_database(username, PASSWORD)++ assert not db_path.exists() and not db_path.is_symlink()+ assert username not in manager.connections+ # The open itself may have created the target, as a plain SQLite open+ # does; no migration may have written a table into it.+ if victim.exists():+ with sqlite3.connect(victim) as conn:+ tables = conn.execute(+ "SELECT count(*) FROM sqlite_master WHERE type = 'table'"+ ).fetchone()[0]+ assert tables == 0+++def test_assert_reserved_file_accepts_the_file_it_reserved(tmp_path):+ path = tmp_path / "ldr_user_reserved.db"+ reserved = _create_private_file(path)+ encrypted_db._assert_reserved_file(path, reserved)+++def test_assert_reserved_file_rejects_a_missing_path(tmp_path):+ path = tmp_path / "ldr_user_gone.db"+ reserved = _create_private_file(path)+ os.unlink(path)+ with pytest.raises(ValueError, match="disappeared"):+ encrypted_db._assert_reserved_file(path, reserved)+++def test_regular_create_still_works(manager):+ engine = manager.create_user_database("plain_create", PASSWORD)+ db_path = manager._get_user_db_path("plain_create")+ assert db_path.is_file() and not db_path.is_symlink()+ assert oct(db_path.stat().st_mode & 0o777) == oct(0o600)+ engine.dispose()+ manager.close_all_databases()+ assert manager.open_user_database("plain_create", PASSWORD) is not None+++def test_create_private_file_reports_a_symlink_as_existing(+ tmp_path, monkeypatch+):+ """O_NOFOLLOW may answer ELOOP before O_EXCL answers EEXIST; callers see one type."""++ def open_with_eloop(*_args, **_kwargs):+ raise OSError(errno.ELOOP, "Too many levels of symbolic links")++ monkeypatch.setattr(os, "open", open_with_eloop)+ with pytest.raises(FileExistsError):+ _create_private_file(tmp_path / "ldr_user_x.db")+++def test_unencrypted_fallback_cleans_up_when_the_engine_cannot_be_built(+ tmp_path, monkeypatch+):+ monkeypatch.setenv("LDR_BOOTSTRAP_ALLOW_UNENCRYPTED", "true")+ with (+ patch(+ "local_deep_research.database.encrypted_db.get_sqlcipher_module",+ side_effect=ImportError("No module named 'sqlcipher3'"),+ ),+ patch(+ "local_deep_research.database.encrypted_db.get_data_directory",+ return_value=tmp_path,+ ),+ patch(+ "local_deep_research.database.encrypted_db.create_engine",+ side_effect=RuntimeError("engine unavailable"),+ ),+ ):+ manager = DatabaseManager()+ db_path = manager._get_user_db_path("engineless_user")+ with pytest.raises(RuntimeError):+ manager.create_user_database("engineless_user", PASSWORD)++ assert not db_path.exists()diff --git a/tests/security/test_db_connection_symlink_toctou.py b/tests/security/test_db_connection_symlink_toctou.pyindex ed7fac5860..2b8f566756 100644--- a/tests/security/test_db_connection_symlink_toctou.py+++ b/tests/security/test_db_connection_symlink_toctou.py@@ -11,20 +11,19 @@ account case.Probed and pinned against the real manager:-* AJ1 dangling .db symlink at create -- OPEN EXPOSURE, asserted as an- ``xfail(strict)`` on the correct behavior. ``Path.exists()`` follows+* AJ1 dangling .db symlink at create -- CLOSED. It was an open+ exposure pinned as an ``xfail(strict)``: ``Path.exists()`` followssymlinks and returns False for a dangling one, so the- ``db_path.exists()`` guard passes, and ``sqlcipher3.connect`` (via- ``create_sqlcipher_connection`` with creation_mode=True) follows the- symlink: the whole database is written AT THE ATTACKER-CHOSEN VICTIM- PATH outside the data dir, while the salt lands inside and the account- looks perfectly normal. The test asserts what MUST happen -- an- ``lstat``/``O_NOFOLLOW`` guard refuses the create and nothing is- written outside the data dir -- so it flips to passing when the guard- lands.-* AJ2 .db symlink to an EXISTING victim -- HOLD: the exists() guard- sees the victim through the link and refuses with "Database already- exists"; the victim's bytes are untouched.+ ``db_path.exists()`` guard passed, and ``sqlcipher3.connect`` (via+ ``create_sqlcipher_connection`` with creation_mode=True) followed the+ symlink -- the whole database written AT THE ATTACKER-CHOSEN VICTIM+ PATH outside the data dir, while the salt landed inside and the+ account looked perfectly normal. ``create_user_database`` now refuses+ a symlink at the .db path before it opens anything, so the test passes+ on its own terms and the marker is gone.+* AJ2 .db symlink to an EXISTING victim -- HOLD, and refused earlier:+ the symlink check names the link instead of the exists() guard seeing+ the victim through it; the victim's bytes are untouched either way.* AJ3 .salt symlink -- HOLD via O_EXCL no-follow: ``create_database_salt`` opens with O_CREAT|O_EXCL, which never follows a symlink (thecanonical mitigation); the FileExistsError flows into create_user_@@ -67,23 +66,17 @@ def manager(tmp_path, monkeypatch):mgr.close_all_databases()- strict=True,- reason=(- "create_user_database must refuse a symlink at the .db path; "- "today Path.exists() returns False for a DANGLING symlink so the "- "existence guard passes and sqlcipher3.connect follows the link, "- "writing the entire database outside the data dir. Fix: lstat / "- "O_NOFOLLOW check on db_path before create "- "(encrypted_db.py::create_user_database)."- ),-)def test_dangling_db_symlink_is_refused_at_create(manager):- """AJ1 (CORRECT behavior, currently unimplemented): a planted- dangling symlink at the .db path must not redirect the create. The- database bytes must never land at an attacker-chosen path outside the- data dir -- an existence guard that follows symlinks (``exists()``)- is exactly the guard a dangling link defeats.+ """AJ1 (now ENFORCED): a planted dangling symlink at the .db path+ must not redirect the create. The database bytes must never land at+ an attacker-chosen path outside the data dir -- an existence guard+ that follows symlinks (``exists()``) is exactly the guard a dangling+ link defeats.++ This carried an ``xfail(strict)`` while the gap was open. The+ ``is_symlink()`` check in ``create_user_database`` closes it, so the+ marker is gone rather than inverted: the assertion below was always+ the correct one and now holds.Precondition is same-user / compromised-account (the data dir is0700), which is why this is a hardening gap rather than a@@ -120,9 +113,17 @@ def test_dangling_db_symlink_is_refused_at_create(manager):def test_db_symlink_to_existing_victim_is_refused(manager):- """AJ2 (HOLD): a .db symlink pointing at an EXISTING victim file- trips the db_path.exists() guard (follows the link) -- ValueError- "Database already exists" -- and the victim's bytes are untouched."""+ """AJ2 (HOLD, and now for a better reason): a .db symlink pointing at+ an EXISTING victim file is refused and the victim's bytes are+ untouched.++ The refusal used to come from the ``db_path.exists()`` guard, which+ saw the victim THROUGH the link and reported "Database already+ exists" -- correct outcome, wrong reason, and the same guard let the+ dangling spelling in AJ1 straight through. The ``is_symlink()`` check+ refuses first and names what it found, so this pins the symlink+ refusal rather than an existence collision that happens to cover it.+ """if not manager.has_encryption:pytest.skip("requires SQLCipher (encrypted mode) to be meaningful")@@ -136,7 +137,7 @@ def test_db_symlink_to_existing_victim_is_refused(manager):# pytest only reclaims on its 3-run rotation -- so the unlink has to be# on the failure path too, not just the happy one.try:- with pytest.raises(ValueError, match="already exists"):+ with pytest.raises(ValueError, match="is a symlink"):manager.create_user_database(username, "ExistingVictimPw1!") # noqa: S105assert victim.read_bytes() == b"existing-precious", (